k8s.io/kubernetes@v1.29.3/pkg/controller/garbagecollector/graph_builder.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package garbagecollector 18 19 import ( 20 "context" 21 "fmt" 22 "reflect" 23 "sync" 24 "time" 25 26 "k8s.io/klog/v2" 27 28 v1 "k8s.io/api/core/v1" 29 eventv1 "k8s.io/api/events/v1" 30 "k8s.io/apimachinery/pkg/api/meta" 31 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 32 "k8s.io/apimachinery/pkg/runtime/schema" 33 "k8s.io/apimachinery/pkg/types" 34 utilerrors "k8s.io/apimachinery/pkg/util/errors" 35 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 36 "k8s.io/apimachinery/pkg/util/sets" 37 "k8s.io/apimachinery/pkg/util/wait" 38 "k8s.io/client-go/metadata" 39 "k8s.io/client-go/tools/cache" 40 "k8s.io/client-go/tools/record" 41 "k8s.io/client-go/util/workqueue" 42 "k8s.io/controller-manager/pkg/informerfactory" 43 44 "k8s.io/kubernetes/pkg/controller/garbagecollector/metaonly" 45 ) 46 47 type eventType int 48 49 func (e eventType) String() string { 50 switch e { 51 case addEvent: 52 return "add" 53 case updateEvent: 54 return "update" 55 case deleteEvent: 56 return "delete" 57 default: 58 return fmt.Sprintf("unknown(%d)", int(e)) 59 } 60 } 61 62 const ( 63 addEvent eventType = iota 64 updateEvent 65 deleteEvent 66 ) 67 68 type event struct { 69 // virtual indicates this event did not come from an informer, but was constructed artificially 70 virtual bool 71 eventType eventType 72 obj interface{} 73 // the update event comes with an old object, but it's not used by the garbage collector. 74 oldObj interface{} 75 gvk schema.GroupVersionKind 76 } 77 78 // GraphBuilder processes events supplied by the informers, updates uidToNode, 79 // a graph that caches the dependencies as we know, and enqueues 80 // items to the attemptToDelete and attemptToOrphan. 81 type GraphBuilder struct { 82 restMapper meta.RESTMapper 83 84 // each monitor list/watches a resource, the results are funneled to the 85 // dependencyGraphBuilder 86 monitors monitors 87 monitorLock sync.RWMutex 88 // informersStarted is closed after after all of the controllers have been initialized and are running. 89 // After that it is safe to start them here, before that it is not. 90 informersStarted <-chan struct{} 91 92 // stopCh drives shutdown. When a receive from it unblocks, monitors will shut down. 93 // This channel is also protected by monitorLock. 94 stopCh <-chan struct{} 95 96 // running tracks whether Run() has been called. 97 // it is protected by monitorLock. 98 running bool 99 100 eventRecorder record.EventRecorder 101 102 metadataClient metadata.Interface 103 // monitors are the producer of the graphChanges queue, graphBuilder alters 104 // the in-memory graph according to the changes. 105 graphChanges workqueue.RateLimitingInterface 106 // uidToNode doesn't require a lock to protect, because only the 107 // single-threaded GraphBuilder.processGraphChanges() reads/writes it. 108 uidToNode *concurrentUIDToNode 109 // GraphBuilder is the producer of attemptToDelete and attemptToOrphan, GC is the consumer. 110 attemptToDelete workqueue.RateLimitingInterface 111 attemptToOrphan workqueue.RateLimitingInterface 112 // GraphBuilder and GC share the absentOwnerCache. Objects that are known to 113 // be non-existent are added to the cached. 114 absentOwnerCache *ReferenceCache 115 sharedInformers informerfactory.InformerFactory 116 ignoredResources map[schema.GroupResource]struct{} 117 } 118 119 // monitor runs a Controller with a local stop channel. 120 type monitor struct { 121 controller cache.Controller 122 store cache.Store 123 124 // stopCh stops Controller. If stopCh is nil, the monitor is considered to be 125 // not yet started. 126 stopCh chan struct{} 127 } 128 129 // Run is intended to be called in a goroutine. Multiple calls of this is an 130 // error. 131 func (m *monitor) Run() { 132 m.controller.Run(m.stopCh) 133 } 134 135 type monitors map[schema.GroupVersionResource]*monitor 136 137 func (gb *GraphBuilder) controllerFor(logger klog.Logger, resource schema.GroupVersionResource, kind schema.GroupVersionKind) (cache.Controller, cache.Store, error) { 138 handlers := cache.ResourceEventHandlerFuncs{ 139 // add the event to the dependencyGraphBuilder's graphChanges. 140 AddFunc: func(obj interface{}) { 141 event := &event{ 142 eventType: addEvent, 143 obj: obj, 144 gvk: kind, 145 } 146 gb.graphChanges.Add(event) 147 }, 148 UpdateFunc: func(oldObj, newObj interface{}) { 149 // TODO: check if there are differences in the ownerRefs, 150 // finalizers, and DeletionTimestamp; if not, ignore the update. 151 event := &event{ 152 eventType: updateEvent, 153 obj: newObj, 154 oldObj: oldObj, 155 gvk: kind, 156 } 157 gb.graphChanges.Add(event) 158 }, 159 DeleteFunc: func(obj interface{}) { 160 // delta fifo may wrap the object in a cache.DeletedFinalStateUnknown, unwrap it 161 if deletedFinalStateUnknown, ok := obj.(cache.DeletedFinalStateUnknown); ok { 162 obj = deletedFinalStateUnknown.Obj 163 } 164 event := &event{ 165 eventType: deleteEvent, 166 obj: obj, 167 gvk: kind, 168 } 169 gb.graphChanges.Add(event) 170 }, 171 } 172 173 shared, err := gb.sharedInformers.ForResource(resource) 174 if err != nil { 175 logger.V(4).Error(err, "unable to use a shared informer", "resource", resource, "kind", kind) 176 return nil, nil, err 177 } 178 logger.V(4).Info("using a shared informer", "resource", resource, "kind", kind) 179 // need to clone because it's from a shared cache 180 shared.Informer().AddEventHandlerWithResyncPeriod(handlers, ResourceResyncTime) 181 return shared.Informer().GetController(), shared.Informer().GetStore(), nil 182 } 183 184 // syncMonitors rebuilds the monitor set according to the supplied resources, 185 // creating or deleting monitors as necessary. It will return any error 186 // encountered, but will make an attempt to create a monitor for each resource 187 // instead of immediately exiting on an error. It may be called before or after 188 // Run. Monitors are NOT started as part of the sync. To ensure all existing 189 // monitors are started, call startMonitors. 190 func (gb *GraphBuilder) syncMonitors(logger klog.Logger, resources map[schema.GroupVersionResource]struct{}) error { 191 gb.monitorLock.Lock() 192 defer gb.monitorLock.Unlock() 193 194 toRemove := gb.monitors 195 if toRemove == nil { 196 toRemove = monitors{} 197 } 198 current := monitors{} 199 errs := []error{} 200 kept := 0 201 added := 0 202 for resource := range resources { 203 if _, ok := gb.ignoredResources[resource.GroupResource()]; ok { 204 continue 205 } 206 if m, ok := toRemove[resource]; ok { 207 current[resource] = m 208 delete(toRemove, resource) 209 kept++ 210 continue 211 } 212 kind, err := gb.restMapper.KindFor(resource) 213 if err != nil { 214 errs = append(errs, fmt.Errorf("couldn't look up resource %q: %v", resource, err)) 215 continue 216 } 217 c, s, err := gb.controllerFor(logger, resource, kind) 218 if err != nil { 219 errs = append(errs, fmt.Errorf("couldn't start monitor for resource %q: %v", resource, err)) 220 continue 221 } 222 current[resource] = &monitor{store: s, controller: c} 223 added++ 224 } 225 gb.monitors = current 226 227 for _, monitor := range toRemove { 228 if monitor.stopCh != nil { 229 close(monitor.stopCh) 230 } 231 } 232 233 logger.V(4).Info("synced monitors", "added", added, "kept", kept, "removed", len(toRemove)) 234 // NewAggregate returns nil if errs is 0-length 235 return utilerrors.NewAggregate(errs) 236 } 237 238 // startMonitors ensures the current set of monitors are running. Any newly 239 // started monitors will also cause shared informers to be started. 240 // 241 // If called before Run, startMonitors does nothing (as there is no stop channel 242 // to support monitor/informer execution). 243 func (gb *GraphBuilder) startMonitors(logger klog.Logger) { 244 gb.monitorLock.Lock() 245 defer gb.monitorLock.Unlock() 246 247 if !gb.running { 248 return 249 } 250 251 // we're waiting until after the informer start that happens once all the controllers are initialized. This ensures 252 // that they don't get unexpected events on their work queues. 253 <-gb.informersStarted 254 255 monitors := gb.monitors 256 started := 0 257 for _, monitor := range monitors { 258 if monitor.stopCh == nil { 259 monitor.stopCh = make(chan struct{}) 260 gb.sharedInformers.Start(gb.stopCh) 261 go monitor.Run() 262 started++ 263 } 264 } 265 logger.V(4).Info("started new monitors", "new", started, "current", len(monitors)) 266 } 267 268 // IsResourceSynced returns true if a monitor exists for the given resource and has synced 269 func (gb *GraphBuilder) IsResourceSynced(resource schema.GroupVersionResource) bool { 270 gb.monitorLock.Lock() 271 defer gb.monitorLock.Unlock() 272 monitor, ok := gb.monitors[resource] 273 return ok && monitor.controller.HasSynced() 274 } 275 276 // IsSynced returns true if any monitors exist AND all those monitors' 277 // controllers HasSynced functions return true. This means IsSynced could return 278 // true at one time, and then later return false if all monitors were 279 // reconstructed. 280 func (gb *GraphBuilder) IsSynced(logger klog.Logger) bool { 281 gb.monitorLock.Lock() 282 defer gb.monitorLock.Unlock() 283 284 if len(gb.monitors) == 0 { 285 logger.V(4).Info("garbage controller monitor not synced: no monitors") 286 return false 287 } 288 289 for resource, monitor := range gb.monitors { 290 if !monitor.controller.HasSynced() { 291 logger.V(4).Info("garbage controller monitor not yet synced", "resource", resource) 292 return false 293 } 294 } 295 return true 296 } 297 298 // Run sets the stop channel and starts monitor execution until stopCh is 299 // closed. Any running monitors will be stopped before Run returns. 300 func (gb *GraphBuilder) Run(ctx context.Context) { 301 logger := klog.FromContext(ctx) 302 logger.Info("Running", "component", "GraphBuilder") 303 defer logger.Info("Stopping", "component", "GraphBuilder") 304 305 // Set up the stop channel. 306 gb.monitorLock.Lock() 307 gb.stopCh = ctx.Done() 308 gb.running = true 309 gb.monitorLock.Unlock() 310 311 // Start monitors and begin change processing until the stop channel is 312 // closed. 313 gb.startMonitors(logger) 314 wait.Until(func() { gb.runProcessGraphChanges(logger) }, 1*time.Second, ctx.Done()) 315 316 // Stop any running monitors. 317 gb.monitorLock.Lock() 318 defer gb.monitorLock.Unlock() 319 monitors := gb.monitors 320 stopped := 0 321 for _, monitor := range monitors { 322 if monitor.stopCh != nil { 323 stopped++ 324 close(monitor.stopCh) 325 } 326 } 327 328 // reset monitors so that the graph builder can be safely re-run/synced. 329 gb.monitors = nil 330 logger.Info("stopped monitors", "stopped", stopped, "total", len(monitors)) 331 } 332 333 var ignoredResources = map[schema.GroupResource]struct{}{ 334 {Group: "", Resource: "events"}: {}, 335 {Group: eventv1.GroupName, Resource: "events"}: {}, 336 } 337 338 // DefaultIgnoredResources returns the default set of resources that the garbage collector controller 339 // should ignore. This is exposed so downstream integrators can have access to the defaults, and add 340 // to them as necessary when constructing the controller. 341 func DefaultIgnoredResources() map[schema.GroupResource]struct{} { 342 return ignoredResources 343 } 344 345 // enqueueVirtualDeleteEvent is used to add a virtual delete event to be processed for virtual nodes 346 // once it is determined they do not have backing objects in storage 347 func (gb *GraphBuilder) enqueueVirtualDeleteEvent(ref objectReference) { 348 gv, _ := schema.ParseGroupVersion(ref.APIVersion) 349 gb.graphChanges.Add(&event{ 350 virtual: true, 351 eventType: deleteEvent, 352 gvk: gv.WithKind(ref.Kind), 353 obj: &metaonly.MetadataOnlyObject{ 354 TypeMeta: metav1.TypeMeta{APIVersion: ref.APIVersion, Kind: ref.Kind}, 355 ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, UID: ref.UID, Name: ref.Name}, 356 }, 357 }) 358 } 359 360 // addDependentToOwners adds n to owners' dependents list. If the owner does not 361 // exist in the gb.uidToNode yet, a "virtual" node will be created to represent 362 // the owner. The "virtual" node will be enqueued to the attemptToDelete, so that 363 // attemptToDeleteItem() will verify if the owner exists according to the API server. 364 func (gb *GraphBuilder) addDependentToOwners(logger klog.Logger, n *node, owners []metav1.OwnerReference) { 365 // track if some of the referenced owners already exist in the graph and have been observed, 366 // and the dependent's ownerRef does not match their observed coordinates 367 hasPotentiallyInvalidOwnerReference := false 368 369 for _, owner := range owners { 370 ownerNode, ok := gb.uidToNode.Read(owner.UID) 371 if !ok { 372 // Create a "virtual" node in the graph for the owner if it doesn't 373 // exist in the graph yet. 374 ownerNode = &node{ 375 identity: objectReference{ 376 OwnerReference: ownerReferenceCoordinates(owner), 377 Namespace: n.identity.Namespace, 378 }, 379 dependents: make(map[*node]struct{}), 380 virtual: true, 381 } 382 logger.V(5).Info("add virtual item", "identity", ownerNode.identity) 383 gb.uidToNode.Write(ownerNode) 384 } 385 ownerNode.addDependent(n) 386 if !ok { 387 // Enqueue the virtual node into attemptToDelete. 388 // The garbage processor will enqueue a virtual delete 389 // event to delete it from the graph if API server confirms this 390 // owner doesn't exist. 391 gb.attemptToDelete.Add(ownerNode) 392 } else if !hasPotentiallyInvalidOwnerReference { 393 ownerIsNamespaced := len(ownerNode.identity.Namespace) > 0 394 if ownerIsNamespaced && ownerNode.identity.Namespace != n.identity.Namespace { 395 if ownerNode.isObserved() { 396 // The owner node has been observed via an informer 397 // the dependent's namespace doesn't match the observed owner's namespace, this is definitely wrong. 398 // cluster-scoped owners can be referenced as an owner from any namespace or cluster-scoped object. 399 logger.V(2).Info("item references an owner but does not match namespaces", "item", n.identity, "owner", ownerNode.identity) 400 gb.reportInvalidNamespaceOwnerRef(n, owner.UID) 401 } 402 hasPotentiallyInvalidOwnerReference = true 403 } else if !ownerReferenceMatchesCoordinates(owner, ownerNode.identity.OwnerReference) { 404 if ownerNode.isObserved() { 405 // The owner node has been observed via an informer 406 // n's owner reference doesn't match the observed identity, this might be wrong. 407 logger.V(2).Info("item references an owner with coordinates that do not match the observed identity", "item", n.identity, "owner", ownerNode.identity) 408 } 409 hasPotentiallyInvalidOwnerReference = true 410 } else if !ownerIsNamespaced && ownerNode.identity.Namespace != n.identity.Namespace && !ownerNode.isObserved() { 411 // the ownerNode is cluster-scoped and virtual, and does not match the child node's namespace. 412 // the owner could be a missing instance of a namespaced type incorrectly referenced by a cluster-scoped child (issue #98040). 413 // enqueue this child to attemptToDelete to verify parent references. 414 hasPotentiallyInvalidOwnerReference = true 415 } 416 } 417 } 418 419 if hasPotentiallyInvalidOwnerReference { 420 // Enqueue the potentially invalid dependent node into attemptToDelete. 421 // The garbage processor will verify whether the owner references are dangling 422 // and delete the dependent if all owner references are confirmed absent. 423 gb.attemptToDelete.Add(n) 424 } 425 } 426 427 func (gb *GraphBuilder) reportInvalidNamespaceOwnerRef(n *node, invalidOwnerUID types.UID) { 428 var invalidOwnerRef metav1.OwnerReference 429 var found = false 430 for _, ownerRef := range n.owners { 431 if ownerRef.UID == invalidOwnerUID { 432 invalidOwnerRef = ownerRef 433 found = true 434 break 435 } 436 } 437 if !found { 438 return 439 } 440 ref := &v1.ObjectReference{ 441 Kind: n.identity.Kind, 442 APIVersion: n.identity.APIVersion, 443 Namespace: n.identity.Namespace, 444 Name: n.identity.Name, 445 UID: n.identity.UID, 446 } 447 invalidIdentity := objectReference{ 448 OwnerReference: metav1.OwnerReference{ 449 Kind: invalidOwnerRef.Kind, 450 APIVersion: invalidOwnerRef.APIVersion, 451 Name: invalidOwnerRef.Name, 452 UID: invalidOwnerRef.UID, 453 }, 454 Namespace: n.identity.Namespace, 455 } 456 gb.eventRecorder.Eventf(ref, v1.EventTypeWarning, "OwnerRefInvalidNamespace", "ownerRef %s does not exist in namespace %q", invalidIdentity, n.identity.Namespace) 457 } 458 459 // insertNode insert the node to gb.uidToNode; then it finds all owners as listed 460 // in n.owners, and adds the node to their dependents list. 461 func (gb *GraphBuilder) insertNode(logger klog.Logger, n *node) { 462 gb.uidToNode.Write(n) 463 gb.addDependentToOwners(logger, n, n.owners) 464 } 465 466 // removeDependentFromOwners remove n from owners' dependents list. 467 func (gb *GraphBuilder) removeDependentFromOwners(n *node, owners []metav1.OwnerReference) { 468 for _, owner := range owners { 469 ownerNode, ok := gb.uidToNode.Read(owner.UID) 470 if !ok { 471 continue 472 } 473 ownerNode.deleteDependent(n) 474 } 475 } 476 477 // removeNode removes the node from gb.uidToNode, then finds all 478 // owners as listed in n.owners, and removes n from their dependents list. 479 func (gb *GraphBuilder) removeNode(n *node) { 480 gb.uidToNode.Delete(n.identity.UID) 481 gb.removeDependentFromOwners(n, n.owners) 482 } 483 484 type ownerRefPair struct { 485 oldRef metav1.OwnerReference 486 newRef metav1.OwnerReference 487 } 488 489 // TODO: profile this function to see if a naive N^2 algorithm performs better 490 // when the number of references is small. 491 func referencesDiffs(old []metav1.OwnerReference, new []metav1.OwnerReference) (added []metav1.OwnerReference, removed []metav1.OwnerReference, changed []ownerRefPair) { 492 oldUIDToRef := make(map[string]metav1.OwnerReference) 493 for _, value := range old { 494 oldUIDToRef[string(value.UID)] = value 495 } 496 oldUIDSet := sets.StringKeySet(oldUIDToRef) 497 for _, value := range new { 498 newUID := string(value.UID) 499 if oldUIDSet.Has(newUID) { 500 if !reflect.DeepEqual(oldUIDToRef[newUID], value) { 501 changed = append(changed, ownerRefPair{oldRef: oldUIDToRef[newUID], newRef: value}) 502 } 503 oldUIDSet.Delete(newUID) 504 } else { 505 added = append(added, value) 506 } 507 } 508 for oldUID := range oldUIDSet { 509 removed = append(removed, oldUIDToRef[oldUID]) 510 } 511 512 return added, removed, changed 513 } 514 515 func deletionStartsWithFinalizer(oldObj interface{}, newAccessor metav1.Object, matchingFinalizer string) bool { 516 // if the new object isn't being deleted, or doesn't have the finalizer we're interested in, return false 517 if !beingDeleted(newAccessor) || !hasFinalizer(newAccessor, matchingFinalizer) { 518 return false 519 } 520 521 // if the old object is nil, or wasn't being deleted, or didn't have the finalizer, return true 522 if oldObj == nil { 523 return true 524 } 525 oldAccessor, err := meta.Accessor(oldObj) 526 if err != nil { 527 utilruntime.HandleError(fmt.Errorf("cannot access oldObj: %v", err)) 528 return false 529 } 530 return !beingDeleted(oldAccessor) || !hasFinalizer(oldAccessor, matchingFinalizer) 531 } 532 533 func beingDeleted(accessor metav1.Object) bool { 534 return accessor.GetDeletionTimestamp() != nil 535 } 536 537 func hasDeleteDependentsFinalizer(accessor metav1.Object) bool { 538 return hasFinalizer(accessor, metav1.FinalizerDeleteDependents) 539 } 540 541 func hasOrphanFinalizer(accessor metav1.Object) bool { 542 return hasFinalizer(accessor, metav1.FinalizerOrphanDependents) 543 } 544 545 func hasFinalizer(accessor metav1.Object, matchingFinalizer string) bool { 546 finalizers := accessor.GetFinalizers() 547 for _, finalizer := range finalizers { 548 if finalizer == matchingFinalizer { 549 return true 550 } 551 } 552 return false 553 } 554 555 // this function takes newAccessor directly because the caller already 556 // instantiates an accessor for the newObj. 557 func startsWaitingForDependentsDeleted(oldObj interface{}, newAccessor metav1.Object) bool { 558 return deletionStartsWithFinalizer(oldObj, newAccessor, metav1.FinalizerDeleteDependents) 559 } 560 561 // this function takes newAccessor directly because the caller already 562 // instantiates an accessor for the newObj. 563 func startsWaitingForDependentsOrphaned(oldObj interface{}, newAccessor metav1.Object) bool { 564 return deletionStartsWithFinalizer(oldObj, newAccessor, metav1.FinalizerOrphanDependents) 565 } 566 567 // if an blocking ownerReference points to an object gets removed, or gets set to 568 // "BlockOwnerDeletion=false", add the object to the attemptToDelete queue. 569 func (gb *GraphBuilder) addUnblockedOwnersToDeleteQueue(logger klog.Logger, removed []metav1.OwnerReference, changed []ownerRefPair) { 570 for _, ref := range removed { 571 if ref.BlockOwnerDeletion != nil && *ref.BlockOwnerDeletion { 572 node, found := gb.uidToNode.Read(ref.UID) 573 if !found { 574 logger.V(5).Info("cannot find uid in uidToNode", "uid", ref.UID) 575 continue 576 } 577 gb.attemptToDelete.Add(node) 578 } 579 } 580 for _, c := range changed { 581 wasBlocked := c.oldRef.BlockOwnerDeletion != nil && *c.oldRef.BlockOwnerDeletion 582 isUnblocked := c.newRef.BlockOwnerDeletion == nil || (c.newRef.BlockOwnerDeletion != nil && !*c.newRef.BlockOwnerDeletion) 583 if wasBlocked && isUnblocked { 584 node, found := gb.uidToNode.Read(c.newRef.UID) 585 if !found { 586 logger.V(5).Info("cannot find uid in uidToNode", "uid", c.newRef.UID) 587 continue 588 } 589 gb.attemptToDelete.Add(node) 590 } 591 } 592 } 593 594 func (gb *GraphBuilder) processTransitions(logger klog.Logger, oldObj interface{}, newAccessor metav1.Object, n *node) { 595 if startsWaitingForDependentsOrphaned(oldObj, newAccessor) { 596 logger.V(5).Info("add item to attemptToOrphan", "item", n.identity) 597 gb.attemptToOrphan.Add(n) 598 return 599 } 600 if startsWaitingForDependentsDeleted(oldObj, newAccessor) { 601 logger.V(2).Info("add item to attemptToDelete, because it's waiting for its dependents to be deleted", "item", n.identity) 602 // if the n is added as a "virtual" node, its deletingDependents field is not properly set, so always set it here. 603 n.markDeletingDependents() 604 for dep := range n.dependents { 605 gb.attemptToDelete.Add(dep) 606 } 607 gb.attemptToDelete.Add(n) 608 } 609 } 610 611 func (gb *GraphBuilder) runProcessGraphChanges(logger klog.Logger) { 612 for gb.processGraphChanges(logger) { 613 } 614 } 615 616 func identityFromEvent(event *event, accessor metav1.Object) objectReference { 617 return objectReference{ 618 OwnerReference: metav1.OwnerReference{ 619 APIVersion: event.gvk.GroupVersion().String(), 620 Kind: event.gvk.Kind, 621 UID: accessor.GetUID(), 622 Name: accessor.GetName(), 623 }, 624 Namespace: accessor.GetNamespace(), 625 } 626 } 627 628 // Dequeueing an event from graphChanges, updating graph, populating dirty_queue. 629 func (gb *GraphBuilder) processGraphChanges(logger klog.Logger) bool { 630 item, quit := gb.graphChanges.Get() 631 if quit { 632 return false 633 } 634 defer gb.graphChanges.Done(item) 635 event, ok := item.(*event) 636 if !ok { 637 utilruntime.HandleError(fmt.Errorf("expect a *event, got %v", item)) 638 return true 639 } 640 obj := event.obj 641 accessor, err := meta.Accessor(obj) 642 if err != nil { 643 utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err)) 644 return true 645 } 646 647 logger.V(5).Info("GraphBuilder process object", 648 "apiVersion", event.gvk.GroupVersion().String(), 649 "kind", event.gvk.Kind, 650 "object", klog.KObj(accessor), 651 "uid", string(accessor.GetUID()), 652 "eventType", event.eventType, 653 "virtual", event.virtual, 654 ) 655 656 // Check if the node already exists 657 existingNode, found := gb.uidToNode.Read(accessor.GetUID()) 658 if found && !event.virtual && !existingNode.isObserved() { 659 // this marks the node as having been observed via an informer event 660 // 1. this depends on graphChanges only containing add/update events from the actual informer 661 // 2. this allows things tracking virtual nodes' existence to stop polling and rely on informer events 662 observedIdentity := identityFromEvent(event, accessor) 663 if observedIdentity != existingNode.identity { 664 // find dependents that don't match the identity we observed 665 _, potentiallyInvalidDependents := partitionDependents(existingNode.getDependents(), observedIdentity) 666 // add those potentially invalid dependents to the attemptToDelete queue. 667 // if their owners are still solid the attemptToDelete will be a no-op. 668 // this covers the bad child -> good parent observation sequence. 669 // the good parent -> bad child observation sequence is handled in addDependentToOwners 670 for _, dep := range potentiallyInvalidDependents { 671 if len(observedIdentity.Namespace) > 0 && dep.identity.Namespace != observedIdentity.Namespace { 672 // Namespace mismatch, this is definitely wrong 673 logger.V(2).Info("item references an owner but does not match namespaces", 674 "item", dep.identity, 675 "owner", observedIdentity, 676 ) 677 gb.reportInvalidNamespaceOwnerRef(dep, observedIdentity.UID) 678 } 679 gb.attemptToDelete.Add(dep) 680 } 681 682 // make a copy (so we don't modify the existing node in place), store the observed identity, and replace the virtual node 683 logger.V(2).Info("replacing virtual item with observed item", 684 "virtual", existingNode.identity, 685 "observed", observedIdentity, 686 ) 687 existingNode = existingNode.clone() 688 existingNode.identity = observedIdentity 689 gb.uidToNode.Write(existingNode) 690 } 691 existingNode.markObserved() 692 } 693 switch { 694 case (event.eventType == addEvent || event.eventType == updateEvent) && !found: 695 newNode := &node{ 696 identity: identityFromEvent(event, accessor), 697 dependents: make(map[*node]struct{}), 698 owners: accessor.GetOwnerReferences(), 699 deletingDependents: beingDeleted(accessor) && hasDeleteDependentsFinalizer(accessor), 700 beingDeleted: beingDeleted(accessor), 701 } 702 gb.insertNode(logger, newNode) 703 // the underlying delta_fifo may combine a creation and a deletion into 704 // one event, so we need to further process the event. 705 gb.processTransitions(logger, event.oldObj, accessor, newNode) 706 case (event.eventType == addEvent || event.eventType == updateEvent) && found: 707 // handle changes in ownerReferences 708 added, removed, changed := referencesDiffs(existingNode.owners, accessor.GetOwnerReferences()) 709 if len(added) != 0 || len(removed) != 0 || len(changed) != 0 { 710 // check if the changed dependency graph unblock owners that are 711 // waiting for the deletion of their dependents. 712 gb.addUnblockedOwnersToDeleteQueue(logger, removed, changed) 713 // update the node itself 714 existingNode.owners = accessor.GetOwnerReferences() 715 // Add the node to its new owners' dependent lists. 716 gb.addDependentToOwners(logger, existingNode, added) 717 // remove the node from the dependent list of node that are no longer in 718 // the node's owners list. 719 gb.removeDependentFromOwners(existingNode, removed) 720 } 721 722 if beingDeleted(accessor) { 723 existingNode.markBeingDeleted() 724 } 725 gb.processTransitions(logger, event.oldObj, accessor, existingNode) 726 case event.eventType == deleteEvent: 727 if !found { 728 logger.V(5).Info("item doesn't exist in the graph, this shouldn't happen", 729 "item", accessor.GetUID(), 730 ) 731 return true 732 } 733 734 removeExistingNode := true 735 736 if event.virtual { 737 // this is a virtual delete event, not one observed from an informer 738 deletedIdentity := identityFromEvent(event, accessor) 739 if existingNode.virtual { 740 741 // our existing node is also virtual, we're not sure of its coordinates. 742 // see if any dependents reference this owner with coordinates other than the one we got a virtual delete event for. 743 if matchingDependents, nonmatchingDependents := partitionDependents(existingNode.getDependents(), deletedIdentity); len(nonmatchingDependents) > 0 { 744 745 // some of our dependents disagree on our coordinates, so do not remove the existing virtual node from the graph 746 removeExistingNode = false 747 748 if len(matchingDependents) > 0 { 749 // mark the observed deleted identity as absent 750 gb.absentOwnerCache.Add(deletedIdentity) 751 // attempt to delete dependents that do match the verified deleted identity 752 for _, dep := range matchingDependents { 753 gb.attemptToDelete.Add(dep) 754 } 755 } 756 757 // if the delete event verified existingNode.identity doesn't exist... 758 if existingNode.identity == deletedIdentity { 759 // find an alternative identity our nonmatching dependents refer to us by 760 replacementIdentity := getAlternateOwnerIdentity(nonmatchingDependents, deletedIdentity) 761 if replacementIdentity != nil { 762 // replace the existing virtual node with a new one with one of our other potential identities 763 replacementNode := existingNode.clone() 764 replacementNode.identity = *replacementIdentity 765 gb.uidToNode.Write(replacementNode) 766 // and add the new virtual node back to the attemptToDelete queue 767 gb.attemptToDelete.AddRateLimited(replacementNode) 768 } 769 } 770 } 771 772 } else if existingNode.identity != deletedIdentity { 773 // do not remove the existing real node from the graph based on a virtual delete event 774 removeExistingNode = false 775 776 // our existing node which was observed via informer disagrees with the virtual delete event's coordinates 777 matchingDependents, _ := partitionDependents(existingNode.getDependents(), deletedIdentity) 778 779 if len(matchingDependents) > 0 { 780 // mark the observed deleted identity as absent 781 gb.absentOwnerCache.Add(deletedIdentity) 782 // attempt to delete dependents that do match the verified deleted identity 783 for _, dep := range matchingDependents { 784 gb.attemptToDelete.Add(dep) 785 } 786 } 787 } 788 } 789 790 if removeExistingNode { 791 // removeNode updates the graph 792 gb.removeNode(existingNode) 793 existingNode.dependentsLock.RLock() 794 defer existingNode.dependentsLock.RUnlock() 795 if len(existingNode.dependents) > 0 { 796 gb.absentOwnerCache.Add(identityFromEvent(event, accessor)) 797 } 798 for dep := range existingNode.dependents { 799 gb.attemptToDelete.Add(dep) 800 } 801 for _, owner := range existingNode.owners { 802 ownerNode, found := gb.uidToNode.Read(owner.UID) 803 if !found || !ownerNode.isDeletingDependents() { 804 continue 805 } 806 // this is to let attempToDeleteItem check if all the owner's 807 // dependents are deleted, if so, the owner will be deleted. 808 gb.attemptToDelete.Add(ownerNode) 809 } 810 } 811 } 812 return true 813 } 814 815 // partitionDependents divides the provided dependents into a list which have an ownerReference matching the provided identity, 816 // and ones which have an ownerReference for the given uid that do not match the provided identity. 817 // Note that a dependent with multiple ownerReferences for the target uid can end up in both lists. 818 func partitionDependents(dependents []*node, matchOwnerIdentity objectReference) (matching, nonmatching []*node) { 819 ownerIsNamespaced := len(matchOwnerIdentity.Namespace) > 0 820 for i := range dependents { 821 dep := dependents[i] 822 foundMatch := false 823 foundMismatch := false 824 // if the dep namespace matches or the owner is cluster scoped ... 825 if ownerIsNamespaced && matchOwnerIdentity.Namespace != dep.identity.Namespace { 826 // all references to the parent do not match, since the dependent namespace does not match the owner 827 foundMismatch = true 828 } else { 829 for _, ownerRef := range dep.owners { 830 // ... find the ownerRef with a matching uid ... 831 if ownerRef.UID == matchOwnerIdentity.UID { 832 // ... and check if it matches all coordinates 833 if ownerReferenceMatchesCoordinates(ownerRef, matchOwnerIdentity.OwnerReference) { 834 foundMatch = true 835 } else { 836 foundMismatch = true 837 } 838 } 839 } 840 } 841 842 if foundMatch { 843 matching = append(matching, dep) 844 } 845 if foundMismatch { 846 nonmatching = append(nonmatching, dep) 847 } 848 } 849 return matching, nonmatching 850 } 851 852 func referenceLessThan(a, b objectReference) bool { 853 // kind/apiVersion are more significant than namespace, 854 // so that we get coherent ordering between kinds 855 // regardless of whether they are cluster-scoped or namespaced 856 if a.Kind != b.Kind { 857 return a.Kind < b.Kind 858 } 859 if a.APIVersion != b.APIVersion { 860 return a.APIVersion < b.APIVersion 861 } 862 // namespace is more significant than name 863 if a.Namespace != b.Namespace { 864 return a.Namespace < b.Namespace 865 } 866 // name is more significant than uid 867 if a.Name != b.Name { 868 return a.Name < b.Name 869 } 870 // uid is included for completeness, but is expected to be identical 871 // when getting alternate identities for an owner since they are keyed by uid 872 if a.UID != b.UID { 873 return a.UID < b.UID 874 } 875 return false 876 } 877 878 // getAlternateOwnerIdentity searches deps for owner references which match 879 // verifiedAbsentIdentity.UID but differ in apiVersion/kind/name or namespace. 880 // The first that follows verifiedAbsentIdentity (according to referenceLessThan) is returned. 881 // If none follow verifiedAbsentIdentity, the first (according to referenceLessThan) is returned. 882 // If no alternate identities are found, nil is returned. 883 func getAlternateOwnerIdentity(deps []*node, verifiedAbsentIdentity objectReference) *objectReference { 884 absentIdentityIsClusterScoped := len(verifiedAbsentIdentity.Namespace) == 0 885 886 seenAlternates := map[objectReference]bool{verifiedAbsentIdentity: true} 887 888 // keep track of the first alternate reference (according to referenceLessThan) 889 var first *objectReference 890 // keep track of the first reference following verifiedAbsentIdentity (according to referenceLessThan) 891 var firstFollowing *objectReference 892 893 for _, dep := range deps { 894 for _, ownerRef := range dep.owners { 895 if ownerRef.UID != verifiedAbsentIdentity.UID { 896 // skip references that aren't the uid we care about 897 continue 898 } 899 900 if ownerReferenceMatchesCoordinates(ownerRef, verifiedAbsentIdentity.OwnerReference) { 901 if absentIdentityIsClusterScoped || verifiedAbsentIdentity.Namespace == dep.identity.Namespace { 902 // skip references that exactly match verifiedAbsentIdentity 903 continue 904 } 905 } 906 907 ref := objectReference{OwnerReference: ownerReferenceCoordinates(ownerRef), Namespace: dep.identity.Namespace} 908 if absentIdentityIsClusterScoped && ref.APIVersion == verifiedAbsentIdentity.APIVersion && ref.Kind == verifiedAbsentIdentity.Kind { 909 // we know this apiVersion/kind is cluster-scoped because of verifiedAbsentIdentity, 910 // so clear the namespace from the alternate identity 911 ref.Namespace = "" 912 } 913 914 if seenAlternates[ref] { 915 // skip references we've already seen 916 continue 917 } 918 seenAlternates[ref] = true 919 920 if first == nil || referenceLessThan(ref, *first) { 921 // this alternate comes first lexically 922 first = &ref 923 } 924 if referenceLessThan(verifiedAbsentIdentity, ref) && (firstFollowing == nil || referenceLessThan(ref, *firstFollowing)) { 925 // this alternate is the first following verifiedAbsentIdentity lexically 926 firstFollowing = &ref 927 } 928 } 929 } 930 931 // return the first alternate identity following the verified absent identity, if there is one 932 if firstFollowing != nil { 933 return firstFollowing 934 } 935 // otherwise return the first alternate identity 936 return first 937 }