k8s.io/kubernetes@v1.29.3/pkg/controller/garbagecollector/graph_builder.go (about)

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package garbagecollector
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"reflect"
    23  	"sync"
    24  	"time"
    25  
    26  	"k8s.io/klog/v2"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	eventv1 "k8s.io/api/events/v1"
    30  	"k8s.io/apimachinery/pkg/api/meta"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/runtime/schema"
    33  	"k8s.io/apimachinery/pkg/types"
    34  	utilerrors "k8s.io/apimachinery/pkg/util/errors"
    35  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    36  	"k8s.io/apimachinery/pkg/util/sets"
    37  	"k8s.io/apimachinery/pkg/util/wait"
    38  	"k8s.io/client-go/metadata"
    39  	"k8s.io/client-go/tools/cache"
    40  	"k8s.io/client-go/tools/record"
    41  	"k8s.io/client-go/util/workqueue"
    42  	"k8s.io/controller-manager/pkg/informerfactory"
    43  
    44  	"k8s.io/kubernetes/pkg/controller/garbagecollector/metaonly"
    45  )
    46  
    47  type eventType int
    48  
    49  func (e eventType) String() string {
    50  	switch e {
    51  	case addEvent:
    52  		return "add"
    53  	case updateEvent:
    54  		return "update"
    55  	case deleteEvent:
    56  		return "delete"
    57  	default:
    58  		return fmt.Sprintf("unknown(%d)", int(e))
    59  	}
    60  }
    61  
    62  const (
    63  	addEvent eventType = iota
    64  	updateEvent
    65  	deleteEvent
    66  )
    67  
    68  type event struct {
    69  	// virtual indicates this event did not come from an informer, but was constructed artificially
    70  	virtual   bool
    71  	eventType eventType
    72  	obj       interface{}
    73  	// the update event comes with an old object, but it's not used by the garbage collector.
    74  	oldObj interface{}
    75  	gvk    schema.GroupVersionKind
    76  }
    77  
    78  // GraphBuilder processes events supplied by the informers, updates uidToNode,
    79  // a graph that caches the dependencies as we know, and enqueues
    80  // items to the attemptToDelete and attemptToOrphan.
    81  type GraphBuilder struct {
    82  	restMapper meta.RESTMapper
    83  
    84  	// each monitor list/watches a resource, the results are funneled to the
    85  	// dependencyGraphBuilder
    86  	monitors    monitors
    87  	monitorLock sync.RWMutex
    88  	// informersStarted is closed after after all of the controllers have been initialized and are running.
    89  	// After that it is safe to start them here, before that it is not.
    90  	informersStarted <-chan struct{}
    91  
    92  	// stopCh drives shutdown. When a receive from it unblocks, monitors will shut down.
    93  	// This channel is also protected by monitorLock.
    94  	stopCh <-chan struct{}
    95  
    96  	// running tracks whether Run() has been called.
    97  	// it is protected by monitorLock.
    98  	running bool
    99  
   100  	eventRecorder record.EventRecorder
   101  
   102  	metadataClient metadata.Interface
   103  	// monitors are the producer of the graphChanges queue, graphBuilder alters
   104  	// the in-memory graph according to the changes.
   105  	graphChanges workqueue.RateLimitingInterface
   106  	// uidToNode doesn't require a lock to protect, because only the
   107  	// single-threaded GraphBuilder.processGraphChanges() reads/writes it.
   108  	uidToNode *concurrentUIDToNode
   109  	// GraphBuilder is the producer of attemptToDelete and attemptToOrphan, GC is the consumer.
   110  	attemptToDelete workqueue.RateLimitingInterface
   111  	attemptToOrphan workqueue.RateLimitingInterface
   112  	// GraphBuilder and GC share the absentOwnerCache. Objects that are known to
   113  	// be non-existent are added to the cached.
   114  	absentOwnerCache *ReferenceCache
   115  	sharedInformers  informerfactory.InformerFactory
   116  	ignoredResources map[schema.GroupResource]struct{}
   117  }
   118  
   119  // monitor runs a Controller with a local stop channel.
   120  type monitor struct {
   121  	controller cache.Controller
   122  	store      cache.Store
   123  
   124  	// stopCh stops Controller. If stopCh is nil, the monitor is considered to be
   125  	// not yet started.
   126  	stopCh chan struct{}
   127  }
   128  
   129  // Run is intended to be called in a goroutine. Multiple calls of this is an
   130  // error.
   131  func (m *monitor) Run() {
   132  	m.controller.Run(m.stopCh)
   133  }
   134  
   135  type monitors map[schema.GroupVersionResource]*monitor
   136  
   137  func (gb *GraphBuilder) controllerFor(logger klog.Logger, resource schema.GroupVersionResource, kind schema.GroupVersionKind) (cache.Controller, cache.Store, error) {
   138  	handlers := cache.ResourceEventHandlerFuncs{
   139  		// add the event to the dependencyGraphBuilder's graphChanges.
   140  		AddFunc: func(obj interface{}) {
   141  			event := &event{
   142  				eventType: addEvent,
   143  				obj:       obj,
   144  				gvk:       kind,
   145  			}
   146  			gb.graphChanges.Add(event)
   147  		},
   148  		UpdateFunc: func(oldObj, newObj interface{}) {
   149  			// TODO: check if there are differences in the ownerRefs,
   150  			// finalizers, and DeletionTimestamp; if not, ignore the update.
   151  			event := &event{
   152  				eventType: updateEvent,
   153  				obj:       newObj,
   154  				oldObj:    oldObj,
   155  				gvk:       kind,
   156  			}
   157  			gb.graphChanges.Add(event)
   158  		},
   159  		DeleteFunc: func(obj interface{}) {
   160  			// delta fifo may wrap the object in a cache.DeletedFinalStateUnknown, unwrap it
   161  			if deletedFinalStateUnknown, ok := obj.(cache.DeletedFinalStateUnknown); ok {
   162  				obj = deletedFinalStateUnknown.Obj
   163  			}
   164  			event := &event{
   165  				eventType: deleteEvent,
   166  				obj:       obj,
   167  				gvk:       kind,
   168  			}
   169  			gb.graphChanges.Add(event)
   170  		},
   171  	}
   172  
   173  	shared, err := gb.sharedInformers.ForResource(resource)
   174  	if err != nil {
   175  		logger.V(4).Error(err, "unable to use a shared informer", "resource", resource, "kind", kind)
   176  		return nil, nil, err
   177  	}
   178  	logger.V(4).Info("using a shared informer", "resource", resource, "kind", kind)
   179  	// need to clone because it's from a shared cache
   180  	shared.Informer().AddEventHandlerWithResyncPeriod(handlers, ResourceResyncTime)
   181  	return shared.Informer().GetController(), shared.Informer().GetStore(), nil
   182  }
   183  
   184  // syncMonitors rebuilds the monitor set according to the supplied resources,
   185  // creating or deleting monitors as necessary. It will return any error
   186  // encountered, but will make an attempt to create a monitor for each resource
   187  // instead of immediately exiting on an error. It may be called before or after
   188  // Run. Monitors are NOT started as part of the sync. To ensure all existing
   189  // monitors are started, call startMonitors.
   190  func (gb *GraphBuilder) syncMonitors(logger klog.Logger, resources map[schema.GroupVersionResource]struct{}) error {
   191  	gb.monitorLock.Lock()
   192  	defer gb.monitorLock.Unlock()
   193  
   194  	toRemove := gb.monitors
   195  	if toRemove == nil {
   196  		toRemove = monitors{}
   197  	}
   198  	current := monitors{}
   199  	errs := []error{}
   200  	kept := 0
   201  	added := 0
   202  	for resource := range resources {
   203  		if _, ok := gb.ignoredResources[resource.GroupResource()]; ok {
   204  			continue
   205  		}
   206  		if m, ok := toRemove[resource]; ok {
   207  			current[resource] = m
   208  			delete(toRemove, resource)
   209  			kept++
   210  			continue
   211  		}
   212  		kind, err := gb.restMapper.KindFor(resource)
   213  		if err != nil {
   214  			errs = append(errs, fmt.Errorf("couldn't look up resource %q: %v", resource, err))
   215  			continue
   216  		}
   217  		c, s, err := gb.controllerFor(logger, resource, kind)
   218  		if err != nil {
   219  			errs = append(errs, fmt.Errorf("couldn't start monitor for resource %q: %v", resource, err))
   220  			continue
   221  		}
   222  		current[resource] = &monitor{store: s, controller: c}
   223  		added++
   224  	}
   225  	gb.monitors = current
   226  
   227  	for _, monitor := range toRemove {
   228  		if monitor.stopCh != nil {
   229  			close(monitor.stopCh)
   230  		}
   231  	}
   232  
   233  	logger.V(4).Info("synced monitors", "added", added, "kept", kept, "removed", len(toRemove))
   234  	// NewAggregate returns nil if errs is 0-length
   235  	return utilerrors.NewAggregate(errs)
   236  }
   237  
   238  // startMonitors ensures the current set of monitors are running. Any newly
   239  // started monitors will also cause shared informers to be started.
   240  //
   241  // If called before Run, startMonitors does nothing (as there is no stop channel
   242  // to support monitor/informer execution).
   243  func (gb *GraphBuilder) startMonitors(logger klog.Logger) {
   244  	gb.monitorLock.Lock()
   245  	defer gb.monitorLock.Unlock()
   246  
   247  	if !gb.running {
   248  		return
   249  	}
   250  
   251  	// we're waiting until after the informer start that happens once all the controllers are initialized.  This ensures
   252  	// that they don't get unexpected events on their work queues.
   253  	<-gb.informersStarted
   254  
   255  	monitors := gb.monitors
   256  	started := 0
   257  	for _, monitor := range monitors {
   258  		if monitor.stopCh == nil {
   259  			monitor.stopCh = make(chan struct{})
   260  			gb.sharedInformers.Start(gb.stopCh)
   261  			go monitor.Run()
   262  			started++
   263  		}
   264  	}
   265  	logger.V(4).Info("started new monitors", "new", started, "current", len(monitors))
   266  }
   267  
   268  // IsResourceSynced returns true if a monitor exists for the given resource and has synced
   269  func (gb *GraphBuilder) IsResourceSynced(resource schema.GroupVersionResource) bool {
   270  	gb.monitorLock.Lock()
   271  	defer gb.monitorLock.Unlock()
   272  	monitor, ok := gb.monitors[resource]
   273  	return ok && monitor.controller.HasSynced()
   274  }
   275  
   276  // IsSynced returns true if any monitors exist AND all those monitors'
   277  // controllers HasSynced functions return true. This means IsSynced could return
   278  // true at one time, and then later return false if all monitors were
   279  // reconstructed.
   280  func (gb *GraphBuilder) IsSynced(logger klog.Logger) bool {
   281  	gb.monitorLock.Lock()
   282  	defer gb.monitorLock.Unlock()
   283  
   284  	if len(gb.monitors) == 0 {
   285  		logger.V(4).Info("garbage controller monitor not synced: no monitors")
   286  		return false
   287  	}
   288  
   289  	for resource, monitor := range gb.monitors {
   290  		if !monitor.controller.HasSynced() {
   291  			logger.V(4).Info("garbage controller monitor not yet synced", "resource", resource)
   292  			return false
   293  		}
   294  	}
   295  	return true
   296  }
   297  
   298  // Run sets the stop channel and starts monitor execution until stopCh is
   299  // closed. Any running monitors will be stopped before Run returns.
   300  func (gb *GraphBuilder) Run(ctx context.Context) {
   301  	logger := klog.FromContext(ctx)
   302  	logger.Info("Running", "component", "GraphBuilder")
   303  	defer logger.Info("Stopping", "component", "GraphBuilder")
   304  
   305  	// Set up the stop channel.
   306  	gb.monitorLock.Lock()
   307  	gb.stopCh = ctx.Done()
   308  	gb.running = true
   309  	gb.monitorLock.Unlock()
   310  
   311  	// Start monitors and begin change processing until the stop channel is
   312  	// closed.
   313  	gb.startMonitors(logger)
   314  	wait.Until(func() { gb.runProcessGraphChanges(logger) }, 1*time.Second, ctx.Done())
   315  
   316  	// Stop any running monitors.
   317  	gb.monitorLock.Lock()
   318  	defer gb.monitorLock.Unlock()
   319  	monitors := gb.monitors
   320  	stopped := 0
   321  	for _, monitor := range monitors {
   322  		if monitor.stopCh != nil {
   323  			stopped++
   324  			close(monitor.stopCh)
   325  		}
   326  	}
   327  
   328  	// reset monitors so that the graph builder can be safely re-run/synced.
   329  	gb.monitors = nil
   330  	logger.Info("stopped monitors", "stopped", stopped, "total", len(monitors))
   331  }
   332  
   333  var ignoredResources = map[schema.GroupResource]struct{}{
   334  	{Group: "", Resource: "events"}:                {},
   335  	{Group: eventv1.GroupName, Resource: "events"}: {},
   336  }
   337  
   338  // DefaultIgnoredResources returns the default set of resources that the garbage collector controller
   339  // should ignore. This is exposed so downstream integrators can have access to the defaults, and add
   340  // to them as necessary when constructing the controller.
   341  func DefaultIgnoredResources() map[schema.GroupResource]struct{} {
   342  	return ignoredResources
   343  }
   344  
   345  // enqueueVirtualDeleteEvent is used to add a virtual delete event to be processed for virtual nodes
   346  // once it is determined they do not have backing objects in storage
   347  func (gb *GraphBuilder) enqueueVirtualDeleteEvent(ref objectReference) {
   348  	gv, _ := schema.ParseGroupVersion(ref.APIVersion)
   349  	gb.graphChanges.Add(&event{
   350  		virtual:   true,
   351  		eventType: deleteEvent,
   352  		gvk:       gv.WithKind(ref.Kind),
   353  		obj: &metaonly.MetadataOnlyObject{
   354  			TypeMeta:   metav1.TypeMeta{APIVersion: ref.APIVersion, Kind: ref.Kind},
   355  			ObjectMeta: metav1.ObjectMeta{Namespace: ref.Namespace, UID: ref.UID, Name: ref.Name},
   356  		},
   357  	})
   358  }
   359  
   360  // addDependentToOwners adds n to owners' dependents list. If the owner does not
   361  // exist in the gb.uidToNode yet, a "virtual" node will be created to represent
   362  // the owner. The "virtual" node will be enqueued to the attemptToDelete, so that
   363  // attemptToDeleteItem() will verify if the owner exists according to the API server.
   364  func (gb *GraphBuilder) addDependentToOwners(logger klog.Logger, n *node, owners []metav1.OwnerReference) {
   365  	// track if some of the referenced owners already exist in the graph and have been observed,
   366  	// and the dependent's ownerRef does not match their observed coordinates
   367  	hasPotentiallyInvalidOwnerReference := false
   368  
   369  	for _, owner := range owners {
   370  		ownerNode, ok := gb.uidToNode.Read(owner.UID)
   371  		if !ok {
   372  			// Create a "virtual" node in the graph for the owner if it doesn't
   373  			// exist in the graph yet.
   374  			ownerNode = &node{
   375  				identity: objectReference{
   376  					OwnerReference: ownerReferenceCoordinates(owner),
   377  					Namespace:      n.identity.Namespace,
   378  				},
   379  				dependents: make(map[*node]struct{}),
   380  				virtual:    true,
   381  			}
   382  			logger.V(5).Info("add virtual item", "identity", ownerNode.identity)
   383  			gb.uidToNode.Write(ownerNode)
   384  		}
   385  		ownerNode.addDependent(n)
   386  		if !ok {
   387  			// Enqueue the virtual node into attemptToDelete.
   388  			// The garbage processor will enqueue a virtual delete
   389  			// event to delete it from the graph if API server confirms this
   390  			// owner doesn't exist.
   391  			gb.attemptToDelete.Add(ownerNode)
   392  		} else if !hasPotentiallyInvalidOwnerReference {
   393  			ownerIsNamespaced := len(ownerNode.identity.Namespace) > 0
   394  			if ownerIsNamespaced && ownerNode.identity.Namespace != n.identity.Namespace {
   395  				if ownerNode.isObserved() {
   396  					// The owner node has been observed via an informer
   397  					// the dependent's namespace doesn't match the observed owner's namespace, this is definitely wrong.
   398  					// cluster-scoped owners can be referenced as an owner from any namespace or cluster-scoped object.
   399  					logger.V(2).Info("item references an owner but does not match namespaces", "item", n.identity, "owner", ownerNode.identity)
   400  					gb.reportInvalidNamespaceOwnerRef(n, owner.UID)
   401  				}
   402  				hasPotentiallyInvalidOwnerReference = true
   403  			} else if !ownerReferenceMatchesCoordinates(owner, ownerNode.identity.OwnerReference) {
   404  				if ownerNode.isObserved() {
   405  					// The owner node has been observed via an informer
   406  					// n's owner reference doesn't match the observed identity, this might be wrong.
   407  					logger.V(2).Info("item references an owner with coordinates that do not match the observed identity", "item", n.identity, "owner", ownerNode.identity)
   408  				}
   409  				hasPotentiallyInvalidOwnerReference = true
   410  			} else if !ownerIsNamespaced && ownerNode.identity.Namespace != n.identity.Namespace && !ownerNode.isObserved() {
   411  				// the ownerNode is cluster-scoped and virtual, and does not match the child node's namespace.
   412  				// the owner could be a missing instance of a namespaced type incorrectly referenced by a cluster-scoped child (issue #98040).
   413  				// enqueue this child to attemptToDelete to verify parent references.
   414  				hasPotentiallyInvalidOwnerReference = true
   415  			}
   416  		}
   417  	}
   418  
   419  	if hasPotentiallyInvalidOwnerReference {
   420  		// Enqueue the potentially invalid dependent node into attemptToDelete.
   421  		// The garbage processor will verify whether the owner references are dangling
   422  		// and delete the dependent if all owner references are confirmed absent.
   423  		gb.attemptToDelete.Add(n)
   424  	}
   425  }
   426  
   427  func (gb *GraphBuilder) reportInvalidNamespaceOwnerRef(n *node, invalidOwnerUID types.UID) {
   428  	var invalidOwnerRef metav1.OwnerReference
   429  	var found = false
   430  	for _, ownerRef := range n.owners {
   431  		if ownerRef.UID == invalidOwnerUID {
   432  			invalidOwnerRef = ownerRef
   433  			found = true
   434  			break
   435  		}
   436  	}
   437  	if !found {
   438  		return
   439  	}
   440  	ref := &v1.ObjectReference{
   441  		Kind:       n.identity.Kind,
   442  		APIVersion: n.identity.APIVersion,
   443  		Namespace:  n.identity.Namespace,
   444  		Name:       n.identity.Name,
   445  		UID:        n.identity.UID,
   446  	}
   447  	invalidIdentity := objectReference{
   448  		OwnerReference: metav1.OwnerReference{
   449  			Kind:       invalidOwnerRef.Kind,
   450  			APIVersion: invalidOwnerRef.APIVersion,
   451  			Name:       invalidOwnerRef.Name,
   452  			UID:        invalidOwnerRef.UID,
   453  		},
   454  		Namespace: n.identity.Namespace,
   455  	}
   456  	gb.eventRecorder.Eventf(ref, v1.EventTypeWarning, "OwnerRefInvalidNamespace", "ownerRef %s does not exist in namespace %q", invalidIdentity, n.identity.Namespace)
   457  }
   458  
   459  // insertNode insert the node to gb.uidToNode; then it finds all owners as listed
   460  // in n.owners, and adds the node to their dependents list.
   461  func (gb *GraphBuilder) insertNode(logger klog.Logger, n *node) {
   462  	gb.uidToNode.Write(n)
   463  	gb.addDependentToOwners(logger, n, n.owners)
   464  }
   465  
   466  // removeDependentFromOwners remove n from owners' dependents list.
   467  func (gb *GraphBuilder) removeDependentFromOwners(n *node, owners []metav1.OwnerReference) {
   468  	for _, owner := range owners {
   469  		ownerNode, ok := gb.uidToNode.Read(owner.UID)
   470  		if !ok {
   471  			continue
   472  		}
   473  		ownerNode.deleteDependent(n)
   474  	}
   475  }
   476  
   477  // removeNode removes the node from gb.uidToNode, then finds all
   478  // owners as listed in n.owners, and removes n from their dependents list.
   479  func (gb *GraphBuilder) removeNode(n *node) {
   480  	gb.uidToNode.Delete(n.identity.UID)
   481  	gb.removeDependentFromOwners(n, n.owners)
   482  }
   483  
   484  type ownerRefPair struct {
   485  	oldRef metav1.OwnerReference
   486  	newRef metav1.OwnerReference
   487  }
   488  
   489  // TODO: profile this function to see if a naive N^2 algorithm performs better
   490  // when the number of references is small.
   491  func referencesDiffs(old []metav1.OwnerReference, new []metav1.OwnerReference) (added []metav1.OwnerReference, removed []metav1.OwnerReference, changed []ownerRefPair) {
   492  	oldUIDToRef := make(map[string]metav1.OwnerReference)
   493  	for _, value := range old {
   494  		oldUIDToRef[string(value.UID)] = value
   495  	}
   496  	oldUIDSet := sets.StringKeySet(oldUIDToRef)
   497  	for _, value := range new {
   498  		newUID := string(value.UID)
   499  		if oldUIDSet.Has(newUID) {
   500  			if !reflect.DeepEqual(oldUIDToRef[newUID], value) {
   501  				changed = append(changed, ownerRefPair{oldRef: oldUIDToRef[newUID], newRef: value})
   502  			}
   503  			oldUIDSet.Delete(newUID)
   504  		} else {
   505  			added = append(added, value)
   506  		}
   507  	}
   508  	for oldUID := range oldUIDSet {
   509  		removed = append(removed, oldUIDToRef[oldUID])
   510  	}
   511  
   512  	return added, removed, changed
   513  }
   514  
   515  func deletionStartsWithFinalizer(oldObj interface{}, newAccessor metav1.Object, matchingFinalizer string) bool {
   516  	// if the new object isn't being deleted, or doesn't have the finalizer we're interested in, return false
   517  	if !beingDeleted(newAccessor) || !hasFinalizer(newAccessor, matchingFinalizer) {
   518  		return false
   519  	}
   520  
   521  	// if the old object is nil, or wasn't being deleted, or didn't have the finalizer, return true
   522  	if oldObj == nil {
   523  		return true
   524  	}
   525  	oldAccessor, err := meta.Accessor(oldObj)
   526  	if err != nil {
   527  		utilruntime.HandleError(fmt.Errorf("cannot access oldObj: %v", err))
   528  		return false
   529  	}
   530  	return !beingDeleted(oldAccessor) || !hasFinalizer(oldAccessor, matchingFinalizer)
   531  }
   532  
   533  func beingDeleted(accessor metav1.Object) bool {
   534  	return accessor.GetDeletionTimestamp() != nil
   535  }
   536  
   537  func hasDeleteDependentsFinalizer(accessor metav1.Object) bool {
   538  	return hasFinalizer(accessor, metav1.FinalizerDeleteDependents)
   539  }
   540  
   541  func hasOrphanFinalizer(accessor metav1.Object) bool {
   542  	return hasFinalizer(accessor, metav1.FinalizerOrphanDependents)
   543  }
   544  
   545  func hasFinalizer(accessor metav1.Object, matchingFinalizer string) bool {
   546  	finalizers := accessor.GetFinalizers()
   547  	for _, finalizer := range finalizers {
   548  		if finalizer == matchingFinalizer {
   549  			return true
   550  		}
   551  	}
   552  	return false
   553  }
   554  
   555  // this function takes newAccessor directly because the caller already
   556  // instantiates an accessor for the newObj.
   557  func startsWaitingForDependentsDeleted(oldObj interface{}, newAccessor metav1.Object) bool {
   558  	return deletionStartsWithFinalizer(oldObj, newAccessor, metav1.FinalizerDeleteDependents)
   559  }
   560  
   561  // this function takes newAccessor directly because the caller already
   562  // instantiates an accessor for the newObj.
   563  func startsWaitingForDependentsOrphaned(oldObj interface{}, newAccessor metav1.Object) bool {
   564  	return deletionStartsWithFinalizer(oldObj, newAccessor, metav1.FinalizerOrphanDependents)
   565  }
   566  
   567  // if an blocking ownerReference points to an object gets removed, or gets set to
   568  // "BlockOwnerDeletion=false", add the object to the attemptToDelete queue.
   569  func (gb *GraphBuilder) addUnblockedOwnersToDeleteQueue(logger klog.Logger, removed []metav1.OwnerReference, changed []ownerRefPair) {
   570  	for _, ref := range removed {
   571  		if ref.BlockOwnerDeletion != nil && *ref.BlockOwnerDeletion {
   572  			node, found := gb.uidToNode.Read(ref.UID)
   573  			if !found {
   574  				logger.V(5).Info("cannot find uid in uidToNode", "uid", ref.UID)
   575  				continue
   576  			}
   577  			gb.attemptToDelete.Add(node)
   578  		}
   579  	}
   580  	for _, c := range changed {
   581  		wasBlocked := c.oldRef.BlockOwnerDeletion != nil && *c.oldRef.BlockOwnerDeletion
   582  		isUnblocked := c.newRef.BlockOwnerDeletion == nil || (c.newRef.BlockOwnerDeletion != nil && !*c.newRef.BlockOwnerDeletion)
   583  		if wasBlocked && isUnblocked {
   584  			node, found := gb.uidToNode.Read(c.newRef.UID)
   585  			if !found {
   586  				logger.V(5).Info("cannot find uid in uidToNode", "uid", c.newRef.UID)
   587  				continue
   588  			}
   589  			gb.attemptToDelete.Add(node)
   590  		}
   591  	}
   592  }
   593  
   594  func (gb *GraphBuilder) processTransitions(logger klog.Logger, oldObj interface{}, newAccessor metav1.Object, n *node) {
   595  	if startsWaitingForDependentsOrphaned(oldObj, newAccessor) {
   596  		logger.V(5).Info("add item to attemptToOrphan", "item", n.identity)
   597  		gb.attemptToOrphan.Add(n)
   598  		return
   599  	}
   600  	if startsWaitingForDependentsDeleted(oldObj, newAccessor) {
   601  		logger.V(2).Info("add item to attemptToDelete, because it's waiting for its dependents to be deleted", "item", n.identity)
   602  		// if the n is added as a "virtual" node, its deletingDependents field is not properly set, so always set it here.
   603  		n.markDeletingDependents()
   604  		for dep := range n.dependents {
   605  			gb.attemptToDelete.Add(dep)
   606  		}
   607  		gb.attemptToDelete.Add(n)
   608  	}
   609  }
   610  
   611  func (gb *GraphBuilder) runProcessGraphChanges(logger klog.Logger) {
   612  	for gb.processGraphChanges(logger) {
   613  	}
   614  }
   615  
   616  func identityFromEvent(event *event, accessor metav1.Object) objectReference {
   617  	return objectReference{
   618  		OwnerReference: metav1.OwnerReference{
   619  			APIVersion: event.gvk.GroupVersion().String(),
   620  			Kind:       event.gvk.Kind,
   621  			UID:        accessor.GetUID(),
   622  			Name:       accessor.GetName(),
   623  		},
   624  		Namespace: accessor.GetNamespace(),
   625  	}
   626  }
   627  
   628  // Dequeueing an event from graphChanges, updating graph, populating dirty_queue.
   629  func (gb *GraphBuilder) processGraphChanges(logger klog.Logger) bool {
   630  	item, quit := gb.graphChanges.Get()
   631  	if quit {
   632  		return false
   633  	}
   634  	defer gb.graphChanges.Done(item)
   635  	event, ok := item.(*event)
   636  	if !ok {
   637  		utilruntime.HandleError(fmt.Errorf("expect a *event, got %v", item))
   638  		return true
   639  	}
   640  	obj := event.obj
   641  	accessor, err := meta.Accessor(obj)
   642  	if err != nil {
   643  		utilruntime.HandleError(fmt.Errorf("cannot access obj: %v", err))
   644  		return true
   645  	}
   646  
   647  	logger.V(5).Info("GraphBuilder process object",
   648  		"apiVersion", event.gvk.GroupVersion().String(),
   649  		"kind", event.gvk.Kind,
   650  		"object", klog.KObj(accessor),
   651  		"uid", string(accessor.GetUID()),
   652  		"eventType", event.eventType,
   653  		"virtual", event.virtual,
   654  	)
   655  
   656  	// Check if the node already exists
   657  	existingNode, found := gb.uidToNode.Read(accessor.GetUID())
   658  	if found && !event.virtual && !existingNode.isObserved() {
   659  		// this marks the node as having been observed via an informer event
   660  		// 1. this depends on graphChanges only containing add/update events from the actual informer
   661  		// 2. this allows things tracking virtual nodes' existence to stop polling and rely on informer events
   662  		observedIdentity := identityFromEvent(event, accessor)
   663  		if observedIdentity != existingNode.identity {
   664  			// find dependents that don't match the identity we observed
   665  			_, potentiallyInvalidDependents := partitionDependents(existingNode.getDependents(), observedIdentity)
   666  			// add those potentially invalid dependents to the attemptToDelete queue.
   667  			// if their owners are still solid the attemptToDelete will be a no-op.
   668  			// this covers the bad child -> good parent observation sequence.
   669  			// the good parent -> bad child observation sequence is handled in addDependentToOwners
   670  			for _, dep := range potentiallyInvalidDependents {
   671  				if len(observedIdentity.Namespace) > 0 && dep.identity.Namespace != observedIdentity.Namespace {
   672  					// Namespace mismatch, this is definitely wrong
   673  					logger.V(2).Info("item references an owner but does not match namespaces",
   674  						"item", dep.identity,
   675  						"owner", observedIdentity,
   676  					)
   677  					gb.reportInvalidNamespaceOwnerRef(dep, observedIdentity.UID)
   678  				}
   679  				gb.attemptToDelete.Add(dep)
   680  			}
   681  
   682  			// make a copy (so we don't modify the existing node in place), store the observed identity, and replace the virtual node
   683  			logger.V(2).Info("replacing virtual item with observed item",
   684  				"virtual", existingNode.identity,
   685  				"observed", observedIdentity,
   686  			)
   687  			existingNode = existingNode.clone()
   688  			existingNode.identity = observedIdentity
   689  			gb.uidToNode.Write(existingNode)
   690  		}
   691  		existingNode.markObserved()
   692  	}
   693  	switch {
   694  	case (event.eventType == addEvent || event.eventType == updateEvent) && !found:
   695  		newNode := &node{
   696  			identity:           identityFromEvent(event, accessor),
   697  			dependents:         make(map[*node]struct{}),
   698  			owners:             accessor.GetOwnerReferences(),
   699  			deletingDependents: beingDeleted(accessor) && hasDeleteDependentsFinalizer(accessor),
   700  			beingDeleted:       beingDeleted(accessor),
   701  		}
   702  		gb.insertNode(logger, newNode)
   703  		// the underlying delta_fifo may combine a creation and a deletion into
   704  		// one event, so we need to further process the event.
   705  		gb.processTransitions(logger, event.oldObj, accessor, newNode)
   706  	case (event.eventType == addEvent || event.eventType == updateEvent) && found:
   707  		// handle changes in ownerReferences
   708  		added, removed, changed := referencesDiffs(existingNode.owners, accessor.GetOwnerReferences())
   709  		if len(added) != 0 || len(removed) != 0 || len(changed) != 0 {
   710  			// check if the changed dependency graph unblock owners that are
   711  			// waiting for the deletion of their dependents.
   712  			gb.addUnblockedOwnersToDeleteQueue(logger, removed, changed)
   713  			// update the node itself
   714  			existingNode.owners = accessor.GetOwnerReferences()
   715  			// Add the node to its new owners' dependent lists.
   716  			gb.addDependentToOwners(logger, existingNode, added)
   717  			// remove the node from the dependent list of node that are no longer in
   718  			// the node's owners list.
   719  			gb.removeDependentFromOwners(existingNode, removed)
   720  		}
   721  
   722  		if beingDeleted(accessor) {
   723  			existingNode.markBeingDeleted()
   724  		}
   725  		gb.processTransitions(logger, event.oldObj, accessor, existingNode)
   726  	case event.eventType == deleteEvent:
   727  		if !found {
   728  			logger.V(5).Info("item doesn't exist in the graph, this shouldn't happen",
   729  				"item", accessor.GetUID(),
   730  			)
   731  			return true
   732  		}
   733  
   734  		removeExistingNode := true
   735  
   736  		if event.virtual {
   737  			// this is a virtual delete event, not one observed from an informer
   738  			deletedIdentity := identityFromEvent(event, accessor)
   739  			if existingNode.virtual {
   740  
   741  				// our existing node is also virtual, we're not sure of its coordinates.
   742  				// see if any dependents reference this owner with coordinates other than the one we got a virtual delete event for.
   743  				if matchingDependents, nonmatchingDependents := partitionDependents(existingNode.getDependents(), deletedIdentity); len(nonmatchingDependents) > 0 {
   744  
   745  					// some of our dependents disagree on our coordinates, so do not remove the existing virtual node from the graph
   746  					removeExistingNode = false
   747  
   748  					if len(matchingDependents) > 0 {
   749  						// mark the observed deleted identity as absent
   750  						gb.absentOwnerCache.Add(deletedIdentity)
   751  						// attempt to delete dependents that do match the verified deleted identity
   752  						for _, dep := range matchingDependents {
   753  							gb.attemptToDelete.Add(dep)
   754  						}
   755  					}
   756  
   757  					// if the delete event verified existingNode.identity doesn't exist...
   758  					if existingNode.identity == deletedIdentity {
   759  						// find an alternative identity our nonmatching dependents refer to us by
   760  						replacementIdentity := getAlternateOwnerIdentity(nonmatchingDependents, deletedIdentity)
   761  						if replacementIdentity != nil {
   762  							// replace the existing virtual node with a new one with one of our other potential identities
   763  							replacementNode := existingNode.clone()
   764  							replacementNode.identity = *replacementIdentity
   765  							gb.uidToNode.Write(replacementNode)
   766  							// and add the new virtual node back to the attemptToDelete queue
   767  							gb.attemptToDelete.AddRateLimited(replacementNode)
   768  						}
   769  					}
   770  				}
   771  
   772  			} else if existingNode.identity != deletedIdentity {
   773  				// do not remove the existing real node from the graph based on a virtual delete event
   774  				removeExistingNode = false
   775  
   776  				// our existing node which was observed via informer disagrees with the virtual delete event's coordinates
   777  				matchingDependents, _ := partitionDependents(existingNode.getDependents(), deletedIdentity)
   778  
   779  				if len(matchingDependents) > 0 {
   780  					// mark the observed deleted identity as absent
   781  					gb.absentOwnerCache.Add(deletedIdentity)
   782  					// attempt to delete dependents that do match the verified deleted identity
   783  					for _, dep := range matchingDependents {
   784  						gb.attemptToDelete.Add(dep)
   785  					}
   786  				}
   787  			}
   788  		}
   789  
   790  		if removeExistingNode {
   791  			// removeNode updates the graph
   792  			gb.removeNode(existingNode)
   793  			existingNode.dependentsLock.RLock()
   794  			defer existingNode.dependentsLock.RUnlock()
   795  			if len(existingNode.dependents) > 0 {
   796  				gb.absentOwnerCache.Add(identityFromEvent(event, accessor))
   797  			}
   798  			for dep := range existingNode.dependents {
   799  				gb.attemptToDelete.Add(dep)
   800  			}
   801  			for _, owner := range existingNode.owners {
   802  				ownerNode, found := gb.uidToNode.Read(owner.UID)
   803  				if !found || !ownerNode.isDeletingDependents() {
   804  					continue
   805  				}
   806  				// this is to let attempToDeleteItem check if all the owner's
   807  				// dependents are deleted, if so, the owner will be deleted.
   808  				gb.attemptToDelete.Add(ownerNode)
   809  			}
   810  		}
   811  	}
   812  	return true
   813  }
   814  
   815  // partitionDependents divides the provided dependents into a list which have an ownerReference matching the provided identity,
   816  // and ones which have an ownerReference for the given uid that do not match the provided identity.
   817  // Note that a dependent with multiple ownerReferences for the target uid can end up in both lists.
   818  func partitionDependents(dependents []*node, matchOwnerIdentity objectReference) (matching, nonmatching []*node) {
   819  	ownerIsNamespaced := len(matchOwnerIdentity.Namespace) > 0
   820  	for i := range dependents {
   821  		dep := dependents[i]
   822  		foundMatch := false
   823  		foundMismatch := false
   824  		// if the dep namespace matches or the owner is cluster scoped ...
   825  		if ownerIsNamespaced && matchOwnerIdentity.Namespace != dep.identity.Namespace {
   826  			// all references to the parent do not match, since the dependent namespace does not match the owner
   827  			foundMismatch = true
   828  		} else {
   829  			for _, ownerRef := range dep.owners {
   830  				// ... find the ownerRef with a matching uid ...
   831  				if ownerRef.UID == matchOwnerIdentity.UID {
   832  					// ... and check if it matches all coordinates
   833  					if ownerReferenceMatchesCoordinates(ownerRef, matchOwnerIdentity.OwnerReference) {
   834  						foundMatch = true
   835  					} else {
   836  						foundMismatch = true
   837  					}
   838  				}
   839  			}
   840  		}
   841  
   842  		if foundMatch {
   843  			matching = append(matching, dep)
   844  		}
   845  		if foundMismatch {
   846  			nonmatching = append(nonmatching, dep)
   847  		}
   848  	}
   849  	return matching, nonmatching
   850  }
   851  
   852  func referenceLessThan(a, b objectReference) bool {
   853  	// kind/apiVersion are more significant than namespace,
   854  	// so that we get coherent ordering between kinds
   855  	// regardless of whether they are cluster-scoped or namespaced
   856  	if a.Kind != b.Kind {
   857  		return a.Kind < b.Kind
   858  	}
   859  	if a.APIVersion != b.APIVersion {
   860  		return a.APIVersion < b.APIVersion
   861  	}
   862  	// namespace is more significant than name
   863  	if a.Namespace != b.Namespace {
   864  		return a.Namespace < b.Namespace
   865  	}
   866  	// name is more significant than uid
   867  	if a.Name != b.Name {
   868  		return a.Name < b.Name
   869  	}
   870  	// uid is included for completeness, but is expected to be identical
   871  	// when getting alternate identities for an owner since they are keyed by uid
   872  	if a.UID != b.UID {
   873  		return a.UID < b.UID
   874  	}
   875  	return false
   876  }
   877  
   878  // getAlternateOwnerIdentity searches deps for owner references which match
   879  // verifiedAbsentIdentity.UID but differ in apiVersion/kind/name or namespace.
   880  // The first that follows verifiedAbsentIdentity (according to referenceLessThan) is returned.
   881  // If none follow verifiedAbsentIdentity, the first (according to referenceLessThan) is returned.
   882  // If no alternate identities are found, nil is returned.
   883  func getAlternateOwnerIdentity(deps []*node, verifiedAbsentIdentity objectReference) *objectReference {
   884  	absentIdentityIsClusterScoped := len(verifiedAbsentIdentity.Namespace) == 0
   885  
   886  	seenAlternates := map[objectReference]bool{verifiedAbsentIdentity: true}
   887  
   888  	// keep track of the first alternate reference (according to referenceLessThan)
   889  	var first *objectReference
   890  	// keep track of the first reference following verifiedAbsentIdentity (according to referenceLessThan)
   891  	var firstFollowing *objectReference
   892  
   893  	for _, dep := range deps {
   894  		for _, ownerRef := range dep.owners {
   895  			if ownerRef.UID != verifiedAbsentIdentity.UID {
   896  				// skip references that aren't the uid we care about
   897  				continue
   898  			}
   899  
   900  			if ownerReferenceMatchesCoordinates(ownerRef, verifiedAbsentIdentity.OwnerReference) {
   901  				if absentIdentityIsClusterScoped || verifiedAbsentIdentity.Namespace == dep.identity.Namespace {
   902  					// skip references that exactly match verifiedAbsentIdentity
   903  					continue
   904  				}
   905  			}
   906  
   907  			ref := objectReference{OwnerReference: ownerReferenceCoordinates(ownerRef), Namespace: dep.identity.Namespace}
   908  			if absentIdentityIsClusterScoped && ref.APIVersion == verifiedAbsentIdentity.APIVersion && ref.Kind == verifiedAbsentIdentity.Kind {
   909  				// we know this apiVersion/kind is cluster-scoped because of verifiedAbsentIdentity,
   910  				// so clear the namespace from the alternate identity
   911  				ref.Namespace = ""
   912  			}
   913  
   914  			if seenAlternates[ref] {
   915  				// skip references we've already seen
   916  				continue
   917  			}
   918  			seenAlternates[ref] = true
   919  
   920  			if first == nil || referenceLessThan(ref, *first) {
   921  				// this alternate comes first lexically
   922  				first = &ref
   923  			}
   924  			if referenceLessThan(verifiedAbsentIdentity, ref) && (firstFollowing == nil || referenceLessThan(ref, *firstFollowing)) {
   925  				// this alternate is the first following verifiedAbsentIdentity lexically
   926  				firstFollowing = &ref
   927  			}
   928  		}
   929  	}
   930  
   931  	// return the first alternate identity following the verified absent identity, if there is one
   932  	if firstFollowing != nil {
   933  		return firstFollowing
   934  	}
   935  	// otherwise return the first alternate identity
   936  	return first
   937  }