github.com/cilium/cilium@v1.16.2/pkg/envoy/xds/ack.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package xds
     5  
     6  import (
     7  	"errors"
     8  
     9  	"github.com/sirupsen/logrus"
    10  	"google.golang.org/protobuf/proto"
    11  
    12  	"github.com/cilium/cilium/pkg/completion"
    13  	"github.com/cilium/cilium/pkg/lock"
    14  	"github.com/cilium/cilium/pkg/logging/logfields"
    15  )
    16  
    17  // ProxyError wraps the error and the detail received from the proxy in to a new type
    18  // that implements the error interface.
    19  type ProxyError struct {
    20  	Err    error
    21  	Detail string
    22  }
    23  
    24  func (pe *ProxyError) Error() string {
    25  	return pe.Err.Error() + ": " + pe.Detail
    26  }
    27  
    28  var ErrNackReceived = errors.New("NACK received")
    29  
    30  // ResourceVersionAckObserver defines the HandleResourceVersionAck method
    31  // which is called whenever a node acknowledges having applied a version of
    32  // the resources of a given type.
    33  type ResourceVersionAckObserver interface {
    34  	// HandleResourceVersionAck notifies that the node with the given NodeIP
    35  	// has acknowledged having applied the resources.
    36  	// Calls to this function must not block.
    37  	HandleResourceVersionAck(ackVersion uint64, nackVersion uint64, nodeIP string, resourceNames []string, typeURL string, detail string)
    38  
    39  	// MarkRestorePending informs the observer about a pending state restoration.
    40  	MarkRestorePending()
    41  
    42  	// MarkRestoreCompleted clears the 'restore' state so that updates are acked normally.
    43  	MarkRestoreCompleted()
    44  }
    45  
    46  // AckingResourceMutatorRevertFunc is a function which reverts the effects of
    47  // an update on a AckingResourceMutator.
    48  // The completion, if not nil, is called back when the new resource update is
    49  // ACKed by the Envoy nodes.
    50  type AckingResourceMutatorRevertFunc func(completion *completion.Completion)
    51  
    52  type AckingResourceMutatorRevertFuncList []AckingResourceMutatorRevertFunc
    53  
    54  func (rl AckingResourceMutatorRevertFuncList) Revert(wg *completion.WaitGroup) {
    55  	// Revert the listed funcions in reverse order
    56  	for i := len(rl) - 1; i >= 0; i-- {
    57  		var c *completion.Completion
    58  		if wg != nil {
    59  			c = wg.AddCompletion()
    60  		}
    61  		rl[i](c)
    62  	}
    63  }
    64  
    65  // AckingResourceMutator is a variant of ResourceMutator which calls back a
    66  // Completion when a resource update is ACKed by a set of Envoy nodes.
    67  type AckingResourceMutator interface {
    68  	// Upsert inserts or updates a resource from this set by name and increases
    69  	// the set's version number atomically if the resource is actually inserted
    70  	// or updated.
    71  	// The completion is called back when the new upserted resources' version is
    72  	// ACKed by the Envoy nodes which IDs are given in nodeIDs.
    73  	// A call to the returned revert function reverts the effects of this
    74  	// method call.
    75  	Upsert(typeURL string, resourceName string, resource proto.Message, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc
    76  
    77  	// DeleteNode frees resources held for the named node
    78  	DeleteNode(nodeID string)
    79  
    80  	// Delete deletes a resource from this set by name and increases the cache's
    81  	// version number atomically if the resource is actually deleted.
    82  	// The completion is called back when the new deleted resources' version is
    83  	// ACKed by the Envoy nodes which IDs are given in nodeIDs.
    84  	// A call to the returned revert function reverts the effects of this
    85  	// method call.
    86  	Delete(typeURL string, resourceName string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc
    87  }
    88  
    89  // AckingResourceMutatorWrapper is an AckingResourceMutator which wraps a
    90  // ResourceMutator to notifies callers when resource updates are ACKed by
    91  // nodes.
    92  // AckingResourceMutatorWrapper also implements ResourceVersionAckObserver in
    93  // order to be notified of ACKs from nodes.
    94  type AckingResourceMutatorWrapper struct {
    95  	// mutator is the wrapped resource mutator.
    96  	mutator ResourceMutator
    97  
    98  	// locker locks all accesses to the remaining fields.
    99  	locker lock.Mutex
   100  
   101  	// Last version stored by 'mutator'
   102  	version uint64
   103  
   104  	// ackedVersions is the last version acked by a node for this cache.
   105  	// The key is the IPv4 address of the Envoy instance in string format.
   106  	// e.g. "127.0.0.1" for the host proxy.
   107  	ackedVersions map[string]uint64
   108  
   109  	// pendingCompletions is the list of updates that are pending completion.
   110  	pendingCompletions map[*completion.Completion]*pendingCompletion
   111  
   112  	// restoring controls waiting for acks. When 'true' updates do not wait for acks from the xDS client,
   113  	// as xDS caches are pre-populated before passing any resources to xDS clients.
   114  	restoring bool
   115  }
   116  
   117  // pendingCompletion is an update that is pending completion.
   118  type pendingCompletion struct {
   119  	// version is the version to be ACKed.
   120  	version uint64
   121  
   122  	// typeURL is the type URL of the resources to be ACKed.
   123  	typeURL string
   124  
   125  	// remainingNodesResources maps each pending node ID to pending resource
   126  	// name.
   127  	remainingNodesResources map[string]map[string]struct{}
   128  }
   129  
   130  // NewAckingResourceMutatorWrapper creates a new AckingResourceMutatorWrapper
   131  // to wrap the given ResourceMutator.
   132  func NewAckingResourceMutatorWrapper(mutator ResourceMutator) *AckingResourceMutatorWrapper {
   133  	return &AckingResourceMutatorWrapper{
   134  		mutator:            mutator,
   135  		ackedVersions:      make(map[string]uint64),
   136  		pendingCompletions: make(map[*completion.Completion]*pendingCompletion),
   137  	}
   138  }
   139  
   140  func (m *AckingResourceMutatorWrapper) MarkRestorePending() {
   141  	m.locker.Lock()
   142  	defer m.locker.Unlock()
   143  
   144  	m.restoring = true
   145  }
   146  
   147  // MarkRestoreCompleted clears the 'restore' state so that updates are acked normally.
   148  func (m *AckingResourceMutatorWrapper) MarkRestoreCompleted() {
   149  	m.locker.Lock()
   150  	defer m.locker.Unlock()
   151  
   152  	m.restoring = false
   153  }
   154  
   155  // AddVersionCompletion adds a completion to wait for any ACK for the
   156  // version and type URL, ignoring the ACKed resource names.
   157  func (m *AckingResourceMutatorWrapper) addVersionCompletion(typeURL string, version uint64, nodeIDs []string, c *completion.Completion) {
   158  	comp := &pendingCompletion{
   159  		version:                 version,
   160  		typeURL:                 typeURL,
   161  		remainingNodesResources: make(map[string]map[string]struct{}, len(nodeIDs)),
   162  	}
   163  	for _, nodeID := range nodeIDs {
   164  		comp.remainingNodesResources[nodeID] = nil
   165  	}
   166  	m.pendingCompletions[c] = comp
   167  }
   168  
   169  // DeleteNode frees resources held for the named nodes
   170  func (m *AckingResourceMutatorWrapper) DeleteNode(nodeID string) {
   171  	m.locker.Lock()
   172  	defer m.locker.Unlock()
   173  
   174  	delete(m.ackedVersions, nodeID)
   175  }
   176  
   177  func (m *AckingResourceMutatorWrapper) Upsert(typeURL string, resourceName string, resource proto.Message, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc {
   178  	m.locker.Lock()
   179  	defer m.locker.Unlock()
   180  
   181  	wait := wg != nil
   182  
   183  	if m.restoring {
   184  		// Do not wait for acks when restoring state
   185  		log.WithFields(logrus.Fields{
   186  			logfields.XDSTypeURL:      typeURL,
   187  			logfields.XDSResourceName: resourceName,
   188  		}).Debug("Upsert: Restoring, skipping wait for ACK")
   189  
   190  		wait = false
   191  	}
   192  
   193  	var updated bool
   194  	var revert ResourceMutatorRevertFunc
   195  	m.version, updated, revert = m.mutator.Upsert(typeURL, resourceName, resource)
   196  
   197  	if !updated {
   198  		if wait {
   199  			m.useCurrent(typeURL, nodeIDs, wg, callback)
   200  		} else if callback != nil {
   201  			callback(nil)
   202  		}
   203  		return func(completion *completion.Completion) {}
   204  	}
   205  
   206  	if wait {
   207  		// Create a new completion
   208  		c := wg.AddCompletionWithCallback(callback)
   209  		if _, found := m.pendingCompletions[c]; found {
   210  			log.WithFields(logrus.Fields{
   211  				logfields.XDSTypeURL:      typeURL,
   212  				logfields.XDSResourceName: resourceName,
   213  			}).Fatalf("attempt to reuse completion to upsert xDS resource: %v", c)
   214  		}
   215  
   216  		comp := &pendingCompletion{
   217  			version:                 m.version,
   218  			typeURL:                 typeURL,
   219  			remainingNodesResources: make(map[string]map[string]struct{}, len(nodeIDs)),
   220  		}
   221  		for _, nodeID := range nodeIDs {
   222  			comp.remainingNodesResources[nodeID] = make(map[string]struct{}, 1)
   223  			comp.remainingNodesResources[nodeID][resourceName] = struct{}{}
   224  		}
   225  		m.pendingCompletions[c] = comp
   226  	} else if callback != nil {
   227  		callback(nil)
   228  	}
   229  
   230  	// Returned revert function locks again, so it can NOT be called from 'callback' directly,
   231  	// as 'callback' is called with the lock already held.
   232  	return func(completion *completion.Completion) {
   233  		m.locker.Lock()
   234  		defer m.locker.Unlock()
   235  
   236  		if revert != nil {
   237  			m.version, _ = revert()
   238  
   239  			if completion != nil {
   240  				// We don't know whether the revert did an Upsert or a Delete, so as a
   241  				// best effort, just wait for any ACK for the version and type URL,
   242  				// and ignore the ACKed resource names, like for a Delete.
   243  				m.addVersionCompletion(typeURL, m.version, nodeIDs, completion)
   244  			}
   245  		}
   246  	}
   247  }
   248  
   249  func (m *AckingResourceMutatorWrapper) useCurrent(typeURL string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) {
   250  	if !m.currentVersionAcked(nodeIDs) {
   251  		// Add a completion object for 'version' so that the caller may wait for the N/ACK
   252  		m.addVersionCompletion(typeURL, m.version, nodeIDs, wg.AddCompletionWithCallback(callback))
   253  	}
   254  }
   255  
   256  func (m *AckingResourceMutatorWrapper) currentVersionAcked(nodeIDs []string) bool {
   257  	for _, node := range nodeIDs {
   258  		if acked, exists := m.ackedVersions[node]; !exists || acked < m.version {
   259  			ackLog := log.WithFields(logrus.Fields{
   260  				logfields.XDSCachedVersion: m.version,
   261  				logfields.XDSAckedVersion:  acked,
   262  				logfields.XDSClientNode:    node,
   263  			})
   264  			ackLog.Debugf("Node has not acked the current cached version yet")
   265  			return false
   266  		}
   267  	}
   268  	return true
   269  }
   270  
   271  func (m *AckingResourceMutatorWrapper) Delete(typeURL string, resourceName string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc {
   272  	m.locker.Lock()
   273  	defer m.locker.Unlock()
   274  
   275  	wait := wg != nil
   276  
   277  	if m.restoring {
   278  		// Do not wait for acks when restoring state
   279  		log.WithFields(logrus.Fields{
   280  			logfields.XDSTypeURL:      typeURL,
   281  			logfields.XDSResourceName: resourceName,
   282  		}).Debug("Delete: Restoring, skipping wait for ACK")
   283  
   284  		wait = false
   285  	}
   286  
   287  	// Always delete the resource, even if the completion's context was
   288  	// canceled before we even started, since we have no way to signal whether
   289  	// the resource is actually deleted.
   290  
   291  	// There is no explicit ACK for resource deletion in the xDS protocol.
   292  	// As a best effort, just wait for any ACK for the version and type URL,
   293  	// and ignore the ACKed resource names.
   294  
   295  	var updated bool
   296  	var revert ResourceMutatorRevertFunc
   297  	m.version, updated, revert = m.mutator.Delete(typeURL, resourceName)
   298  
   299  	if !updated {
   300  		if wait {
   301  			m.useCurrent(typeURL, nodeIDs, wg, callback)
   302  		} else if callback != nil {
   303  			callback(nil)
   304  		}
   305  		return func(completion *completion.Completion) {}
   306  	}
   307  
   308  	if wait {
   309  		c := wg.AddCompletionWithCallback(callback)
   310  		if _, found := m.pendingCompletions[c]; found {
   311  			log.WithFields(logrus.Fields{
   312  				logfields.XDSTypeURL:      typeURL,
   313  				logfields.XDSResourceName: resourceName,
   314  			}).Fatalf("attempt to reuse completion to delete xDS resource: %v", c)
   315  		}
   316  
   317  		m.addVersionCompletion(typeURL, m.version, nodeIDs, c)
   318  	} else if callback != nil {
   319  		callback(nil)
   320  	}
   321  
   322  	return func(completion *completion.Completion) {
   323  		m.locker.Lock()
   324  		defer m.locker.Unlock()
   325  
   326  		if revert != nil {
   327  			m.version, _ = revert()
   328  
   329  			if completion != nil {
   330  				// We don't know whether the revert had any effect at all, so as a
   331  				// best effort, just wait for any ACK for the version and type URL,
   332  				// and ignore the ACKed resource names, like for a Delete.
   333  				m.addVersionCompletion(typeURL, m.version, nodeIDs, completion)
   334  			}
   335  		}
   336  	}
   337  }
   338  
   339  // 'ackVersion' is the last version that was acked. 'nackVersion', if greater than 'nackVersion', is the last version that was NACKed.
   340  func (m *AckingResourceMutatorWrapper) HandleResourceVersionAck(ackVersion uint64, nackVersion uint64, nodeIP string, resourceNames []string, typeURL string, detail string) {
   341  	ackLog := log.WithFields(logrus.Fields{
   342  		logfields.XDSAckedVersion: ackVersion,
   343  		logfields.XDSNonce:        nackVersion,
   344  		logfields.XDSClientNode:   nodeIP,
   345  		logfields.XDSTypeURL:      typeURL,
   346  	})
   347  
   348  	m.locker.Lock()
   349  	defer m.locker.Unlock()
   350  
   351  	// Update the last seen ACKed version if it advances the previously ACKed version.
   352  	// Version 0 is special as it indicates that we have received the first xDS
   353  	// resource request from Envoy. Prior to that we do not have a map entry for the
   354  	// node at all.
   355  	if previouslyAckedVersion, exists := m.ackedVersions[nodeIP]; !exists || previouslyAckedVersion < ackVersion {
   356  		m.ackedVersions[nodeIP] = ackVersion
   357  	}
   358  
   359  	remainingCompletions := make(map[*completion.Completion]*pendingCompletion, len(m.pendingCompletions))
   360  
   361  	for comp, pending := range m.pendingCompletions {
   362  		if comp.Err() != nil {
   363  			// Completion was canceled or timed out.
   364  			// Remove from pending list.
   365  			ackLog.Debugf("completion context was canceled: %v", pending)
   366  			continue
   367  		}
   368  
   369  		if pending.typeURL == typeURL {
   370  			if pending.version <= nackVersion {
   371  				// Get the set of resource names we are still waiting for the node
   372  				// to ACK.
   373  				remainingResourceNames, found := pending.remainingNodesResources[nodeIP]
   374  				if found {
   375  					for _, name := range resourceNames {
   376  						delete(remainingResourceNames, name)
   377  					}
   378  					if len(remainingResourceNames) == 0 {
   379  						delete(pending.remainingNodesResources, nodeIP)
   380  					}
   381  					if len(pending.remainingNodesResources) == 0 {
   382  						// completedComparision. Notify and remove from pending list.
   383  						if pending.version <= ackVersion {
   384  							ackLog.Debugf("completing ACK: %v", pending)
   385  							comp.Complete(nil)
   386  						} else {
   387  							ackLog.Debugf("completing NACK: %v", pending)
   388  							comp.Complete(&ProxyError{Err: ErrNackReceived, Detail: detail})
   389  						}
   390  						continue
   391  					}
   392  				}
   393  			}
   394  		}
   395  
   396  		// Completion didn't match or is still waiting for some ACKs. Keep it
   397  		// in the pending list.
   398  		remainingCompletions[comp] = pending
   399  	}
   400  
   401  	m.pendingCompletions = remainingCompletions
   402  }