github.com/cilium/cilium@v1.16.2/pkg/envoy/xds/ack.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package xds 5 6 import ( 7 "errors" 8 9 "github.com/sirupsen/logrus" 10 "google.golang.org/protobuf/proto" 11 12 "github.com/cilium/cilium/pkg/completion" 13 "github.com/cilium/cilium/pkg/lock" 14 "github.com/cilium/cilium/pkg/logging/logfields" 15 ) 16 17 // ProxyError wraps the error and the detail received from the proxy in to a new type 18 // that implements the error interface. 19 type ProxyError struct { 20 Err error 21 Detail string 22 } 23 24 func (pe *ProxyError) Error() string { 25 return pe.Err.Error() + ": " + pe.Detail 26 } 27 28 var ErrNackReceived = errors.New("NACK received") 29 30 // ResourceVersionAckObserver defines the HandleResourceVersionAck method 31 // which is called whenever a node acknowledges having applied a version of 32 // the resources of a given type. 33 type ResourceVersionAckObserver interface { 34 // HandleResourceVersionAck notifies that the node with the given NodeIP 35 // has acknowledged having applied the resources. 36 // Calls to this function must not block. 37 HandleResourceVersionAck(ackVersion uint64, nackVersion uint64, nodeIP string, resourceNames []string, typeURL string, detail string) 38 39 // MarkRestorePending informs the observer about a pending state restoration. 40 MarkRestorePending() 41 42 // MarkRestoreCompleted clears the 'restore' state so that updates are acked normally. 43 MarkRestoreCompleted() 44 } 45 46 // AckingResourceMutatorRevertFunc is a function which reverts the effects of 47 // an update on a AckingResourceMutator. 48 // The completion, if not nil, is called back when the new resource update is 49 // ACKed by the Envoy nodes. 50 type AckingResourceMutatorRevertFunc func(completion *completion.Completion) 51 52 type AckingResourceMutatorRevertFuncList []AckingResourceMutatorRevertFunc 53 54 func (rl AckingResourceMutatorRevertFuncList) Revert(wg *completion.WaitGroup) { 55 // Revert the listed funcions in reverse order 56 for i := len(rl) - 1; i >= 0; i-- { 57 var c *completion.Completion 58 if wg != nil { 59 c = wg.AddCompletion() 60 } 61 rl[i](c) 62 } 63 } 64 65 // AckingResourceMutator is a variant of ResourceMutator which calls back a 66 // Completion when a resource update is ACKed by a set of Envoy nodes. 67 type AckingResourceMutator interface { 68 // Upsert inserts or updates a resource from this set by name and increases 69 // the set's version number atomically if the resource is actually inserted 70 // or updated. 71 // The completion is called back when the new upserted resources' version is 72 // ACKed by the Envoy nodes which IDs are given in nodeIDs. 73 // A call to the returned revert function reverts the effects of this 74 // method call. 75 Upsert(typeURL string, resourceName string, resource proto.Message, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc 76 77 // DeleteNode frees resources held for the named node 78 DeleteNode(nodeID string) 79 80 // Delete deletes a resource from this set by name and increases the cache's 81 // version number atomically if the resource is actually deleted. 82 // The completion is called back when the new deleted resources' version is 83 // ACKed by the Envoy nodes which IDs are given in nodeIDs. 84 // A call to the returned revert function reverts the effects of this 85 // method call. 86 Delete(typeURL string, resourceName string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc 87 } 88 89 // AckingResourceMutatorWrapper is an AckingResourceMutator which wraps a 90 // ResourceMutator to notifies callers when resource updates are ACKed by 91 // nodes. 92 // AckingResourceMutatorWrapper also implements ResourceVersionAckObserver in 93 // order to be notified of ACKs from nodes. 94 type AckingResourceMutatorWrapper struct { 95 // mutator is the wrapped resource mutator. 96 mutator ResourceMutator 97 98 // locker locks all accesses to the remaining fields. 99 locker lock.Mutex 100 101 // Last version stored by 'mutator' 102 version uint64 103 104 // ackedVersions is the last version acked by a node for this cache. 105 // The key is the IPv4 address of the Envoy instance in string format. 106 // e.g. "127.0.0.1" for the host proxy. 107 ackedVersions map[string]uint64 108 109 // pendingCompletions is the list of updates that are pending completion. 110 pendingCompletions map[*completion.Completion]*pendingCompletion 111 112 // restoring controls waiting for acks. When 'true' updates do not wait for acks from the xDS client, 113 // as xDS caches are pre-populated before passing any resources to xDS clients. 114 restoring bool 115 } 116 117 // pendingCompletion is an update that is pending completion. 118 type pendingCompletion struct { 119 // version is the version to be ACKed. 120 version uint64 121 122 // typeURL is the type URL of the resources to be ACKed. 123 typeURL string 124 125 // remainingNodesResources maps each pending node ID to pending resource 126 // name. 127 remainingNodesResources map[string]map[string]struct{} 128 } 129 130 // NewAckingResourceMutatorWrapper creates a new AckingResourceMutatorWrapper 131 // to wrap the given ResourceMutator. 132 func NewAckingResourceMutatorWrapper(mutator ResourceMutator) *AckingResourceMutatorWrapper { 133 return &AckingResourceMutatorWrapper{ 134 mutator: mutator, 135 ackedVersions: make(map[string]uint64), 136 pendingCompletions: make(map[*completion.Completion]*pendingCompletion), 137 } 138 } 139 140 func (m *AckingResourceMutatorWrapper) MarkRestorePending() { 141 m.locker.Lock() 142 defer m.locker.Unlock() 143 144 m.restoring = true 145 } 146 147 // MarkRestoreCompleted clears the 'restore' state so that updates are acked normally. 148 func (m *AckingResourceMutatorWrapper) MarkRestoreCompleted() { 149 m.locker.Lock() 150 defer m.locker.Unlock() 151 152 m.restoring = false 153 } 154 155 // AddVersionCompletion adds a completion to wait for any ACK for the 156 // version and type URL, ignoring the ACKed resource names. 157 func (m *AckingResourceMutatorWrapper) addVersionCompletion(typeURL string, version uint64, nodeIDs []string, c *completion.Completion) { 158 comp := &pendingCompletion{ 159 version: version, 160 typeURL: typeURL, 161 remainingNodesResources: make(map[string]map[string]struct{}, len(nodeIDs)), 162 } 163 for _, nodeID := range nodeIDs { 164 comp.remainingNodesResources[nodeID] = nil 165 } 166 m.pendingCompletions[c] = comp 167 } 168 169 // DeleteNode frees resources held for the named nodes 170 func (m *AckingResourceMutatorWrapper) DeleteNode(nodeID string) { 171 m.locker.Lock() 172 defer m.locker.Unlock() 173 174 delete(m.ackedVersions, nodeID) 175 } 176 177 func (m *AckingResourceMutatorWrapper) Upsert(typeURL string, resourceName string, resource proto.Message, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc { 178 m.locker.Lock() 179 defer m.locker.Unlock() 180 181 wait := wg != nil 182 183 if m.restoring { 184 // Do not wait for acks when restoring state 185 log.WithFields(logrus.Fields{ 186 logfields.XDSTypeURL: typeURL, 187 logfields.XDSResourceName: resourceName, 188 }).Debug("Upsert: Restoring, skipping wait for ACK") 189 190 wait = false 191 } 192 193 var updated bool 194 var revert ResourceMutatorRevertFunc 195 m.version, updated, revert = m.mutator.Upsert(typeURL, resourceName, resource) 196 197 if !updated { 198 if wait { 199 m.useCurrent(typeURL, nodeIDs, wg, callback) 200 } else if callback != nil { 201 callback(nil) 202 } 203 return func(completion *completion.Completion) {} 204 } 205 206 if wait { 207 // Create a new completion 208 c := wg.AddCompletionWithCallback(callback) 209 if _, found := m.pendingCompletions[c]; found { 210 log.WithFields(logrus.Fields{ 211 logfields.XDSTypeURL: typeURL, 212 logfields.XDSResourceName: resourceName, 213 }).Fatalf("attempt to reuse completion to upsert xDS resource: %v", c) 214 } 215 216 comp := &pendingCompletion{ 217 version: m.version, 218 typeURL: typeURL, 219 remainingNodesResources: make(map[string]map[string]struct{}, len(nodeIDs)), 220 } 221 for _, nodeID := range nodeIDs { 222 comp.remainingNodesResources[nodeID] = make(map[string]struct{}, 1) 223 comp.remainingNodesResources[nodeID][resourceName] = struct{}{} 224 } 225 m.pendingCompletions[c] = comp 226 } else if callback != nil { 227 callback(nil) 228 } 229 230 // Returned revert function locks again, so it can NOT be called from 'callback' directly, 231 // as 'callback' is called with the lock already held. 232 return func(completion *completion.Completion) { 233 m.locker.Lock() 234 defer m.locker.Unlock() 235 236 if revert != nil { 237 m.version, _ = revert() 238 239 if completion != nil { 240 // We don't know whether the revert did an Upsert or a Delete, so as a 241 // best effort, just wait for any ACK for the version and type URL, 242 // and ignore the ACKed resource names, like for a Delete. 243 m.addVersionCompletion(typeURL, m.version, nodeIDs, completion) 244 } 245 } 246 } 247 } 248 249 func (m *AckingResourceMutatorWrapper) useCurrent(typeURL string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) { 250 if !m.currentVersionAcked(nodeIDs) { 251 // Add a completion object for 'version' so that the caller may wait for the N/ACK 252 m.addVersionCompletion(typeURL, m.version, nodeIDs, wg.AddCompletionWithCallback(callback)) 253 } 254 } 255 256 func (m *AckingResourceMutatorWrapper) currentVersionAcked(nodeIDs []string) bool { 257 for _, node := range nodeIDs { 258 if acked, exists := m.ackedVersions[node]; !exists || acked < m.version { 259 ackLog := log.WithFields(logrus.Fields{ 260 logfields.XDSCachedVersion: m.version, 261 logfields.XDSAckedVersion: acked, 262 logfields.XDSClientNode: node, 263 }) 264 ackLog.Debugf("Node has not acked the current cached version yet") 265 return false 266 } 267 } 268 return true 269 } 270 271 func (m *AckingResourceMutatorWrapper) Delete(typeURL string, resourceName string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc { 272 m.locker.Lock() 273 defer m.locker.Unlock() 274 275 wait := wg != nil 276 277 if m.restoring { 278 // Do not wait for acks when restoring state 279 log.WithFields(logrus.Fields{ 280 logfields.XDSTypeURL: typeURL, 281 logfields.XDSResourceName: resourceName, 282 }).Debug("Delete: Restoring, skipping wait for ACK") 283 284 wait = false 285 } 286 287 // Always delete the resource, even if the completion's context was 288 // canceled before we even started, since we have no way to signal whether 289 // the resource is actually deleted. 290 291 // There is no explicit ACK for resource deletion in the xDS protocol. 292 // As a best effort, just wait for any ACK for the version and type URL, 293 // and ignore the ACKed resource names. 294 295 var updated bool 296 var revert ResourceMutatorRevertFunc 297 m.version, updated, revert = m.mutator.Delete(typeURL, resourceName) 298 299 if !updated { 300 if wait { 301 m.useCurrent(typeURL, nodeIDs, wg, callback) 302 } else if callback != nil { 303 callback(nil) 304 } 305 return func(completion *completion.Completion) {} 306 } 307 308 if wait { 309 c := wg.AddCompletionWithCallback(callback) 310 if _, found := m.pendingCompletions[c]; found { 311 log.WithFields(logrus.Fields{ 312 logfields.XDSTypeURL: typeURL, 313 logfields.XDSResourceName: resourceName, 314 }).Fatalf("attempt to reuse completion to delete xDS resource: %v", c) 315 } 316 317 m.addVersionCompletion(typeURL, m.version, nodeIDs, c) 318 } else if callback != nil { 319 callback(nil) 320 } 321 322 return func(completion *completion.Completion) { 323 m.locker.Lock() 324 defer m.locker.Unlock() 325 326 if revert != nil { 327 m.version, _ = revert() 328 329 if completion != nil { 330 // We don't know whether the revert had any effect at all, so as a 331 // best effort, just wait for any ACK for the version and type URL, 332 // and ignore the ACKed resource names, like for a Delete. 333 m.addVersionCompletion(typeURL, m.version, nodeIDs, completion) 334 } 335 } 336 } 337 } 338 339 // 'ackVersion' is the last version that was acked. 'nackVersion', if greater than 'nackVersion', is the last version that was NACKed. 340 func (m *AckingResourceMutatorWrapper) HandleResourceVersionAck(ackVersion uint64, nackVersion uint64, nodeIP string, resourceNames []string, typeURL string, detail string) { 341 ackLog := log.WithFields(logrus.Fields{ 342 logfields.XDSAckedVersion: ackVersion, 343 logfields.XDSNonce: nackVersion, 344 logfields.XDSClientNode: nodeIP, 345 logfields.XDSTypeURL: typeURL, 346 }) 347 348 m.locker.Lock() 349 defer m.locker.Unlock() 350 351 // Update the last seen ACKed version if it advances the previously ACKed version. 352 // Version 0 is special as it indicates that we have received the first xDS 353 // resource request from Envoy. Prior to that we do not have a map entry for the 354 // node at all. 355 if previouslyAckedVersion, exists := m.ackedVersions[nodeIP]; !exists || previouslyAckedVersion < ackVersion { 356 m.ackedVersions[nodeIP] = ackVersion 357 } 358 359 remainingCompletions := make(map[*completion.Completion]*pendingCompletion, len(m.pendingCompletions)) 360 361 for comp, pending := range m.pendingCompletions { 362 if comp.Err() != nil { 363 // Completion was canceled or timed out. 364 // Remove from pending list. 365 ackLog.Debugf("completion context was canceled: %v", pending) 366 continue 367 } 368 369 if pending.typeURL == typeURL { 370 if pending.version <= nackVersion { 371 // Get the set of resource names we are still waiting for the node 372 // to ACK. 373 remainingResourceNames, found := pending.remainingNodesResources[nodeIP] 374 if found { 375 for _, name := range resourceNames { 376 delete(remainingResourceNames, name) 377 } 378 if len(remainingResourceNames) == 0 { 379 delete(pending.remainingNodesResources, nodeIP) 380 } 381 if len(pending.remainingNodesResources) == 0 { 382 // completedComparision. Notify and remove from pending list. 383 if pending.version <= ackVersion { 384 ackLog.Debugf("completing ACK: %v", pending) 385 comp.Complete(nil) 386 } else { 387 ackLog.Debugf("completing NACK: %v", pending) 388 comp.Complete(&ProxyError{Err: ErrNackReceived, Detail: detail}) 389 } 390 continue 391 } 392 } 393 } 394 } 395 396 // Completion didn't match or is still waiting for some ACKs. Keep it 397 // in the pending list. 398 remainingCompletions[comp] = pending 399 } 400 401 m.pendingCompletions = remainingCompletions 402 }