github.com/imran-kn/cilium-fork@v1.6.9/pkg/envoy/xds/ack.go (about) 1 // Copyright 2018 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package xds 16 17 import ( 18 "errors" 19 20 "github.com/cilium/cilium/pkg/completion" 21 "github.com/cilium/cilium/pkg/lock" 22 "github.com/cilium/cilium/pkg/logging/logfields" 23 24 "github.com/golang/protobuf/proto" 25 "github.com/sirupsen/logrus" 26 ) 27 28 // ProxyError wraps the error and the detail received from the proxy in to a new type 29 // that implements the error interface. 30 type ProxyError struct { 31 Err error 32 Detail string 33 } 34 35 func (pe *ProxyError) Error() string { 36 return pe.Err.Error() + ": " + pe.Detail 37 } 38 39 var ( 40 ErrNackReceived error = errors.New("NACK received") 41 ) 42 43 // ResourceVersionAckObserver defines the HandleResourceVersionAck method 44 // which is called whenever a node acknowledges having applied a version of 45 // the resources of a given type. 46 type ResourceVersionAckObserver interface { 47 // HandleResourceVersionAck notifies that the node with the given NodeIP 48 // has acknowledged having applied the resources. 49 // Calls to this function must not block. 50 HandleResourceVersionAck(ackVersion uint64, nackVersion uint64, nodeIP string, resourceNames []string, typeURL string, detail string) 51 } 52 53 // AckingResourceMutatorRevertFunc is a function which reverts the effects of 54 // an update on a AckingResourceMutator. 55 // The completion is called back when the new resource update is 56 // ACKed by the Envoy nodes. 57 type AckingResourceMutatorRevertFunc func(completion *completion.Completion) 58 59 // AckingResourceMutator is a variant of ResourceMutator which calls back a 60 // Completion when a resource update is ACKed by a set of Envoy nodes. 61 type AckingResourceMutator interface { 62 // Upsert inserts or updates a resource from this set by name and increases 63 // the set's version number atomically if the resource is actually inserted 64 // or updated. 65 // The completion is called back when the new upserted resources' version is 66 // ACKed by the Envoy nodes which IDs are given in nodeIDs. 67 // A call to the returned revert function reverts the effects of this 68 // method call. 69 Upsert(typeURL string, resourceName string, resource proto.Message, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc 70 71 // UseCurrent inserts a completion that allows the caller to wait for the current 72 // version of the given typeURL to be ACKed. 73 UseCurrent(typeURL string, nodeIDs []string, wg *completion.WaitGroup) 74 75 // DeleteNode frees resources held for the named node 76 DeleteNode(nodeID string) 77 78 // Delete deletes a resource from this set by name and increases the cache's 79 // version number atomically if the resource is actually deleted. 80 // The completion is called back when the new deleted resources' version is 81 // ACKed by the Envoy nodes which IDs are given in nodeIDs. 82 // A call to the returned revert function reverts the effects of this 83 // method call. 84 Delete(typeURL string, resourceName string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc 85 } 86 87 // AckingResourceMutatorWrapper is an AckingResourceMutator which wraps a 88 // ResourceMutator to notifies callers when resource updates are ACKed by 89 // nodes. 90 // AckingResourceMutatorWrapper also implements ResourceVersionAckObserver in 91 // order to be notified of ACKs from nodes. 92 type AckingResourceMutatorWrapper struct { 93 // mutator is the wrapped resource mutator. 94 mutator ResourceMutator 95 96 // locker locks all accesses to the remaining fields. 97 locker lock.Mutex 98 99 // Last version stored by 'mutator' 100 version uint64 101 102 // ackedVersions is the last version acked by a node for this cache. 103 // The key is the IPv4 address in string format for an Istio sidecar, 104 // or "127.0.0.1" for the host proxy. 105 ackedVersions map[string]uint64 106 107 // pendingCompletions is the list of updates that are pending completion. 108 pendingCompletions map[*completion.Completion]*pendingCompletion 109 } 110 111 // pendingCompletion is an update that is pending completion. 112 type pendingCompletion struct { 113 // version is the version to be ACKed. 114 version uint64 115 116 // typeURL is the type URL of the resources to be ACKed. 117 typeURL string 118 119 // remainingNodesResources maps each pending node ID to pending resource 120 // name. 121 remainingNodesResources map[string]map[string]struct{} 122 } 123 124 // NewAckingResourceMutatorWrapper creates a new AckingResourceMutatorWrapper 125 // to wrap the given ResourceMutator. 126 func NewAckingResourceMutatorWrapper(mutator ResourceMutator) *AckingResourceMutatorWrapper { 127 return &AckingResourceMutatorWrapper{ 128 mutator: mutator, 129 ackedVersions: make(map[string]uint64), 130 pendingCompletions: make(map[*completion.Completion]*pendingCompletion), 131 } 132 } 133 134 // AddVersionCompletion adds a completion to wait for any ACK for the 135 // version and type URL, ignoring the ACKed resource names. 136 func (m *AckingResourceMutatorWrapper) addVersionCompletion(typeURL string, version uint64, nodeIDs []string, c *completion.Completion) { 137 comp := &pendingCompletion{ 138 version: version, 139 typeURL: typeURL, 140 remainingNodesResources: make(map[string]map[string]struct{}, len(nodeIDs)), 141 } 142 for _, nodeID := range nodeIDs { 143 comp.remainingNodesResources[nodeID] = nil 144 } 145 m.pendingCompletions[c] = comp 146 } 147 148 // DeleteNode frees resources held for the named nodes 149 func (m *AckingResourceMutatorWrapper) DeleteNode(nodeID string) { 150 m.locker.Lock() 151 defer m.locker.Unlock() 152 153 delete(m.ackedVersions, nodeID) 154 } 155 156 func (m *AckingResourceMutatorWrapper) Upsert(typeURL string, resourceName string, resource proto.Message, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc { 157 m.locker.Lock() 158 defer m.locker.Unlock() 159 160 var updated bool 161 var revert ResourceMutatorRevertFunc 162 m.version, updated, revert = m.mutator.Upsert(typeURL, resourceName, resource) 163 164 if !updated { 165 if wg != nil { 166 m.useCurrent(typeURL, nodeIDs, wg) 167 } 168 return func(completion *completion.Completion) {} 169 } 170 171 if wg != nil { 172 c := wg.AddCompletionWithCallback(callback) 173 if _, found := m.pendingCompletions[c]; found { 174 log.WithFields(logrus.Fields{ 175 logfields.XDSTypeURL: typeURL, 176 logfields.XDSResourceName: resourceName, 177 }).Fatalf("attempt to reuse completion to upsert xDS resource: %v", c) 178 } 179 180 comp := &pendingCompletion{ 181 version: m.version, 182 typeURL: typeURL, 183 remainingNodesResources: make(map[string]map[string]struct{}, len(nodeIDs)), 184 } 185 for _, nodeID := range nodeIDs { 186 comp.remainingNodesResources[nodeID] = make(map[string]struct{}, 1) 187 comp.remainingNodesResources[nodeID][resourceName] = struct{}{} 188 } 189 m.pendingCompletions[c] = comp 190 } 191 192 return func(completion *completion.Completion) { 193 m.locker.Lock() 194 defer m.locker.Unlock() 195 196 if revert != nil { 197 m.version, _ = revert() 198 199 if completion != nil { 200 // We don't know whether the revert did an Upsert or a Delete, so as a 201 // best effort, just wait for any ACK for the version and type URL, 202 // and ignore the ACKed resource names, like for a Delete. 203 m.addVersionCompletion(typeURL, m.version, nodeIDs, completion) 204 } 205 } 206 } 207 } 208 209 func (m *AckingResourceMutatorWrapper) useCurrent(typeURL string, nodeIDs []string, wg *completion.WaitGroup) { 210 if !m.currentVersionAcked(nodeIDs) { 211 // Add a completion object for 'version' so that the caller may wait for the N/ACK 212 m.addVersionCompletion(typeURL, m.version, nodeIDs, wg.AddCompletion()) 213 } 214 } 215 216 // UseCurrent adds a completion to the WaitGroup if the current 217 // version of the cached resource has not been acked yet, allowing the 218 // caller to wait for the ACK. 219 func (m *AckingResourceMutatorWrapper) UseCurrent(typeURL string, nodeIDs []string, wg *completion.WaitGroup) { 220 m.locker.Lock() 221 defer m.locker.Unlock() 222 223 m.useCurrent(typeURL, nodeIDs, wg) 224 } 225 226 func (m *AckingResourceMutatorWrapper) currentVersionAcked(nodeIDs []string) bool { 227 for _, node := range nodeIDs { 228 if acked, exists := m.ackedVersions[node]; !exists || acked < m.version { 229 ackLog := log.WithFields(logrus.Fields{ 230 logfields.XDSCachedVersion: m.version, 231 logfields.XDSAckedVersion: acked, 232 logfields.XDSClientNode: node, 233 }) 234 ackLog.Debugf("Node has not acked the current cached version yet") 235 return false 236 } 237 } 238 return true 239 } 240 241 func (m *AckingResourceMutatorWrapper) Delete(typeURL string, resourceName string, nodeIDs []string, wg *completion.WaitGroup, callback func(error)) AckingResourceMutatorRevertFunc { 242 m.locker.Lock() 243 defer m.locker.Unlock() 244 245 // Always delete the resource, even if the completion's context was 246 // canceled before we even started, since we have no way to signal whether 247 // the resource is actually deleted. 248 249 // There is no explicit ACK for resource deletion in the xDS protocol. 250 // As a best effort, just wait for any ACK for the version and type URL, 251 // and ignore the ACKed resource names. 252 253 var updated bool 254 var revert ResourceMutatorRevertFunc 255 m.version, updated, revert = m.mutator.Delete(typeURL, resourceName) 256 257 if !updated { 258 if wg != nil { 259 m.useCurrent(typeURL, nodeIDs, wg) 260 } 261 return func(completion *completion.Completion) {} 262 } 263 264 if wg != nil { 265 c := wg.AddCompletionWithCallback(callback) 266 if _, found := m.pendingCompletions[c]; found { 267 log.WithFields(logrus.Fields{ 268 logfields.XDSTypeURL: typeURL, 269 logfields.XDSResourceName: resourceName, 270 }).Fatalf("attempt to reuse completion to delete xDS resource: %v", c) 271 } 272 273 m.addVersionCompletion(typeURL, m.version, nodeIDs, c) 274 } 275 276 return func(completion *completion.Completion) { 277 m.locker.Lock() 278 defer m.locker.Unlock() 279 280 if revert != nil { 281 m.version, _ = revert() 282 283 if completion != nil { 284 // We don't know whether the revert had any effect at all, so as a 285 // best effort, just wait for any ACK for the version and type URL, 286 // and ignore the ACKed resource names, like for a Delete. 287 m.addVersionCompletion(typeURL, m.version, nodeIDs, completion) 288 } 289 } 290 } 291 } 292 293 // 'ackVersion' is the last version that was acked. 'nackVersion', if greater than 'nackVersion', is the last version that was NACKed. 294 func (m *AckingResourceMutatorWrapper) HandleResourceVersionAck(ackVersion uint64, nackVersion uint64, nodeIP string, resourceNames []string, typeURL string, detail string) { 295 ackLog := log.WithFields(logrus.Fields{ 296 logfields.XDSAckedVersion: ackVersion, 297 logfields.XDSNonce: nackVersion, 298 logfields.XDSClientNode: nodeIP, 299 logfields.XDSTypeURL: typeURL, 300 }) 301 302 m.locker.Lock() 303 defer m.locker.Unlock() 304 305 // Update the last seen ACKed version if it advances the previously ACKed version. 306 // Version 0 is special as it indicates that we have received the first xDS 307 // resource request from Envoy. Prior to that we do not have a map entry for the 308 // node at all. 309 if previouslyAckedVersion, exists := m.ackedVersions[nodeIP]; !exists || previouslyAckedVersion < ackVersion { 310 m.ackedVersions[nodeIP] = ackVersion 311 } 312 313 remainingCompletions := make(map[*completion.Completion]*pendingCompletion, len(m.pendingCompletions)) 314 315 for comp, pending := range m.pendingCompletions { 316 if comp.Err() != nil { 317 // Completion was canceled or timed out. 318 // Remove from pending list. 319 ackLog.Debugf("completion context was canceled: %v", pending) 320 continue 321 } 322 323 if pending.typeURL == typeURL { 324 if pending.version <= nackVersion { 325 // Get the set of resource names we are still waiting for the node 326 // to ACK. 327 remainingResourceNames, found := pending.remainingNodesResources[nodeIP] 328 if found { 329 for _, name := range resourceNames { 330 delete(remainingResourceNames, name) 331 } 332 if len(remainingResourceNames) == 0 { 333 delete(pending.remainingNodesResources, nodeIP) 334 } 335 if len(pending.remainingNodesResources) == 0 { 336 // Completed. Notify and remove from pending list. 337 if pending.version <= ackVersion { 338 ackLog.Debugf("completing ACK: %v", pending) 339 comp.Complete(nil) 340 } else { 341 ackLog.Debugf("completing NACK: %v", pending) 342 comp.Complete(&ProxyError{Err: ErrNackReceived, Detail: detail}) 343 } 344 continue 345 } 346 } 347 } 348 } 349 350 // Completion didn't match or is still waiting for some ACKs. Keep it 351 // in the pending list. 352 remainingCompletions[comp] = pending 353 } 354 355 m.pendingCompletions = remainingCompletions 356 }