github.com/zhyoulun/cilium@v1.6.12/operator/k8s_node.go (about) 1 // Copyright 2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "context" 19 "encoding/json" 20 "strings" 21 "sync" 22 "time" 23 24 "github.com/cilium/cilium/pkg/controller" 25 "github.com/cilium/cilium/pkg/k8s" 26 cilium_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" 27 "github.com/cilium/cilium/pkg/k8s/client/clientset/versioned/typed/cilium.io/v2" 28 "github.com/cilium/cilium/pkg/k8s/informer" 29 "github.com/cilium/cilium/pkg/k8s/utils" 30 k8sversion "github.com/cilium/cilium/pkg/k8s/version" 31 "github.com/cilium/cilium/pkg/kvstore/store" 32 "github.com/cilium/cilium/pkg/node" 33 nodeStore "github.com/cilium/cilium/pkg/node/store" 34 "github.com/cilium/cilium/pkg/option" 35 "github.com/cilium/cilium/pkg/serializer" 36 "github.com/cilium/cilium/pkg/source" 37 38 "k8s.io/api/core/v1" 39 core_v1 "k8s.io/api/core/v1" 40 meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 41 "k8s.io/apimachinery/pkg/fields" 42 "k8s.io/apimachinery/pkg/types" 43 "k8s.io/apimachinery/pkg/util/wait" 44 "k8s.io/client-go/tools/cache" 45 ) 46 47 var ( 48 // kvNodeGCInterval duration for which the nodes are GC in the KVStore. 49 kvNodeGCInterval time.Duration 50 enableCNPNodeStatusGC bool 51 ciliumCNPNodeStatusGCInterval time.Duration 52 ) 53 54 func runNodeWatcher() error { 55 log.Info("Starting to synchronize k8s nodes to kvstore...") 56 57 serNodes := serializer.NewFunctionQueue(1024) 58 59 ciliumNodeStore, err := store.JoinSharedStore(store.Configuration{ 60 Prefix: nodeStore.NodeStorePrefix, 61 KeyCreator: nodeStore.KeyCreator, 62 }) 63 if err != nil { 64 return err 65 } 66 67 k8sNodeStore, nodeController := informer.NewInformer( 68 cache.NewListWatchFromClient(k8s.Client().CoreV1().RESTClient(), 69 "nodes", v1.NamespaceAll, fields.Everything()), 70 &v1.Node{}, 71 0, 72 cache.ResourceEventHandlerFuncs{ 73 AddFunc: func(obj interface{}) { 74 if n := k8s.CopyObjToV1Node(obj); n != nil { 75 serNodes.Enqueue(func() error { 76 nodeNew := k8s.ParseNode(n, source.Kubernetes) 77 ciliumNodeStore.UpdateKeySync(nodeNew) 78 return nil 79 }, serializer.NoRetry) 80 } 81 }, 82 UpdateFunc: func(oldObj, newObj interface{}) { 83 if oldNode := k8s.CopyObjToV1Node(oldObj); oldNode != nil { 84 if newNode := k8s.CopyObjToV1Node(newObj); newNode != nil { 85 if k8s.EqualV1Node(oldNode, newNode) { 86 return 87 } 88 89 serNodes.Enqueue(func() error { 90 newNode := k8s.ParseNode(newNode, source.Kubernetes) 91 ciliumNodeStore.UpdateKeySync(newNode) 92 return nil 93 }, serializer.NoRetry) 94 } 95 } 96 }, 97 DeleteFunc: func(obj interface{}) { 98 n := k8s.CopyObjToV1Node(obj) 99 if n == nil { 100 deletedObj, ok := obj.(cache.DeletedFinalStateUnknown) 101 if !ok { 102 return 103 } 104 // Delete was not observed by the watcher but is 105 // removed from kube-apiserver. This is the last 106 // known state and the object no longer exists. 107 n = k8s.CopyObjToV1Node(deletedObj.Obj) 108 if n == nil { 109 return 110 } 111 } 112 serNodes.Enqueue(func() error { 113 deletedNode := k8s.ParseNode(n, source.Kubernetes) 114 ciliumNodeStore.DeleteLocalKey(deletedNode) 115 deleteCiliumNode(n.Name) 116 return nil 117 }, serializer.NoRetry) 118 }, 119 }, 120 k8s.ConvertToNode, 121 ) 122 go nodeController.Run(wait.NeverStop) 123 124 go func() { 125 cache.WaitForCacheSync(wait.NeverStop, nodeController.HasSynced) 126 127 serNodes.Enqueue(func() error { 128 // Since we serialize all events received from k8s we know that 129 // at this point the list in k8sNodeStore should be the source of truth 130 // and we need to delete all nodes in the kvNodeStore that are *not* 131 // present in the k8sNodeStore. 132 133 if enableENI { 134 nodes, err := ciliumK8sClient.CiliumV2().CiliumNodes().List(meta_v1.ListOptions{}) 135 if err != nil { 136 log.WithError(err).Warning("Unable to list CiliumNodes. Won't clean up stale CiliumNodes") 137 } else { 138 for _, node := range nodes.Items { 139 if _, ok, err := k8sNodeStore.GetByKey(node.Name); !ok && err == nil { 140 deleteCiliumNode(node.Name) 141 } 142 } 143 } 144 } 145 146 listOfK8sNodes := k8sNodeStore.ListKeys() 147 148 kvStoreNodes := ciliumNodeStore.SharedKeysMap() 149 for _, k8sNode := range listOfK8sNodes { 150 // The remaining kvStoreNodes are leftovers 151 kvStoreNodeName := node.GetKeyNodeName(option.Config.ClusterName, k8sNode) 152 delete(kvStoreNodes, kvStoreNodeName) 153 } 154 155 for _, kvStoreNode := range kvStoreNodes { 156 if strings.HasPrefix(kvStoreNode.GetKeyName(), option.Config.ClusterName) { 157 ciliumNodeStore.DeleteLocalKey(kvStoreNode) 158 } 159 } 160 161 return nil 162 }, serializer.NoRetry) 163 }() 164 165 go func() { 166 if !enableCNPNodeStatusGC { 167 return 168 } 169 parallelRequests := 4 170 removeNodeFromCNP := make(chan func(), 50) 171 for i := 0; i < parallelRequests; i++ { 172 go func() { 173 for f := range removeNodeFromCNP { 174 f() 175 } 176 }() 177 } 178 controller.NewManager().UpdateController("cnp-node-gc", 179 controller.ControllerParams{ 180 RunInterval: ciliumCNPNodeStatusGCInterval, 181 DoFunc: func(ctx context.Context) error { 182 lastRun := time.Now().Add(-kvNodeGCInterval) 183 k8sCapabilities := k8sversion.Capabilities() 184 continueID := "" 185 wg := sync.WaitGroup{} 186 defer wg.Wait() 187 kvStoreNodes := ciliumNodeStore.SharedKeysMap() 188 for { 189 cnpList, err := ciliumK8sClient.CiliumV2().CiliumNetworkPolicies(core_v1.NamespaceAll).List(meta_v1.ListOptions{ 190 Limit: 10, 191 Continue: continueID, 192 }) 193 if err != nil { 194 return err 195 } 196 197 for _, cnp := range cnpList.Items { 198 needsUpdate := false 199 nodesToDelete := map[string]cilium_v2.Timestamp{} 200 for n, status := range cnp.Status.Nodes { 201 kvStoreNodeName := node.GetKeyNodeName(option.Config.ClusterName, n) 202 if _, exists := kvStoreNodes[kvStoreNodeName]; !exists { 203 // To avoid concurrency issues where a is 204 // created and adds its CNP Status before the operator 205 // node watcher receives an event that the node 206 // was created, we will only delete the node 207 // from the CNP Status if the last time it was 208 // update was before the lastRun. 209 if status.LastUpdated.Before(lastRun) { 210 nodesToDelete[n] = status.LastUpdated 211 delete(cnp.Status.Nodes, n) 212 needsUpdate = true 213 } 214 } 215 } 216 if needsUpdate { 217 wg.Add(1) 218 cnpCpy := cnp.DeepCopy() 219 removeNodeFromCNP <- func() { 220 updateCNP(ciliumK8sClient.CiliumV2(), cnpCpy, nodesToDelete, k8sCapabilities) 221 wg.Done() 222 } 223 } 224 } 225 226 continueID = cnpList.Continue 227 if continueID == "" { 228 break 229 } 230 } 231 232 return nil 233 }, 234 }) 235 }() 236 237 return nil 238 } 239 240 func updateCNP(ciliumClient v2.CiliumV2Interface, cnp *cilium_v2.CiliumNetworkPolicy, nodesToDelete map[string]cilium_v2.Timestamp, capabilities k8sversion.ServerCapabilities) { 241 if len(nodesToDelete) == 0 { 242 return 243 } 244 245 ns := utils.ExtractNamespace(&cnp.ObjectMeta) 246 247 switch { 248 case capabilities.Patch: 249 var removeStatusNode, remainingStatusNode []k8s.JSONPatch 250 for nodeToDelete, timeStamp := range nodesToDelete { 251 removeStatusNode = append(removeStatusNode, 252 // It is really unlikely to happen but if a node reappears 253 // with the same name and updates the CNP Status we will perform 254 // a test to verify if the lastUpdated timestamp is the same to 255 // to avoid accidentally deleting that node. 256 // If any of the nodes fails this test *all* of the JSON patch 257 // will not be executed. 258 k8s.JSONPatch{ 259 OP: "test", 260 Path: "/status/nodes/" + nodeToDelete + "/lastUpdated", 261 Value: timeStamp, 262 }, 263 k8s.JSONPatch{ 264 OP: "remove", 265 Path: "/status/nodes/" + nodeToDelete, 266 }, 267 ) 268 } 269 for { 270 if len(removeStatusNode) > k8s.MaxJSONPatchOperations { 271 remainingStatusNode = removeStatusNode[k8s.MaxJSONPatchOperations:] 272 removeStatusNode = removeStatusNode[:k8s.MaxJSONPatchOperations] 273 } 274 275 removeStatusNodeJSON, err := json.Marshal(removeStatusNode) 276 if err != nil { 277 break 278 } 279 280 _, err = ciliumClient.CiliumNetworkPolicies(ns).Patch(cnp.GetName(), types.JSONPatchType, removeStatusNodeJSON, "status") 281 if err != nil { 282 // We can leave the errors as debug as the GC happens on a best effort 283 log.WithError(err).Debug("Unable to PATCH") 284 } 285 286 removeStatusNode = remainingStatusNode 287 288 if len(remainingStatusNode) == 0 { 289 return 290 } 291 } 292 case capabilities.UpdateStatus: 293 // This should be treat is as best effort, we don't care if the 294 // UpdateStatus fails. 295 _, err := ciliumClient.CiliumNetworkPolicies(ns).UpdateStatus(cnp) 296 if err != nil { 297 // We can leave the errors as debug as the GC happens on a best effort 298 log.WithError(err).Debug("Unable to UpdateStatus with garbage collected nodes") 299 } 300 default: 301 // This should be treat is as best effort, we don't care if the 302 // Update fails. 303 _, err := ciliumClient.CiliumNetworkPolicies(ns).Update(cnp) 304 if err != nil { 305 // We can leave the errors as debug as the GC happens on a best effort 306 log.WithError(err).Debug("Unable to Update CNP with garbage collected nodes") 307 } 308 } 309 }