github.com/kubearmor/cilium@v1.6.12/operator/k8s_service_sync.go (about) 1 // Copyright 2018 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "time" 19 20 "github.com/cilium/cilium/pkg/k8s" 21 "github.com/cilium/cilium/pkg/k8s/informer" 22 "github.com/cilium/cilium/pkg/kvstore/store" 23 "github.com/cilium/cilium/pkg/loadbalancer" 24 "github.com/cilium/cilium/pkg/logging/logfields" 25 "github.com/cilium/cilium/pkg/metrics" 26 "github.com/cilium/cilium/pkg/option" 27 "github.com/cilium/cilium/pkg/service" 28 29 "github.com/sirupsen/logrus" 30 "k8s.io/api/core/v1" 31 "k8s.io/apimachinery/pkg/fields" 32 "k8s.io/apimachinery/pkg/util/wait" 33 "k8s.io/client-go/tools/cache" 34 ) 35 36 var ( 37 k8sSvcCache = k8s.NewServiceCache() 38 // k8sSvcCacheSynced is used do signalize when all services are synced with 39 // k8s. 40 k8sSvcCacheSynced = make(chan struct{}) 41 servicesStore *store.SharedStore 42 ) 43 44 func k8sServiceHandler() { 45 for { 46 event, ok := <-k8sSvcCache.Events 47 if !ok { 48 return 49 } 50 51 svc := k8s.NewClusterService(event.ID, event.Service, event.Endpoints) 52 svc.Cluster = option.Config.ClusterName 53 54 log.WithFields(logrus.Fields{ 55 logfields.K8sSvcName: event.ID.Name, 56 logfields.K8sNamespace: event.ID.Namespace, 57 "action": event.Action.String(), 58 "service": event.Service.String(), 59 "endpoints": event.Endpoints.String(), 60 "shared": event.Service.Shared, 61 }).Debug("Kubernetes service definition changed") 62 63 if !event.Service.Shared { 64 // The annotation may have been added, delete an eventual existing service 65 servicesStore.DeleteLocalKey(&svc) 66 continue 67 } 68 69 switch event.Action { 70 case k8s.UpdateService, k8s.UpdateIngress: 71 servicesStore.UpdateLocalKeySync(&svc) 72 73 case k8s.DeleteService, k8s.DeleteIngress: 74 servicesStore.DeleteLocalKey(&svc) 75 } 76 } 77 } 78 79 func startSynchronizingServices() { 80 log.Info("Starting to synchronize k8s services to kvstore...") 81 82 readyChan := make(chan struct{}, 0) 83 84 go func() { 85 store, err := store.JoinSharedStore(store.Configuration{ 86 Prefix: service.ServiceStorePrefix, 87 KeyCreator: func() store.Key { 88 return &service.ClusterService{} 89 }, 90 SynchronizationInterval: 5 * time.Minute, 91 }) 92 93 if err != nil { 94 log.WithError(err).Fatal("Unable to join kvstore store to announce services") 95 } 96 97 servicesStore = store 98 close(readyChan) 99 }() 100 101 // Watch for v1.Service changes and push changes into ServiceCache 102 _, svcController := informer.NewInformer( 103 cache.NewListWatchFromClient(k8s.Client().CoreV1().RESTClient(), 104 "services", v1.NamespaceAll, fields.Everything()), 105 &v1.Service{}, 106 0, 107 cache.ResourceEventHandlerFuncs{ 108 AddFunc: func(obj interface{}) { 109 metrics.EventTSK8s.SetToCurrentTime() 110 if k8sSvc := k8s.CopyObjToV1Services(obj); k8sSvc != nil { 111 log.Debugf("Received service addition %+v", k8sSvc) 112 k8sSvcCache.UpdateService(k8sSvc) 113 } 114 }, 115 UpdateFunc: func(oldObj, newObj interface{}) { 116 metrics.EventTSK8s.SetToCurrentTime() 117 if oldk8sSvc := k8s.CopyObjToV1Services(oldObj); oldk8sSvc != nil { 118 if newk8sSvc := k8s.CopyObjToV1Services(newObj); newk8sSvc != nil { 119 if k8s.EqualV1Services(oldk8sSvc, newk8sSvc) { 120 return 121 } 122 123 log.Debugf("Received service update %+v", newk8sSvc) 124 k8sSvcCache.UpdateService(newk8sSvc) 125 } 126 } 127 }, 128 DeleteFunc: func(obj interface{}) { 129 metrics.EventTSK8s.SetToCurrentTime() 130 k8sSvc := k8s.CopyObjToV1Services(obj) 131 if k8sSvc == nil { 132 deletedObj, ok := obj.(cache.DeletedFinalStateUnknown) 133 if !ok { 134 return 135 } 136 // Delete was not observed by the watcher but is 137 // removed from kube-apiserver. This is the last 138 // known state and the object no longer exists. 139 k8sSvc = k8s.CopyObjToV1Services(deletedObj.Obj) 140 if k8sSvc == nil { 141 return 142 } 143 } 144 log.Debugf("Received service deletion %+v", k8sSvc) 145 k8sSvcCache.DeleteService(k8sSvc) 146 }, 147 }, 148 k8s.ConvertToK8sService, 149 ) 150 151 go svcController.Run(wait.NeverStop) 152 153 // Watch for v1.Endpoints changes and push changes into ServiceCache 154 _, endpointController := informer.NewInformer( 155 cache.NewListWatchFromClient(k8s.Client().CoreV1().RESTClient(), 156 "endpoints", v1.NamespaceAll, 157 // Don't get any events from kubernetes endpoints. 158 fields.ParseSelectorOrDie("metadata.name!=kube-scheduler,metadata.name!=kube-controller-manager"), 159 ), 160 &v1.Endpoints{}, 161 0, 162 cache.ResourceEventHandlerFuncs{ 163 AddFunc: func(obj interface{}) { 164 metrics.EventTSK8s.SetToCurrentTime() 165 if k8sEP := k8s.CopyObjToV1Endpoints(obj); k8sEP != nil { 166 k8sSvcCache.UpdateEndpoints(k8sEP) 167 } 168 }, 169 UpdateFunc: func(oldObj, newObj interface{}) { 170 metrics.EventTSK8s.SetToCurrentTime() 171 if oldk8sEP := k8s.CopyObjToV1Endpoints(oldObj); oldk8sEP != nil { 172 if newk8sEP := k8s.CopyObjToV1Endpoints(newObj); newk8sEP != nil { 173 if k8s.EqualV1Endpoints(oldk8sEP, newk8sEP) { 174 return 175 } 176 k8sSvcCache.UpdateEndpoints(newk8sEP) 177 } 178 } 179 }, 180 DeleteFunc: func(obj interface{}) { 181 metrics.EventTSK8s.SetToCurrentTime() 182 k8sEP := k8s.CopyObjToV1Endpoints(obj) 183 if k8sEP == nil { 184 deletedObj, ok := obj.(cache.DeletedFinalStateUnknown) 185 if !ok { 186 return 187 } 188 // Delete was not observed by the watcher but is 189 // removed from kube-apiserver. This is the last 190 // known state and the object no longer exists. 191 k8sEP = k8s.CopyObjToV1Endpoints(deletedObj.Obj) 192 if k8sEP == nil { 193 return 194 } 195 } 196 k8sSvcCache.DeleteEndpoints(k8sEP) 197 }, 198 }, 199 k8s.ConvertToK8sEndpoints, 200 ) 201 202 go endpointController.Run(wait.NeverStop) 203 204 go func() { 205 cache.WaitForCacheSync(wait.NeverStop, svcController.HasSynced) 206 close(k8sSvcCacheSynced) 207 208 cache.WaitForCacheSync(wait.NeverStop, endpointController.HasSynced) 209 }() 210 211 go func() { 212 <-readyChan 213 log.Info("Starting to synchronize Kubernetes services to kvstore") 214 k8sServiceHandler() 215 }() 216 } 217 218 // serviceGetter is a wrapper for 2 k8sCaches, its intention is for 219 // `shortCutK8sCache` to be used until `k8sSvcCacheSynced` is closed, for which 220 // `k8sCache` is started to be used. 221 type serviceGetter struct { 222 shortCutK8sCache k8s.ServiceIPGetter 223 k8sCache k8s.ServiceIPGetter 224 } 225 226 // GetServiceIP returns the result of GetServiceIP for `s.shortCutK8sCache` 227 // until `k8sSvcCacheSynced` is closed. This is helpful as we can have a 228 // shortcut of `s.k8sCache` since we can pre-populate `s.shortCutK8sCache` with 229 // the entries that we need until `s.k8sCache` is synchronized with kubernetes. 230 func (s *serviceGetter) GetServiceIP(svcID k8s.ServiceID) *loadbalancer.L3n4Addr { 231 select { 232 case <-k8sSvcCacheSynced: 233 return s.k8sCache.GetServiceIP(svcID) 234 default: 235 return s.shortCutK8sCache.GetServiceIP(svcID) 236 } 237 }