github.com/kubewharf/katalyst-core@v0.5.3/pkg/controller/kcc/target/target_handler.go (about) 1 /* 2 Copyright 2022 The Katalyst Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package target 18 19 import ( 20 "context" 21 "fmt" 22 "sync" 23 24 apierrors "k8s.io/apimachinery/pkg/api/errors" 25 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 "k8s.io/apimachinery/pkg/labels" 27 utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 "k8s.io/apimachinery/pkg/util/sets" 29 "k8s.io/client-go/tools/cache" 30 "k8s.io/klog/v2" 31 32 configapis "github.com/kubewharf/katalyst-api/pkg/apis/config/v1alpha1" 33 configinformers "github.com/kubewharf/katalyst-api/pkg/client/informers/externalversions/config/v1alpha1" 34 kcclient "github.com/kubewharf/katalyst-core/pkg/client" 35 "github.com/kubewharf/katalyst-core/pkg/config/controller" 36 "github.com/kubewharf/katalyst-core/pkg/util/native" 37 ) 38 39 type KatalystCustomConfigTargetHandler struct { 40 mu sync.RWMutex 41 42 ctx context.Context 43 client *kcclient.GenericClientSet 44 kccConfig *controller.KCCConfig 45 46 syncedFunc []cache.InformerSynced 47 48 // map gvr to kcc key set; actually, it's invalid to hold more than one kcc for 49 // one individual gvr, and should be alerted 50 gvrKatalystCustomConfigMap map[metav1.GroupVersionResource]sets.String 51 // katalystCustomConfigGVRMap map kcc key to gvr; since the gvc in kcc may be unexpected changed 52 // by cases, store in cache to make sure we can still find ite original mapping 53 katalystCustomConfigGVRMap map[string]metav1.GroupVersionResource 54 // map gvr to kcc target accessor 55 targetAccessorMap map[metav1.GroupVersionResource]KatalystCustomConfigTargetAccessor 56 // targetHandlerFuncMap stores those handler functions for all controllers 57 // that are interested in kcc-target changes 58 targetHandlerFuncMap map[string]KatalystCustomConfigTargetHandlerFunc 59 } 60 61 func NewKatalystCustomConfigTargetHandler(ctx context.Context, client *kcclient.GenericClientSet, kccConfig *controller.KCCConfig, 62 katalystCustomConfigInformer configinformers.KatalystCustomConfigInformer, 63 ) *KatalystCustomConfigTargetHandler { 64 k := &KatalystCustomConfigTargetHandler{ 65 ctx: ctx, 66 client: client, 67 kccConfig: kccConfig, 68 syncedFunc: []cache.InformerSynced{ 69 katalystCustomConfigInformer.Informer().HasSynced, 70 }, 71 gvrKatalystCustomConfigMap: make(map[metav1.GroupVersionResource]sets.String), 72 katalystCustomConfigGVRMap: make(map[string]metav1.GroupVersionResource), 73 targetHandlerFuncMap: make(map[string]KatalystCustomConfigTargetHandlerFunc), 74 targetAccessorMap: make(map[metav1.GroupVersionResource]KatalystCustomConfigTargetAccessor), 75 } 76 77 katalystCustomConfigInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ 78 AddFunc: k.addKatalystCustomConfigEventHandle, 79 UpdateFunc: k.updateKatalystCustomConfigEventHandle, 80 DeleteFunc: k.deleteKatalystCustomConfigEventHandle, 81 }) 82 return k 83 } 84 85 // HasSynced whether all cache has synced 86 func (k *KatalystCustomConfigTargetHandler) HasSynced() bool { 87 for _, hasSynced := range k.syncedFunc { 88 if !hasSynced() { 89 return false 90 } 91 } 92 return true 93 } 94 95 func (k *KatalystCustomConfigTargetHandler) Run() { 96 defer k.shutDown() 97 <-k.ctx.Done() 98 } 99 100 // RegisterTargetHandler is used to register handler functions for the given gvr 101 func (k *KatalystCustomConfigTargetHandler) RegisterTargetHandler(name string, handlerFunc KatalystCustomConfigTargetHandlerFunc) { 102 k.mu.Lock() 103 defer k.mu.Unlock() 104 105 k.targetHandlerFuncMap[name] = handlerFunc 106 } 107 108 // GetKCCKeyListByGVR get kcc keyList by gvr. 109 func (k *KatalystCustomConfigTargetHandler) GetKCCKeyListByGVR(gvr metav1.GroupVersionResource) []string { 110 k.mu.RLock() 111 defer k.mu.RUnlock() 112 113 kccKeys, ok := k.gvrKatalystCustomConfigMap[gvr] 114 if ok { 115 return kccKeys.List() 116 } 117 return nil 118 } 119 120 func (k *KatalystCustomConfigTargetHandler) GetTargetAccessorByGVR(gvr metav1.GroupVersionResource) (KatalystCustomConfigTargetAccessor, bool) { 121 k.mu.RLock() 122 defer k.mu.RUnlock() 123 124 accessor, ok := k.targetAccessorMap[gvr] 125 if ok { 126 return accessor, true 127 } 128 return nil, false 129 } 130 131 // RangeGVRTargetAccessor is used to walk through all accessors and perform the given function 132 func (k *KatalystCustomConfigTargetHandler) RangeGVRTargetAccessor(f func(gvr metav1.GroupVersionResource, accessor KatalystCustomConfigTargetAccessor) bool) { 133 k.mu.RLock() 134 defer k.mu.RUnlock() 135 136 for gvr, a := range k.targetAccessorMap { 137 ret := f(gvr, a) 138 if !ret { 139 return 140 } 141 } 142 } 143 144 func (k *KatalystCustomConfigTargetHandler) addKatalystCustomConfigEventHandle(obj interface{}) { 145 kcc, ok := obj.(*configapis.KatalystCustomConfig) 146 if !ok { 147 klog.Errorf("cannot convert obj to *KatalystCustomConfig: %v", obj) 148 return 149 } 150 151 key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(kcc) 152 if err != nil { 153 utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", kcc, err)) 154 return 155 } 156 157 _, err = k.addOrUpdateGVRAndKCC(kcc.Spec.TargetType, key) 158 if err != nil { 159 klog.Errorf("cannot convert add or update gvr %s kcc %s: %v", kcc.Spec.TargetType, key, err) 160 return 161 } 162 } 163 164 func (k *KatalystCustomConfigTargetHandler) updateKatalystCustomConfigEventHandle(old interface{}, new interface{}) { 165 oldKCC, ok := old.(*configapis.KatalystCustomConfig) 166 if !ok { 167 klog.Errorf("cannot convert obj to *KatalystCustomConfig: %v", new) 168 return 169 } 170 171 newKCC, ok := new.(*configapis.KatalystCustomConfig) 172 if !ok { 173 klog.Errorf("cannot convert obj to *KatalystCustomConfig: %v", new) 174 return 175 } 176 177 key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(newKCC) 178 if err != nil { 179 utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", newKCC, err)) 180 return 181 } 182 183 _, err = k.addOrUpdateGVRAndKCC(newKCC.Spec.TargetType, key) 184 if err != nil { 185 klog.Errorf("cannot convert add or update gvr %s kcc %s: %v", newKCC.Spec.TargetType, key, err) 186 return 187 } 188 189 // if kcc has updated, it needs trigger all related kcc target to reconcile 190 if newKCC.GetGeneration() == newKCC.Status.ObservedGeneration && 191 oldKCC.Status.ObservedGeneration != newKCC.Status.ObservedGeneration { 192 accessor, ok := k.GetTargetAccessorByGVR(newKCC.Spec.TargetType) 193 if ok { 194 kccTargets, err := accessor.List(labels.Everything()) 195 if err != nil { 196 klog.Errorf("list gvr %s kcc target failed: %v", newKCC.Spec.TargetType, err) 197 return 198 } 199 200 for _, target := range kccTargets { 201 accessor.Enqueue("", target) 202 } 203 } 204 } 205 } 206 207 func (k *KatalystCustomConfigTargetHandler) deleteKatalystCustomConfigEventHandle(obj interface{}) { 208 kcc, ok := obj.(*configapis.KatalystCustomConfig) 209 if !ok { 210 klog.Errorf("cannot convert obj to *KatalystCustomConfig: %v", obj) 211 return 212 } 213 214 key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(kcc) 215 if err != nil { 216 utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", kcc, err)) 217 return 218 } 219 220 k.deleteGVRAndKCCKey(kcc.Spec.TargetType, key) 221 222 // when some kcc of a gvr has been deleted, we need reconcile its kcc target immediately 223 accessor, ok := k.GetTargetAccessorByGVR(kcc.Spec.TargetType) 224 if ok { 225 kccTargets, err := accessor.List(labels.Everything()) 226 if err != nil { 227 klog.Errorf("list gvr %s kcc target failed: %v", kcc.Spec.TargetType, err) 228 return 229 } 230 231 for _, target := range kccTargets { 232 accessor.Enqueue("", target) 233 } 234 } 235 } 236 237 // addOrUpdateGVRAndKCC add gvr and kcc key to cache and return current kcc keys which use this gvr. 238 func (k *KatalystCustomConfigTargetHandler) addOrUpdateGVRAndKCC(gvr metav1.GroupVersionResource, key string) (KatalystCustomConfigTargetAccessor, error) { 239 k.mu.Lock() 240 defer k.mu.Unlock() 241 old, ok := k.katalystCustomConfigGVRMap[key] 242 if ok && old != gvr { 243 k.deleteGVRAndKCCKeyWithoutLock(old, key) 244 } 245 246 return k.addGVRAndKCCKeyWithoutLock(gvr, key) 247 } 248 249 // deleteGVRAndKCCKey delete gvr and kcc key, return whether it is empty after delete that 250 func (k *KatalystCustomConfigTargetHandler) deleteGVRAndKCCKey(gvr metav1.GroupVersionResource, key string) { 251 k.mu.Lock() 252 defer k.mu.Unlock() 253 k.deleteGVRAndKCCKeyWithoutLock(gvr, key) 254 } 255 256 func (k *KatalystCustomConfigTargetHandler) deleteGVRAndKCCKeyWithoutLock(gvr metav1.GroupVersionResource, key string) { 257 kccKeys, ok := k.gvrKatalystCustomConfigMap[gvr] 258 if ok { 259 delete(kccKeys, key) 260 delete(k.katalystCustomConfigGVRMap, key) 261 } 262 263 if len(kccKeys) == 0 { 264 if accessor, ok := k.targetAccessorMap[gvr]; ok { 265 accessor.Stop() 266 } 267 delete(k.targetAccessorMap, gvr) 268 delete(k.gvrKatalystCustomConfigMap, gvr) 269 } 270 } 271 272 func (k *KatalystCustomConfigTargetHandler) addGVRAndKCCKeyWithoutLock(gvr metav1.GroupVersionResource, key string) (KatalystCustomConfigTargetAccessor, error) { 273 if err := k.checkGVRValid(gvr); err != nil { 274 return nil, err 275 } 276 277 _, ok := k.gvrKatalystCustomConfigMap[gvr] 278 if !ok { 279 _, ok := k.targetAccessorMap[gvr] 280 if !ok { 281 accessor, err := NewRealKatalystCustomConfigTargetAccessor(gvr, 282 k.client.DynamicClient, k.targetHandlerFuncMap) 283 if err != nil { 284 return nil, err 285 } 286 accessor.Start() 287 k.targetAccessorMap[gvr] = accessor 288 } else { 289 klog.Fatalf("gvr of targetAccessorMap %s not exist", gvr.String()) 290 } 291 k.gvrKatalystCustomConfigMap[gvr] = sets.NewString() 292 } 293 k.gvrKatalystCustomConfigMap[gvr].Insert(key) 294 k.katalystCustomConfigGVRMap[key] = gvr 295 return k.targetAccessorMap[gvr], nil 296 } 297 298 // checkGVRValid is used to check whether the given gvr is valid, skip to create corresponding 299 // target accessor otherwise 300 func (k *KatalystCustomConfigTargetHandler) checkGVRValid(gvr metav1.GroupVersionResource) error { 301 if !k.kccConfig.ValidAPIGroupSet.Has(gvr.Group) { 302 return fmt.Errorf("gvr %s is not in valid api group set", gvr.String()) 303 } 304 305 schemaGVR := native.ToSchemaGVR(gvr.Group, gvr.Version, gvr.Resource) 306 resourceList, err := k.client.DiscoveryClient.ServerResourcesForGroupVersion(schemaGVR.GroupVersion().String()) 307 if err != nil { 308 return err 309 } 310 311 for _, resource := range resourceList.APIResources { 312 if resource.Name == gvr.Resource { 313 return nil 314 } 315 } 316 317 return apierrors.NewNotFound(schemaGVR.GroupResource(), schemaGVR.Resource) 318 } 319 320 func (k *KatalystCustomConfigTargetHandler) shutDown() { 321 k.mu.Lock() 322 defer k.mu.Unlock() 323 324 for _, accessor := range k.targetAccessorMap { 325 accessor.Stop() 326 } 327 328 // clear all maps 329 k.targetAccessorMap = make(map[metav1.GroupVersionResource]KatalystCustomConfigTargetAccessor) 330 k.gvrKatalystCustomConfigMap = make(map[metav1.GroupVersionResource]sets.String) 331 k.katalystCustomConfigGVRMap = make(map[string]metav1.GroupVersionResource) 332 }