github.com/looshlee/cilium@v1.6.12/daemon/loadbalancer.go (about) 1 // Copyright 2016-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "fmt" 19 "time" 20 21 "github.com/cilium/cilium/api/v1/models" 22 . "github.com/cilium/cilium/api/v1/server/restapi/service" 23 "github.com/cilium/cilium/pkg/api" 24 "github.com/cilium/cilium/pkg/loadbalancer" 25 "github.com/cilium/cilium/pkg/logging/logfields" 26 "github.com/cilium/cilium/pkg/maps/lbmap" 27 "github.com/cilium/cilium/pkg/option" 28 "github.com/cilium/cilium/pkg/service" 29 30 "github.com/go-openapi/runtime/middleware" 31 "github.com/sirupsen/logrus" 32 ) 33 34 type errSVCNotFound struct { 35 frontend string 36 } 37 38 func (e *errSVCNotFound) Error() string { 39 return fmt.Sprintf("Service frontend not found: %s", e.frontend) 40 } 41 42 // addSVC2BPFMap adds the given bpf service to the bpf maps. If addRevNAT is set, adds the 43 // RevNAT value (feCilium.L3n4Addr) to the lb's RevNAT map for the given feCilium.ID. 44 func (d *Daemon) addSVC2BPFMap(feCilium loadbalancer.L3n4AddrID, feBPF lbmap.ServiceKey, 45 besBPF []lbmap.ServiceValue, 46 svcKeyV2 lbmap.ServiceKeyV2, svcValuesV2 []lbmap.ServiceValueV2, backendsV2 []lbmap.Backend, 47 addRevNAT bool) error { 48 log.WithField(logfields.ServiceName, feCilium.String()).Debug("adding service to BPF maps") 49 50 revNATID := int(feCilium.ID) 51 52 if err := lbmap.UpdateService(feBPF, besBPF, addRevNAT, revNATID, 53 service.AcquireBackendID, service.DeleteBackendID); err != nil { 54 if addRevNAT { 55 delete(d.loadBalancer.RevNATMap, loadbalancer.ServiceID(feCilium.ID)) 56 } 57 return err 58 } 59 60 if addRevNAT { 61 log.WithField(logfields.ServiceName, feCilium.String()).Debug("adding service to RevNATMap") 62 d.loadBalancer.RevNATMap[loadbalancer.ServiceID(feCilium.ID)] = *feCilium.L3n4Addr.DeepCopy() 63 } 64 return nil 65 } 66 67 // SVCAdd is the public method to add services. We assume the ID provided is not in 68 // sync with the KVStore. If that's the, case the service won't be used and an error is 69 // returned to the caller. 70 // 71 // Returns true if service was created. 72 func (d *Daemon) SVCAdd(feL3n4Addr loadbalancer.L3n4AddrID, be []loadbalancer.LBBackEnd, addRevNAT bool) (bool, error) { 73 log.WithField(logfields.ServiceID, feL3n4Addr.String()).Debug("adding service") 74 if feL3n4Addr.ID == 0 { 75 return false, fmt.Errorf("invalid service ID 0") 76 } 77 // Check if the service is already registered with this ID. 78 feAddr, err := service.GetID(uint32(feL3n4Addr.ID)) 79 if err != nil { 80 return false, fmt.Errorf("unable to get service %d: %s", feL3n4Addr.ID, err) 81 } 82 if feAddr == nil { 83 feAddr, err = service.AcquireID(feL3n4Addr.L3n4Addr, uint32(feL3n4Addr.ID)) 84 if err != nil { 85 return false, fmt.Errorf("unable to store service %s in kvstore: %s", feL3n4Addr.String(), err) 86 } 87 // This won't be atomic so we need to check if the baseID, feL3n4Addr.ID was given to the service 88 if feAddr.ID != feL3n4Addr.ID { 89 return false, fmt.Errorf("the service provided %s is already registered with ID %d, please use that ID instead of %d", feL3n4Addr.L3n4Addr.String(), feAddr.ID, feL3n4Addr.ID) 90 } 91 } 92 93 feAddr256Sum := feAddr.L3n4Addr.SHA256Sum() 94 feL3n4Addr256Sum := feL3n4Addr.L3n4Addr.SHA256Sum() 95 96 if feAddr256Sum != feL3n4Addr256Sum { 97 return false, fmt.Errorf("service ID %d is already registered to L3n4Addr %s, please choose a different ID", feL3n4Addr.ID, feAddr.String()) 98 } 99 100 return d.svcAdd(feL3n4Addr, be, addRevNAT, false) 101 } 102 103 // svcAdd adds a service from the given feL3n4Addr (frontend) and LBBackEnd (backends). 104 // If addRevNAT is set, the RevNAT entry is also created for this particular service. 105 // If any of the backend addresses set in bes have a different L3 address type than the 106 // one set in fe, it returns an error without modifying the bpf LB map. If any backend 107 // entry fails while updating the LB map, the frontend won't be inserted in the LB map 108 // therefore there won't be any traffic going to the given backends. 109 // All of the backends added will be DeepCopied to the internal load balancer map. 110 func (d *Daemon) svcAdd( 111 feL3n4Addr loadbalancer.L3n4AddrID, bes []loadbalancer.LBBackEnd, 112 addRevNAT, nodePort bool) (bool, error) { 113 114 scopedLog := log.WithFields(logrus.Fields{ 115 logfields.ServiceID: feL3n4Addr.String(), 116 logfields.Object: logfields.Repr(bes), 117 }) 118 scopedLog.Debug("adding service") 119 120 // Move the slice to the loadbalancer map which has a mutex. If we don't 121 // copy the slice we might risk changing memory that should be locked. 122 beCpy := []loadbalancer.LBBackEnd{} 123 for _, v := range bes { 124 beCpy = append(beCpy, v) 125 } 126 127 svc := loadbalancer.LBSVC{ 128 FE: feL3n4Addr, 129 BES: beCpy, 130 Sha256: feL3n4Addr.L3n4Addr.SHA256Sum(), 131 NodePort: nodePort, 132 } 133 134 fe, besValues, err := lbmap.LBSVC2ServiceKeynValue(svc) 135 if err != nil { 136 return false, err 137 } 138 139 svcKeyV2, svcValuesV2, backendsV2, err := lbmap.LBSVC2ServiceKeynValuenBackendV2(&svc) 140 if err != nil { 141 return false, err 142 } 143 144 d.loadBalancer.BPFMapMU.Lock() 145 defer d.loadBalancer.BPFMapMU.Unlock() 146 147 err = d.addSVC2BPFMap(feL3n4Addr, fe, besValues, svcKeyV2, svcValuesV2, backendsV2, addRevNAT) 148 if err != nil { 149 return false, err 150 } 151 152 // Fill the just acquired backend IDs to ensure the consistent ordering 153 // of the backends when listing services. This step will go away once 154 // we start acquiring backend IDs in this module. 155 for i, be := range svc.BES { 156 id, err := service.LookupBackendID(be.L3n4Addr) 157 if err != nil { 158 scopedLog.WithField(logfields.BackendName, be.L3n4Addr).WithError(err). 159 Warning("Unable to lookup backend ID") 160 continue 161 } 162 svc.BES[i].ID = id 163 } 164 165 return d.loadBalancer.AddService(svc), nil 166 } 167 168 type putServiceID struct { 169 d *Daemon 170 } 171 172 func NewPutServiceIDHandler(d *Daemon) PutServiceIDHandler { 173 return &putServiceID{d: d} 174 } 175 176 func (h *putServiceID) Handle(params PutServiceIDParams) middleware.Responder { 177 log.WithField(logfields.Params, logfields.Repr(params)).Debug("PUT /service/{id} request") 178 179 f, err := loadbalancer.NewL3n4AddrFromModel(params.Config.FrontendAddress) 180 if err != nil { 181 return api.Error(PutServiceIDInvalidFrontendCode, err) 182 } 183 184 frontend := loadbalancer.L3n4AddrID{ 185 L3n4Addr: *f, 186 ID: loadbalancer.ID(params.Config.ID), 187 } 188 189 backends := []loadbalancer.LBBackEnd{} 190 for _, v := range params.Config.BackendAddresses { 191 b, err := loadbalancer.NewLBBackEndFromBackendModel(v) 192 if err != nil { 193 return api.Error(PutServiceIDInvalidBackendCode, err) 194 } 195 backends = append(backends, *b) 196 } 197 198 revnat := false 199 if params.Config.Flags != nil { 200 revnat = params.Config.Flags.DirectServerReturn 201 } 202 203 // FIXME 204 // Add flag to indicate whether service should be registered in 205 // global key value store 206 207 if created, err := h.d.SVCAdd(frontend, backends, revnat); err != nil { 208 return api.Error(PutServiceIDFailureCode, err) 209 } else if created { 210 return NewPutServiceIDCreated() 211 } else { 212 return NewPutServiceIDOK() 213 } 214 } 215 216 type deleteServiceID struct { 217 d *Daemon 218 } 219 220 func NewDeleteServiceIDHandler(d *Daemon) DeleteServiceIDHandler { 221 return &deleteServiceID{d: d} 222 } 223 224 func (h *deleteServiceID) Handle(params DeleteServiceIDParams) middleware.Responder { 225 log.WithField(logfields.Params, logfields.Repr(params)).Debug("DELETE /service/{id} request") 226 227 d := h.d 228 d.loadBalancer.BPFMapMU.Lock() 229 defer d.loadBalancer.BPFMapMU.Unlock() 230 231 svc, ok := d.loadBalancer.SVCMapID[loadbalancer.ServiceID(params.ID)] 232 233 if !ok { 234 return NewDeleteServiceIDNotFound() 235 } 236 237 // FIXME: How to handle error? 238 err := service.DeleteID(uint32(params.ID)) 239 240 if err != nil { 241 log.WithError(err).Warn("error, DeleteL3n4AddrIDByUUID failed") 242 } 243 244 if err := h.d.svcDelete(svc); err != nil { 245 log.WithError(err).WithField(logfields.Object, logfields.Repr(svc)).Warn("DELETE /service/{id}: error deleting service") 246 return api.Error(DeleteServiceIDFailureCode, err) 247 } 248 249 return NewDeleteServiceIDOK() 250 } 251 252 func (d *Daemon) svcDeleteByFrontendLocked(frontend *loadbalancer.L3n4Addr) error { 253 svc, ok := d.loadBalancer.SVCMap[frontend.SHA256Sum()] 254 if !ok { 255 return &errSVCNotFound{frontend: frontend.String()} 256 } 257 return d.svcDelete(&svc) 258 } 259 260 // Deletes a service by the frontend address 261 func (d *Daemon) svcDeleteByFrontend(frontend *loadbalancer.L3n4Addr) error { 262 d.loadBalancer.BPFMapMU.Lock() 263 defer d.loadBalancer.BPFMapMU.Unlock() 264 265 return d.svcDeleteByFrontendLocked(frontend) 266 } 267 268 func (d *Daemon) svcDelete(svc *loadbalancer.LBSVC) error { 269 if err := d.svcDeleteBPF(svc.FE); err != nil { 270 return err 271 } 272 273 d.loadBalancer.DeleteService(svc) 274 275 return nil 276 } 277 278 func (d *Daemon) svcDeleteBPF(svc loadbalancer.L3n4AddrID) error { 279 if err := lbmap.DeleteServiceV2(svc, service.DeleteBackendID); err != nil { 280 return fmt.Errorf("Deleting service from BPF maps failed: %s", err) 281 } 282 283 lbmap.DeleteServiceCache(svc) 284 285 return nil 286 } 287 288 type getServiceID struct { 289 daemon *Daemon 290 } 291 292 func NewGetServiceIDHandler(d *Daemon) GetServiceIDHandler { 293 return &getServiceID{daemon: d} 294 } 295 296 func (h *getServiceID) Handle(params GetServiceIDParams) middleware.Responder { 297 log.WithField(logfields.Params, logfields.Repr(params)).Debug("GET /service/{id} request") 298 299 d := h.daemon 300 301 d.loadBalancer.BPFMapMU.RLock() 302 defer d.loadBalancer.BPFMapMU.RUnlock() 303 304 if svc, ok := d.loadBalancer.SVCMapID[loadbalancer.ServiceID(params.ID)]; ok { 305 return NewGetServiceIDOK().WithPayload(svc.GetModel()) 306 } 307 return NewGetServiceIDNotFound() 308 } 309 310 // SVCGetBySHA256Sum returns a DeepCopied frontend with its backends. 311 func (d *Daemon) svcGetBySHA256Sum(feL3n4SHA256Sum string) *loadbalancer.LBSVC { 312 d.loadBalancer.BPFMapMU.RLock() 313 defer d.loadBalancer.BPFMapMU.RUnlock() 314 315 v, ok := d.loadBalancer.SVCMap[feL3n4SHA256Sum] 316 if !ok { 317 return nil 318 } 319 // We will move the slice from the loadbalancer map which has a mutex. If 320 // we don't copy the slice we might risk changing memory that should be 321 // locked. 322 beCpy := []loadbalancer.LBBackEnd{} 323 for _, v := range v.BES { 324 beCpy = append(beCpy, v) 325 } 326 return &loadbalancer.LBSVC{ 327 FE: *v.FE.DeepCopy(), 328 BES: beCpy, 329 } 330 } 331 332 type getService struct { 333 d *Daemon 334 } 335 336 func NewGetServiceHandler(d *Daemon) GetServiceHandler { 337 return &getService{d: d} 338 } 339 340 func (h *getService) Handle(params GetServiceParams) middleware.Responder { 341 log.WithField(logfields.Params, logfields.Repr(params)).Debug("GET /service request") 342 list := h.d.GetServiceList() 343 return NewGetServiceOK().WithPayload(list) 344 } 345 346 // RevNATAdd deep copies the given revNAT address to the cilium lbmap with the given id. 347 func (d *Daemon) RevNATAdd(id loadbalancer.ServiceID, revNAT loadbalancer.L3n4Addr) error { 348 revNATK, revNATV := lbmap.L3n4Addr2RevNatKeynValue(id, revNAT) 349 350 d.loadBalancer.BPFMapMU.Lock() 351 defer d.loadBalancer.BPFMapMU.Unlock() 352 353 err := lbmap.UpdateRevNat(revNATK, revNATV) 354 if err != nil { 355 return err 356 } 357 358 d.loadBalancer.RevNATMap[id] = *revNAT.DeepCopy() 359 return nil 360 } 361 362 // RevNATDelete deletes the revNatKey from the local bpf map. 363 func (d *Daemon) RevNATDelete(id loadbalancer.ServiceID) error { 364 d.loadBalancer.BPFMapMU.Lock() 365 defer d.loadBalancer.BPFMapMU.Unlock() 366 367 revNAT, ok := d.loadBalancer.RevNATMap[id] 368 if !ok { 369 return nil 370 } 371 372 err := lbmap.DeleteRevNATBPF(id, revNAT.IsIPv6()) 373 374 // TODO should we delete even if err is != nil? 375 if err == nil { 376 delete(d.loadBalancer.RevNATMap, id) 377 } 378 return err 379 } 380 381 // RevNATDeleteAll deletes all RevNAT4, if IPv4 is enabled on daemon, and all RevNAT6 382 // stored on the daemon and on the bpf maps. 383 // 384 // Must be called with d.loadBalancer.BPFMapMU locked. 385 func (d *Daemon) RevNATDeleteAll() error { 386 if option.Config.EnableIPv4 { 387 if err := lbmap.RevNat4Map.DeleteAll(); err != nil { 388 return err 389 } 390 } 391 392 if option.Config.EnableIPv6 { 393 if err := lbmap.RevNat6Map.DeleteAll(); err != nil { 394 return err 395 } 396 } 397 398 // TODO should we delete even if err is != nil? 399 d.loadBalancer.RevNATMap = map[loadbalancer.ServiceID]loadbalancer.L3n4Addr{} 400 return nil 401 } 402 403 // RevNATGet returns a DeepCopy of the revNAT found with the given ID or nil if not found. 404 func (d *Daemon) RevNATGet(id loadbalancer.ServiceID) (*loadbalancer.L3n4Addr, error) { 405 d.loadBalancer.BPFMapMU.RLock() 406 defer d.loadBalancer.BPFMapMU.RUnlock() 407 408 revNAT, ok := d.loadBalancer.RevNATMap[id] 409 if !ok { 410 return nil, nil 411 } 412 return revNAT.DeepCopy(), nil 413 } 414 415 // RevNATDump dumps a DeepCopy of the cilium's loadbalancer. 416 func (d *Daemon) RevNATDump() ([]loadbalancer.L3n4AddrID, error) { 417 dump := []loadbalancer.L3n4AddrID{} 418 419 d.loadBalancer.BPFMapMU.RLock() 420 defer d.loadBalancer.BPFMapMU.RUnlock() 421 422 for k, v := range d.loadBalancer.RevNATMap { 423 dump = append(dump, loadbalancer.L3n4AddrID{ 424 ID: loadbalancer.ID(k), 425 L3n4Addr: *v.DeepCopy(), 426 }) 427 } 428 429 return dump, nil 430 } 431 432 func openServiceMaps() error { 433 if err := lbmap.RemoveDeprecatedMaps(); err != nil { 434 return err 435 } 436 437 if option.Config.EnableIPv6 { 438 if _, err := lbmap.Service6MapV2.OpenOrCreate(); err != nil { 439 return err 440 } 441 if _, err := lbmap.Backend6Map.OpenOrCreate(); err != nil { 442 return err 443 } 444 if _, err := lbmap.RevNat6Map.OpenOrCreate(); err != nil { 445 return err 446 } 447 if _, err := lbmap.RRSeq6MapV2.OpenOrCreate(); err != nil { 448 return err 449 } 450 } 451 452 if option.Config.EnableIPv4 { 453 if _, err := lbmap.Service4MapV2.OpenOrCreate(); err != nil { 454 return err 455 } 456 if _, err := lbmap.Backend4Map.OpenOrCreate(); err != nil { 457 return err 458 } 459 if _, err := lbmap.RevNat4Map.OpenOrCreate(); err != nil { 460 return err 461 } 462 if _, err := lbmap.RRSeq4MapV2.OpenOrCreate(); err != nil { 463 return err 464 } 465 } 466 467 return nil 468 } 469 470 // SyncLBMap syncs the bpf lbmap with the daemon's lb map. All bpf entries will overwrite 471 // the daemon's LB map. If the bpf lbmap entry has a different service ID than the 472 // KVStore's ID, that entry will be updated on the bpf map accordingly with the new ID 473 // retrieved from the KVStore. 474 func (d *Daemon) SyncLBMap() error { 475 // Don't bother syncing if we are in dry mode. 476 if option.Config.DryMode { 477 return nil 478 } 479 480 log.Info("Restoring services from BPF maps...") 481 482 d.loadBalancer.BPFMapMU.Lock() 483 defer d.loadBalancer.BPFMapMU.Unlock() 484 485 newSVCMapID := loadbalancer.SVCMapID{} 486 newRevNATMap := loadbalancer.RevNATMap{} 487 failedSyncSVC := []loadbalancer.LBSVC{} 488 failedSyncRevNAT := map[loadbalancer.ServiceID]loadbalancer.L3n4Addr{} 489 490 addSVC2BPFMap := func(oldID loadbalancer.ServiceID, svc loadbalancer.LBSVC) error { 491 scopedLog := log.WithFields(logrus.Fields{ 492 logfields.ServiceID: oldID, 493 logfields.SHA: svc.FE.SHA256Sum(), 494 }) 495 scopedLog.Debug("adding service ID with SHA") 496 497 // check if the ID for revNat is present in the bpf map, update the 498 // reverse nat key, delete the old one. 499 revNAT, ok := newRevNATMap[oldID] 500 if ok { 501 scopedLog.Debug("Service ID is present in BPF map, updating revnat key") 502 revNATK, revNATV := lbmap.L3n4Addr2RevNatKeynValue( 503 loadbalancer.ServiceID(svc.FE.ID), revNAT) 504 err := lbmap.UpdateRevNat(revNATK, revNATV) 505 if err != nil { 506 return fmt.Errorf("Unable to add revNAT: %s: %s."+ 507 " This entry will be removed from the bpf's LB map.", revNAT.String(), err) 508 } 509 510 // Remove the old entry from the bpf map. 511 revNATK, _ = lbmap.L3n4Addr2RevNatKeynValue(oldID, revNAT) 512 if err := lbmap.DeleteRevNat(revNATK); err != nil { 513 scopedLog.WithError(err).Warn("Unable to remove old rev NAT entry") 514 } 515 516 scopedLog.Debug("deleting old ID from newRevNATMap") 517 delete(newRevNATMap, oldID) 518 519 log.WithFields(logrus.Fields{ 520 logfields.ServiceName: svc.FE.String(), 521 "revNAT": revNAT, 522 }).Debug("adding service --> revNAT to newRevNATMap") 523 newRevNATMap[loadbalancer.ServiceID(svc.FE.ID)] = revNAT 524 } 525 526 fe, besValues, err := lbmap.LBSVC2ServiceKeynValue(svc) 527 if err != nil { 528 return fmt.Errorf("Unable to create a BPF key and values for service FE: %s and backends: %+v. Error: %s."+ 529 " This entry will be removed from the bpf's LB map.", svc.FE.String(), svc.BES, err) 530 } 531 532 svcKeyV2, svcValuesV2, backendsV2, err := lbmap.LBSVC2ServiceKeynValuenBackendV2(&svc) 533 if err != nil { 534 return fmt.Errorf("Unable to create a BPF key and values for service v2 FE: %s and backends: %+v. Error: %s."+ 535 " This entry will be removed from the bpf's LB map.", svc.FE.String(), svc.BES, err) 536 } 537 538 err = d.addSVC2BPFMap(svc.FE, fe, besValues, svcKeyV2, svcValuesV2, backendsV2, false) 539 if err != nil { 540 return fmt.Errorf("Unable to add service FE: %s: %s."+ 541 " This entry will be removed from the bpf's LB map.", svc.FE.String(), err) 542 } 543 return nil 544 } 545 546 newSVCMap, newSVCList, lbmapDumpErrors := lbmap.DumpServiceMapsToUserspaceV2() 547 for _, err := range lbmapDumpErrors { 548 log.WithError(err).Warn("Unable to list services in services BPF map") 549 } 550 newRevNATMap, revNATMapDumpErrors := lbmap.DumpRevNATMapsToUserspace() 551 for _, err := range revNATMapDumpErrors { 552 log.WithError(err).Warn("Unable to list services in RevNat BPF map") 553 } 554 555 // Need to do this outside of parseSVCEntries to avoid deadlock, because we 556 // are modifying the BPF maps, and calling Dump on a Map RLocks the maps. 557 for _, svc := range newSVCList { 558 scopedLog := log.WithField(logfields.Object, logfields.Repr(svc)) 559 kvL3n4AddrID, err := service.RestoreID(svc.FE.L3n4Addr, uint32(svc.FE.ID)) 560 if err != nil { 561 scopedLog.WithError(err).Error("Unable to restore service ID") 562 failedSyncSVC = append(failedSyncSVC, *svc) 563 delete(newSVCMap, svc.Sha256) 564 // Don't update the maps of services since the service failed to 565 // sync. 566 continue 567 } 568 569 // Mismatch detected between BPF Maps and KVstore, so we need to update 570 // the ID in the BPF Maps to reflect the ID of the KVstore. 571 if svc.FE.ID != kvL3n4AddrID.ID { 572 scopedLog = scopedLog.WithField(logfields.ServiceID+".new", kvL3n4AddrID.ID) 573 scopedLog.WithError(err).Warning("Service ID in BPF map is out of sync with KVStore. Acquired new ID") 574 575 oldID := loadbalancer.ServiceID(svc.FE.ID) 576 svc.FE.ID = kvL3n4AddrID.ID 577 // If we cannot add the service to the BPF maps, update the list of 578 // services that failed to sync. 579 if err := addSVC2BPFMap(oldID, *svc); err != nil { 580 scopedLog.WithError(err).Error("Unable to synchronize service to BPF map") 581 582 failedSyncSVC = append(failedSyncSVC, *svc) 583 delete(newSVCMap, svc.Sha256) 584 585 revNAT, ok := newRevNATMap[loadbalancer.ServiceID(svc.FE.ID)] 586 if ok { 587 // Revert the old revNAT 588 newRevNATMap[oldID] = revNAT 589 failedSyncRevNAT[loadbalancer.ServiceID(svc.FE.ID)] = revNAT 590 delete(newRevNATMap, loadbalancer.ServiceID(svc.FE.ID)) 591 } 592 // Don't update the maps of services since the service failed to 593 // sync. 594 continue 595 } 596 } 597 newSVCMapID[loadbalancer.ServiceID(svc.FE.ID)] = svc 598 } 599 600 // Clean services and rev nats from BPF maps that failed to be restored. 601 for _, svc := range failedSyncSVC { 602 if err := d.svcDeleteBPF(svc.FE); err != nil { 603 log.WithError(err).WithField(logfields.Object, logfields.Repr(svc)). 604 Warn("Unable to remove unrestorable service from BPF map") 605 } 606 } 607 608 for id, revNAT := range failedSyncRevNAT { 609 var revNATK lbmap.RevNatKey 610 if !revNAT.IsIPv6() { 611 revNATK = lbmap.NewRevNat4Key(uint16(id)) 612 } else { 613 revNATK = lbmap.NewRevNat6Key(uint16(id)) 614 } 615 616 if err := lbmap.DeleteRevNat(revNATK); err != nil { 617 log.WithError(err).WithFields(logrus.Fields{ 618 logfields.ServiceID: id, 619 "revNAT": revNAT, 620 }).Warn("Unable to clean rev NAT from BPF map") 621 } 622 } 623 624 log.WithFields(logrus.Fields{ 625 "restoredServices": len(newSVCMap), 626 "restoredRevNat": len(newRevNATMap), 627 "failedServices": len(failedSyncSVC), 628 "failedRevNat": len(failedSyncRevNAT), 629 }).Info("Restored services from BPF maps") 630 631 d.loadBalancer.SVCMap = newSVCMap 632 d.loadBalancer.SVCMapID = newSVCMapID 633 d.loadBalancer.RevNATMap = newRevNATMap 634 635 return nil 636 } 637 638 // syncLBMapsWithK8s ensures that the only contents of all BPF maps related to 639 // services (loadbalancer, RevNAT - for IPv4 and IPv6) are those that are 640 // sent to Cilium via K8s. This function is intended to be ran as part of a 641 // controller by the daemon when bootstrapping, although it could be called 642 // elsewhere it needed. Returns an error if any issues occur dumping BPF maps 643 // or deleting entries from BPF maps. 644 func (d *Daemon) syncLBMapsWithK8s() error { 645 k8sDeletedServices := map[string]loadbalancer.L3n4AddrID{} 646 alreadyChecked := map[string]struct{}{} 647 648 // Maps service IDs to whether they are IPv6 (true) or IPv4 (false). 649 k8sDeletedRevNATS := make(map[loadbalancer.ServiceID]bool) 650 651 // Set of L3n4Addrs in string form for storage as a key in map. 652 k8sServicesFrontendAddresses := d.k8sSvcCache.UniqueServiceFrontends() 653 654 // NOTE: loadBalancer.BPFMapMU should be taken after k8sSvcCache.Mutex 655 // has been released, otherwise a deadlock can happen: See GH-8764. 656 d.loadBalancer.BPFMapMU.Lock() 657 defer d.loadBalancer.BPFMapMU.Unlock() 658 659 log.Debugf("dumping BPF service maps to userspace") 660 // At this point the creation of the v2 svc from the corresponding legacy 661 // one has already happened, so it's safe to rely on the v2 when dumping 662 _, newSVCList, lbmapDumpErrors := lbmap.DumpServiceMapsToUserspaceV2() 663 664 if len(lbmapDumpErrors) > 0 { 665 errorStrings := "" 666 for _, err := range lbmapDumpErrors { 667 errorStrings = fmt.Sprintf("%s, %s", err, errorStrings) 668 } 669 return fmt.Errorf("error(s): %s", errorStrings) 670 } 671 672 newRevNATMap, revNATMapDumpErrors := lbmap.DumpRevNATMapsToUserspace() 673 if len(revNATMapDumpErrors) > 0 { 674 errorStrings := "" 675 for _, err := range revNATMapDumpErrors { 676 errorStrings = fmt.Sprintf("%s, %s", err, errorStrings) 677 } 678 return fmt.Errorf("error(s): %s", errorStrings) 679 } 680 681 // Check whether services in service and revNAT BPF maps exist in the 682 // in-memory K8s service maps. If not, mark them for deletion. 683 for _, svc := range newSVCList { 684 id := svc.FE.L3n4Addr.StringWithProtocol() 685 if _, ok := alreadyChecked[id]; ok { 686 continue 687 } 688 689 alreadyChecked[id] = struct{}{} 690 691 scopedLog := log.WithFields(logrus.Fields{ 692 logfields.ServiceID: svc.FE.ID, 693 logfields.L3n4Addr: logfields.Repr(svc.FE.L3n4Addr)}) 694 695 if !k8sServicesFrontendAddresses.LooseMatch(svc.FE.L3n4Addr) { 696 scopedLog.Warning("Deleting no longer present service in datapath") 697 k8sDeletedServices[id] = svc.FE 698 continue 699 } 700 701 scopedLog.Debug("Found matching k8s service for service in BPF map") 702 } 703 704 for serviceID, serviceInfo := range newRevNATMap { 705 if _, ok := d.loadBalancer.RevNATMap[serviceID]; !ok { 706 log.WithFields(logrus.Fields{ 707 logfields.ServiceID: serviceID, 708 logfields.L3n4Addr: logfields.Repr(serviceInfo)}).Debug("revNAT ID read from BPF maps is not managed by K8s; will delete it from BPF maps") 709 // Map service ID to whether service is IPv4 or IPv6. 710 if serviceInfo.IP.To4() == nil { 711 k8sDeletedRevNATS[serviceID] = true 712 } else { 713 k8sDeletedRevNATS[serviceID] = false 714 } 715 } 716 } 717 718 bpfDeleteErrors := []error{} 719 720 // Delete map entries from BPF which don't exist in list of Kubernetes 721 // services. 722 for _, svc := range k8sDeletedServices { 723 svcLogger := log.WithField(logfields.Object, logfields.Repr(svc)) 724 svcLogger.Debug("removing service because it was not synced from Kubernetes") 725 if err := d.svcDeleteBPF(svc); err != nil { 726 bpfDeleteErrors = append(bpfDeleteErrors, err) 727 } 728 } 729 730 for serviceID, isIPv6 := range k8sDeletedRevNATS { 731 log.WithFields(logrus.Fields{logfields.ServiceID: serviceID, "isIPv6": isIPv6}).Debug("removing revNAT because it was not synced from Kubernetes") 732 if err := lbmap.DeleteRevNATBPF(serviceID, isIPv6); err != nil { 733 bpfDeleteErrors = append(bpfDeleteErrors, err) 734 } 735 } 736 737 if len(bpfDeleteErrors) > 0 { 738 bpfErrorsString := "" 739 for _, err := range bpfDeleteErrors { 740 bpfErrorsString = fmt.Sprintf("%s, %s", err, bpfErrorsString) 741 } 742 return fmt.Errorf("Errors deleting BPF map entries: %s", bpfErrorsString) 743 } 744 745 log.Debugf("successfully synced BPF loadbalancer and revNAT maps with in-memory Kubernetes service maps") 746 747 return nil 748 } 749 750 func restoreBackendIDs() (map[lbmap.BackendAddrID]lbmap.BackendKey, error) { 751 lbBackends, err := lbmap.DumpBackendMapsToUserspace() 752 if err != nil { 753 return nil, fmt.Errorf("Unable to dump LB backend maps: %s", err) 754 } 755 756 restoredBackendIDs := map[lbmap.BackendAddrID]lbmap.BackendKey{} 757 758 for addrID, lbBackend := range lbBackends { 759 err := service.RestoreBackendID(lbBackend.L3n4Addr, lbBackend.ID) 760 if err != nil { 761 return nil, err 762 } 763 be, err := lbmap.LBBackEnd2Backend(*lbBackend) 764 if err != nil { 765 return nil, err 766 } 767 restoredBackendIDs[addrID] = be.GetKey() 768 } 769 770 log.WithField(logfields.BackendIDs, restoredBackendIDs). 771 Debug("Restored backend IDs") 772 773 return restoredBackendIDs, nil 774 } 775 776 func restoreServices() { 777 before := time.Now() 778 779 // Restore Backend IDs first, otherwise they can get taken by subsequent 780 // calls to UpdateService 781 restoredBackendIDs, err := restoreBackendIDs() 782 if err != nil { 783 log.WithError(err).Warning("Error occurred while restoring backend IDs") 784 } 785 lbmap.AddBackendIDsToCache(restoredBackendIDs) 786 787 failed, restored, skipped, removed := 0, 0, 0, 0 788 svcIDs := make(map[loadbalancer.ID]struct{}) 789 790 svcMapV2, _, errors := lbmap.DumpServiceMapsToUserspaceV2() 791 for _, err := range errors { 792 log.WithError(err).Warning("Error occurred while dumping service v2 table from datapath") 793 } 794 795 for _, svc := range svcMapV2 { 796 scopedLog := log.WithFields(logrus.Fields{ 797 logfields.ServiceID: svc.FE.ID, 798 logfields.ServiceIP: svc.FE.L3n4Addr.String(), 799 }) 800 // Services where the service ID was missing in the BPF map 801 // cannot be restored 802 if uint32(svc.FE.ID) == uint32(0) { 803 skipped++ 804 continue 805 } 806 807 svcIDs[svc.FE.ID] = struct{}{} 808 809 // The service ID can only be restored when global service IDs 810 // are disabled. Global service IDs require kvstore access but 811 // service load-balancing needs to be enabled before the 812 // kvstore is guaranteed to be connected 813 if option.Config.LBInterface == "" { 814 _, err := service.RestoreID(svc.FE.L3n4Addr, uint32(svc.FE.ID)) 815 if err != nil { 816 failed++ 817 scopedLog.WithError(err).Warning("Unable to restore service ID from datapath") 818 } else { 819 restored++ 820 scopedLog.Debug("Restored service ID from datapath") 821 } 822 } 823 824 // Restore the service cache to guarantee backend ordering 825 // across restarts 826 if err := lbmap.RestoreService(svc); err != nil { 827 scopedLog.WithError(err).Warning("Unable to restore service in cache") 828 failed++ 829 continue 830 } 831 } 832 833 // Remove backend entries which are not used by any service. 834 if errs := lbmap.DeleteOrphanBackends(service.DeleteBackendID); errs != nil && len(errs) > 0 { 835 for _, err := range errs { 836 log.WithError(err).Warning("Unable to remove orphan backend") 837 } 838 } 839 840 log.WithFields(logrus.Fields{ 841 logfields.Duration: time.Now().Sub(before), 842 "restored": restored, 843 "failed": failed, 844 "skipped": skipped, 845 "removed": removed, 846 }).Info("Restore service IDs from BPF maps") 847 } 848 849 // GetServiceList returns list of services 850 func (d *Daemon) GetServiceList() []*models.Service { 851 list := []*models.Service{} 852 853 d.loadBalancer.BPFMapMU.RLock() 854 defer d.loadBalancer.BPFMapMU.RUnlock() 855 856 for _, v := range d.loadBalancer.SVCMap { 857 list = append(list, v.GetModel()) 858 } 859 return list 860 }