github.com/kubearmor/cilium@v1.6.12/pkg/loadbalancer/loadbalancer.go (about) 1 // Copyright 2016-2017 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package loadbalancer 16 17 import ( 18 "crypto/sha512" 19 "fmt" 20 "net" 21 "sort" 22 "strings" 23 24 "github.com/cilium/cilium/api/v1/models" 25 "github.com/cilium/cilium/pkg/lock" 26 "github.com/cilium/cilium/pkg/logging" 27 "github.com/cilium/cilium/pkg/logging/logfields" 28 "github.com/cilium/cilium/pkg/metrics" 29 30 "github.com/sirupsen/logrus" 31 ) 32 33 var ( 34 log = logging.DefaultLogger.WithField(logfields.LogSubsys, "loadbalancer") 35 36 updateMetric = metrics.ServicesCount.WithLabelValues("update") 37 deleteMetric = metrics.ServicesCount.WithLabelValues("delete") 38 addMetric = metrics.ServicesCount.WithLabelValues("add") 39 ) 40 41 const ( 42 NONE = L4Type("NONE") 43 // TCP type. 44 TCP = L4Type("TCP") 45 // UDP type. 46 UDP = L4Type("UDP") 47 ) 48 49 var ( 50 // AllProtocols is the list of all supported L4 protocols 51 AllProtocols = []L4Type{TCP, UDP} 52 ) 53 54 // L4Type name. 55 type L4Type string 56 57 // FEPortName is the name of the frontend's port. 58 type FEPortName string 59 60 // ServiceID is the service's ID. 61 type ServiceID uint16 62 63 // BackendID is the backend's ID. 64 type BackendID uint16 65 66 // ID is the ID of L3n4Addr endpoint (either service or backend). 67 type ID uint32 68 69 // LBBackEnd represents load balancer backend. 70 type LBBackEnd struct { 71 ID BackendID 72 L3n4Addr 73 Weight uint16 74 } 75 76 func (lbbe *LBBackEnd) String() string { 77 return fmt.Sprintf("%s, weight: %d", lbbe.L3n4Addr.String(), lbbe.Weight) 78 } 79 80 // LBSVC is essentially used for the REST API. 81 type LBSVC struct { 82 Sha256 string 83 FE L3n4AddrID 84 BES []LBBackEnd 85 NodePort bool 86 } 87 88 type backendPlacement struct { 89 pos int 90 id BackendID 91 } 92 93 func (s *LBSVC) GetModel() *models.Service { 94 if s == nil { 95 return nil 96 } 97 98 id := int64(s.FE.ID) 99 spec := &models.ServiceSpec{ 100 ID: id, 101 FrontendAddress: s.FE.GetModel(), 102 BackendAddresses: make([]*models.BackendAddress, len(s.BES)), 103 Flags: &models.ServiceSpecFlags{ 104 NodePort: s.NodePort, 105 }, 106 } 107 108 placements := make([]backendPlacement, len(s.BES)) 109 for i, be := range s.BES { 110 placements[i] = backendPlacement{pos: i, id: be.ID} 111 } 112 sort.Slice(placements, 113 func(i, j int) bool { return placements[i].id < placements[j].id }) 114 for i, placement := range placements { 115 spec.BackendAddresses[i] = s.BES[placement.pos].GetBackendModel() 116 } 117 118 return &models.Service{ 119 Spec: spec, 120 Status: &models.ServiceStatus{ 121 Realized: spec, 122 }, 123 } 124 } 125 126 // SVCMap is a map of the daemon's services. The key is the sha256sum of the LBSVC's FE 127 // and the value the LBSVC. 128 type SVCMap map[string]LBSVC 129 130 // SVCMapID maps service IDs to service structures. 131 type SVCMapID map[ServiceID]*LBSVC 132 133 // RevNATMap is a map of the daemon's RevNATs. 134 type RevNATMap map[ServiceID]L3n4Addr 135 136 // LoadBalancer is the internal representation of the loadbalancer in the local cilium 137 // daemon. 138 type LoadBalancer struct { 139 BPFMapMU lock.RWMutex 140 SVCMap SVCMap 141 SVCMapID SVCMapID 142 RevNATMap RevNATMap 143 } 144 145 // AddService adds a service to list of loadbalancers and returns true if created. 146 func (lb *LoadBalancer) AddService(svc LBSVC) bool { 147 scopedLog := log.WithFields(logrus.Fields{ 148 logfields.ServiceName: svc.FE.String(), 149 logfields.SHA: svc.Sha256, 150 }) 151 152 oldSvc, ok := lb.SVCMapID[ServiceID(svc.FE.ID)] 153 if ok { 154 // If service already existed, remove old entry from Cilium's map 155 scopedLog.Debug("service is already in lb.SVCMapID; deleting old entry and updating it with new entry") 156 delete(lb.SVCMap, oldSvc.Sha256) 157 updateMetric.Inc() 158 } else { 159 addMetric.Inc() 160 } 161 scopedLog.Debug("adding service to loadbalancer") 162 lb.SVCMap[svc.Sha256] = svc 163 lb.SVCMapID[ServiceID(svc.FE.ID)] = &svc 164 return !ok 165 } 166 167 // DeleteService deletes svc from lb's SVCMap and SVCMapID. 168 func (lb *LoadBalancer) DeleteService(svc *LBSVC) { 169 log.WithFields(logrus.Fields{ 170 logfields.ServiceName: svc.FE.String(), 171 logfields.SHA: svc.Sha256, 172 }).Debug("deleting service from loadbalancer") 173 delete(lb.SVCMap, svc.Sha256) 174 delete(lb.SVCMapID, ServiceID(svc.FE.ID)) 175 deleteMetric.Inc() 176 } 177 178 func NewL4Type(name string) (L4Type, error) { 179 switch strings.ToLower(name) { 180 case "tcp": 181 return TCP, nil 182 case "udp": 183 return UDP, nil 184 default: 185 return "", fmt.Errorf("unknown L4 protocol") 186 } 187 } 188 189 // NewLoadBalancer returns a LoadBalancer with all maps initialized. 190 func NewLoadBalancer() *LoadBalancer { 191 return &LoadBalancer{ 192 SVCMap: SVCMap{}, 193 SVCMapID: SVCMapID{}, 194 RevNATMap: RevNATMap{}, 195 } 196 } 197 198 // L4Addr is an abstraction for the backend port with a L4Type, usually tcp or udp, and 199 // the Port number. 200 type L4Addr struct { 201 Protocol L4Type 202 Port uint16 203 } 204 205 // NewL4Addr creates a new L4Addr. 206 func NewL4Addr(protocol L4Type, number uint16) *L4Addr { 207 return &L4Addr{Protocol: protocol, Port: number} 208 } 209 210 // Equals returns true if both L4Addr are considered equal. 211 func (l *L4Addr) Equals(o *L4Addr) bool { 212 switch { 213 case (l == nil) != (o == nil): 214 return false 215 case (l == nil) && (o == nil): 216 return true 217 } 218 return l.Port == o.Port && l.Protocol == o.Protocol 219 } 220 221 // DeepCopy returns a DeepCopy of the given L4Addr. 222 func (l *L4Addr) DeepCopy() *L4Addr { 223 return &L4Addr{ 224 Port: l.Port, 225 Protocol: l.Protocol, 226 } 227 } 228 229 // FEPort represents a frontend port with its ID and the L4Addr's inheritance. 230 type FEPort struct { 231 *L4Addr 232 ID ServiceID 233 } 234 235 // NewFEPort creates a new FEPort with the ID set to 0. 236 func NewFEPort(protocol L4Type, portNumber uint16) *FEPort { 237 return &FEPort{L4Addr: NewL4Addr(protocol, portNumber)} 238 } 239 240 // EqualsIgnoreID returns true if both L4Addr are considered equal without 241 // comparing its ID. 242 func (f *FEPort) EqualsIgnoreID(o *FEPort) bool { 243 switch { 244 case (f == nil) != (o == nil): 245 return false 246 case (f == nil) && (o == nil): 247 return true 248 } 249 return f.L4Addr.Equals(o.L4Addr) 250 } 251 252 // Equals returns true if both L4Addr are considered equal. 253 func (f *FEPort) Equals(o *FEPort) bool { 254 switch { 255 case (f == nil) != (o == nil): 256 return false 257 case (f == nil) && (o == nil): 258 return true 259 } 260 return f.EqualsIgnoreID(o) && f.ID == o.ID 261 } 262 263 // L3n4Addr is used to store, as an unique L3+L4 address in the KVStore. 264 type L3n4Addr struct { 265 IP net.IP 266 L4Addr 267 } 268 269 // NewL3n4Addr creates a new L3n4Addr. 270 func NewL3n4Addr(protocol L4Type, ip net.IP, portNumber uint16) *L3n4Addr { 271 lbport := NewL4Addr(protocol, portNumber) 272 273 addr := L3n4Addr{IP: ip, L4Addr: *lbport} 274 log.WithField(logfields.IPAddr, addr).Debug("created new L3n4Addr") 275 276 return &addr 277 } 278 279 func NewL3n4AddrFromModel(base *models.FrontendAddress) (*L3n4Addr, error) { 280 if base == nil { 281 return nil, nil 282 } 283 284 if base.IP == "" { 285 return nil, fmt.Errorf("missing IP address") 286 } 287 288 proto := NONE 289 if base.Protocol != "" { 290 p, err := NewL4Type(base.Protocol) 291 if err != nil { 292 return nil, err 293 } 294 proto = p 295 } 296 297 l4addr := NewL4Addr(proto, base.Port) 298 ip := net.ParseIP(base.IP) 299 if ip == nil { 300 return nil, fmt.Errorf("invalid IP address \"%s\"", base.IP) 301 } 302 303 return &L3n4Addr{IP: ip, L4Addr: *l4addr}, nil 304 } 305 306 // NewLBBackEnd creates the LBBackEnd struct instance from given params. 307 func NewLBBackEnd(id BackendID, protocol L4Type, ip net.IP, portNumber uint16, weight uint16) *LBBackEnd { 308 lbport := NewL4Addr(protocol, portNumber) 309 lbbe := LBBackEnd{ 310 ID: BackendID(id), 311 L3n4Addr: L3n4Addr{IP: ip, L4Addr: *lbport}, 312 Weight: weight, 313 } 314 log.WithField("backend", lbbe).Debug("created new LBBackend") 315 316 return &lbbe 317 } 318 319 func NewLBBackEndFromBackendModel(base *models.BackendAddress) (*LBBackEnd, error) { 320 if base.IP == nil { 321 return nil, fmt.Errorf("missing IP address") 322 } 323 324 // FIXME: Should this be NONE ? 325 l4addr := NewL4Addr(NONE, base.Port) 326 ip := net.ParseIP(*base.IP) 327 if ip == nil { 328 return nil, fmt.Errorf("invalid IP address \"%s\"", *base.IP) 329 } 330 331 return &LBBackEnd{ 332 L3n4Addr: L3n4Addr{IP: ip, L4Addr: *l4addr}, 333 Weight: base.Weight, 334 }, nil 335 } 336 337 func NewL3n4AddrFromBackendModel(base *models.BackendAddress) (*L3n4Addr, error) { 338 if base.IP == nil { 339 return nil, fmt.Errorf("missing IP address") 340 } 341 342 // FIXME: Should this be NONE ? 343 l4addr := NewL4Addr(NONE, base.Port) 344 ip := net.ParseIP(*base.IP) 345 if ip == nil { 346 return nil, fmt.Errorf("invalid IP address \"%s\"", *base.IP) 347 } 348 return &L3n4Addr{IP: ip, L4Addr: *l4addr}, nil 349 } 350 351 func (a *L3n4Addr) GetModel() *models.FrontendAddress { 352 if a == nil { 353 return nil 354 } 355 356 return &models.FrontendAddress{ 357 IP: a.IP.String(), 358 Port: a.Port, 359 } 360 } 361 362 func (b *LBBackEnd) GetBackendModel() *models.BackendAddress { 363 if b == nil { 364 return nil 365 } 366 367 ip := b.IP.String() 368 return &models.BackendAddress{ 369 IP: &ip, 370 Port: b.Port, 371 Weight: b.Weight, 372 } 373 } 374 375 // String returns the L3n4Addr in the "IPv4:Port" format for IPv4 and 376 // "[IPv6]:Port" format for IPv6. 377 func (a *L3n4Addr) String() string { 378 if a.IsIPv6() { 379 return fmt.Sprintf("[%s]:%d", a.IP.String(), a.Port) 380 } 381 return fmt.Sprintf("%s:%d", a.IP.String(), a.Port) 382 } 383 384 // StringWithProtocol returns the L3n4Addr in the "IPv4:Port/Protocol" format 385 // for IPv4 and "[IPv6]:Port/Protocol" format for IPv6. 386 func (a *L3n4Addr) StringWithProtocol() string { 387 if a.IsIPv6() { 388 return fmt.Sprintf("[%s]:%d/%s", a.IP.String(), a.Port, a.Protocol) 389 } 390 return fmt.Sprintf("%s:%d/%s", a.IP.String(), a.Port, a.Protocol) 391 } 392 393 // StringID returns the L3n4Addr as string to be used for unique identification 394 func (a *L3n4Addr) StringID() string { 395 // This does not include the protocol right now as the datapath does 396 // not include the protocol in the lookup of the service IP. 397 return a.String() 398 } 399 400 // DeepCopy returns a DeepCopy of the given L3n4Addr. 401 func (a *L3n4Addr) DeepCopy() *L3n4Addr { 402 copyIP := make(net.IP, len(a.IP)) 403 copy(copyIP, a.IP) 404 return &L3n4Addr{ 405 IP: copyIP, 406 L4Addr: *a.L4Addr.DeepCopy(), 407 } 408 } 409 410 // SHA256Sum calculates L3n4Addr's internal SHA256Sum. 411 func (a L3n4Addr) SHA256Sum() string { 412 // FIXME: Remove Protocol's omission once we care about protocols. 413 protoBak := a.Protocol 414 a.Protocol = "" 415 defer func() { 416 a.Protocol = protoBak 417 }() 418 419 str := []byte(fmt.Sprintf("%+v", a)) 420 return fmt.Sprintf("%x", sha512.Sum512_256(str)) 421 } 422 423 // IsIPv6 returns true if the IP address in the given L3n4Addr is IPv6 or not. 424 func (a *L3n4Addr) IsIPv6() bool { 425 return a.IP.To4() == nil 426 } 427 428 // L3n4AddrID is used to store, as an unique L3+L4 plus the assigned ID, in the 429 // KVStore. 430 type L3n4AddrID struct { 431 L3n4Addr 432 ID ID 433 } 434 435 // NewL3n4AddrID creates a new L3n4AddrID. 436 func NewL3n4AddrID(protocol L4Type, ip net.IP, portNumber uint16, id ID) *L3n4AddrID { 437 l3n4Addr := NewL3n4Addr(protocol, ip, portNumber) 438 return &L3n4AddrID{L3n4Addr: *l3n4Addr, ID: id} 439 } 440 441 // DeepCopy returns a DeepCopy of the given L3n4AddrID. 442 func (l *L3n4AddrID) DeepCopy() *L3n4AddrID { 443 return &L3n4AddrID{ 444 L3n4Addr: *l.L3n4Addr.DeepCopy(), 445 ID: l.ID, 446 } 447 448 } 449 450 // IsIPv6 returns true if the IP address in L3n4Addr's L3n4AddrID is IPv6 or not. 451 func (l *L3n4AddrID) IsIPv6() bool { 452 return l.L3n4Addr.IsIPv6() 453 } 454 455 // Equals checks equality of both given addresses. 456 func (l *L3n4AddrID) Equals(o *L3n4AddrID) bool { 457 switch { 458 case (l == nil) != (o == nil): 459 return false 460 case (l == nil) && (o == nil): 461 return true 462 } 463 464 if l.ID != o.ID { 465 return false 466 } 467 if !l.IP.Equal(o.IP) { 468 return false 469 } 470 if !l.L4Addr.Equals(&o.L4Addr) { 471 return false 472 } 473 474 return true 475 } 476 477 // AddFEnBE adds the given 'fe' and 'be' to the SVCMap. If 'fe' exists and beIndex is 0, 478 // the new 'be' will be appended to the list of existing backends. If beIndex is bigger 479 // than the size of existing backends slice, it will be created a new array with size of 480 // beIndex and the new 'be' will be inserted on index beIndex-1 of that new array. All 481 // remaining be elements will be kept on the same index and, in case the new array is 482 // larger than the number of backends, some elements will be empty. 483 func (svcs SVCMap) AddFEnBE(fe *L3n4AddrID, be *LBBackEnd, beIndex int) *LBSVC { 484 log.WithFields(logrus.Fields{ 485 "frontend": fe, 486 "backend": be, 487 "backendIndex": beIndex, 488 }).Debug("adding frontend and backend to SVCMap") 489 sha := fe.SHA256Sum() 490 491 var lbsvc LBSVC 492 lbsvc, ok := svcs[sha] 493 if !ok { 494 var bes []LBBackEnd 495 if beIndex == 0 { 496 bes = make([]LBBackEnd, 1) 497 bes[0] = *be 498 } else { 499 bes = make([]LBBackEnd, beIndex) 500 bes[beIndex-1] = *be 501 } 502 lbsvc = LBSVC{ 503 FE: *fe, 504 BES: bes, 505 } 506 } else { 507 var bes []LBBackEnd 508 if len(lbsvc.BES) < beIndex { 509 bes = make([]LBBackEnd, beIndex) 510 copy(bes, lbsvc.BES) 511 lbsvc.BES = bes 512 } 513 if beIndex == 0 { 514 lbsvc.BES = append(lbsvc.BES, *be) 515 } else { 516 lbsvc.BES[beIndex-1] = *be 517 } 518 } 519 520 lbsvc.Sha256 = sha 521 svcs[sha] = lbsvc 522 return &lbsvc 523 }