github.com/cilium/cilium@v1.16.2/pkg/alibabacloud/eni/node.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package eni 5 6 import ( 7 "context" 8 "fmt" 9 10 "github.com/sirupsen/logrus" 11 12 "github.com/cilium/cilium/pkg/alibabacloud/eni/limits" 13 eniTypes "github.com/cilium/cilium/pkg/alibabacloud/eni/types" 14 "github.com/cilium/cilium/pkg/alibabacloud/utils" 15 "github.com/cilium/cilium/pkg/defaults" 16 "github.com/cilium/cilium/pkg/ipam" 17 "github.com/cilium/cilium/pkg/ipam/stats" 18 ipamTypes "github.com/cilium/cilium/pkg/ipam/types" 19 v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" 20 "github.com/cilium/cilium/pkg/lock" 21 "github.com/cilium/cilium/pkg/math" 22 ) 23 24 // The following error constants represent the error conditions for 25 // CreateInterface without additional context embedded in order to make them 26 // usable for metrics accounting purposes. 27 const ( 28 errUnableToDetermineLimits = "unable to determine limits" 29 unableToDetermineLimits = "unableToDetermineLimits" 30 errUnableToGetSecurityGroups = "unable to get security groups" 31 unableToGetSecurityGroups = "unableToGetSecurityGroups" 32 errUnableToCreateENI = "unable to create ENI" 33 unableToCreateENI = "unableToCreateENI" 34 errUnableToAttachENI = "unable to attach ENI" 35 unableToAttachENI = "unableToAttachENI" 36 unableToFindSubnet = "unableToFindSubnet" 37 ) 38 39 const ( 40 maxENIIPCreate = 10 41 42 maxENIPerNode = 50 43 ) 44 45 type ipamNodeActions interface { 46 InstanceID() string 47 } 48 49 type Node struct { 50 // node contains the general purpose fields of a node 51 node ipamNodeActions 52 53 // mutex protects members below this field 54 mutex lock.RWMutex 55 56 // enis is the list of ENIs attached to the node indexed by ENI ID. 57 // Protected by Node.mutex. 58 enis map[string]eniTypes.ENI 59 60 // k8sObj is the CiliumNode custom resource representing the node 61 k8sObj *v2.CiliumNode 62 63 // manager is the ecs node manager responsible for this node 64 manager *InstancesManager 65 66 // instanceID of the node 67 instanceID string 68 } 69 70 // UpdatedNode is called when an update to the CiliumNode is received. 71 func (n *Node) UpdatedNode(obj *v2.CiliumNode) { 72 n.mutex.Lock() 73 defer n.mutex.Unlock() 74 n.k8sObj = obj 75 } 76 77 // PopulateStatusFields fills in the status field of the CiliumNode custom 78 // resource with ENI specific information 79 func (n *Node) PopulateStatusFields(resource *v2.CiliumNode) { 80 resource.Status.AlibabaCloud.ENIs = map[string]eniTypes.ENI{} 81 82 n.manager.ForeachInstance(n.node.InstanceID(), 83 func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error { 84 e, ok := rev.Resource.(*eniTypes.ENI) 85 if ok { 86 resource.Status.AlibabaCloud.ENIs[interfaceID] = *e.DeepCopy() 87 } 88 return nil 89 }) 90 } 91 92 // CreateInterface creates an additional interface with the instance and 93 // attaches it to the instance as specified by the CiliumNode. neededAddresses 94 // of secondary IPs are assigned to the interface up to the maximum number of 95 // addresses as allowed by the instance. 96 func (n *Node) CreateInterface(ctx context.Context, allocation *ipam.AllocationAction, scopedLog *logrus.Entry) (int, string, error) { 97 l, limitsAvailable := n.getLimits() 98 if !limitsAvailable { 99 return 0, unableToDetermineLimits, fmt.Errorf(errUnableToDetermineLimits) 100 } 101 102 n.mutex.RLock() 103 resource := *n.k8sObj 104 n.mutex.RUnlock() 105 106 // Must allocate secondary ENI IPs as needed, up to ENI instance limit 107 toAllocate := math.IntMin(allocation.IPv4.MaxIPsToAllocate, l.IPv4) 108 toAllocate = math.IntMin(maxENIIPCreate, toAllocate) // in first alloc no more than 10 109 // Validate whether request has already been fulfilled in the meantime 110 if toAllocate == 0 { 111 return 0, "", nil 112 } 113 114 bestSubnet := n.manager.FindOneVSwitch(resource.Spec.AlibabaCloud, toAllocate) 115 if bestSubnet == nil { 116 return 0, 117 unableToFindSubnet, 118 fmt.Errorf( 119 "no matching vSwitch available for interface creation (VPC=%s AZ=%s SubnetTags=%s", 120 resource.Spec.AlibabaCloud.VPCID, 121 resource.Spec.AlibabaCloud.AvailabilityZone, 122 resource.Spec.AlibabaCloud.VSwitchTags, 123 ) 124 } 125 allocation.PoolID = ipamTypes.PoolID(bestSubnet.ID) 126 127 securityGroupIDs, err := n.getSecurityGroupIDs(ctx, resource.Spec.AlibabaCloud) 128 if err != nil { 129 return 0, 130 unableToGetSecurityGroups, 131 fmt.Errorf("%s: %w", errUnableToGetSecurityGroups, err) 132 } 133 134 scopedLog = scopedLog.WithFields(logrus.Fields{ 135 "securityGroupIDs": securityGroupIDs, 136 "vSwitchID": bestSubnet.ID, 137 "toAllocate": toAllocate, 138 }) 139 scopedLog.Info("No more IPs available, creating new ENI") 140 141 instanceID := n.node.InstanceID() 142 n.mutex.Lock() 143 defer n.mutex.Unlock() 144 index, err := n.allocENIIndex() 145 if err != nil { 146 scopedLog.WithField("instanceID", instanceID).Error(err) 147 return 0, "", err 148 } 149 eniID, eni, err := n.manager.api.CreateNetworkInterface(ctx, toAllocate-1, bestSubnet.ID, securityGroupIDs, 150 utils.FillTagWithENIIndex(map[string]string{}, index)) 151 if err != nil { 152 return 0, unableToCreateENI, fmt.Errorf("%s: %w", errUnableToCreateENI, err) 153 } 154 155 scopedLog = scopedLog.WithField(fieldENIID, eniID) 156 scopedLog.Info("Created new ENI") 157 158 if bestSubnet.CIDR != nil { 159 eni.VSwitch.CIDRBlock = bestSubnet.CIDR.String() 160 } 161 162 err = n.manager.api.AttachNetworkInterface(ctx, instanceID, eniID) 163 if err != nil { 164 err2 := n.manager.api.DeleteNetworkInterface(ctx, eniID) 165 if err2 != nil { 166 scopedLog.Errorf("Failed to release ENI after failure to attach, %s", err2.Error()) 167 } 168 return 0, unableToAttachENI, fmt.Errorf("%s: %w", errUnableToAttachENI, err) 169 } 170 _, err = n.manager.api.WaitENIAttached(ctx, eniID) 171 if err != nil { 172 err2 := n.manager.api.DeleteNetworkInterface(ctx, eniID) 173 if err2 != nil { 174 scopedLog.Errorf("Failed to release ENI after failure to attach, %s", err2.Error()) 175 } 176 return 0, unableToAttachENI, fmt.Errorf("%s: %w", errUnableToAttachENI, err) 177 } 178 179 n.enis[eniID] = *eni 180 scopedLog.Info("Attached ENI to instance") 181 182 // Add the information of the created ENI to the instances manager 183 n.manager.UpdateENI(instanceID, eni) 184 return toAllocate, "", nil 185 } 186 187 // ResyncInterfacesAndIPs is called to retrieve and ENIs and IPs as known to 188 // the AlibabaCloud API and return them 189 func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Entry) (available ipamTypes.AllocationMap, stats stats.InterfaceStats, err error) { 190 limits, limitsAvailable := n.getLimits() 191 if !limitsAvailable { 192 return nil, stats, fmt.Errorf(errUnableToDetermineLimits) 193 } 194 195 // During preparation of IP allocations, the primary NIC is not considered 196 // for allocation, so we don't need to consider it for capacity calculation. 197 stats.NodeCapacity = limits.IPv4 * (limits.Adapters - 1) 198 199 instanceID := n.node.InstanceID() 200 available = ipamTypes.AllocationMap{} 201 202 n.mutex.Lock() 203 defer n.mutex.Unlock() 204 n.enis = map[string]eniTypes.ENI{} 205 206 n.manager.ForeachInstance(instanceID, 207 func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error { 208 e, ok := rev.Resource.(*eniTypes.ENI) 209 if !ok { 210 return nil 211 } 212 213 n.enis[e.NetworkInterfaceID] = *e 214 if e.Type == eniTypes.ENITypePrimary { 215 return nil 216 } 217 218 // We exclude all "primary" IPs from the capacity. 219 primaryAllocated := 0 220 for _, ip := range e.PrivateIPSets { 221 if ip.Primary { 222 primaryAllocated++ 223 } 224 } 225 stats.NodeCapacity -= primaryAllocated 226 227 availableOnENI := math.IntMax(limits.IPv4-len(e.PrivateIPSets), 0) 228 if availableOnENI > 0 { 229 stats.RemainingAvailableInterfaceCount++ 230 } 231 232 for _, ip := range e.PrivateIPSets { 233 available[ip.PrivateIpAddress] = ipamTypes.AllocationIP{Resource: e.NetworkInterfaceID} 234 } 235 return nil 236 }) 237 enis := len(n.enis) 238 239 // An ECS instance has at least one ENI attached, no ENI found implies instance not found. 240 if enis == 0 { 241 scopedLog.Warning("Instance not found! Please delete corresponding ciliumnode if instance has already been deleted.") 242 return nil, stats, fmt.Errorf("unable to retrieve ENIs") 243 } 244 245 stats.RemainingAvailableInterfaceCount += limits.Adapters - len(n.enis) 246 return available, stats, nil 247 } 248 249 // PrepareIPAllocation returns the number of ENI IPs and interfaces that can be 250 // allocated/created. 251 func (n *Node) PrepareIPAllocation(scopedLog *logrus.Entry) (*ipam.AllocationAction, error) { 252 l, limitsAvailable := n.getLimits() 253 if !limitsAvailable { 254 return nil, fmt.Errorf(errUnableToDetermineLimits) 255 } 256 a := &ipam.AllocationAction{} 257 258 n.mutex.RLock() 259 defer n.mutex.RUnlock() 260 261 for key, e := range n.enis { 262 if e.Type != eniTypes.ENITypeSecondary { 263 continue 264 } 265 scopedLog.WithFields(logrus.Fields{ 266 fieldENIID: e.NetworkInterfaceID, 267 "ipv4Limit": l.IPv4, 268 "allocated": len(e.PrivateIPSets), 269 }).Debug("Considering ENI for allocation") 270 271 // limit 272 availableOnENI := math.IntMax(l.IPv4-len(e.PrivateIPSets), 0) 273 if availableOnENI <= 0 { 274 continue 275 } else { 276 a.IPv4.InterfaceCandidates++ 277 } 278 279 scopedLog.WithFields(logrus.Fields{ 280 fieldENIID: e.NetworkInterfaceID, 281 "availableOnENI": availableOnENI, 282 }).Debug("ENI has IPs available") 283 284 if subnet := n.manager.GetVSwitch(e.VSwitch.VSwitchID); subnet != nil { 285 if subnet.AvailableAddresses > 0 && a.InterfaceID == "" { 286 scopedLog.WithFields(logrus.Fields{ 287 "vSwitchID": e.VSwitch.VSwitchID, 288 "availableAddresses": subnet.AvailableAddresses, 289 }).Debug("Subnet has IPs available") 290 291 a.InterfaceID = key 292 a.PoolID = ipamTypes.PoolID(subnet.ID) 293 a.IPv4.AvailableForAllocation = math.IntMin(subnet.AvailableAddresses, availableOnENI) 294 } 295 } 296 } 297 a.EmptyInterfaceSlots = l.Adapters - len(n.enis) 298 return a, nil 299 } 300 301 // AllocateIPs performs the ENI allocation operation 302 func (n *Node) AllocateIPs(ctx context.Context, a *ipam.AllocationAction) error { 303 _, err := n.manager.api.AssignPrivateIPAddresses(ctx, a.InterfaceID, a.IPv4.AvailableForAllocation) 304 return err 305 } 306 307 // PrepareIPRelease prepares the release of ENI IPs. 308 func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *logrus.Entry) *ipam.ReleaseAction { 309 r := &ipam.ReleaseAction{} 310 311 n.mutex.Lock() 312 defer n.mutex.Unlock() 313 314 // Iterate over ENIs on this node, select the ENI with the most 315 // addresses available for release 316 for key, e := range n.enis { 317 if e.Type != eniTypes.ENITypeSecondary { 318 continue 319 } 320 scopedLog.WithFields(logrus.Fields{ 321 fieldENIID: e.NetworkInterfaceID, 322 "numAddresses": len(e.PrivateIPSets), 323 }).Debug("Considering ENI for IP release") 324 325 // Count free IP addresses on this ENI 326 ipsOnENI := n.k8sObj.Status.AlibabaCloud.ENIs[e.NetworkInterfaceID].PrivateIPSets 327 freeIpsOnENI := []string{} 328 for _, ip := range ipsOnENI { 329 // exclude primary IPs 330 if ip.Primary { 331 continue 332 } 333 _, ipUsed := n.k8sObj.Status.IPAM.Used[ip.PrivateIpAddress] 334 if !ipUsed { 335 freeIpsOnENI = append(freeIpsOnENI, ip.PrivateIpAddress) 336 } 337 } 338 freeOnENICount := len(freeIpsOnENI) 339 340 if freeOnENICount <= 0 { 341 continue 342 } 343 344 scopedLog.WithFields(logrus.Fields{ 345 fieldENIID: e.NetworkInterfaceID, 346 "excessIPs": excessIPs, 347 "freeOnENICount": freeOnENICount, 348 }).Debug("ENI has unused IPs that can be released") 349 maxReleaseOnENI := math.IntMin(freeOnENICount, excessIPs) 350 351 r.InterfaceID = key 352 r.PoolID = ipamTypes.PoolID(e.VPC.VPCID) 353 r.IPsToRelease = freeIpsOnENI[:maxReleaseOnENI] 354 } 355 356 return r 357 } 358 359 // ReleaseIPs performs the ENI IP release operation 360 func (n *Node) ReleaseIPs(ctx context.Context, r *ipam.ReleaseAction) error { 361 return n.manager.api.UnassignPrivateIPAddresses(ctx, r.InterfaceID, r.IPsToRelease) 362 } 363 364 // GetMaximumAllocatableIPv4 returns the maximum amount of IPv4 addresses 365 // that can be allocated to the instance 366 func (n *Node) GetMaximumAllocatableIPv4() int { 367 n.mutex.RLock() 368 defer n.mutex.RUnlock() 369 370 // Retrieve l for the instance type 371 l, limitsAvailable := n.getLimitsLocked() 372 if !limitsAvailable { 373 return 0 374 } 375 376 // Return the maximum amount of IP addresses allocatable on the instance 377 // reserve Primary eni 378 return (l.Adapters - 1) * l.IPv4 379 } 380 381 // GetMinimumAllocatableIPv4 returns the minimum amount of IPv4 addresses that 382 // must be allocated to the instance. 383 func (n *Node) GetMinimumAllocatableIPv4() int { 384 return defaults.IPAMPreAllocation 385 } 386 387 func (n *Node) loggerLocked() *logrus.Entry { 388 if n == nil || n.instanceID == "" { 389 return log 390 } 391 392 return log.WithField("instanceID", n.instanceID) 393 } 394 395 func (n *Node) IsPrefixDelegated() bool { 396 return false 397 } 398 399 func (n *Node) GetUsedIPWithPrefixes() int { 400 if n.k8sObj == nil { 401 return 0 402 } 403 return len(n.k8sObj.Status.IPAM.Used) 404 } 405 406 // getLimits returns the interface and IP limits of this node 407 func (n *Node) getLimits() (ipamTypes.Limits, bool) { 408 n.mutex.RLock() 409 l, b := n.getLimitsLocked() 410 n.mutex.RUnlock() 411 return l, b 412 } 413 414 // getLimitsLocked is the same function as getLimits, but assumes the n.mutex 415 // is read locked. 416 func (n *Node) getLimitsLocked() (ipamTypes.Limits, bool) { 417 return limits.Get(n.k8sObj.Spec.AlibabaCloud.InstanceType) 418 } 419 420 func (n *Node) getSecurityGroupIDs(ctx context.Context, eniSpec eniTypes.Spec) ([]string, error) { 421 // ENI must have at least one security group 422 // 1. use security group defined by user 423 // 2. use security group used by primary ENI (eth0) 424 425 if len(eniSpec.SecurityGroups) > 0 { 426 return eniSpec.SecurityGroups, nil 427 } 428 429 if len(eniSpec.SecurityGroupTags) > 0 { 430 securityGroups := n.manager.FindSecurityGroupByTags(eniSpec.VPCID, eniSpec.SecurityGroupTags) 431 if len(securityGroups) == 0 { 432 n.loggerLocked().WithFields(logrus.Fields{ 433 "vpcID": eniSpec.VPCID, 434 "tags": eniSpec.SecurityGroupTags, 435 }).Warn("No security groups match required VPC ID and tags, using primary ENI's security groups") 436 } else { 437 groups := make([]string, 0, len(securityGroups)) 438 for _, secGroup := range securityGroups { 439 groups = append(groups, secGroup.ID) 440 } 441 return groups, nil 442 } 443 } 444 445 var securityGroups []string 446 447 n.manager.ForeachInstance(n.node.InstanceID(), 448 func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error { 449 e, ok := rev.Resource.(*eniTypes.ENI) 450 if ok && e.Type == eniTypes.ENITypePrimary { 451 securityGroups = append(securityGroups, e.SecurityGroupIDs...) 452 } 453 return nil 454 }) 455 456 if len(securityGroups) <= 0 { 457 return nil, fmt.Errorf("failed to get security group ids") 458 } 459 460 return securityGroups, nil 461 } 462 463 // allocENIIndex will alloc an monotonically increased index for each ENI on this instance. 464 // The index generated the first time this ENI is created, and stored in ENI.Tags. 465 func (n *Node) allocENIIndex() (int, error) { 466 // alloc index for each created ENI 467 used := make([]bool, maxENIPerNode) 468 for _, v := range n.enis { 469 index := utils.GetENIIndexFromTags(v.Tags) 470 if index > maxENIPerNode || index < 0 { 471 return 0, fmt.Errorf("ENI index(%d) is out of range", index) 472 } 473 used[index] = true 474 } 475 // ECS has at least 1 ENI, 0 is reserved for eth0 476 i := 1 477 for ; i < maxENIPerNode; i++ { 478 if !used[i] { 479 break 480 } 481 } 482 return i, nil 483 }