github.com/cilium/cilium@v1.16.2/pkg/alibabacloud/eni/node.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package eni
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  
    10  	"github.com/sirupsen/logrus"
    11  
    12  	"github.com/cilium/cilium/pkg/alibabacloud/eni/limits"
    13  	eniTypes "github.com/cilium/cilium/pkg/alibabacloud/eni/types"
    14  	"github.com/cilium/cilium/pkg/alibabacloud/utils"
    15  	"github.com/cilium/cilium/pkg/defaults"
    16  	"github.com/cilium/cilium/pkg/ipam"
    17  	"github.com/cilium/cilium/pkg/ipam/stats"
    18  	ipamTypes "github.com/cilium/cilium/pkg/ipam/types"
    19  	v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
    20  	"github.com/cilium/cilium/pkg/lock"
    21  	"github.com/cilium/cilium/pkg/math"
    22  )
    23  
    24  // The following error constants represent the error conditions for
    25  // CreateInterface without additional context embedded in order to make them
    26  // usable for metrics accounting purposes.
    27  const (
    28  	errUnableToDetermineLimits   = "unable to determine limits"
    29  	unableToDetermineLimits      = "unableToDetermineLimits"
    30  	errUnableToGetSecurityGroups = "unable to get security groups"
    31  	unableToGetSecurityGroups    = "unableToGetSecurityGroups"
    32  	errUnableToCreateENI         = "unable to create ENI"
    33  	unableToCreateENI            = "unableToCreateENI"
    34  	errUnableToAttachENI         = "unable to attach ENI"
    35  	unableToAttachENI            = "unableToAttachENI"
    36  	unableToFindSubnet           = "unableToFindSubnet"
    37  )
    38  
    39  const (
    40  	maxENIIPCreate = 10
    41  
    42  	maxENIPerNode = 50
    43  )
    44  
    45  type ipamNodeActions interface {
    46  	InstanceID() string
    47  }
    48  
    49  type Node struct {
    50  	// node contains the general purpose fields of a node
    51  	node ipamNodeActions
    52  
    53  	// mutex protects members below this field
    54  	mutex lock.RWMutex
    55  
    56  	// enis is the list of ENIs attached to the node indexed by ENI ID.
    57  	// Protected by Node.mutex.
    58  	enis map[string]eniTypes.ENI
    59  
    60  	// k8sObj is the CiliumNode custom resource representing the node
    61  	k8sObj *v2.CiliumNode
    62  
    63  	// manager is the ecs node manager responsible for this node
    64  	manager *InstancesManager
    65  
    66  	// instanceID of the node
    67  	instanceID string
    68  }
    69  
    70  // UpdatedNode is called when an update to the CiliumNode is received.
    71  func (n *Node) UpdatedNode(obj *v2.CiliumNode) {
    72  	n.mutex.Lock()
    73  	defer n.mutex.Unlock()
    74  	n.k8sObj = obj
    75  }
    76  
    77  // PopulateStatusFields fills in the status field of the CiliumNode custom
    78  // resource with ENI specific information
    79  func (n *Node) PopulateStatusFields(resource *v2.CiliumNode) {
    80  	resource.Status.AlibabaCloud.ENIs = map[string]eniTypes.ENI{}
    81  
    82  	n.manager.ForeachInstance(n.node.InstanceID(),
    83  		func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error {
    84  			e, ok := rev.Resource.(*eniTypes.ENI)
    85  			if ok {
    86  				resource.Status.AlibabaCloud.ENIs[interfaceID] = *e.DeepCopy()
    87  			}
    88  			return nil
    89  		})
    90  }
    91  
    92  // CreateInterface creates an additional interface with the instance and
    93  // attaches it to the instance as specified by the CiliumNode. neededAddresses
    94  // of secondary IPs are assigned to the interface up to the maximum number of
    95  // addresses as allowed by the instance.
    96  func (n *Node) CreateInterface(ctx context.Context, allocation *ipam.AllocationAction, scopedLog *logrus.Entry) (int, string, error) {
    97  	l, limitsAvailable := n.getLimits()
    98  	if !limitsAvailable {
    99  		return 0, unableToDetermineLimits, fmt.Errorf(errUnableToDetermineLimits)
   100  	}
   101  
   102  	n.mutex.RLock()
   103  	resource := *n.k8sObj
   104  	n.mutex.RUnlock()
   105  
   106  	// Must allocate secondary ENI IPs as needed, up to ENI instance limit
   107  	toAllocate := math.IntMin(allocation.IPv4.MaxIPsToAllocate, l.IPv4)
   108  	toAllocate = math.IntMin(maxENIIPCreate, toAllocate) // in first alloc no more than 10
   109  	// Validate whether request has already been fulfilled in the meantime
   110  	if toAllocate == 0 {
   111  		return 0, "", nil
   112  	}
   113  
   114  	bestSubnet := n.manager.FindOneVSwitch(resource.Spec.AlibabaCloud, toAllocate)
   115  	if bestSubnet == nil {
   116  		return 0,
   117  			unableToFindSubnet,
   118  			fmt.Errorf(
   119  				"no matching vSwitch available for interface creation (VPC=%s AZ=%s SubnetTags=%s",
   120  				resource.Spec.AlibabaCloud.VPCID,
   121  				resource.Spec.AlibabaCloud.AvailabilityZone,
   122  				resource.Spec.AlibabaCloud.VSwitchTags,
   123  			)
   124  	}
   125  	allocation.PoolID = ipamTypes.PoolID(bestSubnet.ID)
   126  
   127  	securityGroupIDs, err := n.getSecurityGroupIDs(ctx, resource.Spec.AlibabaCloud)
   128  	if err != nil {
   129  		return 0,
   130  			unableToGetSecurityGroups,
   131  			fmt.Errorf("%s: %w", errUnableToGetSecurityGroups, err)
   132  	}
   133  
   134  	scopedLog = scopedLog.WithFields(logrus.Fields{
   135  		"securityGroupIDs": securityGroupIDs,
   136  		"vSwitchID":        bestSubnet.ID,
   137  		"toAllocate":       toAllocate,
   138  	})
   139  	scopedLog.Info("No more IPs available, creating new ENI")
   140  
   141  	instanceID := n.node.InstanceID()
   142  	n.mutex.Lock()
   143  	defer n.mutex.Unlock()
   144  	index, err := n.allocENIIndex()
   145  	if err != nil {
   146  		scopedLog.WithField("instanceID", instanceID).Error(err)
   147  		return 0, "", err
   148  	}
   149  	eniID, eni, err := n.manager.api.CreateNetworkInterface(ctx, toAllocate-1, bestSubnet.ID, securityGroupIDs,
   150  		utils.FillTagWithENIIndex(map[string]string{}, index))
   151  	if err != nil {
   152  		return 0, unableToCreateENI, fmt.Errorf("%s: %w", errUnableToCreateENI, err)
   153  	}
   154  
   155  	scopedLog = scopedLog.WithField(fieldENIID, eniID)
   156  	scopedLog.Info("Created new ENI")
   157  
   158  	if bestSubnet.CIDR != nil {
   159  		eni.VSwitch.CIDRBlock = bestSubnet.CIDR.String()
   160  	}
   161  
   162  	err = n.manager.api.AttachNetworkInterface(ctx, instanceID, eniID)
   163  	if err != nil {
   164  		err2 := n.manager.api.DeleteNetworkInterface(ctx, eniID)
   165  		if err2 != nil {
   166  			scopedLog.Errorf("Failed to release ENI after failure to attach, %s", err2.Error())
   167  		}
   168  		return 0, unableToAttachENI, fmt.Errorf("%s: %w", errUnableToAttachENI, err)
   169  	}
   170  	_, err = n.manager.api.WaitENIAttached(ctx, eniID)
   171  	if err != nil {
   172  		err2 := n.manager.api.DeleteNetworkInterface(ctx, eniID)
   173  		if err2 != nil {
   174  			scopedLog.Errorf("Failed to release ENI after failure to attach, %s", err2.Error())
   175  		}
   176  		return 0, unableToAttachENI, fmt.Errorf("%s: %w", errUnableToAttachENI, err)
   177  	}
   178  
   179  	n.enis[eniID] = *eni
   180  	scopedLog.Info("Attached ENI to instance")
   181  
   182  	// Add the information of the created ENI to the instances manager
   183  	n.manager.UpdateENI(instanceID, eni)
   184  	return toAllocate, "", nil
   185  }
   186  
   187  // ResyncInterfacesAndIPs is called to retrieve and ENIs and IPs as known to
   188  // the AlibabaCloud API and return them
   189  func (n *Node) ResyncInterfacesAndIPs(ctx context.Context, scopedLog *logrus.Entry) (available ipamTypes.AllocationMap, stats stats.InterfaceStats, err error) {
   190  	limits, limitsAvailable := n.getLimits()
   191  	if !limitsAvailable {
   192  		return nil, stats, fmt.Errorf(errUnableToDetermineLimits)
   193  	}
   194  
   195  	// During preparation of IP allocations, the primary NIC is not considered
   196  	// for allocation, so we don't need to consider it for capacity calculation.
   197  	stats.NodeCapacity = limits.IPv4 * (limits.Adapters - 1)
   198  
   199  	instanceID := n.node.InstanceID()
   200  	available = ipamTypes.AllocationMap{}
   201  
   202  	n.mutex.Lock()
   203  	defer n.mutex.Unlock()
   204  	n.enis = map[string]eniTypes.ENI{}
   205  
   206  	n.manager.ForeachInstance(instanceID,
   207  		func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error {
   208  			e, ok := rev.Resource.(*eniTypes.ENI)
   209  			if !ok {
   210  				return nil
   211  			}
   212  
   213  			n.enis[e.NetworkInterfaceID] = *e
   214  			if e.Type == eniTypes.ENITypePrimary {
   215  				return nil
   216  			}
   217  
   218  			// We exclude all "primary" IPs from the capacity.
   219  			primaryAllocated := 0
   220  			for _, ip := range e.PrivateIPSets {
   221  				if ip.Primary {
   222  					primaryAllocated++
   223  				}
   224  			}
   225  			stats.NodeCapacity -= primaryAllocated
   226  
   227  			availableOnENI := math.IntMax(limits.IPv4-len(e.PrivateIPSets), 0)
   228  			if availableOnENI > 0 {
   229  				stats.RemainingAvailableInterfaceCount++
   230  			}
   231  
   232  			for _, ip := range e.PrivateIPSets {
   233  				available[ip.PrivateIpAddress] = ipamTypes.AllocationIP{Resource: e.NetworkInterfaceID}
   234  			}
   235  			return nil
   236  		})
   237  	enis := len(n.enis)
   238  
   239  	// An ECS instance has at least one ENI attached, no ENI found implies instance not found.
   240  	if enis == 0 {
   241  		scopedLog.Warning("Instance not found! Please delete corresponding ciliumnode if instance has already been deleted.")
   242  		return nil, stats, fmt.Errorf("unable to retrieve ENIs")
   243  	}
   244  
   245  	stats.RemainingAvailableInterfaceCount += limits.Adapters - len(n.enis)
   246  	return available, stats, nil
   247  }
   248  
   249  // PrepareIPAllocation returns the number of ENI IPs and interfaces that can be
   250  // allocated/created.
   251  func (n *Node) PrepareIPAllocation(scopedLog *logrus.Entry) (*ipam.AllocationAction, error) {
   252  	l, limitsAvailable := n.getLimits()
   253  	if !limitsAvailable {
   254  		return nil, fmt.Errorf(errUnableToDetermineLimits)
   255  	}
   256  	a := &ipam.AllocationAction{}
   257  
   258  	n.mutex.RLock()
   259  	defer n.mutex.RUnlock()
   260  
   261  	for key, e := range n.enis {
   262  		if e.Type != eniTypes.ENITypeSecondary {
   263  			continue
   264  		}
   265  		scopedLog.WithFields(logrus.Fields{
   266  			fieldENIID:  e.NetworkInterfaceID,
   267  			"ipv4Limit": l.IPv4,
   268  			"allocated": len(e.PrivateIPSets),
   269  		}).Debug("Considering ENI for allocation")
   270  
   271  		// limit
   272  		availableOnENI := math.IntMax(l.IPv4-len(e.PrivateIPSets), 0)
   273  		if availableOnENI <= 0 {
   274  			continue
   275  		} else {
   276  			a.IPv4.InterfaceCandidates++
   277  		}
   278  
   279  		scopedLog.WithFields(logrus.Fields{
   280  			fieldENIID:       e.NetworkInterfaceID,
   281  			"availableOnENI": availableOnENI,
   282  		}).Debug("ENI has IPs available")
   283  
   284  		if subnet := n.manager.GetVSwitch(e.VSwitch.VSwitchID); subnet != nil {
   285  			if subnet.AvailableAddresses > 0 && a.InterfaceID == "" {
   286  				scopedLog.WithFields(logrus.Fields{
   287  					"vSwitchID":          e.VSwitch.VSwitchID,
   288  					"availableAddresses": subnet.AvailableAddresses,
   289  				}).Debug("Subnet has IPs available")
   290  
   291  				a.InterfaceID = key
   292  				a.PoolID = ipamTypes.PoolID(subnet.ID)
   293  				a.IPv4.AvailableForAllocation = math.IntMin(subnet.AvailableAddresses, availableOnENI)
   294  			}
   295  		}
   296  	}
   297  	a.EmptyInterfaceSlots = l.Adapters - len(n.enis)
   298  	return a, nil
   299  }
   300  
   301  // AllocateIPs performs the ENI allocation operation
   302  func (n *Node) AllocateIPs(ctx context.Context, a *ipam.AllocationAction) error {
   303  	_, err := n.manager.api.AssignPrivateIPAddresses(ctx, a.InterfaceID, a.IPv4.AvailableForAllocation)
   304  	return err
   305  }
   306  
   307  // PrepareIPRelease prepares the release of ENI IPs.
   308  func (n *Node) PrepareIPRelease(excessIPs int, scopedLog *logrus.Entry) *ipam.ReleaseAction {
   309  	r := &ipam.ReleaseAction{}
   310  
   311  	n.mutex.Lock()
   312  	defer n.mutex.Unlock()
   313  
   314  	// Iterate over ENIs on this node, select the ENI with the most
   315  	// addresses available for release
   316  	for key, e := range n.enis {
   317  		if e.Type != eniTypes.ENITypeSecondary {
   318  			continue
   319  		}
   320  		scopedLog.WithFields(logrus.Fields{
   321  			fieldENIID:     e.NetworkInterfaceID,
   322  			"numAddresses": len(e.PrivateIPSets),
   323  		}).Debug("Considering ENI for IP release")
   324  
   325  		// Count free IP addresses on this ENI
   326  		ipsOnENI := n.k8sObj.Status.AlibabaCloud.ENIs[e.NetworkInterfaceID].PrivateIPSets
   327  		freeIpsOnENI := []string{}
   328  		for _, ip := range ipsOnENI {
   329  			// exclude primary IPs
   330  			if ip.Primary {
   331  				continue
   332  			}
   333  			_, ipUsed := n.k8sObj.Status.IPAM.Used[ip.PrivateIpAddress]
   334  			if !ipUsed {
   335  				freeIpsOnENI = append(freeIpsOnENI, ip.PrivateIpAddress)
   336  			}
   337  		}
   338  		freeOnENICount := len(freeIpsOnENI)
   339  
   340  		if freeOnENICount <= 0 {
   341  			continue
   342  		}
   343  
   344  		scopedLog.WithFields(logrus.Fields{
   345  			fieldENIID:       e.NetworkInterfaceID,
   346  			"excessIPs":      excessIPs,
   347  			"freeOnENICount": freeOnENICount,
   348  		}).Debug("ENI has unused IPs that can be released")
   349  		maxReleaseOnENI := math.IntMin(freeOnENICount, excessIPs)
   350  
   351  		r.InterfaceID = key
   352  		r.PoolID = ipamTypes.PoolID(e.VPC.VPCID)
   353  		r.IPsToRelease = freeIpsOnENI[:maxReleaseOnENI]
   354  	}
   355  
   356  	return r
   357  }
   358  
   359  // ReleaseIPs performs the ENI IP release operation
   360  func (n *Node) ReleaseIPs(ctx context.Context, r *ipam.ReleaseAction) error {
   361  	return n.manager.api.UnassignPrivateIPAddresses(ctx, r.InterfaceID, r.IPsToRelease)
   362  }
   363  
   364  // GetMaximumAllocatableIPv4 returns the maximum amount of IPv4 addresses
   365  // that can be allocated to the instance
   366  func (n *Node) GetMaximumAllocatableIPv4() int {
   367  	n.mutex.RLock()
   368  	defer n.mutex.RUnlock()
   369  
   370  	// Retrieve l for the instance type
   371  	l, limitsAvailable := n.getLimitsLocked()
   372  	if !limitsAvailable {
   373  		return 0
   374  	}
   375  
   376  	// Return the maximum amount of IP addresses allocatable on the instance
   377  	// reserve Primary eni
   378  	return (l.Adapters - 1) * l.IPv4
   379  }
   380  
   381  // GetMinimumAllocatableIPv4 returns the minimum amount of IPv4 addresses that
   382  // must be allocated to the instance.
   383  func (n *Node) GetMinimumAllocatableIPv4() int {
   384  	return defaults.IPAMPreAllocation
   385  }
   386  
   387  func (n *Node) loggerLocked() *logrus.Entry {
   388  	if n == nil || n.instanceID == "" {
   389  		return log
   390  	}
   391  
   392  	return log.WithField("instanceID", n.instanceID)
   393  }
   394  
   395  func (n *Node) IsPrefixDelegated() bool {
   396  	return false
   397  }
   398  
   399  func (n *Node) GetUsedIPWithPrefixes() int {
   400  	if n.k8sObj == nil {
   401  		return 0
   402  	}
   403  	return len(n.k8sObj.Status.IPAM.Used)
   404  }
   405  
   406  // getLimits returns the interface and IP limits of this node
   407  func (n *Node) getLimits() (ipamTypes.Limits, bool) {
   408  	n.mutex.RLock()
   409  	l, b := n.getLimitsLocked()
   410  	n.mutex.RUnlock()
   411  	return l, b
   412  }
   413  
   414  // getLimitsLocked is the same function as getLimits, but assumes the n.mutex
   415  // is read locked.
   416  func (n *Node) getLimitsLocked() (ipamTypes.Limits, bool) {
   417  	return limits.Get(n.k8sObj.Spec.AlibabaCloud.InstanceType)
   418  }
   419  
   420  func (n *Node) getSecurityGroupIDs(ctx context.Context, eniSpec eniTypes.Spec) ([]string, error) {
   421  	// ENI must have at least one security group
   422  	// 1. use security group defined by user
   423  	// 2. use security group used by primary ENI (eth0)
   424  
   425  	if len(eniSpec.SecurityGroups) > 0 {
   426  		return eniSpec.SecurityGroups, nil
   427  	}
   428  
   429  	if len(eniSpec.SecurityGroupTags) > 0 {
   430  		securityGroups := n.manager.FindSecurityGroupByTags(eniSpec.VPCID, eniSpec.SecurityGroupTags)
   431  		if len(securityGroups) == 0 {
   432  			n.loggerLocked().WithFields(logrus.Fields{
   433  				"vpcID": eniSpec.VPCID,
   434  				"tags":  eniSpec.SecurityGroupTags,
   435  			}).Warn("No security groups match required VPC ID and tags, using primary ENI's security groups")
   436  		} else {
   437  			groups := make([]string, 0, len(securityGroups))
   438  			for _, secGroup := range securityGroups {
   439  				groups = append(groups, secGroup.ID)
   440  			}
   441  			return groups, nil
   442  		}
   443  	}
   444  
   445  	var securityGroups []string
   446  
   447  	n.manager.ForeachInstance(n.node.InstanceID(),
   448  		func(instanceID, interfaceID string, rev ipamTypes.InterfaceRevision) error {
   449  			e, ok := rev.Resource.(*eniTypes.ENI)
   450  			if ok && e.Type == eniTypes.ENITypePrimary {
   451  				securityGroups = append(securityGroups, e.SecurityGroupIDs...)
   452  			}
   453  			return nil
   454  		})
   455  
   456  	if len(securityGroups) <= 0 {
   457  		return nil, fmt.Errorf("failed to get security group ids")
   458  	}
   459  
   460  	return securityGroups, nil
   461  }
   462  
   463  // allocENIIndex will alloc an monotonically increased index for each ENI on this instance.
   464  // The index generated the first time this ENI is created, and stored in ENI.Tags.
   465  func (n *Node) allocENIIndex() (int, error) {
   466  	// alloc index for each created ENI
   467  	used := make([]bool, maxENIPerNode)
   468  	for _, v := range n.enis {
   469  		index := utils.GetENIIndexFromTags(v.Tags)
   470  		if index > maxENIPerNode || index < 0 {
   471  			return 0, fmt.Errorf("ENI index(%d) is out of range", index)
   472  		}
   473  		used[index] = true
   474  	}
   475  	// ECS has at least 1 ENI, 0 is reserved for eth0
   476  	i := 1
   477  	for ; i < maxENIPerNode; i++ {
   478  		if !used[i] {
   479  			break
   480  		}
   481  	}
   482  	return i, nil
   483  }