github.com/cilium/cilium@v1.16.2/pkg/alibabacloud/eni/instances.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package eni 5 6 import ( 7 "context" 8 9 "github.com/sirupsen/logrus" 10 11 eniTypes "github.com/cilium/cilium/pkg/alibabacloud/eni/types" 12 "github.com/cilium/cilium/pkg/alibabacloud/types" 13 "github.com/cilium/cilium/pkg/ipam" 14 ipamTypes "github.com/cilium/cilium/pkg/ipam/types" 15 v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" 16 "github.com/cilium/cilium/pkg/lock" 17 "github.com/cilium/cilium/pkg/time" 18 ) 19 20 // AlibabaCloudAPI is the API surface used of the ECS API 21 type AlibabaCloudAPI interface { 22 GetInstance(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap, instanceID string) (*ipamTypes.Instance, error) 23 GetInstances(ctx context.Context, vpcs ipamTypes.VirtualNetworkMap, subnets ipamTypes.SubnetMap) (*ipamTypes.InstanceMap, error) 24 GetVSwitches(ctx context.Context) (ipamTypes.SubnetMap, error) 25 GetVPC(ctx context.Context, vpcID string) (*ipamTypes.VirtualNetwork, error) 26 GetVPCs(ctx context.Context) (ipamTypes.VirtualNetworkMap, error) 27 GetSecurityGroups(ctx context.Context) (types.SecurityGroupMap, error) 28 CreateNetworkInterface(ctx context.Context, secondaryPrivateIPCount int, vSwitchID string, groups []string, tags map[string]string) (string, *eniTypes.ENI, error) 29 AttachNetworkInterface(ctx context.Context, instanceID, eniID string) error 30 WaitENIAttached(ctx context.Context, eniID string) (string, error) 31 DeleteNetworkInterface(ctx context.Context, eniID string) error 32 AssignPrivateIPAddresses(ctx context.Context, eniID string, toAllocate int) ([]string, error) 33 UnassignPrivateIPAddresses(ctx context.Context, eniID string, addresses []string) error 34 } 35 36 // InstancesManager maintains the list of instances. It must be kept up to date 37 // by calling resync() regularly. 38 type InstancesManager struct { 39 mutex lock.RWMutex 40 resyncLock lock.RWMutex 41 instances *ipamTypes.InstanceMap 42 vSwitches ipamTypes.SubnetMap 43 vpcs ipamTypes.VirtualNetworkMap 44 securityGroups types.SecurityGroupMap 45 api AlibabaCloudAPI 46 } 47 48 // NewInstancesManager returns a new instances manager 49 func NewInstancesManager(api AlibabaCloudAPI) *InstancesManager { 50 return &InstancesManager{ 51 instances: ipamTypes.NewInstanceMap(), 52 api: api, 53 } 54 } 55 56 // CreateNode 57 func (m *InstancesManager) CreateNode(obj *v2.CiliumNode, node *ipam.Node) ipam.NodeOperations { 58 return &Node{k8sObj: obj, manager: m, node: node, instanceID: node.InstanceID()} 59 } 60 61 // HasInstance returns whether the instance is in instances 62 func (m *InstancesManager) HasInstance(instanceID string) bool { 63 m.mutex.RLock() 64 defer m.mutex.RUnlock() 65 return m.instances.Exists(instanceID) 66 } 67 68 // GetPoolQuota returns the number of available IPs in all IP pools 69 func (m *InstancesManager) GetPoolQuota() ipamTypes.PoolQuotaMap { 70 pool := ipamTypes.PoolQuotaMap{} 71 for subnetID, subnet := range m.GetVSwitches() { 72 pool[ipamTypes.PoolID(subnetID)] = ipamTypes.PoolQuota{ 73 AvailabilityZone: subnet.AvailabilityZone, 74 AvailableIPs: subnet.AvailableAddresses, 75 } 76 } 77 return pool 78 } 79 80 // Resync fetches the list of ECS instances and vSwitches and updates the local 81 // cache in the instanceManager. It returns the time when the resync has 82 // started or time.Time{} if it did not complete. 83 func (m *InstancesManager) Resync(ctx context.Context) time.Time { 84 // Full API resync should block the instance incremental resync from all nodes. 85 m.resyncLock.Lock() 86 defer m.resyncLock.Unlock() 87 // An empty instanceID indicates the full resync. 88 return m.resync(ctx, "") 89 } 90 91 // InstanceSync fetches the ECS instance by the given ID and vSwitches and updates the local 92 // cache in the instanceManager. It returns the time when the resync has 93 // started or time.Time{} if it did not complete. 94 func (m *InstancesManager) InstanceSync(ctx context.Context, instanceID string) time.Time { 95 // Instance incremental resync from different nodes should be executed in parallel, 96 // but must block the full API resync. 97 m.resyncLock.RLock() 98 defer m.resyncLock.RUnlock() 99 return m.resync(ctx, instanceID) 100 } 101 102 func (m *InstancesManager) resync(ctx context.Context, instanceID string) time.Time { 103 resyncStart := time.Now() 104 105 vpcs, err := m.api.GetVPCs(ctx) 106 if err != nil { 107 log.WithError(err).Warning("Unable to synchronize VPC list") 108 return time.Time{} 109 } 110 111 vSwitches, err := m.api.GetVSwitches(ctx) 112 if err != nil { 113 log.WithError(err).Warning("Unable to retrieve VPC vSwitches list") 114 return time.Time{} 115 } 116 117 securityGroups, err := m.api.GetSecurityGroups(ctx) 118 if err != nil { 119 log.WithError(err).Warning("Unable to retrieve ECS security group list") 120 return time.Time{} 121 } 122 123 // An empty instanceID indicates that this is full resync, ENIs from all instances 124 // will be refetched from ECS API and updated to the local cache. Otherwise only 125 // the given instance will be updated. 126 if instanceID == "" { 127 instances, err := m.api.GetInstances(ctx, vpcs, vSwitches) 128 if err != nil { 129 log.WithError(err).Warning("Unable to synchronize ECS interface list") 130 return time.Time{} 131 } 132 133 log.WithFields(logrus.Fields{ 134 "numInstances": instances.NumInstances(), 135 "numVPCs": len(vpcs), 136 "numVSwitches": len(vSwitches), 137 "numSecurityGroups": len(securityGroups), 138 }).Info("Synchronized ENI information") 139 140 m.mutex.Lock() 141 defer m.mutex.Unlock() 142 m.instances = instances 143 } else { 144 instance, err := m.api.GetInstance(ctx, vpcs, vSwitches, instanceID) 145 if err != nil { 146 log.WithError(err).Warning("Unable to synchronize ECS interface list") 147 return time.Time{} 148 } 149 150 log.WithFields(logrus.Fields{ 151 "instance": instanceID, 152 "numVPCs": len(vpcs), 153 "numVSwitches": len(vSwitches), 154 "numSecurityGroups": len(securityGroups), 155 }).Info("Synchronized ENI information for the corresponding instance") 156 157 m.mutex.Lock() 158 defer m.mutex.Unlock() 159 m.instances.UpdateInstance(instanceID, instance) 160 } 161 162 m.vSwitches = vSwitches 163 m.vpcs = vpcs 164 m.securityGroups = securityGroups 165 166 return resyncStart 167 } 168 169 // GetVSwitches returns all the tracked vSwitches 170 // The returned subnetMap is immutable so it can be safely accessed 171 func (m *InstancesManager) GetVSwitches() ipamTypes.SubnetMap { 172 m.mutex.RLock() 173 defer m.mutex.RUnlock() 174 175 subnetsCopy := make(ipamTypes.SubnetMap) 176 for k, v := range m.vSwitches { 177 subnetsCopy[k] = v 178 } 179 180 return subnetsCopy 181 } 182 183 // GetVSwitch return vSwitch by id 184 func (m *InstancesManager) GetVSwitch(id string) *ipamTypes.Subnet { 185 m.mutex.RLock() 186 defer m.mutex.RUnlock() 187 return m.vSwitches[id] 188 } 189 190 // ForeachInstance will iterate over each instance inside `instances`, and call 191 // `fn`. This function is read-locked for the entire execution. 192 func (m *InstancesManager) ForeachInstance(instanceID string, fn ipamTypes.InterfaceIterator) { 193 m.mutex.RLock() 194 defer m.mutex.RUnlock() 195 m.instances.ForeachInterface(instanceID, fn) 196 } 197 198 // UpdateENI updates the ENI definition of an ENI for a particular instance. If 199 // the ENI is already known, the definition is updated, otherwise the ENI is 200 // added to the instance. 201 func (m *InstancesManager) UpdateENI(instanceID string, eni *eniTypes.ENI) { 202 m.mutex.Lock() 203 defer m.mutex.Unlock() 204 eniRevision := ipamTypes.InterfaceRevision{Resource: eni} 205 m.instances.Update(instanceID, eniRevision) 206 } 207 208 // FindOneVSwitch returns the vSwitch with the most available addresses, matching vpc and az. 209 // If we have explicit ID or tag constraints, chose a matching vSwitch. ID constraints take 210 // precedence. 211 func (m *InstancesManager) FindOneVSwitch(spec eniTypes.Spec, toAllocate int) *ipamTypes.Subnet { 212 if len(spec.VSwitches) > 0 { 213 return m.FindVSwitchByIDs(spec, toAllocate) 214 } 215 var bestSubnet *ipamTypes.Subnet 216 for _, vSwitch := range m.GetVSwitches() { 217 if vSwitch.VirtualNetworkID != spec.VPCID { 218 continue 219 } 220 if vSwitch.AvailabilityZone != spec.AvailabilityZone { 221 continue 222 } 223 if vSwitch.AvailableAddresses < toAllocate { 224 continue 225 } 226 if !vSwitch.Tags.Match(spec.VSwitchTags) { 227 continue 228 } 229 if bestSubnet == nil || bestSubnet.AvailableAddresses < vSwitch.AvailableAddresses { 230 bestSubnet = vSwitch 231 } 232 } 233 return bestSubnet 234 } 235 236 // FindVSwitchByIDs returns the vSwitch within a provided list of vSwitch IDs with the most available addresses, 237 // matching vpc and az. 238 func (m *InstancesManager) FindVSwitchByIDs(spec eniTypes.Spec, toAllocate int) *ipamTypes.Subnet { 239 m.mutex.RLock() 240 defer m.mutex.RUnlock() 241 242 var bestSubnet *ipamTypes.Subnet 243 for _, vSwitch := range m.vSwitches { 244 if vSwitch.VirtualNetworkID != spec.VPCID || vSwitch.AvailabilityZone != spec.AvailabilityZone { 245 continue 246 } 247 if vSwitch.AvailableAddresses < toAllocate { 248 continue 249 } 250 for _, vSwitchID := range spec.VSwitches { 251 if vSwitch.ID != vSwitchID { 252 continue 253 } 254 if bestSubnet == nil || bestSubnet.AvailableAddresses < vSwitch.AvailableAddresses { 255 bestSubnet = vSwitch 256 } 257 } 258 } 259 return bestSubnet 260 } 261 262 // FindSecurityGroupByTags returns the security groups matching VPC ID and all required tags 263 // The returned security groups slice is immutable so it can be safely accessed 264 func (m *InstancesManager) FindSecurityGroupByTags(vpcID string, required ipamTypes.Tags) []*types.SecurityGroup { 265 m.mutex.RLock() 266 defer m.mutex.RUnlock() 267 268 securityGroups := []*types.SecurityGroup{} 269 for _, securityGroup := range m.securityGroups { 270 if securityGroup.VPCID == vpcID && securityGroup.Tags.Match(required) { 271 securityGroups = append(securityGroups, securityGroup) 272 } 273 } 274 275 return securityGroups 276 } 277 278 // DeleteInstance delete instance from m.instances 279 func (m *InstancesManager) DeleteInstance(instanceID string) { 280 m.mutex.Lock() 281 defer m.mutex.Unlock() 282 m.instances.Delete(instanceID) 283 }