github.com/kubearmor/cilium@v1.6.12/operator/eni.go (about) 1 // Copyright 2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "context" 19 "fmt" 20 "reflect" 21 "time" 22 23 ec2shim "github.com/cilium/cilium/pkg/aws/ec2" 24 "github.com/cilium/cilium/pkg/aws/eni" 25 "github.com/cilium/cilium/pkg/aws/eni/metrics" 26 "github.com/cilium/cilium/pkg/controller" 27 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" 28 k8sversion "github.com/cilium/cilium/pkg/k8s/version" 29 "github.com/cilium/cilium/pkg/trigger" 30 31 "github.com/aws/aws-sdk-go-v2/aws/ec2metadata" 32 "github.com/aws/aws-sdk-go-v2/aws/external" 33 "github.com/aws/aws-sdk-go-v2/service/ec2" 34 "github.com/sirupsen/logrus" 35 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 36 ) 37 38 var nodeManager *eni.NodeManager 39 40 type k8sAPI struct{} 41 42 func (k *k8sAPI) Get(node string) (*v2.CiliumNode, error) { 43 return ciliumK8sClient.CiliumV2().CiliumNodes().Get(node, metav1.GetOptions{}) 44 } 45 46 func (k *k8sAPI) UpdateStatus(node, origNode *v2.CiliumNode) (*v2.CiliumNode, error) { 47 // If k8s supports status as a sub-resource, then we need to update the status separately 48 k8sCapabilities := k8sversion.Capabilities() 49 switch { 50 case k8sCapabilities.UpdateStatus: 51 if !reflect.DeepEqual(origNode.Status, node.Status) { 52 return ciliumK8sClient.CiliumV2().CiliumNodes().UpdateStatus(node) 53 } 54 default: 55 if !reflect.DeepEqual(origNode.Status, node.Status) { 56 return ciliumK8sClient.CiliumV2().CiliumNodes().Update(node) 57 } 58 } 59 60 return nil, nil 61 } 62 63 func (k *k8sAPI) Update(node, origNode *v2.CiliumNode) (*v2.CiliumNode, error) { 64 // If k8s supports status as a sub-resource, then we need to update the status separately 65 k8sCapabilities := k8sversion.Capabilities() 66 switch { 67 case k8sCapabilities.UpdateStatus: 68 if !reflect.DeepEqual(origNode.Spec, node.Spec) { 69 return ciliumK8sClient.CiliumV2().CiliumNodes().Update(node) 70 } 71 default: 72 if !reflect.DeepEqual(origNode, node) { 73 return ciliumK8sClient.CiliumV2().CiliumNodes().Update(node) 74 } 75 } 76 77 return nil, nil 78 } 79 80 func ciliumNodeUpdated(resource *v2.CiliumNode) { 81 if nodeManager != nil { 82 nodeManager.Update(resource) 83 } 84 } 85 86 func ciliumNodeDeleted(nodeName string) { 87 if nodeManager != nil { 88 nodeManager.Delete(nodeName) 89 } 90 } 91 92 // startENIAllocator kicks of ENI allocation, the initial connection to AWS 93 // APIs is done in a blocking manner, given that is successful, a controller is 94 // started to manage allocation based on CiliumNode custom resources 95 func startENIAllocator(awsClientQPSLimit float64, awsClientBurst int) error { 96 log.Info("Starting ENI allocator...") 97 98 cfg, err := external.LoadDefaultAWSConfig() 99 if err != nil { 100 return fmt.Errorf("unable to load AWS configuration: %s", err) 101 } 102 103 log.Info("Retrieving own metadata from EC2 metadata server...") 104 metadataClient := ec2metadata.New(cfg) 105 instance, err := metadataClient.GetInstanceIdentityDocument() 106 if err != nil { 107 return fmt.Errorf("unable to retrieve instance identity document: %s", err) 108 } 109 110 log.WithFields(logrus.Fields{ 111 "instance": instance.InstanceID, 112 "region": instance.Region, 113 }).Info("Connected to EC2 metadata server") 114 115 cfg.Region = instance.Region 116 117 var ( 118 ec2Client *ec2shim.Client 119 instances *eni.InstancesManager 120 ) 121 122 if enableMetrics { 123 eniMetrics := metrics.NewPrometheusMetrics(metricNamespace, registry) 124 ec2Client = ec2shim.NewClient(ec2.New(cfg), eniMetrics, awsClientQPSLimit, awsClientBurst) 125 log.Info("Connected to EC2 service API") 126 instances = eni.NewInstancesManager(ec2Client, eniMetrics) 127 nodeManager, err = eni.NewNodeManager(instances, ec2Client, &k8sAPI{}, eniMetrics, eniParallelWorkers) 128 if err != nil { 129 return fmt.Errorf("unable to initialize ENI node manager: %s", err) 130 } 131 } else { 132 // Inject dummy metrics operations that do nothing so we don't panic if 133 // metrics aren't enabled 134 noOpMetric := &noOpMetrics{} 135 ec2Client = ec2shim.NewClient(ec2.New(cfg), noOpMetric, awsClientQPSLimit, awsClientBurst) 136 log.Info("Connected to EC2 service API") 137 instances = eni.NewInstancesManager(ec2Client, noOpMetric) 138 nodeManager, err = eni.NewNodeManager(instances, ec2Client, &k8sAPI{}, noOpMetric, eniParallelWorkers) 139 if err != nil { 140 return fmt.Errorf("unable to initialize ENI node manager: %s", err) 141 } 142 } 143 144 // Initial blocking synchronization of all ENIs and subnets 145 instances.Resync() 146 147 // Start an interval based background resync for safety, it will 148 // synchronize the state regularly and resolve eventual deficit if the 149 // event driven trigger fails, and also release excess IP addresses 150 // if release-excess-ips is enabled 151 go func() { 152 time.Sleep(time.Minute) 153 mngr := controller.NewManager() 154 mngr.UpdateController("eni-refresh", 155 controller.ControllerParams{ 156 RunInterval: time.Minute, 157 DoFunc: func(_ context.Context) error { 158 syncTime := instances.Resync() 159 nodeManager.Resync(syncTime) 160 return nil 161 }, 162 }) 163 }() 164 165 return nil 166 } 167 168 // The below are types which fulfill various interfaces which are needed by the 169 // eni / ec2 functions we have which do nothing if metrics are disabled. 170 171 type noOpMetricsObserver struct{} 172 173 // MetricsObserver implementation 174 func (m *noOpMetricsObserver) PostRun(callDuration, latency time.Duration, folds int) {} 175 func (m *noOpMetricsObserver) QueueEvent(reason string) {} 176 177 type noOpMetrics struct{} 178 179 // eni metricsAPI interface implementation 180 func (m *noOpMetrics) IncENIAllocationAttempt(status, subnetID string) {} 181 func (m *noOpMetrics) AddIPAllocation(subnetID string, allocated int64) {} 182 func (m *noOpMetrics) AddIPRelease(subnetID string, released int64) {} 183 func (m *noOpMetrics) SetAllocatedIPs(typ string, allocated int) {} 184 func (m *noOpMetrics) SetAvailableENIs(available int) {} 185 func (m *noOpMetrics) SetAvailableIPsPerSubnet(subnetID, availabilityZone string, available int) {} 186 func (m *noOpMetrics) SetNodes(category string, nodes int) {} 187 func (m *noOpMetrics) IncResyncCount() {} 188 func (m *noOpMetrics) PoolMaintainerTrigger() trigger.MetricsObserver { 189 return &noOpMetricsObserver{} 190 } 191 func (m *noOpMetrics) K8sSyncTrigger() trigger.MetricsObserver { 192 return &noOpMetricsObserver{} 193 } 194 func (m *noOpMetrics) ResyncTrigger() trigger.MetricsObserver { 195 return &noOpMetricsObserver{} 196 } 197 198 // ec2 metricsAPI interface implementation 199 func (m *noOpMetrics) ObserveEC2APICall(call, status string, duration float64) {} 200 func (m *noOpMetrics) ObserveEC2RateLimit(operation string, duration time.Duration) {}