k8s.io/kubernetes@v1.29.3/test/e2e/framework/providers/gce/gce.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package gce 18 19 import ( 20 "context" 21 "fmt" 22 "math/rand" 23 "net/http" 24 "os/exec" 25 "regexp" 26 "strings" 27 "time" 28 29 compute "google.golang.org/api/compute/v1" 30 "google.golang.org/api/googleapi" 31 v1 "k8s.io/api/core/v1" 32 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 33 "k8s.io/apimachinery/pkg/util/uuid" 34 "k8s.io/apimachinery/pkg/util/wait" 35 clientset "k8s.io/client-go/kubernetes" 36 "k8s.io/kubernetes/test/e2e/framework" 37 e2epv "k8s.io/kubernetes/test/e2e/framework/pv" 38 e2eservice "k8s.io/kubernetes/test/e2e/framework/service" 39 gcecloud "k8s.io/legacy-cloud-providers/gce" 40 ) 41 42 func init() { 43 framework.RegisterProvider("gce", factory) 44 framework.RegisterProvider("gke", factory) 45 } 46 47 func factory() (framework.ProviderInterface, error) { 48 framework.Logf("Fetching cloud provider for %q\r", framework.TestContext.Provider) 49 zone := framework.TestContext.CloudConfig.Zone 50 region := framework.TestContext.CloudConfig.Region 51 allowedZones := framework.TestContext.CloudConfig.Zones 52 53 // ensure users don't specify a zone outside of the requested zones 54 if len(zone) > 0 && len(allowedZones) > 0 { 55 var found bool 56 for _, allowedZone := range allowedZones { 57 if zone == allowedZone { 58 found = true 59 break 60 } 61 } 62 if !found { 63 return nil, fmt.Errorf("the provided zone %q must be included in the list of allowed zones %v", zone, allowedZones) 64 } 65 } 66 67 var err error 68 if region == "" { 69 region, err = gcecloud.GetGCERegion(zone) 70 if err != nil { 71 return nil, fmt.Errorf("error parsing GCE/GKE region from zone %q: %w", zone, err) 72 } 73 } 74 managedZones := []string{} // Manage all zones in the region 75 if !framework.TestContext.CloudConfig.MultiZone { 76 managedZones = []string{zone} 77 } 78 if len(allowedZones) > 0 { 79 managedZones = allowedZones 80 } 81 82 gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{ 83 APIEndpoint: framework.TestContext.CloudConfig.APIEndpoint, 84 ProjectID: framework.TestContext.CloudConfig.ProjectID, 85 Region: region, 86 Zone: zone, 87 ManagedZones: managedZones, 88 NetworkName: "", // TODO: Change this to use framework.TestContext.CloudConfig.Network? 89 SubnetworkName: "", 90 NodeTags: nil, 91 NodeInstancePrefix: "", 92 TokenSource: nil, 93 UseMetadataServer: false, 94 AlphaFeatureGate: gcecloud.NewAlphaFeatureGate([]string{}), 95 }) 96 97 if err != nil { 98 return nil, fmt.Errorf("Error building GCE/GKE provider: %w", err) 99 } 100 101 // Arbitrarily pick one of the zones we have nodes in, looking at prepopulated zones first. 102 if framework.TestContext.CloudConfig.Zone == "" && len(managedZones) > 0 { 103 framework.TestContext.CloudConfig.Zone = managedZones[rand.Intn(len(managedZones))] 104 } 105 if framework.TestContext.CloudConfig.Zone == "" && framework.TestContext.CloudConfig.MultiZone { 106 zones, err := gceCloud.GetAllZonesFromCloudProvider() 107 if err != nil { 108 return nil, err 109 } 110 111 framework.TestContext.CloudConfig.Zone, _ = zones.PopAny() 112 } 113 114 return NewProvider(gceCloud), nil 115 } 116 117 // NewProvider returns a cloud provider interface for GCE 118 func NewProvider(gceCloud *gcecloud.Cloud) framework.ProviderInterface { 119 return &Provider{ 120 gceCloud: gceCloud, 121 } 122 } 123 124 // Provider is a structure to handle GCE clouds for e2e testing 125 type Provider struct { 126 framework.NullProvider 127 gceCloud *gcecloud.Cloud 128 } 129 130 // ResizeGroup resizes an instance group 131 func (p *Provider) ResizeGroup(group string, size int32) error { 132 // TODO: make this hit the compute API directly instead of shelling out to gcloud. 133 // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic 134 zone, err := getGCEZoneForGroup(group) 135 if err != nil { 136 return err 137 } 138 output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "resize", 139 group, fmt.Sprintf("--size=%v", size), 140 "--project="+framework.TestContext.CloudConfig.ProjectID, "--zone="+zone).CombinedOutput() 141 if err != nil { 142 return fmt.Errorf("Failed to resize node instance group %s: %s", group, output) 143 } 144 return nil 145 } 146 147 // GetGroupNodes returns a node name for the specified node group 148 func (p *Provider) GetGroupNodes(group string) ([]string, error) { 149 // TODO: make this hit the compute API directly instead of shelling out to gcloud. 150 // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic 151 zone, err := getGCEZoneForGroup(group) 152 if err != nil { 153 return nil, err 154 } 155 output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", 156 "list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID, 157 "--zone="+zone).CombinedOutput() 158 if err != nil { 159 return nil, fmt.Errorf("Failed to get nodes in instance group %s: %s", group, output) 160 } 161 re := regexp.MustCompile(".*RUNNING") 162 lines := re.FindAllString(string(output), -1) 163 for i, line := range lines { 164 lines[i] = line[:strings.Index(line, " ")] 165 } 166 return lines, nil 167 } 168 169 // GroupSize returns the size of an instance group 170 func (p *Provider) GroupSize(group string) (int, error) { 171 // TODO: make this hit the compute API directly instead of shelling out to gcloud. 172 // TODO: make gce/gke implement InstanceGroups, so we can eliminate the per-provider logic 173 zone, err := getGCEZoneForGroup(group) 174 if err != nil { 175 return -1, err 176 } 177 output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", 178 "list-instances", group, "--project="+framework.TestContext.CloudConfig.ProjectID, 179 "--zone="+zone).CombinedOutput() 180 if err != nil { 181 return -1, fmt.Errorf("Failed to get group size for group %s: %s", group, output) 182 } 183 re := regexp.MustCompile("RUNNING") 184 return len(re.FindAllString(string(output), -1)), nil 185 } 186 187 // EnsureLoadBalancerResourcesDeleted ensures that cloud load balancer resources that were created 188 func (p *Provider) EnsureLoadBalancerResourcesDeleted(ctx context.Context, ip, portRange string) error { 189 project := framework.TestContext.CloudConfig.ProjectID 190 region, err := gcecloud.GetGCERegion(framework.TestContext.CloudConfig.Zone) 191 if err != nil { 192 return fmt.Errorf("could not get region for zone %q: %w", framework.TestContext.CloudConfig.Zone, err) 193 } 194 195 return wait.PollWithContext(ctx, 10*time.Second, 5*time.Minute, func(ctx context.Context) (bool, error) { 196 computeservice := p.gceCloud.ComputeServices().GA 197 list, err := computeservice.ForwardingRules.List(project, region).Do() 198 if err != nil { 199 return false, err 200 } 201 for _, item := range list.Items { 202 if item.PortRange == portRange && item.IPAddress == ip { 203 framework.Logf("found a load balancer: %v", item) 204 return false, nil 205 } 206 } 207 return true, nil 208 }) 209 } 210 211 func getGCEZoneForGroup(group string) (string, error) { 212 output, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "list", 213 "--project="+framework.TestContext.CloudConfig.ProjectID, "--format=value(zone)", "--filter=name="+group).Output() 214 if err != nil { 215 return "", fmt.Errorf("Failed to get zone for node group %s: %s", group, output) 216 } 217 return strings.TrimSpace(string(output)), nil 218 } 219 220 // DeleteNode deletes a node which is specified as the argument 221 func (p *Provider) DeleteNode(node *v1.Node) error { 222 zone := framework.TestContext.CloudConfig.Zone 223 project := framework.TestContext.CloudConfig.ProjectID 224 225 return p.gceCloud.DeleteInstance(project, zone, node.Name) 226 } 227 228 func (p *Provider) CreateShare() (string, string, string, error) { 229 return "", "", "", nil 230 } 231 232 func (p *Provider) DeleteShare(accountName, shareName string) error { 233 return nil 234 } 235 236 // CreatePD creates a persistent volume 237 func (p *Provider) CreatePD(zone string) (string, error) { 238 pdName := fmt.Sprintf("%s-%s", framework.TestContext.Prefix, string(uuid.NewUUID())) 239 240 if zone == "" && framework.TestContext.CloudConfig.MultiZone { 241 zones, err := p.gceCloud.GetAllZonesFromCloudProvider() 242 if err != nil { 243 return "", err 244 } 245 zone, _ = zones.PopAny() 246 } 247 248 tags := map[string]string{} 249 if _, err := p.gceCloud.CreateDisk(pdName, gcecloud.DiskTypeStandard, zone, 2 /* sizeGb */, tags); err != nil { 250 return "", err 251 } 252 return pdName, nil 253 } 254 255 // DeletePD deletes a persistent volume 256 func (p *Provider) DeletePD(pdName string) error { 257 err := p.gceCloud.DeleteDisk(pdName) 258 259 if err != nil { 260 if gerr, ok := err.(*googleapi.Error); ok && len(gerr.Errors) > 0 && gerr.Errors[0].Reason == "notFound" { 261 // PD already exists, ignore error. 262 return nil 263 } 264 265 framework.Logf("error deleting PD %q: %v", pdName, err) 266 } 267 return err 268 } 269 270 // CreatePVSource creates a persistent volume source 271 func (p *Provider) CreatePVSource(ctx context.Context, zone, diskName string) (*v1.PersistentVolumeSource, error) { 272 return &v1.PersistentVolumeSource{ 273 GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{ 274 PDName: diskName, 275 FSType: "ext3", 276 ReadOnly: false, 277 }, 278 }, nil 279 } 280 281 // DeletePVSource deletes a persistent volume source 282 func (p *Provider) DeletePVSource(ctx context.Context, pvSource *v1.PersistentVolumeSource) error { 283 return e2epv.DeletePDWithRetry(ctx, pvSource.GCEPersistentDisk.PDName) 284 } 285 286 // CleanupServiceResources cleans up GCE Service Type=LoadBalancer resources with 287 // the given name. The name is usually the UUID of the Service prefixed with an 288 // alpha-numeric character ('a') to work around cloudprovider rules. 289 func (p *Provider) CleanupServiceResources(ctx context.Context, c clientset.Interface, loadBalancerName, region, zone string) { 290 if pollErr := wait.PollWithContext(ctx, 5*time.Second, e2eservice.LoadBalancerCleanupTimeout, func(ctx context.Context) (bool, error) { 291 if err := p.cleanupGCEResources(ctx, c, loadBalancerName, region, zone); err != nil { 292 framework.Logf("Still waiting for glbc to cleanup: %v", err) 293 return false, nil 294 } 295 return true, nil 296 }); pollErr != nil { 297 framework.Failf("Failed to cleanup service GCE resources.") 298 } 299 } 300 301 func (p *Provider) cleanupGCEResources(ctx context.Context, c clientset.Interface, loadBalancerName, region, zone string) (retErr error) { 302 if region == "" { 303 // Attempt to parse region from zone if no region is given. 304 var err error 305 region, err = gcecloud.GetGCERegion(zone) 306 if err != nil { 307 return fmt.Errorf("error parsing GCE/GKE region from zone %q: %w", zone, err) 308 } 309 } 310 if err := p.gceCloud.DeleteFirewall(gcecloud.MakeFirewallName(loadBalancerName)); err != nil && 311 !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { 312 retErr = err 313 } 314 if err := p.gceCloud.DeleteRegionForwardingRule(loadBalancerName, region); err != nil && 315 !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { 316 retErr = fmt.Errorf("%v\n%v", retErr, err) 317 318 } 319 if err := p.gceCloud.DeleteRegionAddress(loadBalancerName, region); err != nil && 320 !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { 321 retErr = fmt.Errorf("%v\n%v", retErr, err) 322 } 323 clusterID, err := GetClusterID(ctx, c) 324 if err != nil { 325 retErr = fmt.Errorf("%v\n%v", retErr, err) 326 return 327 } 328 hcNames := []string{gcecloud.MakeNodesHealthCheckName(clusterID)} 329 hc, getErr := p.gceCloud.GetHTTPHealthCheck(loadBalancerName) 330 if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) { 331 retErr = fmt.Errorf("%v\n%v", retErr, getErr) 332 return 333 } 334 if hc != nil { 335 hcNames = append(hcNames, hc.Name) 336 } 337 if err := p.gceCloud.DeleteExternalTargetPoolAndChecks(&v1.Service{}, loadBalancerName, region, clusterID, hcNames...); err != nil && 338 !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { 339 retErr = fmt.Errorf("%v\n%v", retErr, err) 340 } 341 return 342 } 343 344 // L4LoadBalancerSrcRanges contains the ranges of ips used by the GCE L4 load 345 // balancers for proxying client requests and performing health checks. 346 func (p *Provider) L4LoadBalancerSrcRanges() []string { 347 return gcecloud.L4LoadBalancerSrcRanges() 348 } 349 350 // EnableAndDisableInternalLB returns functions for both enabling and disabling internal Load Balancer 351 func (p *Provider) EnableAndDisableInternalLB() (enable, disable func(svc *v1.Service)) { 352 enable = func(svc *v1.Service) { 353 svc.ObjectMeta.Annotations = map[string]string{gcecloud.ServiceAnnotationLoadBalancerType: string(gcecloud.LBTypeInternal)} 354 } 355 disable = func(svc *v1.Service) { 356 delete(svc.ObjectMeta.Annotations, gcecloud.ServiceAnnotationLoadBalancerType) 357 } 358 return 359 } 360 361 // GetInstanceTags gets tags from GCE instance with given name. 362 func GetInstanceTags(cloudConfig framework.CloudConfig, instanceName string) *compute.Tags { 363 gceCloud := cloudConfig.Provider.(*Provider).gceCloud 364 res, err := gceCloud.ComputeServices().GA.Instances.Get(cloudConfig.ProjectID, cloudConfig.Zone, 365 instanceName).Do() 366 if err != nil { 367 framework.Failf("Failed to get instance tags for %v: %v", instanceName, err) 368 } 369 return res.Tags 370 } 371 372 // SetInstanceTags sets tags on GCE instance with given name. 373 func SetInstanceTags(cloudConfig framework.CloudConfig, instanceName, zone string, tags []string) []string { 374 gceCloud := cloudConfig.Provider.(*Provider).gceCloud 375 // Re-get instance everytime because we need the latest fingerprint for updating metadata 376 resTags := GetInstanceTags(cloudConfig, instanceName) 377 _, err := gceCloud.ComputeServices().GA.Instances.SetTags( 378 cloudConfig.ProjectID, zone, instanceName, 379 &compute.Tags{Fingerprint: resTags.Fingerprint, Items: tags}).Do() 380 if err != nil { 381 framework.Failf("failed to set instance tags: %v", err) 382 } 383 framework.Logf("Sent request to set tags %v on instance: %v", tags, instanceName) 384 return resTags.Items 385 } 386 387 // IsGoogleAPIHTTPErrorCode returns true if the error is a google api 388 // error matching the corresponding HTTP error code. 389 func IsGoogleAPIHTTPErrorCode(err error, code int) bool { 390 apiErr, ok := err.(*googleapi.Error) 391 return ok && apiErr.Code == code 392 } 393 394 // GetGCECloud returns GCE cloud provider 395 func GetGCECloud() (*gcecloud.Cloud, error) { 396 p, ok := framework.TestContext.CloudConfig.Provider.(*Provider) 397 if !ok { 398 return nil, fmt.Errorf("failed to convert CloudConfig.Provider to GCE provider: %#v", framework.TestContext.CloudConfig.Provider) 399 } 400 return p.gceCloud, nil 401 } 402 403 // GetClusterID returns cluster ID 404 func GetClusterID(ctx context.Context, c clientset.Interface) (string, error) { 405 cm, err := c.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(ctx, gcecloud.UIDConfigMapName, metav1.GetOptions{}) 406 if err != nil || cm == nil { 407 return "", fmt.Errorf("error getting cluster ID: %w", err) 408 } 409 clusterID, clusterIDExists := cm.Data[gcecloud.UIDCluster] 410 providerID, providerIDExists := cm.Data[gcecloud.UIDProvider] 411 if !clusterIDExists { 412 return "", fmt.Errorf("cluster ID not set") 413 } 414 if providerIDExists { 415 return providerID, nil 416 } 417 return clusterID, nil 418 }