sigs.k8s.io/cluster-api-provider-aws@v1.5.5/pkg/cloud/services/gc/cleanup_test.go (about)

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package gc
    18  
    19  import (
    20  	"context"
    21  	"testing"
    22  
    23  	"github.com/aws/aws-sdk-go/aws"
    24  	"github.com/aws/aws-sdk-go/aws/request"
    25  	"github.com/aws/aws-sdk-go/service/ec2"
    26  	"github.com/aws/aws-sdk-go/service/elb"
    27  	"github.com/aws/aws-sdk-go/service/elbv2"
    28  	rgapi "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
    29  	"github.com/golang/mock/gomock"
    30  	. "github.com/onsi/gomega"
    31  	corev1 "k8s.io/api/core/v1"
    32  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    33  	"k8s.io/apimachinery/pkg/runtime"
    34  	"sigs.k8s.io/controller-runtime/pkg/client"
    35  	"sigs.k8s.io/controller-runtime/pkg/client/fake"
    36  
    37  	infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta1"
    38  	ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/controlplane/eks/api/v1beta1"
    39  	expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/exp/api/v1beta1"
    40  	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud"
    41  	"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/scope"
    42  	"sigs.k8s.io/cluster-api-provider-aws/test/mocks"
    43  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    44  )
    45  
    46  func TestReconcileDelete(t *testing.T) {
    47  	testCases := []struct {
    48  		name         string
    49  		clusterScope cloud.ClusterScoper
    50  		elbMocks     func(m *mocks.MockELBAPIMockRecorder)
    51  		elbv2Mocks   func(m *mocks.MockELBV2APIMockRecorder)
    52  		rgAPIMocks   func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder)
    53  		ec2Mocks     func(m *mocks.MockEC2APIMockRecorder)
    54  		expectErr    bool
    55  	}{
    56  		{
    57  			name:         "eks with cluster opt-out",
    58  			clusterScope: createManageScope(t, "false"),
    59  			rgAPIMocks:   func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {},
    60  			elbMocks:     func(m *mocks.MockELBAPIMockRecorder) {},
    61  			elbv2Mocks:   func(m *mocks.MockELBV2APIMockRecorder) {},
    62  			ec2Mocks:     func(m *mocks.MockEC2APIMockRecorder) {},
    63  			expectErr:    false,
    64  		},
    65  		{
    66  			name:         "eks with no Service load balancers",
    67  			clusterScope: createManageScope(t, ""),
    68  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
    69  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
    70  					TagFilters: []*rgapi.TagFilter{
    71  						{
    72  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
    73  							Values: []*string{aws.String("owned")},
    74  						},
    75  					},
    76  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
    77  					return &rgapi.GetResourcesOutput{
    78  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{},
    79  					}, nil
    80  				})
    81  			},
    82  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
    83  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
    84  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
    85  			expectErr:  false,
    86  		},
    87  		{
    88  			name:         "eks with no Service load balancers and explicit opt-in",
    89  			clusterScope: createManageScope(t, "true"),
    90  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
    91  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
    92  					TagFilters: []*rgapi.TagFilter{
    93  						{
    94  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
    95  							Values: []*string{aws.String("owned")},
    96  						},
    97  					},
    98  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
    99  					return &rgapi.GetResourcesOutput{
   100  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{},
   101  					}, nil
   102  				})
   103  			},
   104  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
   105  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   106  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   107  			expectErr:  false,
   108  		},
   109  		{
   110  			name:         "ec2 cluster with no Service load balancers",
   111  			clusterScope: createUnManageScope(t, ""),
   112  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   113  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   114  					TagFilters: []*rgapi.TagFilter{
   115  						{
   116  							Key:    aws.String("kubernetes.io/cluster/cluster1"),
   117  							Values: []*string{aws.String("owned")},
   118  						},
   119  					},
   120  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   121  					return &rgapi.GetResourcesOutput{
   122  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{},
   123  					}, nil
   124  				})
   125  			},
   126  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
   127  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   128  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   129  			expectErr:  false,
   130  		},
   131  		{
   132  			name:         "eks with non-Service load balancer",
   133  			clusterScope: createManageScope(t, ""),
   134  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   135  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   136  					TagFilters: []*rgapi.TagFilter{
   137  						{
   138  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   139  							Values: []*string{aws.String("owned")},
   140  						},
   141  					},
   142  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   143  					return &rgapi.GetResourcesOutput{
   144  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   145  							{
   146  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/aec24434cd2ce4630bd14a955413ee37"),
   147  								Tags: []*rgapi.Tag{
   148  									{
   149  										Key:   aws.String("kubernetes.io/cluster/eks-test-cluster"),
   150  										Value: aws.String("owned"),
   151  									},
   152  								},
   153  							},
   154  						},
   155  					}, nil
   156  				})
   157  			},
   158  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
   159  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   160  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   161  			expectErr:  false,
   162  		},
   163  		{
   164  			name:         "ec2 cluster with non-Service load balancer",
   165  			clusterScope: createUnManageScope(t, ""),
   166  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   167  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   168  					TagFilters: []*rgapi.TagFilter{
   169  						{
   170  							Key:    aws.String("kubernetes.io/cluster/cluster1"),
   171  							Values: []*string{aws.String("owned")},
   172  						},
   173  					},
   174  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   175  					return &rgapi.GetResourcesOutput{
   176  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   177  							{
   178  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/aec24434cd2ce4630bd14a955413ee37"),
   179  								Tags: []*rgapi.Tag{
   180  									{
   181  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   182  										Value: aws.String("owned"),
   183  									},
   184  								},
   185  							},
   186  						},
   187  					}, nil
   188  				})
   189  			},
   190  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
   191  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   192  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   193  			expectErr:  false,
   194  		},
   195  		{
   196  			name:         "eks with ELB Service load balancer",
   197  			clusterScope: createManageScope(t, ""),
   198  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   199  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   200  					TagFilters: []*rgapi.TagFilter{
   201  						{
   202  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   203  							Values: []*string{aws.String("owned")},
   204  						},
   205  					},
   206  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   207  					return &rgapi.GetResourcesOutput{
   208  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   209  							{
   210  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/aec24434cd2ce4630bd14a955413ee37"),
   211  								Tags: []*rgapi.Tag{
   212  									{
   213  										Key:   aws.String("kubernetes.io/cluster/eks-test-cluster"),
   214  										Value: aws.String("owned"),
   215  									},
   216  									{
   217  										Key:   aws.String(serviceNameTag),
   218  										Value: aws.String("default/svc1"),
   219  									},
   220  								},
   221  							},
   222  						},
   223  					}, nil
   224  				})
   225  			},
   226  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {
   227  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elb.DeleteLoadBalancerInput{
   228  					LoadBalancerName: aws.String("aec24434cd2ce4630bd14a955413ee37"),
   229  				}).Return(&elb.DeleteLoadBalancerOutput{}, nil)
   230  			},
   231  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   232  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   233  			expectErr:  false,
   234  		},
   235  		{
   236  			name:         "ec2 cluster with ELB Service load balancer",
   237  			clusterScope: createUnManageScope(t, ""),
   238  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   239  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   240  					TagFilters: []*rgapi.TagFilter{
   241  						{
   242  							Key:    aws.String("kubernetes.io/cluster/cluster1"),
   243  							Values: []*string{aws.String("owned")},
   244  						},
   245  					},
   246  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   247  					return &rgapi.GetResourcesOutput{
   248  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   249  							{
   250  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/aec24434cd2ce4630bd14a955413ee37"),
   251  								Tags: []*rgapi.Tag{
   252  									{
   253  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   254  										Value: aws.String("owned"),
   255  									},
   256  									{
   257  										Key:   aws.String(serviceNameTag),
   258  										Value: aws.String("default/svc1"),
   259  									},
   260  								},
   261  							},
   262  						},
   263  					}, nil
   264  				})
   265  			},
   266  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {
   267  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elb.DeleteLoadBalancerInput{
   268  					LoadBalancerName: aws.String("aec24434cd2ce4630bd14a955413ee37"),
   269  				}).Return(&elb.DeleteLoadBalancerOutput{}, nil)
   270  			},
   271  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   272  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   273  			expectErr:  false,
   274  		},
   275  		{
   276  			name:         "eks with NLB Service load balancer",
   277  			clusterScope: createManageScope(t, ""),
   278  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   279  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   280  					TagFilters: []*rgapi.TagFilter{
   281  						{
   282  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   283  							Values: []*string{aws.String("owned")},
   284  						},
   285  					},
   286  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   287  					return &rgapi.GetResourcesOutput{
   288  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   289  							{
   290  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/net/aec24434cd2ce4630bd14a955413ee37"),
   291  								Tags: []*rgapi.Tag{
   292  									{
   293  										Key:   aws.String("kubernetes.io/cluster/eks-test-cluster"),
   294  										Value: aws.String("owned"),
   295  									},
   296  									{
   297  										Key:   aws.String(serviceNameTag),
   298  										Value: aws.String("default/svc1"),
   299  									},
   300  								},
   301  							},
   302  						},
   303  					}, nil
   304  				})
   305  			},
   306  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {},
   307  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {
   308  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elbv2.DeleteLoadBalancerInput{
   309  					LoadBalancerArn: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/net/aec24434cd2ce4630bd14a955413ee37"),
   310  				}).Return(&elbv2.DeleteLoadBalancerOutput{}, nil)
   311  			},
   312  			ec2Mocks:  func(m *mocks.MockEC2APIMockRecorder) {},
   313  			expectErr: false,
   314  		},
   315  		{
   316  			name:         "ec2 cluster with NLB Service load balancer",
   317  			clusterScope: createUnManageScope(t, ""),
   318  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   319  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   320  					TagFilters: []*rgapi.TagFilter{
   321  						{
   322  							Key:    aws.String("kubernetes.io/cluster/cluster1"),
   323  							Values: []*string{aws.String("owned")},
   324  						},
   325  					},
   326  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   327  					return &rgapi.GetResourcesOutput{
   328  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   329  							{
   330  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/net/aec24434cd2ce4630bd14a955413ee37"),
   331  								Tags: []*rgapi.Tag{
   332  									{
   333  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   334  										Value: aws.String("owned"),
   335  									},
   336  									{
   337  										Key:   aws.String(serviceNameTag),
   338  										Value: aws.String("default/svc1"),
   339  									},
   340  								},
   341  							},
   342  						},
   343  					}, nil
   344  				})
   345  			},
   346  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {},
   347  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {
   348  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elbv2.DeleteLoadBalancerInput{
   349  					LoadBalancerArn: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/net/aec24434cd2ce4630bd14a955413ee37"),
   350  				}).Return(&elbv2.DeleteLoadBalancerOutput{}, nil)
   351  			},
   352  			ec2Mocks:  func(m *mocks.MockEC2APIMockRecorder) {},
   353  			expectErr: false,
   354  		},
   355  		{
   356  			name:         "eks with ALB Service load balancer",
   357  			clusterScope: createManageScope(t, ""),
   358  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   359  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   360  					TagFilters: []*rgapi.TagFilter{
   361  						{
   362  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   363  							Values: []*string{aws.String("owned")},
   364  						},
   365  					},
   366  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   367  					return &rgapi.GetResourcesOutput{
   368  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   369  							{
   370  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/app/aec24434cd2ce4630bd14a955413ee37"),
   371  								Tags: []*rgapi.Tag{
   372  									{
   373  										Key:   aws.String("kubernetes.io/cluster/eks-test-cluster"),
   374  										Value: aws.String("owned"),
   375  									},
   376  									{
   377  										Key:   aws.String(serviceNameTag),
   378  										Value: aws.String("default/svc1"),
   379  									},
   380  								},
   381  							},
   382  						},
   383  					}, nil
   384  				})
   385  			},
   386  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {},
   387  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {
   388  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elbv2.DeleteLoadBalancerInput{
   389  					LoadBalancerArn: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/app/aec24434cd2ce4630bd14a955413ee37"),
   390  				}).Return(&elbv2.DeleteLoadBalancerOutput{}, nil)
   391  			},
   392  			ec2Mocks:  func(m *mocks.MockEC2APIMockRecorder) {},
   393  			expectErr: false,
   394  		},
   395  		{
   396  			name:         "ec2 cluster with ALB Service load balancer",
   397  			clusterScope: createUnManageScope(t, ""),
   398  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   399  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   400  					TagFilters: []*rgapi.TagFilter{
   401  						{
   402  							Key:    aws.String("kubernetes.io/cluster/cluster1"),
   403  							Values: []*string{aws.String("owned")},
   404  						},
   405  					},
   406  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   407  					return &rgapi.GetResourcesOutput{
   408  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   409  							{
   410  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/app/aec24434cd2ce4630bd14a955413ee37"),
   411  								Tags: []*rgapi.Tag{
   412  									{
   413  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   414  										Value: aws.String("owned"),
   415  									},
   416  									{
   417  										Key:   aws.String(serviceNameTag),
   418  										Value: aws.String("default/svc1"),
   419  									},
   420  								},
   421  							},
   422  						},
   423  					}, nil
   424  				})
   425  			},
   426  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {},
   427  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {
   428  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elbv2.DeleteLoadBalancerInput{
   429  					LoadBalancerArn: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/app/aec24434cd2ce4630bd14a955413ee37"),
   430  				}).Return(&elbv2.DeleteLoadBalancerOutput{}, nil)
   431  			},
   432  			ec2Mocks:  func(m *mocks.MockEC2APIMockRecorder) {},
   433  			expectErr: false,
   434  		},
   435  		{
   436  			name:         "eks cluster with different resource types",
   437  			clusterScope: createManageScope(t, ""),
   438  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   439  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   440  					TagFilters: []*rgapi.TagFilter{
   441  						{
   442  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   443  							Values: []*string{aws.String("owned")},
   444  						},
   445  					},
   446  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   447  					return &rgapi.GetResourcesOutput{
   448  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   449  							{
   450  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:targetgroup/k8s-default-podinfo-2c868b281a/e979fe9bd6825433"),
   451  								Tags: []*rgapi.Tag{
   452  									{
   453  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   454  										Value: aws.String("owned"),
   455  									},
   456  									{
   457  										Key:   aws.String(serviceNameTag),
   458  										Value: aws.String("default/svc1"),
   459  									},
   460  								},
   461  							},
   462  							{
   463  								ResourceARN: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:loadbalancer/aec24434cd2ce4630bd14a955413ee37"),
   464  								Tags: []*rgapi.Tag{
   465  									{
   466  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   467  										Value: aws.String("owned"),
   468  									},
   469  									{
   470  										Key:   aws.String(serviceNameTag),
   471  										Value: aws.String("default/svc1"),
   472  									},
   473  								},
   474  							},
   475  							{
   476  								ResourceARN: aws.String("arn:aws:ec2:eu-west-2:1234567890:security-group/sg-123456"),
   477  								Tags: []*rgapi.Tag{
   478  									{
   479  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   480  										Value: aws.String("owned"),
   481  									},
   482  									{
   483  										Key:   aws.String(serviceNameTag),
   484  										Value: aws.String("default/svc1"),
   485  									},
   486  								},
   487  							},
   488  						},
   489  					}, nil
   490  				})
   491  			},
   492  			elbMocks: func(m *mocks.MockELBAPIMockRecorder) {
   493  				m.DeleteLoadBalancerWithContext(gomock.Any(), &elb.DeleteLoadBalancerInput{
   494  					LoadBalancerName: aws.String("aec24434cd2ce4630bd14a955413ee37"),
   495  				}).Return(&elb.DeleteLoadBalancerOutput{}, nil)
   496  			},
   497  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {
   498  				m.DeleteTargetGroupWithContext(gomock.Any(), &elbv2.DeleteTargetGroupInput{
   499  					TargetGroupArn: aws.String("arn:aws:elasticloadbalancing:eu-west-2:1234567890:targetgroup/k8s-default-podinfo-2c868b281a/e979fe9bd6825433"),
   500  				})
   501  			},
   502  			ec2Mocks: func(m *mocks.MockEC2APIMockRecorder) {
   503  				m.DeleteSecurityGroupWithContext(gomock.Any(), &ec2.DeleteSecurityGroupInput{
   504  					GroupId: aws.String("sg-123456"),
   505  				})
   506  			},
   507  			expectErr: false,
   508  		},
   509  		{
   510  			name:         "eks should ignore unhandled resources",
   511  			clusterScope: createManageScope(t, ""),
   512  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   513  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   514  					TagFilters: []*rgapi.TagFilter{
   515  						{
   516  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   517  							Values: []*string{aws.String("owned")},
   518  						},
   519  					},
   520  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   521  					return &rgapi.GetResourcesOutput{
   522  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   523  							{
   524  								ResourceARN: aws.String("arn:aws:ec2:eu-west-2:217426147237:s3/somebucket"),
   525  								Tags: []*rgapi.Tag{
   526  									{
   527  										Key:   aws.String("kubernetes.io/cluster/eks-test-cluster"),
   528  										Value: aws.String("owned"),
   529  									},
   530  									{
   531  										Key:   aws.String(serviceNameTag),
   532  										Value: aws.String("default/svc1"),
   533  									},
   534  									{
   535  										Key:   aws.String("Name"),
   536  										Value: aws.String("eks-cluster-sg-default_capi-managed-test-control-plane-10156951"),
   537  									},
   538  								},
   539  							},
   540  						},
   541  					}, nil
   542  				})
   543  			},
   544  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
   545  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   546  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   547  			expectErr:  false,
   548  		},
   549  		{
   550  			name:         "eks with security group created by EKS",
   551  			clusterScope: createManageScope(t, ""),
   552  			rgAPIMocks: func(m *mocks.MockResourceGroupsTaggingAPIAPIMockRecorder) {
   553  				m.GetResourcesWithContext(gomock.Any(), &rgapi.GetResourcesInput{
   554  					TagFilters: []*rgapi.TagFilter{
   555  						{
   556  							Key:    aws.String("kubernetes.io/cluster/eks-test-cluster"),
   557  							Values: []*string{aws.String("owned")},
   558  						},
   559  					},
   560  				}).DoAndReturn(func(awsCtx context.Context, input *rgapi.GetResourcesInput, opts ...request.Option) (*rgapi.GetResourcesOutput, error) {
   561  					return &rgapi.GetResourcesOutput{
   562  						ResourceTagMappingList: []*rgapi.ResourceTagMapping{
   563  							{
   564  								ResourceARN: aws.String("arn:aws:ec2:eu-west-2:1234567890:security-group/sg-123456"),
   565  								Tags: []*rgapi.Tag{
   566  									{
   567  										Key:   aws.String("kubernetes.io/cluster/cluster1"),
   568  										Value: aws.String("owned"),
   569  									},
   570  									{
   571  										Key:   aws.String(serviceNameTag),
   572  										Value: aws.String("default/svc1"),
   573  									},
   574  									{
   575  										Key:   aws.String(eksClusterNameTag),
   576  										Value: aws.String("default_eks_test_cluster"),
   577  									},
   578  								},
   579  							},
   580  						},
   581  					}, nil
   582  				})
   583  			},
   584  			elbMocks:   func(m *mocks.MockELBAPIMockRecorder) {},
   585  			elbv2Mocks: func(m *mocks.MockELBV2APIMockRecorder) {},
   586  			ec2Mocks:   func(m *mocks.MockEC2APIMockRecorder) {},
   587  			expectErr:  false,
   588  		},
   589  	}
   590  
   591  	for _, tc := range testCases {
   592  		t.Run(tc.name, func(t *testing.T) {
   593  			g := NewWithT(t)
   594  			mockCtrl := gomock.NewController(t)
   595  			defer mockCtrl.Finish()
   596  
   597  			rgapiMock := mocks.NewMockResourceGroupsTaggingAPIAPI(mockCtrl)
   598  			elbapiMock := mocks.NewMockELBAPI(mockCtrl)
   599  			elbv2Mock := mocks.NewMockELBV2API(mockCtrl)
   600  			ec2Mock := mocks.NewMockEC2API(mockCtrl)
   601  
   602  			tc.rgAPIMocks(rgapiMock.EXPECT())
   603  			tc.elbMocks(elbapiMock.EXPECT())
   604  			tc.elbv2Mocks(elbv2Mock.EXPECT())
   605  			tc.ec2Mocks(ec2Mock.EXPECT())
   606  
   607  			ctx := context.TODO()
   608  
   609  			opts := []ServiceOption{
   610  				withELBClient(elbapiMock),
   611  				withELBv2Client(elbv2Mock),
   612  				withResourceTaggingClient(rgapiMock),
   613  				withEC2Client(ec2Mock),
   614  			}
   615  			wkSvc := NewService(tc.clusterScope, opts...)
   616  			err := wkSvc.ReconcileDelete(ctx)
   617  
   618  			if tc.expectErr {
   619  				g.Expect(err).NotTo(BeNil())
   620  				return
   621  			}
   622  
   623  			g.Expect(err).To(BeNil())
   624  		})
   625  	}
   626  }
   627  
   628  func createManageScope(t *testing.T, annotationValue string) *scope.ManagedControlPlaneScope {
   629  	t.Helper()
   630  	g := NewWithT(t)
   631  
   632  	cluster := createEKSCluster()
   633  	cp := createManagedControlPlane(annotationValue)
   634  	objs := []client.Object{cluster, cp}
   635  
   636  	scheme := createScheme()
   637  	client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build()
   638  
   639  	managedScope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{
   640  		Client:         client,
   641  		Cluster:        cluster,
   642  		ControlPlane:   cp,
   643  		ControllerName: "test-controller",
   644  	})
   645  	g.Expect(err).NotTo(HaveOccurred())
   646  
   647  	return managedScope
   648  }
   649  
   650  func createUnManageScope(t *testing.T, annotationValue string) *scope.ClusterScope {
   651  	t.Helper()
   652  	g := NewWithT(t)
   653  
   654  	cluster := createUnmanagedCluster()
   655  	awsCluster := createAWSCluser(annotationValue)
   656  	objs := []client.Object{cluster, awsCluster}
   657  
   658  	scheme := createScheme()
   659  	client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build()
   660  
   661  	clusterScope, err := scope.NewClusterScope(scope.ClusterScopeParams{
   662  		Client:         client,
   663  		Cluster:        cluster,
   664  		AWSCluster:     awsCluster,
   665  		ControllerName: "test-controller",
   666  	})
   667  	g.Expect(err).NotTo(HaveOccurred())
   668  
   669  	return clusterScope
   670  }
   671  
   672  func createScheme() *runtime.Scheme {
   673  	scheme := runtime.NewScheme()
   674  	_ = corev1.AddToScheme(scheme)
   675  	_ = ekscontrolplanev1.AddToScheme(scheme)
   676  	_ = infrav1.AddToScheme(scheme)
   677  	_ = clusterv1.AddToScheme(scheme)
   678  
   679  	return scheme
   680  }
   681  
   682  func createEKSCluster() *clusterv1.Cluster {
   683  	return &clusterv1.Cluster{
   684  		ObjectMeta: metav1.ObjectMeta{
   685  			Name:      "cluster1",
   686  			Namespace: "default",
   687  		},
   688  		Spec: clusterv1.ClusterSpec{
   689  			InfrastructureRef: &corev1.ObjectReference{
   690  				Kind:       "AWSManagedControlPlane",
   691  				APIVersion: ekscontrolplanev1.GroupVersion.String(),
   692  				Name:       "cp1",
   693  				Namespace:  "default",
   694  			},
   695  		},
   696  	}
   697  }
   698  
   699  func createManagedControlPlane(annotationValue string) *ekscontrolplanev1.AWSManagedControlPlane {
   700  	cp := &ekscontrolplanev1.AWSManagedControlPlane{
   701  		TypeMeta: metav1.TypeMeta{
   702  			Kind:       "AWSManagedControlPlane",
   703  			APIVersion: ekscontrolplanev1.GroupVersion.String(),
   704  		},
   705  		ObjectMeta: metav1.ObjectMeta{
   706  			Name:      "cp1",
   707  			Namespace: "default",
   708  		},
   709  		Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{
   710  			EKSClusterName: "eks-test-cluster",
   711  		},
   712  	}
   713  
   714  	if annotationValue != "" {
   715  		cp.ObjectMeta.Annotations = map[string]string{
   716  			expinfrav1.ExternalResourceGCAnnotation: annotationValue,
   717  		}
   718  	}
   719  
   720  	return cp
   721  }
   722  
   723  func createAWSCluser(annotationValue string) *infrav1.AWSCluster {
   724  	awsc := &infrav1.AWSCluster{
   725  		TypeMeta: metav1.TypeMeta{
   726  			Kind:       "AWSCluster",
   727  			APIVersion: infrav1.GroupVersion.String(),
   728  		},
   729  		ObjectMeta: metav1.ObjectMeta{
   730  			Name:      "cluster1",
   731  			Namespace: "default",
   732  		},
   733  		Spec: infrav1.AWSClusterSpec{},
   734  	}
   735  
   736  	if annotationValue != "" {
   737  		awsc.ObjectMeta.Annotations = map[string]string{
   738  			expinfrav1.ExternalResourceGCAnnotation: annotationValue,
   739  		}
   740  	}
   741  
   742  	return awsc
   743  }
   744  
   745  func createUnmanagedCluster() *clusterv1.Cluster {
   746  	return &clusterv1.Cluster{
   747  		ObjectMeta: metav1.ObjectMeta{
   748  			Name:      "cluster1",
   749  			Namespace: "default",
   750  		},
   751  		Spec: clusterv1.ClusterSpec{
   752  			InfrastructureRef: &corev1.ObjectReference{
   753  				Kind:       "AWSCluster",
   754  				APIVersion: infrav1.GroupVersion.String(),
   755  				Name:       "cluster1",
   756  				Namespace:  "default",
   757  			},
   758  		},
   759  	}
   760  }