k8s.io/apiserver@v0.31.1/pkg/reconcilers/peer_endpoint_lease_test.go

/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package reconcilers

import (
	"reflect"
	"sort"
	"testing"
	"time"

	"github.com/google/uuid"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/apitesting"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apiserver/pkg/features"
	"k8s.io/apiserver/pkg/storage"
	etcd3testing "k8s.io/apiserver/pkg/storage/etcd3/testing"
	"k8s.io/apiserver/pkg/storage/storagebackend/factory"
	utilfeature "k8s.io/apiserver/pkg/util/feature"
	featuregatetesting "k8s.io/component-base/featuregate/testing"
)

func init() {
	var scheme = runtime.NewScheme()

	metav1.AddToGroupVersion(scheme, metav1.SchemeGroupVersion)
	utilruntime.Must(corev1.AddToScheme(scheme))
	utilruntime.Must(scheme.SetVersionPriority(corev1.SchemeGroupVersion))

	codecs = serializer.NewCodecFactory(scheme)
}

var codecs serializer.CodecFactory

type serverInfo struct {
	existingIP     string
	id             string
	ports          []corev1.EndpointPort
	newIP          string
	removeLease    bool
	expectEndpoint string
}

func NewFakePeerEndpointReconciler(t *testing.T, s storage.Interface) peerEndpointLeaseReconciler {
	// use the same base key used by the controlplane, but add a random
	// prefix so we can reuse the etcd instance for subtests independently.
	base := "/" + uuid.New().String() + "/peerserverleases/"
	return peerEndpointLeaseReconciler{serverLeases: &peerEndpointLeases{
		storage:   s,
		destroyFn: func() {},
		baseKey:   base,
		leaseTime: 1 * time.Minute, // avoid the lease to timeout on tests
	}}
}

func (f *peerEndpointLeaseReconciler) SetKeys(servers []serverInfo) error {
	for _, server := range servers {
		if err := f.UpdateLease(server.id, server.existingIP, server.ports); err != nil {
			return err
		}
	}
	return nil
}

func TestPeerEndpointLeaseReconciler(t *testing.T) {
	// enable feature flags
	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)
	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)

	server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
	t.Cleanup(func() { server.Terminate(t) })

	newFunc := func() runtime.Object { return &corev1.Endpoints{} }
	newListFunc := func() runtime.Object { return &corev1.EndpointsList{} }
	sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)

	s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "endpoints"}), newFunc, newListFunc, "")
	if err != nil {
		t.Fatalf("Error creating storage: %v", err)
	}
	t.Cleanup(dFunc)

	tests := []struct {
		testName     string
		servers      []serverInfo
		expectLeases []string
	}{
		{
			testName: "existing IP satisfy",
			servers: []serverInfo{{
				existingIP:     "4.3.2.1",
				id:             "server-1",
				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
				expectEndpoint: "4.3.2.1:8080",
			}, {
				existingIP:     "1.2.3.4",
				id:             "server-2",
				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
				expectEndpoint: "1.2.3.4:8080",
			}},
			expectLeases: []string{"4.3.2.1", "1.2.3.4"},
		},
		{
			testName: "existing IP + new IP = should return the new IP",
			servers: []serverInfo{{
				existingIP:     "4.3.2.2",
				id:             "server-1",
				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
				newIP:          "4.3.2.1",
				expectEndpoint: "4.3.2.1:8080",
			}, {
				existingIP:     "1.2.3.4",
				id:             "server-2",
				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
				newIP:          "1.1.1.1",
				expectEndpoint: "1.1.1.1:8080",
			}},
			expectLeases: []string{"4.3.2.1", "1.1.1.1"},
		},
		{
			testName: "no existing IP, should return new IP",
			servers: []serverInfo{{
				id:             "server-1",
				ports:          []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
				newIP:          "1.2.3.4",
				expectEndpoint: "1.2.3.4:8080",
			}},
			expectLeases: []string{"1.2.3.4"},
		},
	}
	for _, test := range tests {
		t.Run(test.testName, func(t *testing.T) {
			fakeReconciler := NewFakePeerEndpointReconciler(t, s)
			err := fakeReconciler.SetKeys(test.servers)
			if err != nil {
				t.Errorf("unexpected error creating keys: %v", err)
			}

			for _, server := range test.servers {
				if server.newIP != "" {
					err = fakeReconciler.UpdateLease(server.id, server.newIP, server.ports)
					if err != nil {
						t.Errorf("unexpected error reconciling: %v", err)
					}
				}
			}

			leases, err := fakeReconciler.ListLeases()
			if err != nil {
				t.Errorf("unexpected error: %v", err)
			}
			// sort for comparison
			sort.Strings(leases)
			sort.Strings(test.expectLeases)
			if !reflect.DeepEqual(leases, test.expectLeases) {
				t.Errorf("expected %v got: %v", test.expectLeases, leases)
			}

			for _, server := range test.servers {
				endpoint, err := fakeReconciler.GetLease(server.id)
				if err != nil {
					t.Errorf("unexpected error: %v", err)
				}
				if endpoint != server.expectEndpoint {
					t.Errorf("expected %v got: %v", server.expectEndpoint, endpoint)
				}
			}
		})
	}
}

func TestPeerLeaseRemoveEndpoints(t *testing.T) {
	// enable feature flags
	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)
	featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.StorageVersionAPI, true)

	server, sc := etcd3testing.NewUnsecuredEtcd3TestClientServer(t)
	t.Cleanup(func() { server.Terminate(t) })

	newFunc := func() runtime.Object { return &corev1.Endpoints{} }
	newListFunc := func() runtime.Object { return &corev1.EndpointsList{} }
	sc.Codec = apitesting.TestStorageCodec(codecs, corev1.SchemeGroupVersion)

	s, dFunc, err := factory.Create(*sc.ForResource(schema.GroupResource{Resource: "pods"}), newFunc, newListFunc, "")
	if err != nil {
		t.Fatalf("Error creating storage: %v", err)
	}
	t.Cleanup(dFunc)

	stopTests := []struct {
		testName         string
		servers          []serverInfo
		expectLeases     []string
		apiServerStartup bool
	}{
		{
			testName: "successful remove previous endpoints before apiserver starts",
			servers: []serverInfo{
				{
					existingIP:  "1.2.3.4",
					id:          "test-server-1",
					ports:       []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
					removeLease: true,
				},
				{
					existingIP: "2.4.6.8",
					id:         "test-server-2",
					ports:      []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
				}},
			expectLeases:     []string{"2.4.6.8"},
			apiServerStartup: true,
		},
		{
			testName: "stop reconciling with new IP not in existing ip list",
			servers: []serverInfo{{
				existingIP: "1.2.3.4",
				newIP:      "4.6.8.9",
				id:         "test-server-1",
				ports:      []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
			},
				{
					existingIP:  "2.4.6.8",
					id:          "test-server-2",
					ports:       []corev1.EndpointPort{{Name: "foo", Port: 8080, Protocol: "TCP"}},
					removeLease: true,
				}},
			expectLeases: []string{"1.2.3.4"},
		},
	}
	for _, test := range stopTests {
		t.Run(test.testName, func(t *testing.T) {
			fakeReconciler := NewFakePeerEndpointReconciler(t, s)
			err := fakeReconciler.SetKeys(test.servers)
			if err != nil {
				t.Errorf("unexpected error creating keys: %v", err)
			}
			if !test.apiServerStartup {
				fakeReconciler.StopReconciling()
			}
			for _, server := range test.servers {
				if server.removeLease {
					err = fakeReconciler.RemoveLease(server.id)
					// if the ip is not on the endpoints, it must return an storage error and stop reconciling
					if err != nil {
						t.Errorf("unexpected error reconciling: %v", err)
					}
				}
			}

			leases, err := fakeReconciler.ListLeases()
			if err != nil {
				t.Errorf("unexpected error: %v", err)
			}
			// sort for comparison
			sort.Strings(leases)
			sort.Strings(test.expectLeases)
			if !reflect.DeepEqual(leases, test.expectLeases) {
				t.Errorf("expected %v got: %v", test.expectLeases, leases)
			}
		})
	}
}