k8s.io/client-go@v0.31.1/tools/leaderelection/leaderelection_test.go (about)

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package leaderelection
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"sync"
    24  	"testing"
    25  	"time"
    26  
    27  	"github.com/google/go-cmp/cmp"
    28  	"github.com/stretchr/testify/assert"
    29  	coordinationv1 "k8s.io/api/coordination/v1"
    30  	corev1 "k8s.io/api/core/v1"
    31  	"k8s.io/apimachinery/pkg/api/equality"
    32  	"k8s.io/apimachinery/pkg/api/errors"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/runtime"
    35  	"k8s.io/apimachinery/pkg/util/wait"
    36  	"k8s.io/client-go/kubernetes/fake"
    37  	fakeclient "k8s.io/client-go/testing"
    38  	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
    39  	"k8s.io/client-go/tools/record"
    40  	"k8s.io/utils/clock"
    41  )
    42  
    43  func createLockObject(t *testing.T, objectType, namespace, name string, record *rl.LeaderElectionRecord) (obj runtime.Object) {
    44  	objectMeta := metav1.ObjectMeta{
    45  		Namespace: namespace,
    46  		Name:      name,
    47  	}
    48  	if record != nil {
    49  		recordBytes, _ := json.Marshal(record)
    50  		objectMeta.Annotations = map[string]string{
    51  			rl.LeaderElectionRecordAnnotationKey: string(recordBytes),
    52  		}
    53  	}
    54  	switch objectType {
    55  	case "endpoints":
    56  		obj = &corev1.Endpoints{ObjectMeta: objectMeta}
    57  	case "configmaps":
    58  		obj = &corev1.ConfigMap{ObjectMeta: objectMeta}
    59  	case "leases":
    60  		var spec coordinationv1.LeaseSpec
    61  		if record != nil {
    62  			spec = rl.LeaderElectionRecordToLeaseSpec(record)
    63  		}
    64  		obj = &coordinationv1.Lease{ObjectMeta: objectMeta, Spec: spec}
    65  	default:
    66  		t.Fatal("unexpected objType:" + objectType)
    67  	}
    68  	return
    69  }
    70  
    71  type Reactor struct {
    72  	verb       string
    73  	objectType string
    74  	reaction   fakeclient.ReactionFunc
    75  }
    76  
    77  func testTryAcquireOrRenew(t *testing.T, objectType string) {
    78  	clock := clock.RealClock{}
    79  	future := clock.Now().Add(1000 * time.Hour)
    80  	past := clock.Now().Add(-1000 * time.Hour)
    81  
    82  	tests := []struct {
    83  		name           string
    84  		observedRecord rl.LeaderElectionRecord
    85  		observedTime   time.Time
    86  		retryAfter     time.Duration
    87  		reactors       []Reactor
    88  		expectedEvents []string
    89  
    90  		expectSuccess    bool
    91  		transitionLeader bool
    92  		outHolder        string
    93  	}{
    94  		{
    95  			name: "acquire from no object",
    96  			reactors: []Reactor{
    97  				{
    98  					verb: "get",
    99  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   100  						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
   101  					},
   102  				},
   103  				{
   104  					verb: "create",
   105  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   106  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   107  					},
   108  				},
   109  			},
   110  			expectSuccess: true,
   111  			outHolder:     "baz",
   112  		},
   113  		{
   114  			name: "acquire from object without annotations",
   115  			reactors: []Reactor{
   116  				{
   117  					verb: "get",
   118  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   119  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), nil), nil
   120  					},
   121  				},
   122  				{
   123  					verb: "update",
   124  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   125  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   126  					},
   127  				},
   128  			},
   129  			expectSuccess:    true,
   130  			transitionLeader: true,
   131  			outHolder:        "baz",
   132  		},
   133  		{
   134  			name: "acquire from led object with the lease duration seconds",
   135  			reactors: []Reactor{
   136  				{
   137  					verb: "get",
   138  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   139  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil
   140  					},
   141  				},
   142  				{
   143  					verb: "get",
   144  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   145  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil
   146  					},
   147  				},
   148  				{
   149  					verb: "update",
   150  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   151  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   152  					},
   153  				},
   154  			},
   155  			retryAfter:       3 * time.Second,
   156  			expectSuccess:    true,
   157  			transitionLeader: true,
   158  			outHolder:        "baz",
   159  		},
   160  		{
   161  			name: "acquire from unled object",
   162  			reactors: []Reactor{
   163  				{
   164  					verb: "get",
   165  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   166  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil
   167  					},
   168  				},
   169  				{
   170  					verb: "update",
   171  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   172  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   173  					},
   174  				},
   175  			},
   176  
   177  			expectSuccess:    true,
   178  			transitionLeader: true,
   179  			outHolder:        "baz",
   180  		},
   181  		{
   182  			name: "acquire from led, unacked object",
   183  			reactors: []Reactor{
   184  				{
   185  					verb: "get",
   186  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   187  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
   188  					},
   189  				},
   190  				{
   191  					verb: "update",
   192  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   193  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   194  					},
   195  				},
   196  			},
   197  			observedRecord: rl.LeaderElectionRecord{HolderIdentity: "bing"},
   198  			observedTime:   past,
   199  
   200  			expectSuccess:    true,
   201  			transitionLeader: true,
   202  			outHolder:        "baz",
   203  		},
   204  		{
   205  			name: "acquire from empty led, acked object",
   206  			reactors: []Reactor{
   207  				{
   208  					verb: "get",
   209  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   210  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: ""}), nil
   211  					},
   212  				},
   213  				{
   214  					verb: "update",
   215  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   216  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   217  					},
   218  				},
   219  			},
   220  			observedTime: future,
   221  
   222  			expectSuccess:    true,
   223  			transitionLeader: true,
   224  			outHolder:        "baz",
   225  		},
   226  		{
   227  			name: "don't acquire from led, acked object",
   228  			reactors: []Reactor{
   229  				{
   230  					verb: "get",
   231  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   232  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
   233  					},
   234  				},
   235  			},
   236  			observedTime: future,
   237  
   238  			expectSuccess: false,
   239  			outHolder:     "bing",
   240  		},
   241  		{
   242  			name: "renew already acquired object",
   243  			reactors: []Reactor{
   244  				{
   245  					verb: "get",
   246  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   247  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
   248  					},
   249  				},
   250  				{
   251  					verb: "update",
   252  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   253  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   254  					},
   255  				},
   256  			},
   257  			observedTime:   future,
   258  			observedRecord: rl.LeaderElectionRecord{HolderIdentity: "baz"},
   259  
   260  			expectSuccess: true,
   261  			outHolder:     "baz",
   262  		},
   263  	}
   264  
   265  	for i := range tests {
   266  		test := &tests[i]
   267  		t.Run(test.name, func(t *testing.T) {
   268  			// OnNewLeader is called async so we have to wait for it.
   269  			var wg sync.WaitGroup
   270  			wg.Add(1)
   271  			var reportedLeader string
   272  			var lock rl.Interface
   273  
   274  			objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"}
   275  			recorder := record.NewFakeRecorder(100)
   276  			resourceLockConfig := rl.ResourceLockConfig{
   277  				Identity:      "baz",
   278  				EventRecorder: recorder,
   279  			}
   280  			c := &fake.Clientset{}
   281  			for _, reactor := range test.reactors {
   282  				c.AddReactor(reactor.verb, objectType, reactor.reaction)
   283  			}
   284  			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
   285  				t.Errorf("unreachable action. testclient called too many times: %+v", action)
   286  				return true, nil, fmt.Errorf("unreachable action")
   287  			})
   288  
   289  			switch objectType {
   290  			case "leases":
   291  				lock = &rl.LeaseLock{
   292  					LeaseMeta:  objectMeta,
   293  					LockConfig: resourceLockConfig,
   294  					Client:     c.CoordinationV1(),
   295  				}
   296  			default:
   297  				t.Fatalf("Unknown objectType: %v", objectType)
   298  			}
   299  
   300  			lec := LeaderElectionConfig{
   301  				Lock:          lock,
   302  				LeaseDuration: 10 * time.Second,
   303  				Callbacks: LeaderCallbacks{
   304  					OnNewLeader: func(l string) {
   305  						defer wg.Done()
   306  						reportedLeader = l
   307  					},
   308  				},
   309  			}
   310  			observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord)
   311  			le := &LeaderElector{
   312  				config:            lec,
   313  				observedRecord:    test.observedRecord,
   314  				observedRawRecord: observedRawRecord,
   315  				observedTime:      test.observedTime,
   316  				clock:             clock,
   317  				metrics:           globalMetricsFactory.newLeaderMetrics(),
   318  			}
   319  			if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) {
   320  				if test.retryAfter != 0 {
   321  					time.Sleep(test.retryAfter)
   322  					if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) {
   323  						t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess)
   324  					}
   325  				} else {
   326  					t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess)
   327  				}
   328  			}
   329  
   330  			le.observedRecord.AcquireTime = metav1.Time{}
   331  			le.observedRecord.RenewTime = metav1.Time{}
   332  			if le.observedRecord.HolderIdentity != test.outHolder {
   333  				t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity)
   334  			}
   335  			if len(test.reactors) != len(c.Actions()) {
   336  				t.Errorf("wrong number of api interactions")
   337  			}
   338  			if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 {
   339  				t.Errorf("leader should have transitioned but did not")
   340  			}
   341  			if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 {
   342  				t.Errorf("leader should not have transitioned but did")
   343  			}
   344  
   345  			le.maybeReportTransition()
   346  			wg.Wait()
   347  			if reportedLeader != test.outHolder {
   348  				t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader)
   349  			}
   350  			assertEqualEvents(t, test.expectedEvents, recorder.Events)
   351  		})
   352  	}
   353  }
   354  
   355  func TestTryCoordinatedRenew(t *testing.T) {
   356  	objectType := "leases"
   357  	clock := clock.RealClock{}
   358  	future := clock.Now().Add(1000 * time.Hour)
   359  
   360  	tests := []struct {
   361  		name           string
   362  		observedRecord rl.LeaderElectionRecord
   363  		observedTime   time.Time
   364  		retryAfter     time.Duration
   365  		reactors       []Reactor
   366  		expectedEvents []string
   367  
   368  		expectSuccess    bool
   369  		transitionLeader bool
   370  		outHolder        string
   371  	}{
   372  		{
   373  			name: "don't acquire from led, acked object",
   374  			reactors: []Reactor{
   375  				{
   376  					verb: "get",
   377  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   378  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil
   379  					},
   380  				},
   381  			},
   382  			observedTime: future,
   383  
   384  			expectSuccess: false,
   385  			outHolder:     "bing",
   386  		},
   387  		{
   388  			name: "renew already acquired object",
   389  			reactors: []Reactor{
   390  				{
   391  					verb: "get",
   392  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   393  						return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil
   394  					},
   395  				},
   396  				{
   397  					verb: "update",
   398  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   399  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   400  					},
   401  				},
   402  			},
   403  			observedTime:   future,
   404  			observedRecord: rl.LeaderElectionRecord{HolderIdentity: "baz"},
   405  
   406  			expectSuccess: true,
   407  			outHolder:     "baz",
   408  		},
   409  	}
   410  
   411  	for i := range tests {
   412  		test := &tests[i]
   413  		t.Run(test.name, func(t *testing.T) {
   414  			// OnNewLeader is called async so we have to wait for it.
   415  			var wg sync.WaitGroup
   416  			wg.Add(1)
   417  			var reportedLeader string
   418  			var lock rl.Interface
   419  
   420  			objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"}
   421  			recorder := record.NewFakeRecorder(100)
   422  			resourceLockConfig := rl.ResourceLockConfig{
   423  				Identity:      "baz",
   424  				EventRecorder: recorder,
   425  			}
   426  			c := &fake.Clientset{}
   427  			for _, reactor := range test.reactors {
   428  				c.AddReactor(reactor.verb, objectType, reactor.reaction)
   429  			}
   430  			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
   431  				t.Errorf("unreachable action. testclient called too many times: %+v", action)
   432  				return true, nil, fmt.Errorf("unreachable action")
   433  			})
   434  
   435  			lock = &rl.LeaseLock{
   436  				LeaseMeta:  objectMeta,
   437  				LockConfig: resourceLockConfig,
   438  				Client:     c.CoordinationV1(),
   439  			}
   440  			lec := LeaderElectionConfig{
   441  				Lock:          lock,
   442  				LeaseDuration: 10 * time.Second,
   443  				Callbacks: LeaderCallbacks{
   444  					OnNewLeader: func(l string) {
   445  						defer wg.Done()
   446  						reportedLeader = l
   447  					},
   448  				},
   449  				Coordinated: true,
   450  			}
   451  			observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord)
   452  			le := &LeaderElector{
   453  				config:            lec,
   454  				observedRecord:    test.observedRecord,
   455  				observedRawRecord: observedRawRecord,
   456  				observedTime:      test.observedTime,
   457  				clock:             clock,
   458  				metrics:           globalMetricsFactory.newLeaderMetrics(),
   459  			}
   460  			if test.expectSuccess != le.tryCoordinatedRenew(context.Background()) {
   461  				if test.retryAfter != 0 {
   462  					time.Sleep(test.retryAfter)
   463  					if test.expectSuccess != le.tryCoordinatedRenew(context.Background()) {
   464  						t.Errorf("unexpected result of tryCoordinatedRenew: [succeeded=%v]", !test.expectSuccess)
   465  					}
   466  				} else {
   467  					t.Errorf("unexpected result of gryCoordinatedRenew: [succeeded=%v]", !test.expectSuccess)
   468  				}
   469  			}
   470  
   471  			le.observedRecord.AcquireTime = metav1.Time{}
   472  			le.observedRecord.RenewTime = metav1.Time{}
   473  			if le.observedRecord.HolderIdentity != test.outHolder {
   474  				t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity)
   475  			}
   476  			if len(test.reactors) != len(c.Actions()) {
   477  				t.Errorf("wrong number of api interactions")
   478  			}
   479  			if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 {
   480  				t.Errorf("leader should have transitioned but did not")
   481  			}
   482  			if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 {
   483  				t.Errorf("leader should not have transitioned but did")
   484  			}
   485  
   486  			le.maybeReportTransition()
   487  			wg.Wait()
   488  			if reportedLeader != test.outHolder {
   489  				t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader)
   490  			}
   491  			assertEqualEvents(t, test.expectedEvents, recorder.Events)
   492  		})
   493  	}
   494  }
   495  
   496  // Will test leader election using lease as the resource
   497  func TestTryAcquireOrRenewLeases(t *testing.T) {
   498  	testTryAcquireOrRenew(t, "leases")
   499  }
   500  
   501  func TestLeaseSpecToLeaderElectionRecordRoundTrip(t *testing.T) {
   502  	holderIdentity := "foo"
   503  	leaseDurationSeconds := int32(10)
   504  	leaseTransitions := int32(1)
   505  	oldSpec := coordinationv1.LeaseSpec{
   506  		HolderIdentity:       &holderIdentity,
   507  		LeaseDurationSeconds: &leaseDurationSeconds,
   508  		AcquireTime:          &metav1.MicroTime{Time: time.Now()},
   509  		RenewTime:            &metav1.MicroTime{Time: time.Now()},
   510  		LeaseTransitions:     &leaseTransitions,
   511  	}
   512  
   513  	oldRecord := rl.LeaseSpecToLeaderElectionRecord(&oldSpec)
   514  	newSpec := rl.LeaderElectionRecordToLeaseSpec(oldRecord)
   515  
   516  	if !equality.Semantic.DeepEqual(oldSpec, newSpec) {
   517  		t.Errorf("diff: %v", cmp.Diff(oldSpec, newSpec))
   518  	}
   519  
   520  	newRecord := rl.LeaseSpecToLeaderElectionRecord(&newSpec)
   521  
   522  	if !equality.Semantic.DeepEqual(oldRecord, newRecord) {
   523  		t.Errorf("diff: %v", cmp.Diff(oldRecord, newRecord))
   524  	}
   525  }
   526  
   527  func GetRawRecordOrDie(t *testing.T, objectType string, ler rl.LeaderElectionRecord) (ret []byte) {
   528  	var err error
   529  	switch objectType {
   530  	case "leases":
   531  		ret, err = json.Marshal(ler)
   532  		if err != nil {
   533  			t.Fatalf("lock %s get raw record %v failed: %v", objectType, ler, err)
   534  		}
   535  	default:
   536  		t.Fatal("unexpected objType:" + objectType)
   537  	}
   538  	return
   539  }
   540  
   541  func testReleaseLease(t *testing.T, objectType string) {
   542  	tests := []struct {
   543  		name           string
   544  		observedRecord rl.LeaderElectionRecord
   545  		observedTime   time.Time
   546  		reactors       []Reactor
   547  		expectedEvents []string
   548  
   549  		expectSuccess    bool
   550  		transitionLeader bool
   551  		outHolder        string
   552  	}{
   553  		{
   554  			name: "release acquired lock from no object",
   555  			reactors: []Reactor{
   556  				{
   557  					verb:       "get",
   558  					objectType: objectType,
   559  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   560  						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
   561  					},
   562  				},
   563  				{
   564  					verb:       "create",
   565  					objectType: objectType,
   566  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   567  						return true, action.(fakeclient.CreateAction).GetObject(), nil
   568  					},
   569  				},
   570  				{
   571  					verb:       "update",
   572  					objectType: objectType,
   573  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   574  						return true, action.(fakeclient.UpdateAction).GetObject(), nil
   575  					},
   576  				},
   577  			},
   578  			expectSuccess: true,
   579  			outHolder:     "",
   580  		},
   581  	}
   582  
   583  	for i := range tests {
   584  		test := &tests[i]
   585  		t.Run(test.name, func(t *testing.T) {
   586  			// OnNewLeader is called async so we have to wait for it.
   587  			var wg sync.WaitGroup
   588  			wg.Add(1)
   589  			var reportedLeader string
   590  			var lock rl.Interface
   591  
   592  			objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"}
   593  			recorder := record.NewFakeRecorder(100)
   594  			resourceLockConfig := rl.ResourceLockConfig{
   595  				Identity:      "baz",
   596  				EventRecorder: recorder,
   597  			}
   598  			c := &fake.Clientset{}
   599  			for _, reactor := range test.reactors {
   600  				c.AddReactor(reactor.verb, objectType, reactor.reaction)
   601  			}
   602  			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
   603  				t.Errorf("unreachable action. testclient called too many times: %+v", action)
   604  				return true, nil, fmt.Errorf("unreachable action")
   605  			})
   606  
   607  			switch objectType {
   608  			case "leases":
   609  				lock = &rl.LeaseLock{
   610  					LeaseMeta:  objectMeta,
   611  					LockConfig: resourceLockConfig,
   612  					Client:     c.CoordinationV1(),
   613  				}
   614  			default:
   615  				t.Fatalf("Unknown objectType: %v", objectType)
   616  			}
   617  
   618  			lec := LeaderElectionConfig{
   619  				Lock:          lock,
   620  				LeaseDuration: 10 * time.Second,
   621  				Callbacks: LeaderCallbacks{
   622  					OnNewLeader: func(l string) {
   623  						defer wg.Done()
   624  						reportedLeader = l
   625  					},
   626  				},
   627  			}
   628  			observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord)
   629  			le := &LeaderElector{
   630  				config:            lec,
   631  				observedRecord:    test.observedRecord,
   632  				observedRawRecord: observedRawRecord,
   633  				observedTime:      test.observedTime,
   634  				clock:             clock.RealClock{},
   635  				metrics:           globalMetricsFactory.newLeaderMetrics(),
   636  			}
   637  			if !le.tryAcquireOrRenew(context.Background()) {
   638  				t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", true)
   639  			}
   640  
   641  			le.maybeReportTransition()
   642  
   643  			// Wait for a response to the leader transition, and add 1 so that we can track the final transition.
   644  			wg.Wait()
   645  			wg.Add(1)
   646  
   647  			if test.expectSuccess != le.release() {
   648  				t.Errorf("unexpected result of release: [succeeded=%v]", !test.expectSuccess)
   649  			}
   650  
   651  			le.observedRecord.AcquireTime = metav1.Time{}
   652  			le.observedRecord.RenewTime = metav1.Time{}
   653  			if le.observedRecord.HolderIdentity != test.outHolder {
   654  				t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity)
   655  			}
   656  			if len(test.reactors) != len(c.Actions()) {
   657  				t.Errorf("wrong number of api interactions")
   658  			}
   659  			if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 {
   660  				t.Errorf("leader should have transitioned but did not")
   661  			}
   662  			if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 {
   663  				t.Errorf("leader should not have transitioned but did")
   664  			}
   665  			le.maybeReportTransition()
   666  			wg.Wait()
   667  			if reportedLeader != test.outHolder {
   668  				t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader)
   669  			}
   670  			assertEqualEvents(t, test.expectedEvents, recorder.Events)
   671  		})
   672  	}
   673  }
   674  
   675  // Will test leader election using endpoints as the resource
   676  func TestReleaseLeaseLeases(t *testing.T) {
   677  	testReleaseLease(t, "leases")
   678  }
   679  
   680  func TestReleaseOnCancellation_Leases(t *testing.T) {
   681  	testReleaseOnCancellation(t, "leases")
   682  }
   683  
   684  func testReleaseOnCancellation(t *testing.T, objectType string) {
   685  	var (
   686  		onNewLeader   = make(chan struct{})
   687  		onRenewCalled = make(chan struct{})
   688  		onRenewResume = make(chan struct{})
   689  		onRelease     = make(chan struct{})
   690  
   691  		lockObj runtime.Object
   692  		gets    int
   693  		updates int
   694  		wg      sync.WaitGroup
   695  	)
   696  	resetVars := func() {
   697  		onNewLeader = make(chan struct{})
   698  		onRenewCalled = make(chan struct{})
   699  		onRenewResume = make(chan struct{})
   700  		onRelease = make(chan struct{})
   701  
   702  		lockObj = nil
   703  		gets = 0
   704  		updates = 0
   705  	}
   706  	lec := LeaderElectionConfig{
   707  		LeaseDuration: 15 * time.Second,
   708  		RenewDeadline: 2 * time.Second,
   709  		RetryPeriod:   1 * time.Second,
   710  
   711  		// This is what we're testing
   712  		ReleaseOnCancel: true,
   713  
   714  		Callbacks: LeaderCallbacks{
   715  			OnNewLeader:      func(identity string) {},
   716  			OnStoppedLeading: func() {},
   717  			OnStartedLeading: func(context.Context) {
   718  				close(onNewLeader)
   719  			},
   720  		},
   721  	}
   722  
   723  	tests := []struct {
   724  		name           string
   725  		reactors       []Reactor
   726  		expectedEvents []string
   727  	}{
   728  		{
   729  			name: "release acquired lock on cancellation of update",
   730  			reactors: []Reactor{
   731  				{
   732  					verb:       "get",
   733  					objectType: objectType,
   734  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   735  						gets++
   736  						if lockObj != nil {
   737  							return true, lockObj, nil
   738  						}
   739  						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
   740  					},
   741  				},
   742  				{
   743  					verb:       "create",
   744  					objectType: objectType,
   745  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   746  						lockObj = action.(fakeclient.CreateAction).GetObject()
   747  						return true, lockObj, nil
   748  					},
   749  				},
   750  				{
   751  					verb:       "update",
   752  					objectType: objectType,
   753  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   754  						updates++
   755  						// Skip initial two fast path renews
   756  						if updates%2 == 1 && updates < 5 {
   757  							return true, nil, context.Canceled
   758  						}
   759  
   760  						// Second update (first renew) should return our canceled error
   761  						// FakeClient doesn't do anything with the context so we're doing this ourselves
   762  						if updates == 4 {
   763  							close(onRenewCalled)
   764  							<-onRenewResume
   765  							return true, nil, context.Canceled
   766  						} else if updates == 5 {
   767  							// We update the lock after the cancellation to release it
   768  							// This wg is to avoid the data race on lockObj
   769  							defer wg.Done()
   770  							close(onRelease)
   771  						}
   772  
   773  						lockObj = action.(fakeclient.UpdateAction).GetObject()
   774  						return true, lockObj, nil
   775  					},
   776  				},
   777  			},
   778  			expectedEvents: []string{
   779  				"Normal LeaderElection baz became leader",
   780  				"Normal LeaderElection baz stopped leading",
   781  			},
   782  		},
   783  		{
   784  			name: "release acquired lock on cancellation of get",
   785  			reactors: []Reactor{
   786  				{
   787  					verb:       "get",
   788  					objectType: objectType,
   789  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   790  						gets++
   791  						if lockObj != nil {
   792  							// Third and more get (first create, second renew) should return our canceled error
   793  							// FakeClient doesn't do anything with the context so we're doing this ourselves
   794  							if gets >= 3 {
   795  								close(onRenewCalled)
   796  								<-onRenewResume
   797  								return true, nil, context.Canceled
   798  							}
   799  							return true, lockObj, nil
   800  						}
   801  						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
   802  					},
   803  				},
   804  				{
   805  					verb:       "create",
   806  					objectType: objectType,
   807  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   808  						lockObj = action.(fakeclient.CreateAction).GetObject()
   809  						return true, lockObj, nil
   810  					},
   811  				},
   812  				{
   813  					verb:       "update",
   814  					objectType: objectType,
   815  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
   816  						updates++
   817  						// Always skip fast path renew
   818  						if updates%2 == 1 {
   819  							return true, nil, context.Canceled
   820  						}
   821  						// Second update (first renew) should release the lock
   822  						if updates == 4 {
   823  							// We update the lock after the cancellation to release it
   824  							// This wg is to avoid the data race on lockObj
   825  							defer wg.Done()
   826  							close(onRelease)
   827  						}
   828  
   829  						lockObj = action.(fakeclient.UpdateAction).GetObject()
   830  						return true, lockObj, nil
   831  					},
   832  				},
   833  			},
   834  			expectedEvents: []string{
   835  				"Normal LeaderElection baz became leader",
   836  				"Normal LeaderElection baz stopped leading",
   837  			},
   838  		},
   839  	}
   840  
   841  	for i := range tests {
   842  		test := &tests[i]
   843  		t.Run(test.name, func(t *testing.T) {
   844  			wg.Add(1)
   845  			resetVars()
   846  
   847  			recorder := record.NewFakeRecorder(100)
   848  			resourceLockConfig := rl.ResourceLockConfig{
   849  				Identity:      "baz",
   850  				EventRecorder: recorder,
   851  			}
   852  			c := &fake.Clientset{}
   853  			for _, reactor := range test.reactors {
   854  				c.AddReactor(reactor.verb, objectType, reactor.reaction)
   855  			}
   856  			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
   857  				t.Errorf("unreachable action. testclient called too many times: %+v", action)
   858  				return true, nil, fmt.Errorf("unreachable action")
   859  			})
   860  			lock, err := rl.New(objectType, "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig)
   861  			if err != nil {
   862  				t.Fatal("resourcelock.New() = ", err)
   863  			}
   864  
   865  			lec.Lock = lock
   866  			elector, err := NewLeaderElector(lec)
   867  			if err != nil {
   868  				t.Fatal("Failed to create leader elector: ", err)
   869  			}
   870  
   871  			ctx, cancel := context.WithCancel(context.Background())
   872  
   873  			go elector.Run(ctx)
   874  
   875  			// Wait for us to become the leader
   876  			select {
   877  			case <-onNewLeader:
   878  			case <-time.After(10 * time.Second):
   879  				t.Fatal("failed to become the leader")
   880  			}
   881  
   882  			// Wait for renew (update) to be invoked
   883  			select {
   884  			case <-onRenewCalled:
   885  			case <-time.After(10 * time.Second):
   886  				t.Fatal("the elector failed to renew the lock")
   887  			}
   888  
   889  			// Cancel the context - stopping the elector while
   890  			// it's running
   891  			cancel()
   892  
   893  			// Resume the tryAcquireOrRenew call to return the cancellation
   894  			// which should trigger the release flow
   895  			close(onRenewResume)
   896  
   897  			select {
   898  			case <-onRelease:
   899  			case <-time.After(10 * time.Second):
   900  				t.Fatal("the lock was not released")
   901  			}
   902  			wg.Wait()
   903  			assertEqualEvents(t, test.expectedEvents, recorder.Events)
   904  		})
   905  	}
   906  }
   907  
   908  func TestLeaderElectionConfigValidation(t *testing.T) {
   909  	resourceLockConfig := rl.ResourceLockConfig{
   910  		Identity: "baz",
   911  	}
   912  
   913  	lock := &rl.LeaseLock{
   914  		LockConfig: resourceLockConfig,
   915  	}
   916  
   917  	lec := LeaderElectionConfig{
   918  		Lock:          lock,
   919  		LeaseDuration: 15 * time.Second,
   920  		RenewDeadline: 2 * time.Second,
   921  		RetryPeriod:   1 * time.Second,
   922  
   923  		ReleaseOnCancel: true,
   924  
   925  		Callbacks: LeaderCallbacks{
   926  			OnNewLeader:      func(identity string) {},
   927  			OnStoppedLeading: func() {},
   928  			OnStartedLeading: func(context.Context) {},
   929  		},
   930  	}
   931  
   932  	_, err := NewLeaderElector(lec)
   933  	assert.NoError(t, err)
   934  
   935  	// Invalid lock identity
   936  	resourceLockConfig.Identity = ""
   937  	lock.LockConfig = resourceLockConfig
   938  	lec.Lock = lock
   939  	_, err = NewLeaderElector(lec)
   940  	assert.Error(t, err, fmt.Errorf("Lock identity is empty"))
   941  }
   942  
   943  func assertEqualEvents(t *testing.T, expected []string, actual <-chan string) {
   944  	c := time.After(wait.ForeverTestTimeout)
   945  	for _, e := range expected {
   946  		select {
   947  		case a := <-actual:
   948  			if e != a {
   949  				t.Errorf("Expected event %q, got %q", e, a)
   950  				return
   951  			}
   952  		case <-c:
   953  			t.Errorf("Expected event %q, got nothing", e)
   954  			// continue iterating to print all expected events
   955  		}
   956  	}
   957  	for {
   958  		select {
   959  		case a := <-actual:
   960  			t.Errorf("Unexpected event: %q", a)
   961  		default:
   962  			return // No more events, as expected.
   963  		}
   964  	}
   965  }
   966  
   967  func TestFastPathLeaderElection(t *testing.T) {
   968  	objectType := "leases"
   969  	var (
   970  		lockObj    runtime.Object
   971  		updates    int
   972  		lockOps    []string
   973  		cancelFunc func()
   974  	)
   975  	resetVars := func() {
   976  		lockObj = nil
   977  		updates = 0
   978  		lockOps = []string{}
   979  		cancelFunc = nil
   980  	}
   981  	lec := LeaderElectionConfig{
   982  		LeaseDuration: 15 * time.Second,
   983  		RenewDeadline: 2 * time.Second,
   984  		RetryPeriod:   1 * time.Second,
   985  
   986  		Callbacks: LeaderCallbacks{
   987  			OnNewLeader:      func(identity string) {},
   988  			OnStoppedLeading: func() {},
   989  			OnStartedLeading: func(context.Context) {
   990  			},
   991  		},
   992  	}
   993  
   994  	tests := []struct {
   995  		name            string
   996  		reactors        []Reactor
   997  		expectedLockOps []string
   998  	}{
   999  		{
  1000  			name: "Exercise fast path after lock acquired",
  1001  			reactors: []Reactor{
  1002  				{
  1003  					verb:       "get",
  1004  					objectType: objectType,
  1005  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
  1006  						lockOps = append(lockOps, "get")
  1007  						if lockObj != nil {
  1008  							return true, lockObj, nil
  1009  						}
  1010  						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
  1011  					},
  1012  				},
  1013  				{
  1014  					verb:       "create",
  1015  					objectType: objectType,
  1016  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
  1017  						lockOps = append(lockOps, "create")
  1018  						lockObj = action.(fakeclient.CreateAction).GetObject()
  1019  						return true, lockObj, nil
  1020  					},
  1021  				},
  1022  				{
  1023  					verb:       "update",
  1024  					objectType: objectType,
  1025  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
  1026  						updates++
  1027  						lockOps = append(lockOps, "update")
  1028  						if updates == 2 {
  1029  							cancelFunc()
  1030  						}
  1031  						lockObj = action.(fakeclient.UpdateAction).GetObject()
  1032  						return true, lockObj, nil
  1033  					},
  1034  				},
  1035  			},
  1036  			expectedLockOps: []string{"get", "create", "update", "update"},
  1037  		},
  1038  		{
  1039  			name: "Fallback to slow path after fast path fails",
  1040  			reactors: []Reactor{
  1041  				{
  1042  					verb:       "get",
  1043  					objectType: objectType,
  1044  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
  1045  						lockOps = append(lockOps, "get")
  1046  						if lockObj != nil {
  1047  							return true, lockObj, nil
  1048  						}
  1049  						return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName())
  1050  					},
  1051  				},
  1052  				{
  1053  					verb:       "create",
  1054  					objectType: objectType,
  1055  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
  1056  						lockOps = append(lockOps, "create")
  1057  						lockObj = action.(fakeclient.CreateAction).GetObject()
  1058  						return true, lockObj, nil
  1059  					},
  1060  				},
  1061  				{
  1062  					verb:       "update",
  1063  					objectType: objectType,
  1064  					reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) {
  1065  						updates++
  1066  						lockOps = append(lockOps, "update")
  1067  						switch updates {
  1068  						case 2:
  1069  							return true, nil, errors.NewConflict(action.(fakeclient.UpdateAction).GetResource().GroupResource(), "fake conflict", nil)
  1070  						case 4:
  1071  							cancelFunc()
  1072  						}
  1073  						lockObj = action.(fakeclient.UpdateAction).GetObject()
  1074  						return true, lockObj, nil
  1075  					},
  1076  				},
  1077  			},
  1078  			expectedLockOps: []string{"get", "create", "update", "update", "get", "update", "update"},
  1079  		},
  1080  	}
  1081  
  1082  	for i := range tests {
  1083  		test := &tests[i]
  1084  		t.Run(test.name, func(t *testing.T) {
  1085  			resetVars()
  1086  
  1087  			recorder := record.NewFakeRecorder(100)
  1088  			resourceLockConfig := rl.ResourceLockConfig{
  1089  				Identity:      "baz",
  1090  				EventRecorder: recorder,
  1091  			}
  1092  			c := &fake.Clientset{}
  1093  			for _, reactor := range test.reactors {
  1094  				c.AddReactor(reactor.verb, objectType, reactor.reaction)
  1095  			}
  1096  			c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) {
  1097  				t.Errorf("unreachable action. testclient called too many times: %+v", action)
  1098  				return true, nil, fmt.Errorf("unreachable action")
  1099  			})
  1100  			lock, err := rl.New("leases", "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig)
  1101  			if err != nil {
  1102  				t.Fatal("resourcelock.New() = ", err)
  1103  			}
  1104  
  1105  			lec.Lock = lock
  1106  			elector, err := NewLeaderElector(lec)
  1107  			if err != nil {
  1108  				t.Fatal("Failed to create leader elector: ", err)
  1109  			}
  1110  
  1111  			ctx, cancel := context.WithCancel(context.Background())
  1112  			cancelFunc = cancel
  1113  
  1114  			elector.Run(ctx)
  1115  			assert.Equal(t, test.expectedLockOps, lockOps, "Expected lock ops %q, got %q", test.expectedLockOps, lockOps)
  1116  		})
  1117  	}
  1118  }