k8s.io/client-go@v0.31.1/tools/leaderelection/leaderelection_test.go (about) 1 /* 2 Copyright 2015 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package leaderelection 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "sync" 24 "testing" 25 "time" 26 27 "github.com/google/go-cmp/cmp" 28 "github.com/stretchr/testify/assert" 29 coordinationv1 "k8s.io/api/coordination/v1" 30 corev1 "k8s.io/api/core/v1" 31 "k8s.io/apimachinery/pkg/api/equality" 32 "k8s.io/apimachinery/pkg/api/errors" 33 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 34 "k8s.io/apimachinery/pkg/runtime" 35 "k8s.io/apimachinery/pkg/util/wait" 36 "k8s.io/client-go/kubernetes/fake" 37 fakeclient "k8s.io/client-go/testing" 38 rl "k8s.io/client-go/tools/leaderelection/resourcelock" 39 "k8s.io/client-go/tools/record" 40 "k8s.io/utils/clock" 41 ) 42 43 func createLockObject(t *testing.T, objectType, namespace, name string, record *rl.LeaderElectionRecord) (obj runtime.Object) { 44 objectMeta := metav1.ObjectMeta{ 45 Namespace: namespace, 46 Name: name, 47 } 48 if record != nil { 49 recordBytes, _ := json.Marshal(record) 50 objectMeta.Annotations = map[string]string{ 51 rl.LeaderElectionRecordAnnotationKey: string(recordBytes), 52 } 53 } 54 switch objectType { 55 case "endpoints": 56 obj = &corev1.Endpoints{ObjectMeta: objectMeta} 57 case "configmaps": 58 obj = &corev1.ConfigMap{ObjectMeta: objectMeta} 59 case "leases": 60 var spec coordinationv1.LeaseSpec 61 if record != nil { 62 spec = rl.LeaderElectionRecordToLeaseSpec(record) 63 } 64 obj = &coordinationv1.Lease{ObjectMeta: objectMeta, Spec: spec} 65 default: 66 t.Fatal("unexpected objType:" + objectType) 67 } 68 return 69 } 70 71 type Reactor struct { 72 verb string 73 objectType string 74 reaction fakeclient.ReactionFunc 75 } 76 77 func testTryAcquireOrRenew(t *testing.T, objectType string) { 78 clock := clock.RealClock{} 79 future := clock.Now().Add(1000 * time.Hour) 80 past := clock.Now().Add(-1000 * time.Hour) 81 82 tests := []struct { 83 name string 84 observedRecord rl.LeaderElectionRecord 85 observedTime time.Time 86 retryAfter time.Duration 87 reactors []Reactor 88 expectedEvents []string 89 90 expectSuccess bool 91 transitionLeader bool 92 outHolder string 93 }{ 94 { 95 name: "acquire from no object", 96 reactors: []Reactor{ 97 { 98 verb: "get", 99 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 100 return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) 101 }, 102 }, 103 { 104 verb: "create", 105 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 106 return true, action.(fakeclient.CreateAction).GetObject(), nil 107 }, 108 }, 109 }, 110 expectSuccess: true, 111 outHolder: "baz", 112 }, 113 { 114 name: "acquire from object without annotations", 115 reactors: []Reactor{ 116 { 117 verb: "get", 118 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 119 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), nil), nil 120 }, 121 }, 122 { 123 verb: "update", 124 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 125 return true, action.(fakeclient.CreateAction).GetObject(), nil 126 }, 127 }, 128 }, 129 expectSuccess: true, 130 transitionLeader: true, 131 outHolder: "baz", 132 }, 133 { 134 name: "acquire from led object with the lease duration seconds", 135 reactors: []Reactor{ 136 { 137 verb: "get", 138 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 139 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil 140 }, 141 }, 142 { 143 verb: "get", 144 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 145 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing", LeaseDurationSeconds: 3}), nil 146 }, 147 }, 148 { 149 verb: "update", 150 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 151 return true, action.(fakeclient.CreateAction).GetObject(), nil 152 }, 153 }, 154 }, 155 retryAfter: 3 * time.Second, 156 expectSuccess: true, 157 transitionLeader: true, 158 outHolder: "baz", 159 }, 160 { 161 name: "acquire from unled object", 162 reactors: []Reactor{ 163 { 164 verb: "get", 165 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 166 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{}), nil 167 }, 168 }, 169 { 170 verb: "update", 171 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 172 return true, action.(fakeclient.CreateAction).GetObject(), nil 173 }, 174 }, 175 }, 176 177 expectSuccess: true, 178 transitionLeader: true, 179 outHolder: "baz", 180 }, 181 { 182 name: "acquire from led, unacked object", 183 reactors: []Reactor{ 184 { 185 verb: "get", 186 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 187 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil 188 }, 189 }, 190 { 191 verb: "update", 192 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 193 return true, action.(fakeclient.CreateAction).GetObject(), nil 194 }, 195 }, 196 }, 197 observedRecord: rl.LeaderElectionRecord{HolderIdentity: "bing"}, 198 observedTime: past, 199 200 expectSuccess: true, 201 transitionLeader: true, 202 outHolder: "baz", 203 }, 204 { 205 name: "acquire from empty led, acked object", 206 reactors: []Reactor{ 207 { 208 verb: "get", 209 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 210 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: ""}), nil 211 }, 212 }, 213 { 214 verb: "update", 215 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 216 return true, action.(fakeclient.CreateAction).GetObject(), nil 217 }, 218 }, 219 }, 220 observedTime: future, 221 222 expectSuccess: true, 223 transitionLeader: true, 224 outHolder: "baz", 225 }, 226 { 227 name: "don't acquire from led, acked object", 228 reactors: []Reactor{ 229 { 230 verb: "get", 231 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 232 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil 233 }, 234 }, 235 }, 236 observedTime: future, 237 238 expectSuccess: false, 239 outHolder: "bing", 240 }, 241 { 242 name: "renew already acquired object", 243 reactors: []Reactor{ 244 { 245 verb: "get", 246 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 247 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil 248 }, 249 }, 250 { 251 verb: "update", 252 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 253 return true, action.(fakeclient.CreateAction).GetObject(), nil 254 }, 255 }, 256 }, 257 observedTime: future, 258 observedRecord: rl.LeaderElectionRecord{HolderIdentity: "baz"}, 259 260 expectSuccess: true, 261 outHolder: "baz", 262 }, 263 } 264 265 for i := range tests { 266 test := &tests[i] 267 t.Run(test.name, func(t *testing.T) { 268 // OnNewLeader is called async so we have to wait for it. 269 var wg sync.WaitGroup 270 wg.Add(1) 271 var reportedLeader string 272 var lock rl.Interface 273 274 objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"} 275 recorder := record.NewFakeRecorder(100) 276 resourceLockConfig := rl.ResourceLockConfig{ 277 Identity: "baz", 278 EventRecorder: recorder, 279 } 280 c := &fake.Clientset{} 281 for _, reactor := range test.reactors { 282 c.AddReactor(reactor.verb, objectType, reactor.reaction) 283 } 284 c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { 285 t.Errorf("unreachable action. testclient called too many times: %+v", action) 286 return true, nil, fmt.Errorf("unreachable action") 287 }) 288 289 switch objectType { 290 case "leases": 291 lock = &rl.LeaseLock{ 292 LeaseMeta: objectMeta, 293 LockConfig: resourceLockConfig, 294 Client: c.CoordinationV1(), 295 } 296 default: 297 t.Fatalf("Unknown objectType: %v", objectType) 298 } 299 300 lec := LeaderElectionConfig{ 301 Lock: lock, 302 LeaseDuration: 10 * time.Second, 303 Callbacks: LeaderCallbacks{ 304 OnNewLeader: func(l string) { 305 defer wg.Done() 306 reportedLeader = l 307 }, 308 }, 309 } 310 observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord) 311 le := &LeaderElector{ 312 config: lec, 313 observedRecord: test.observedRecord, 314 observedRawRecord: observedRawRecord, 315 observedTime: test.observedTime, 316 clock: clock, 317 metrics: globalMetricsFactory.newLeaderMetrics(), 318 } 319 if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { 320 if test.retryAfter != 0 { 321 time.Sleep(test.retryAfter) 322 if test.expectSuccess != le.tryAcquireOrRenew(context.Background()) { 323 t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) 324 } 325 } else { 326 t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", !test.expectSuccess) 327 } 328 } 329 330 le.observedRecord.AcquireTime = metav1.Time{} 331 le.observedRecord.RenewTime = metav1.Time{} 332 if le.observedRecord.HolderIdentity != test.outHolder { 333 t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity) 334 } 335 if len(test.reactors) != len(c.Actions()) { 336 t.Errorf("wrong number of api interactions") 337 } 338 if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 { 339 t.Errorf("leader should have transitioned but did not") 340 } 341 if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 { 342 t.Errorf("leader should not have transitioned but did") 343 } 344 345 le.maybeReportTransition() 346 wg.Wait() 347 if reportedLeader != test.outHolder { 348 t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader) 349 } 350 assertEqualEvents(t, test.expectedEvents, recorder.Events) 351 }) 352 } 353 } 354 355 func TestTryCoordinatedRenew(t *testing.T) { 356 objectType := "leases" 357 clock := clock.RealClock{} 358 future := clock.Now().Add(1000 * time.Hour) 359 360 tests := []struct { 361 name string 362 observedRecord rl.LeaderElectionRecord 363 observedTime time.Time 364 retryAfter time.Duration 365 reactors []Reactor 366 expectedEvents []string 367 368 expectSuccess bool 369 transitionLeader bool 370 outHolder string 371 }{ 372 { 373 name: "don't acquire from led, acked object", 374 reactors: []Reactor{ 375 { 376 verb: "get", 377 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 378 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "bing"}), nil 379 }, 380 }, 381 }, 382 observedTime: future, 383 384 expectSuccess: false, 385 outHolder: "bing", 386 }, 387 { 388 name: "renew already acquired object", 389 reactors: []Reactor{ 390 { 391 verb: "get", 392 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 393 return true, createLockObject(t, objectType, action.GetNamespace(), action.(fakeclient.GetAction).GetName(), &rl.LeaderElectionRecord{HolderIdentity: "baz"}), nil 394 }, 395 }, 396 { 397 verb: "update", 398 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 399 return true, action.(fakeclient.CreateAction).GetObject(), nil 400 }, 401 }, 402 }, 403 observedTime: future, 404 observedRecord: rl.LeaderElectionRecord{HolderIdentity: "baz"}, 405 406 expectSuccess: true, 407 outHolder: "baz", 408 }, 409 } 410 411 for i := range tests { 412 test := &tests[i] 413 t.Run(test.name, func(t *testing.T) { 414 // OnNewLeader is called async so we have to wait for it. 415 var wg sync.WaitGroup 416 wg.Add(1) 417 var reportedLeader string 418 var lock rl.Interface 419 420 objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"} 421 recorder := record.NewFakeRecorder(100) 422 resourceLockConfig := rl.ResourceLockConfig{ 423 Identity: "baz", 424 EventRecorder: recorder, 425 } 426 c := &fake.Clientset{} 427 for _, reactor := range test.reactors { 428 c.AddReactor(reactor.verb, objectType, reactor.reaction) 429 } 430 c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { 431 t.Errorf("unreachable action. testclient called too many times: %+v", action) 432 return true, nil, fmt.Errorf("unreachable action") 433 }) 434 435 lock = &rl.LeaseLock{ 436 LeaseMeta: objectMeta, 437 LockConfig: resourceLockConfig, 438 Client: c.CoordinationV1(), 439 } 440 lec := LeaderElectionConfig{ 441 Lock: lock, 442 LeaseDuration: 10 * time.Second, 443 Callbacks: LeaderCallbacks{ 444 OnNewLeader: func(l string) { 445 defer wg.Done() 446 reportedLeader = l 447 }, 448 }, 449 Coordinated: true, 450 } 451 observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord) 452 le := &LeaderElector{ 453 config: lec, 454 observedRecord: test.observedRecord, 455 observedRawRecord: observedRawRecord, 456 observedTime: test.observedTime, 457 clock: clock, 458 metrics: globalMetricsFactory.newLeaderMetrics(), 459 } 460 if test.expectSuccess != le.tryCoordinatedRenew(context.Background()) { 461 if test.retryAfter != 0 { 462 time.Sleep(test.retryAfter) 463 if test.expectSuccess != le.tryCoordinatedRenew(context.Background()) { 464 t.Errorf("unexpected result of tryCoordinatedRenew: [succeeded=%v]", !test.expectSuccess) 465 } 466 } else { 467 t.Errorf("unexpected result of gryCoordinatedRenew: [succeeded=%v]", !test.expectSuccess) 468 } 469 } 470 471 le.observedRecord.AcquireTime = metav1.Time{} 472 le.observedRecord.RenewTime = metav1.Time{} 473 if le.observedRecord.HolderIdentity != test.outHolder { 474 t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity) 475 } 476 if len(test.reactors) != len(c.Actions()) { 477 t.Errorf("wrong number of api interactions") 478 } 479 if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 { 480 t.Errorf("leader should have transitioned but did not") 481 } 482 if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 { 483 t.Errorf("leader should not have transitioned but did") 484 } 485 486 le.maybeReportTransition() 487 wg.Wait() 488 if reportedLeader != test.outHolder { 489 t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader) 490 } 491 assertEqualEvents(t, test.expectedEvents, recorder.Events) 492 }) 493 } 494 } 495 496 // Will test leader election using lease as the resource 497 func TestTryAcquireOrRenewLeases(t *testing.T) { 498 testTryAcquireOrRenew(t, "leases") 499 } 500 501 func TestLeaseSpecToLeaderElectionRecordRoundTrip(t *testing.T) { 502 holderIdentity := "foo" 503 leaseDurationSeconds := int32(10) 504 leaseTransitions := int32(1) 505 oldSpec := coordinationv1.LeaseSpec{ 506 HolderIdentity: &holderIdentity, 507 LeaseDurationSeconds: &leaseDurationSeconds, 508 AcquireTime: &metav1.MicroTime{Time: time.Now()}, 509 RenewTime: &metav1.MicroTime{Time: time.Now()}, 510 LeaseTransitions: &leaseTransitions, 511 } 512 513 oldRecord := rl.LeaseSpecToLeaderElectionRecord(&oldSpec) 514 newSpec := rl.LeaderElectionRecordToLeaseSpec(oldRecord) 515 516 if !equality.Semantic.DeepEqual(oldSpec, newSpec) { 517 t.Errorf("diff: %v", cmp.Diff(oldSpec, newSpec)) 518 } 519 520 newRecord := rl.LeaseSpecToLeaderElectionRecord(&newSpec) 521 522 if !equality.Semantic.DeepEqual(oldRecord, newRecord) { 523 t.Errorf("diff: %v", cmp.Diff(oldRecord, newRecord)) 524 } 525 } 526 527 func GetRawRecordOrDie(t *testing.T, objectType string, ler rl.LeaderElectionRecord) (ret []byte) { 528 var err error 529 switch objectType { 530 case "leases": 531 ret, err = json.Marshal(ler) 532 if err != nil { 533 t.Fatalf("lock %s get raw record %v failed: %v", objectType, ler, err) 534 } 535 default: 536 t.Fatal("unexpected objType:" + objectType) 537 } 538 return 539 } 540 541 func testReleaseLease(t *testing.T, objectType string) { 542 tests := []struct { 543 name string 544 observedRecord rl.LeaderElectionRecord 545 observedTime time.Time 546 reactors []Reactor 547 expectedEvents []string 548 549 expectSuccess bool 550 transitionLeader bool 551 outHolder string 552 }{ 553 { 554 name: "release acquired lock from no object", 555 reactors: []Reactor{ 556 { 557 verb: "get", 558 objectType: objectType, 559 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 560 return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) 561 }, 562 }, 563 { 564 verb: "create", 565 objectType: objectType, 566 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 567 return true, action.(fakeclient.CreateAction).GetObject(), nil 568 }, 569 }, 570 { 571 verb: "update", 572 objectType: objectType, 573 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 574 return true, action.(fakeclient.UpdateAction).GetObject(), nil 575 }, 576 }, 577 }, 578 expectSuccess: true, 579 outHolder: "", 580 }, 581 } 582 583 for i := range tests { 584 test := &tests[i] 585 t.Run(test.name, func(t *testing.T) { 586 // OnNewLeader is called async so we have to wait for it. 587 var wg sync.WaitGroup 588 wg.Add(1) 589 var reportedLeader string 590 var lock rl.Interface 591 592 objectMeta := metav1.ObjectMeta{Namespace: "foo", Name: "bar"} 593 recorder := record.NewFakeRecorder(100) 594 resourceLockConfig := rl.ResourceLockConfig{ 595 Identity: "baz", 596 EventRecorder: recorder, 597 } 598 c := &fake.Clientset{} 599 for _, reactor := range test.reactors { 600 c.AddReactor(reactor.verb, objectType, reactor.reaction) 601 } 602 c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { 603 t.Errorf("unreachable action. testclient called too many times: %+v", action) 604 return true, nil, fmt.Errorf("unreachable action") 605 }) 606 607 switch objectType { 608 case "leases": 609 lock = &rl.LeaseLock{ 610 LeaseMeta: objectMeta, 611 LockConfig: resourceLockConfig, 612 Client: c.CoordinationV1(), 613 } 614 default: 615 t.Fatalf("Unknown objectType: %v", objectType) 616 } 617 618 lec := LeaderElectionConfig{ 619 Lock: lock, 620 LeaseDuration: 10 * time.Second, 621 Callbacks: LeaderCallbacks{ 622 OnNewLeader: func(l string) { 623 defer wg.Done() 624 reportedLeader = l 625 }, 626 }, 627 } 628 observedRawRecord := GetRawRecordOrDie(t, objectType, test.observedRecord) 629 le := &LeaderElector{ 630 config: lec, 631 observedRecord: test.observedRecord, 632 observedRawRecord: observedRawRecord, 633 observedTime: test.observedTime, 634 clock: clock.RealClock{}, 635 metrics: globalMetricsFactory.newLeaderMetrics(), 636 } 637 if !le.tryAcquireOrRenew(context.Background()) { 638 t.Errorf("unexpected result of tryAcquireOrRenew: [succeeded=%v]", true) 639 } 640 641 le.maybeReportTransition() 642 643 // Wait for a response to the leader transition, and add 1 so that we can track the final transition. 644 wg.Wait() 645 wg.Add(1) 646 647 if test.expectSuccess != le.release() { 648 t.Errorf("unexpected result of release: [succeeded=%v]", !test.expectSuccess) 649 } 650 651 le.observedRecord.AcquireTime = metav1.Time{} 652 le.observedRecord.RenewTime = metav1.Time{} 653 if le.observedRecord.HolderIdentity != test.outHolder { 654 t.Errorf("expected holder:\n\t%+v\ngot:\n\t%+v", test.outHolder, le.observedRecord.HolderIdentity) 655 } 656 if len(test.reactors) != len(c.Actions()) { 657 t.Errorf("wrong number of api interactions") 658 } 659 if test.transitionLeader && le.observedRecord.LeaderTransitions != 1 { 660 t.Errorf("leader should have transitioned but did not") 661 } 662 if !test.transitionLeader && le.observedRecord.LeaderTransitions != 0 { 663 t.Errorf("leader should not have transitioned but did") 664 } 665 le.maybeReportTransition() 666 wg.Wait() 667 if reportedLeader != test.outHolder { 668 t.Errorf("reported leader was not the new leader. expected %q, got %q", test.outHolder, reportedLeader) 669 } 670 assertEqualEvents(t, test.expectedEvents, recorder.Events) 671 }) 672 } 673 } 674 675 // Will test leader election using endpoints as the resource 676 func TestReleaseLeaseLeases(t *testing.T) { 677 testReleaseLease(t, "leases") 678 } 679 680 func TestReleaseOnCancellation_Leases(t *testing.T) { 681 testReleaseOnCancellation(t, "leases") 682 } 683 684 func testReleaseOnCancellation(t *testing.T, objectType string) { 685 var ( 686 onNewLeader = make(chan struct{}) 687 onRenewCalled = make(chan struct{}) 688 onRenewResume = make(chan struct{}) 689 onRelease = make(chan struct{}) 690 691 lockObj runtime.Object 692 gets int 693 updates int 694 wg sync.WaitGroup 695 ) 696 resetVars := func() { 697 onNewLeader = make(chan struct{}) 698 onRenewCalled = make(chan struct{}) 699 onRenewResume = make(chan struct{}) 700 onRelease = make(chan struct{}) 701 702 lockObj = nil 703 gets = 0 704 updates = 0 705 } 706 lec := LeaderElectionConfig{ 707 LeaseDuration: 15 * time.Second, 708 RenewDeadline: 2 * time.Second, 709 RetryPeriod: 1 * time.Second, 710 711 // This is what we're testing 712 ReleaseOnCancel: true, 713 714 Callbacks: LeaderCallbacks{ 715 OnNewLeader: func(identity string) {}, 716 OnStoppedLeading: func() {}, 717 OnStartedLeading: func(context.Context) { 718 close(onNewLeader) 719 }, 720 }, 721 } 722 723 tests := []struct { 724 name string 725 reactors []Reactor 726 expectedEvents []string 727 }{ 728 { 729 name: "release acquired lock on cancellation of update", 730 reactors: []Reactor{ 731 { 732 verb: "get", 733 objectType: objectType, 734 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 735 gets++ 736 if lockObj != nil { 737 return true, lockObj, nil 738 } 739 return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) 740 }, 741 }, 742 { 743 verb: "create", 744 objectType: objectType, 745 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 746 lockObj = action.(fakeclient.CreateAction).GetObject() 747 return true, lockObj, nil 748 }, 749 }, 750 { 751 verb: "update", 752 objectType: objectType, 753 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 754 updates++ 755 // Skip initial two fast path renews 756 if updates%2 == 1 && updates < 5 { 757 return true, nil, context.Canceled 758 } 759 760 // Second update (first renew) should return our canceled error 761 // FakeClient doesn't do anything with the context so we're doing this ourselves 762 if updates == 4 { 763 close(onRenewCalled) 764 <-onRenewResume 765 return true, nil, context.Canceled 766 } else if updates == 5 { 767 // We update the lock after the cancellation to release it 768 // This wg is to avoid the data race on lockObj 769 defer wg.Done() 770 close(onRelease) 771 } 772 773 lockObj = action.(fakeclient.UpdateAction).GetObject() 774 return true, lockObj, nil 775 }, 776 }, 777 }, 778 expectedEvents: []string{ 779 "Normal LeaderElection baz became leader", 780 "Normal LeaderElection baz stopped leading", 781 }, 782 }, 783 { 784 name: "release acquired lock on cancellation of get", 785 reactors: []Reactor{ 786 { 787 verb: "get", 788 objectType: objectType, 789 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 790 gets++ 791 if lockObj != nil { 792 // Third and more get (first create, second renew) should return our canceled error 793 // FakeClient doesn't do anything with the context so we're doing this ourselves 794 if gets >= 3 { 795 close(onRenewCalled) 796 <-onRenewResume 797 return true, nil, context.Canceled 798 } 799 return true, lockObj, nil 800 } 801 return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) 802 }, 803 }, 804 { 805 verb: "create", 806 objectType: objectType, 807 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 808 lockObj = action.(fakeclient.CreateAction).GetObject() 809 return true, lockObj, nil 810 }, 811 }, 812 { 813 verb: "update", 814 objectType: objectType, 815 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 816 updates++ 817 // Always skip fast path renew 818 if updates%2 == 1 { 819 return true, nil, context.Canceled 820 } 821 // Second update (first renew) should release the lock 822 if updates == 4 { 823 // We update the lock after the cancellation to release it 824 // This wg is to avoid the data race on lockObj 825 defer wg.Done() 826 close(onRelease) 827 } 828 829 lockObj = action.(fakeclient.UpdateAction).GetObject() 830 return true, lockObj, nil 831 }, 832 }, 833 }, 834 expectedEvents: []string{ 835 "Normal LeaderElection baz became leader", 836 "Normal LeaderElection baz stopped leading", 837 }, 838 }, 839 } 840 841 for i := range tests { 842 test := &tests[i] 843 t.Run(test.name, func(t *testing.T) { 844 wg.Add(1) 845 resetVars() 846 847 recorder := record.NewFakeRecorder(100) 848 resourceLockConfig := rl.ResourceLockConfig{ 849 Identity: "baz", 850 EventRecorder: recorder, 851 } 852 c := &fake.Clientset{} 853 for _, reactor := range test.reactors { 854 c.AddReactor(reactor.verb, objectType, reactor.reaction) 855 } 856 c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { 857 t.Errorf("unreachable action. testclient called too many times: %+v", action) 858 return true, nil, fmt.Errorf("unreachable action") 859 }) 860 lock, err := rl.New(objectType, "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig) 861 if err != nil { 862 t.Fatal("resourcelock.New() = ", err) 863 } 864 865 lec.Lock = lock 866 elector, err := NewLeaderElector(lec) 867 if err != nil { 868 t.Fatal("Failed to create leader elector: ", err) 869 } 870 871 ctx, cancel := context.WithCancel(context.Background()) 872 873 go elector.Run(ctx) 874 875 // Wait for us to become the leader 876 select { 877 case <-onNewLeader: 878 case <-time.After(10 * time.Second): 879 t.Fatal("failed to become the leader") 880 } 881 882 // Wait for renew (update) to be invoked 883 select { 884 case <-onRenewCalled: 885 case <-time.After(10 * time.Second): 886 t.Fatal("the elector failed to renew the lock") 887 } 888 889 // Cancel the context - stopping the elector while 890 // it's running 891 cancel() 892 893 // Resume the tryAcquireOrRenew call to return the cancellation 894 // which should trigger the release flow 895 close(onRenewResume) 896 897 select { 898 case <-onRelease: 899 case <-time.After(10 * time.Second): 900 t.Fatal("the lock was not released") 901 } 902 wg.Wait() 903 assertEqualEvents(t, test.expectedEvents, recorder.Events) 904 }) 905 } 906 } 907 908 func TestLeaderElectionConfigValidation(t *testing.T) { 909 resourceLockConfig := rl.ResourceLockConfig{ 910 Identity: "baz", 911 } 912 913 lock := &rl.LeaseLock{ 914 LockConfig: resourceLockConfig, 915 } 916 917 lec := LeaderElectionConfig{ 918 Lock: lock, 919 LeaseDuration: 15 * time.Second, 920 RenewDeadline: 2 * time.Second, 921 RetryPeriod: 1 * time.Second, 922 923 ReleaseOnCancel: true, 924 925 Callbacks: LeaderCallbacks{ 926 OnNewLeader: func(identity string) {}, 927 OnStoppedLeading: func() {}, 928 OnStartedLeading: func(context.Context) {}, 929 }, 930 } 931 932 _, err := NewLeaderElector(lec) 933 assert.NoError(t, err) 934 935 // Invalid lock identity 936 resourceLockConfig.Identity = "" 937 lock.LockConfig = resourceLockConfig 938 lec.Lock = lock 939 _, err = NewLeaderElector(lec) 940 assert.Error(t, err, fmt.Errorf("Lock identity is empty")) 941 } 942 943 func assertEqualEvents(t *testing.T, expected []string, actual <-chan string) { 944 c := time.After(wait.ForeverTestTimeout) 945 for _, e := range expected { 946 select { 947 case a := <-actual: 948 if e != a { 949 t.Errorf("Expected event %q, got %q", e, a) 950 return 951 } 952 case <-c: 953 t.Errorf("Expected event %q, got nothing", e) 954 // continue iterating to print all expected events 955 } 956 } 957 for { 958 select { 959 case a := <-actual: 960 t.Errorf("Unexpected event: %q", a) 961 default: 962 return // No more events, as expected. 963 } 964 } 965 } 966 967 func TestFastPathLeaderElection(t *testing.T) { 968 objectType := "leases" 969 var ( 970 lockObj runtime.Object 971 updates int 972 lockOps []string 973 cancelFunc func() 974 ) 975 resetVars := func() { 976 lockObj = nil 977 updates = 0 978 lockOps = []string{} 979 cancelFunc = nil 980 } 981 lec := LeaderElectionConfig{ 982 LeaseDuration: 15 * time.Second, 983 RenewDeadline: 2 * time.Second, 984 RetryPeriod: 1 * time.Second, 985 986 Callbacks: LeaderCallbacks{ 987 OnNewLeader: func(identity string) {}, 988 OnStoppedLeading: func() {}, 989 OnStartedLeading: func(context.Context) { 990 }, 991 }, 992 } 993 994 tests := []struct { 995 name string 996 reactors []Reactor 997 expectedLockOps []string 998 }{ 999 { 1000 name: "Exercise fast path after lock acquired", 1001 reactors: []Reactor{ 1002 { 1003 verb: "get", 1004 objectType: objectType, 1005 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 1006 lockOps = append(lockOps, "get") 1007 if lockObj != nil { 1008 return true, lockObj, nil 1009 } 1010 return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) 1011 }, 1012 }, 1013 { 1014 verb: "create", 1015 objectType: objectType, 1016 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 1017 lockOps = append(lockOps, "create") 1018 lockObj = action.(fakeclient.CreateAction).GetObject() 1019 return true, lockObj, nil 1020 }, 1021 }, 1022 { 1023 verb: "update", 1024 objectType: objectType, 1025 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 1026 updates++ 1027 lockOps = append(lockOps, "update") 1028 if updates == 2 { 1029 cancelFunc() 1030 } 1031 lockObj = action.(fakeclient.UpdateAction).GetObject() 1032 return true, lockObj, nil 1033 }, 1034 }, 1035 }, 1036 expectedLockOps: []string{"get", "create", "update", "update"}, 1037 }, 1038 { 1039 name: "Fallback to slow path after fast path fails", 1040 reactors: []Reactor{ 1041 { 1042 verb: "get", 1043 objectType: objectType, 1044 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 1045 lockOps = append(lockOps, "get") 1046 if lockObj != nil { 1047 return true, lockObj, nil 1048 } 1049 return true, nil, errors.NewNotFound(action.(fakeclient.GetAction).GetResource().GroupResource(), action.(fakeclient.GetAction).GetName()) 1050 }, 1051 }, 1052 { 1053 verb: "create", 1054 objectType: objectType, 1055 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 1056 lockOps = append(lockOps, "create") 1057 lockObj = action.(fakeclient.CreateAction).GetObject() 1058 return true, lockObj, nil 1059 }, 1060 }, 1061 { 1062 verb: "update", 1063 objectType: objectType, 1064 reaction: func(action fakeclient.Action) (handled bool, ret runtime.Object, err error) { 1065 updates++ 1066 lockOps = append(lockOps, "update") 1067 switch updates { 1068 case 2: 1069 return true, nil, errors.NewConflict(action.(fakeclient.UpdateAction).GetResource().GroupResource(), "fake conflict", nil) 1070 case 4: 1071 cancelFunc() 1072 } 1073 lockObj = action.(fakeclient.UpdateAction).GetObject() 1074 return true, lockObj, nil 1075 }, 1076 }, 1077 }, 1078 expectedLockOps: []string{"get", "create", "update", "update", "get", "update", "update"}, 1079 }, 1080 } 1081 1082 for i := range tests { 1083 test := &tests[i] 1084 t.Run(test.name, func(t *testing.T) { 1085 resetVars() 1086 1087 recorder := record.NewFakeRecorder(100) 1088 resourceLockConfig := rl.ResourceLockConfig{ 1089 Identity: "baz", 1090 EventRecorder: recorder, 1091 } 1092 c := &fake.Clientset{} 1093 for _, reactor := range test.reactors { 1094 c.AddReactor(reactor.verb, objectType, reactor.reaction) 1095 } 1096 c.AddReactor("*", "*", func(action fakeclient.Action) (bool, runtime.Object, error) { 1097 t.Errorf("unreachable action. testclient called too many times: %+v", action) 1098 return true, nil, fmt.Errorf("unreachable action") 1099 }) 1100 lock, err := rl.New("leases", "foo", "bar", c.CoreV1(), c.CoordinationV1(), resourceLockConfig) 1101 if err != nil { 1102 t.Fatal("resourcelock.New() = ", err) 1103 } 1104 1105 lec.Lock = lock 1106 elector, err := NewLeaderElector(lec) 1107 if err != nil { 1108 t.Fatal("Failed to create leader elector: ", err) 1109 } 1110 1111 ctx, cancel := context.WithCancel(context.Background()) 1112 cancelFunc = cancel 1113 1114 elector.Run(ctx) 1115 assert.Equal(t, test.expectedLockOps, lockOps, "Expected lock ops %q, got %q", test.expectedLockOps, lockOps) 1116 }) 1117 } 1118 }