github.com/lfch/etcd-io/tests/v3@v3.0.0-20221004140520-eac99acd3e9d/integration/v3_leadership_test.go (about) 1 // Copyright 2017 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 "fmt" 20 "strings" 21 "testing" 22 "time" 23 24 pb "github.com/lfch/etcd-io/api/v3/etcdserverpb" 25 "github.com/lfch/etcd-io/api/v3/v3rpc/rpctypes" 26 "github.com/lfch/etcd-io/tests/v3/framework/integration" 27 "golang.org/x/sync/errgroup" 28 ) 29 30 func TestMoveLeader(t *testing.T) { testMoveLeader(t, true) } 31 func TestMoveLeaderService(t *testing.T) { testMoveLeader(t, false) } 32 33 func testMoveLeader(t *testing.T, auto bool) { 34 integration.BeforeTest(t) 35 36 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 37 defer clus.Terminate(t) 38 39 oldLeadIdx := clus.WaitLeader(t) 40 oldLeadID := uint64(clus.Members[oldLeadIdx].Server.MemberId()) 41 42 // ensure followers go through leader transition while leadership transfer 43 idc := make(chan uint64) 44 stopc := make(chan struct{}) 45 defer close(stopc) 46 47 for i := range clus.Members { 48 if oldLeadIdx != i { 49 go func(m *integration.Member) { 50 select { 51 case idc <- integration.CheckLeaderTransition(m, oldLeadID): 52 case <-stopc: 53 } 54 }(clus.Members[i]) 55 } 56 } 57 58 target := uint64(clus.Members[(oldLeadIdx+1)%3].Server.MemberId()) 59 if auto { 60 err := clus.Members[oldLeadIdx].Server.TransferLeadership() 61 if err != nil { 62 t.Fatal(err) 63 } 64 } else { 65 mvc := integration.ToGRPC(clus.Client(oldLeadIdx)).Maintenance 66 _, err := mvc.MoveLeader(context.TODO(), &pb.MoveLeaderRequest{TargetID: target}) 67 if err != nil { 68 t.Fatal(err) 69 } 70 } 71 72 // wait until leader transitions have happened 73 var newLeadIDs [2]uint64 74 for i := range newLeadIDs { 75 select { 76 case newLeadIDs[i] = <-idc: 77 case <-time.After(time.Second): 78 t.Fatal("timed out waiting for leader transition") 79 } 80 } 81 82 // remaining members must agree on the same leader 83 if newLeadIDs[0] != newLeadIDs[1] { 84 t.Fatalf("expected same new leader %d == %d", newLeadIDs[0], newLeadIDs[1]) 85 } 86 87 // new leader must be different than the old leader 88 if oldLeadID == newLeadIDs[0] { 89 t.Fatalf("expected old leader %d != new leader %d", oldLeadID, newLeadIDs[0]) 90 } 91 92 // if move-leader were used, new leader must match transferee 93 if !auto { 94 if newLeadIDs[0] != target { 95 t.Fatalf("expected new leader %d != target %d", newLeadIDs[0], target) 96 } 97 } 98 } 99 100 // TestMoveLeaderError ensures that request to non-leader fail. 101 func TestMoveLeaderError(t *testing.T) { 102 integration.BeforeTest(t) 103 104 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3}) 105 defer clus.Terminate(t) 106 107 oldLeadIdx := clus.WaitLeader(t) 108 followerIdx := (oldLeadIdx + 1) % 3 109 110 target := uint64(clus.Members[(oldLeadIdx+2)%3].Server.MemberId()) 111 112 mvc := integration.ToGRPC(clus.Client(followerIdx)).Maintenance 113 _, err := mvc.MoveLeader(context.TODO(), &pb.MoveLeaderRequest{TargetID: target}) 114 if !eqErrGRPC(err, rpctypes.ErrGRPCNotLeader) { 115 t.Errorf("err = %v, want %v", err, rpctypes.ErrGRPCNotLeader) 116 } 117 } 118 119 // TestMoveLeaderToLearnerError ensures that leader transfer to learner member will fail. 120 func TestMoveLeaderToLearnerError(t *testing.T) { 121 integration.BeforeTest(t) 122 123 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) 124 defer clus.Terminate(t) 125 126 // we have to add and launch learner member after initial cluster was created, because 127 // bootstrapping a cluster with learner member is not supported. 128 clus.AddAndLaunchLearnerMember(t) 129 130 learners, err := clus.GetLearnerMembers() 131 if err != nil { 132 t.Fatalf("failed to get the learner members in Cluster: %v", err) 133 } 134 if len(learners) != 1 { 135 t.Fatalf("added 1 learner to Cluster, got %d", len(learners)) 136 } 137 138 learnerID := learners[0].ID 139 leaderIdx := clus.WaitLeader(t) 140 cli := clus.Client(leaderIdx) 141 _, err = cli.MoveLeader(context.Background(), learnerID) 142 if err == nil { 143 t.Fatalf("expecting leader transfer to learner to fail, got no error") 144 } 145 expectedErrKeywords := "bad leader transferee" 146 if !strings.Contains(err.Error(), expectedErrKeywords) { 147 t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) 148 } 149 } 150 151 // TestTransferLeadershipWithLearner ensures TransferLeadership does not timeout due to learner is 152 // automatically picked by leader as transferee. 153 func TestTransferLeadershipWithLearner(t *testing.T) { 154 integration.BeforeTest(t) 155 156 clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) 157 defer clus.Terminate(t) 158 159 clus.AddAndLaunchLearnerMember(t) 160 161 learners, err := clus.GetLearnerMembers() 162 if err != nil { 163 t.Fatalf("failed to get the learner members in Cluster: %v", err) 164 } 165 if len(learners) != 1 { 166 t.Fatalf("added 1 learner to Cluster, got %d", len(learners)) 167 } 168 169 leaderIdx := clus.WaitLeader(t) 170 errCh := make(chan error, 1) 171 go func() { 172 // note that this cluster has 1 leader and 1 learner. TransferLeadership should return nil. 173 // Leadership transfer is skipped in cluster with 1 voting member. 174 errCh <- clus.Members[leaderIdx].Server.TransferLeadership() 175 }() 176 select { 177 case err := <-errCh: 178 if err != nil { 179 t.Errorf("got error during leadership transfer: %v", err) 180 } 181 case <-time.After(5 * time.Second): 182 t.Error("timed out waiting for leader transition") 183 } 184 } 185 186 func TestFirstCommitNotification(t *testing.T) { 187 integration.BeforeTest(t) 188 ctx := context.Background() 189 clusterSize := 3 190 cluster := integration.NewCluster(t, &integration.ClusterConfig{Size: clusterSize}) 191 defer cluster.Terminate(t) 192 193 oldLeaderIdx := cluster.WaitLeader(t) 194 oldLeaderClient := cluster.Client(oldLeaderIdx) 195 196 newLeaderIdx := (oldLeaderIdx + 1) % clusterSize 197 newLeaderId := uint64(cluster.Members[newLeaderIdx].ID()) 198 199 notifiers := make(map[int]<-chan struct{}, clusterSize) 200 for i, clusterMember := range cluster.Members { 201 notifiers[i] = clusterMember.Server.FirstCommitInTermNotify() 202 } 203 204 _, err := oldLeaderClient.MoveLeader(context.Background(), newLeaderId) 205 206 if err != nil { 207 t.Errorf("got error during leadership transfer: %v", err) 208 } 209 210 t.Logf("Leadership transferred.") 211 t.Logf("Submitting write to make sure empty and 'foo' index entry was already flushed") 212 cli := cluster.RandClient() 213 214 if _, err := cli.Put(ctx, "foo", "bar"); err != nil { 215 t.Fatalf("Failed to put kv pair.") 216 } 217 218 // It's guaranteed now that leader contains the 'foo'->'bar' index entry. 219 leaderAppliedIndex := cluster.Members[newLeaderIdx].Server.AppliedIndex() 220 221 ctx, cancel := context.WithTimeout(ctx, 5*time.Second) 222 defer cancel() 223 224 group, groupContext := errgroup.WithContext(ctx) 225 226 for i, notifier := range notifiers { 227 member, notifier := cluster.Members[i], notifier 228 group.Go(func() error { 229 return checkFirstCommitNotification(groupContext, t, member, leaderAppliedIndex, notifier) 230 }) 231 } 232 233 err = group.Wait() 234 if err != nil { 235 t.Error(err) 236 } 237 } 238 239 func checkFirstCommitNotification( 240 ctx context.Context, 241 t testing.TB, 242 member *integration.Member, 243 leaderAppliedIndex uint64, 244 notifier <-chan struct{}, 245 ) error { 246 // wait until server applies all the changes of leader 247 for member.Server.AppliedIndex() < leaderAppliedIndex { 248 t.Logf("member.Server.AppliedIndex():%v <= leaderAppliedIndex:%v", member.Server.AppliedIndex(), leaderAppliedIndex) 249 select { 250 case <-ctx.Done(): 251 return ctx.Err() 252 default: 253 time.Sleep(100 * time.Millisecond) 254 } 255 } 256 select { 257 case msg, ok := <-notifier: 258 if ok { 259 return fmt.Errorf( 260 "member with ID %d got message via notifier, msg: %v", 261 member.ID(), 262 msg, 263 ) 264 } 265 default: 266 t.Logf("member.Server.AppliedIndex():%v >= leaderAppliedIndex:%v", member.Server.AppliedIndex(), leaderAppliedIndex) 267 return fmt.Errorf( 268 "notification was not triggered, member ID: %d", 269 member.ID(), 270 ) 271 } 272 273 return nil 274 }