go.etcd.io/etcd@v3.3.27+incompatible/integration/v3_election_test.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package integration 16 17 import ( 18 "context" 19 "fmt" 20 "testing" 21 "time" 22 23 "github.com/coreos/etcd/clientv3" 24 "github.com/coreos/etcd/clientv3/concurrency" 25 ) 26 27 // TestElectionWait tests if followers can correctly wait for elections. 28 func TestElectionWait(t *testing.T) { 29 clus := NewClusterV3(t, &ClusterConfig{Size: 3}) 30 defer clus.Terminate(t) 31 32 leaders := 3 33 followers := 3 34 var clients []*clientv3.Client 35 newClient := makeMultiNodeClients(t, clus.cluster, &clients) 36 37 electedc := make(chan string) 38 nextc := []chan struct{}{} 39 40 // wait for all elections 41 donec := make(chan struct{}) 42 for i := 0; i < followers; i++ { 43 nextc = append(nextc, make(chan struct{})) 44 go func(ch chan struct{}) { 45 for j := 0; j < leaders; j++ { 46 session, err := concurrency.NewSession(newClient()) 47 if err != nil { 48 t.Error(err) 49 } 50 b := concurrency.NewElection(session, "test-election") 51 52 cctx, cancel := context.WithCancel(context.TODO()) 53 defer cancel() 54 s, ok := <-b.Observe(cctx) 55 if !ok { 56 t.Fatalf("could not observe election; channel closed") 57 } 58 electedc <- string(s.Kvs[0].Value) 59 // wait for next election round 60 <-ch 61 session.Orphan() 62 } 63 donec <- struct{}{} 64 }(nextc[i]) 65 } 66 67 // elect some leaders 68 for i := 0; i < leaders; i++ { 69 go func() { 70 session, err := concurrency.NewSession(newClient()) 71 if err != nil { 72 t.Error(err) 73 } 74 defer session.Orphan() 75 76 e := concurrency.NewElection(session, "test-election") 77 ev := fmt.Sprintf("electval-%v", time.Now().UnixNano()) 78 if err := e.Campaign(context.TODO(), ev); err != nil { 79 t.Fatalf("failed volunteer (%v)", err) 80 } 81 // wait for followers to accept leadership 82 for j := 0; j < followers; j++ { 83 s := <-electedc 84 if s != ev { 85 t.Errorf("wrong election value got %s, wanted %s", s, ev) 86 } 87 } 88 // let next leader take over 89 if err := e.Resign(context.TODO()); err != nil { 90 t.Fatalf("failed resign (%v)", err) 91 } 92 // tell followers to start listening for next leader 93 for j := 0; j < followers; j++ { 94 nextc[j] <- struct{}{} 95 } 96 }() 97 } 98 99 // wait on followers 100 for i := 0; i < followers; i++ { 101 <-donec 102 } 103 104 closeClients(t, clients) 105 } 106 107 // TestElectionFailover tests that an election will 108 func TestElectionFailover(t *testing.T) { 109 clus := NewClusterV3(t, &ClusterConfig{Size: 3}) 110 defer clus.Terminate(t) 111 112 cctx, cancel := context.WithCancel(context.TODO()) 113 defer cancel() 114 115 ss := make([]*concurrency.Session, 3) 116 117 for i := 0; i < 3; i++ { 118 var err error 119 ss[i], err = concurrency.NewSession(clus.clients[i]) 120 if err != nil { 121 t.Error(err) 122 } 123 defer ss[i].Orphan() 124 } 125 126 // first leader (elected) 127 e := concurrency.NewElection(ss[0], "test-election") 128 if err := e.Campaign(context.TODO(), "foo"); err != nil { 129 t.Fatalf("failed volunteer (%v)", err) 130 } 131 132 // check first leader 133 resp, ok := <-e.Observe(cctx) 134 if !ok { 135 t.Fatalf("could not wait for first election; channel closed") 136 } 137 s := string(resp.Kvs[0].Value) 138 if s != "foo" { 139 t.Fatalf("wrong election result. got %s, wanted foo", s) 140 } 141 142 // next leader 143 electedc := make(chan struct{}) 144 go func() { 145 ee := concurrency.NewElection(ss[1], "test-election") 146 if eer := ee.Campaign(context.TODO(), "bar"); eer != nil { 147 t.Fatal(eer) 148 } 149 electedc <- struct{}{} 150 }() 151 152 // invoke leader failover 153 if err := ss[0].Close(); err != nil { 154 t.Fatal(err) 155 } 156 157 // check new leader 158 e = concurrency.NewElection(ss[2], "test-election") 159 resp, ok = <-e.Observe(cctx) 160 if !ok { 161 t.Fatalf("could not wait for second election; channel closed") 162 } 163 s = string(resp.Kvs[0].Value) 164 if s != "bar" { 165 t.Fatalf("wrong election result. got %s, wanted bar", s) 166 } 167 168 // leader must ack election (otherwise, Campaign may see closed conn) 169 <-electedc 170 } 171 172 // TestElectionSessionRelock ensures that campaigning twice on the same election 173 // with the same lock will Proclaim instead of deadlocking. 174 func TestElectionSessionRecampaign(t *testing.T) { 175 clus := NewClusterV3(t, &ClusterConfig{Size: 1}) 176 defer clus.Terminate(t) 177 cli := clus.RandClient() 178 179 session, err := concurrency.NewSession(cli) 180 if err != nil { 181 t.Error(err) 182 } 183 defer session.Orphan() 184 185 e := concurrency.NewElection(session, "test-elect") 186 if err := e.Campaign(context.TODO(), "abc"); err != nil { 187 t.Fatal(err) 188 } 189 e2 := concurrency.NewElection(session, "test-elect") 190 if err := e2.Campaign(context.TODO(), "def"); err != nil { 191 t.Fatal(err) 192 } 193 194 ctx, cancel := context.WithCancel(context.TODO()) 195 defer cancel() 196 if resp := <-e.Observe(ctx); len(resp.Kvs) == 0 || string(resp.Kvs[0].Value) != "def" { 197 t.Fatalf("expected value=%q, got response %v", "def", resp) 198 } 199 } 200 201 // TestElectionOnPrefixOfExistingKey checks that a single 202 // candidate can be elected on a new key that is a prefix 203 // of an existing key. To wit, check for regression 204 // of bug #6278. https://github.com/coreos/etcd/issues/6278 205 // 206 func TestElectionOnPrefixOfExistingKey(t *testing.T) { 207 clus := NewClusterV3(t, &ClusterConfig{Size: 1}) 208 defer clus.Terminate(t) 209 210 cli := clus.RandClient() 211 if _, err := cli.Put(context.TODO(), "testa", "value"); err != nil { 212 t.Fatal(err) 213 } 214 s, serr := concurrency.NewSession(cli) 215 if serr != nil { 216 t.Fatal(serr) 217 } 218 e := concurrency.NewElection(s, "test") 219 ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) 220 err := e.Campaign(ctx, "abc") 221 cancel() 222 if err != nil { 223 // after 5 seconds, deadlock results in 224 // 'context deadline exceeded' here. 225 t.Fatal(err) 226 } 227 } 228 229 // TestElectionOnSessionRestart tests that a quick restart of leader (resulting 230 // in a new session with the same lease id) does not result in loss of 231 // leadership. 232 func TestElectionOnSessionRestart(t *testing.T) { 233 clus := NewClusterV3(t, &ClusterConfig{Size: 1}) 234 defer clus.Terminate(t) 235 cli := clus.RandClient() 236 237 session, err := concurrency.NewSession(cli) 238 if err != nil { 239 t.Fatal(err) 240 } 241 242 e := concurrency.NewElection(session, "test-elect") 243 if cerr := e.Campaign(context.TODO(), "abc"); cerr != nil { 244 t.Fatal(cerr) 245 } 246 247 // ensure leader is not lost to waiter on fail-over 248 waitSession, werr := concurrency.NewSession(cli) 249 if werr != nil { 250 t.Fatal(werr) 251 } 252 defer waitSession.Orphan() 253 waitCtx, waitCancel := context.WithTimeout(context.TODO(), 5*time.Second) 254 defer waitCancel() 255 go concurrency.NewElection(waitSession, "test-elect").Campaign(waitCtx, "123") 256 257 // simulate restart by reusing the lease from the old session 258 newSession, nerr := concurrency.NewSession(cli, concurrency.WithLease(session.Lease())) 259 if nerr != nil { 260 t.Fatal(nerr) 261 } 262 defer newSession.Orphan() 263 264 newElection := concurrency.NewElection(newSession, "test-elect") 265 if ncerr := newElection.Campaign(context.TODO(), "def"); ncerr != nil { 266 t.Fatal(ncerr) 267 } 268 269 ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) 270 defer cancel() 271 if resp := <-newElection.Observe(ctx); len(resp.Kvs) == 0 || string(resp.Kvs[0].Value) != "def" { 272 t.Errorf("expected value=%q, got response %v", "def", resp) 273 } 274 } 275 276 // TestElectionObserveCompacted checks that observe can tolerate 277 // a leader key with a modrev less than the compaction revision. 278 func TestElectionObserveCompacted(t *testing.T) { 279 clus := NewClusterV3(t, &ClusterConfig{Size: 1}) 280 defer clus.Terminate(t) 281 282 cli := clus.Client(0) 283 284 session, err := concurrency.NewSession(cli) 285 if err != nil { 286 t.Fatal(err) 287 } 288 defer session.Orphan() 289 290 e := concurrency.NewElection(session, "test-elect") 291 if cerr := e.Campaign(context.TODO(), "abc"); cerr != nil { 292 t.Fatal(cerr) 293 } 294 295 presp, perr := cli.Put(context.TODO(), "foo", "bar") 296 if perr != nil { 297 t.Fatal(perr) 298 } 299 if _, cerr := cli.Compact(context.TODO(), presp.Header.Revision); cerr != nil { 300 t.Fatal(cerr) 301 } 302 303 v, ok := <-e.Observe(context.TODO()) 304 if !ok { 305 t.Fatal("failed to observe on compacted revision") 306 } 307 if string(v.Kvs[0].Value) != "abc" { 308 t.Fatalf(`expected leader value "abc", got %q`, string(v.Kvs[0].Value)) 309 } 310 }