github.com/lfch/etcd-io/tests/v3@v3.0.0-20221004140520-eac99acd3e9d/integration/clientv3/cluster_test.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package clientv3test 16 17 import ( 18 "context" 19 "fmt" 20 "math/rand" 21 "reflect" 22 "strings" 23 "testing" 24 "time" 25 26 "github.com/lfch/etcd-io/client/pkg/v3/types" 27 integration2 "github.com/lfch/etcd-io/tests/v3/framework/integration" 28 ) 29 30 func TestMemberList(t *testing.T) { 31 integration2.BeforeTest(t) 32 33 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) 34 defer clus.Terminate(t) 35 36 capi := clus.RandClient() 37 38 resp, err := capi.MemberList(context.Background()) 39 if err != nil { 40 t.Fatalf("failed to list member %v", err) 41 } 42 43 if len(resp.Members) != 3 { 44 t.Errorf("number of members = %d, want %d", len(resp.Members), 3) 45 } 46 } 47 48 func TestMemberAdd(t *testing.T) { 49 integration2.BeforeTest(t) 50 51 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) 52 defer clus.Terminate(t) 53 54 capi := clus.RandClient() 55 56 urls := []string{"http://127.0.0.1:1234"} 57 resp, err := capi.MemberAdd(context.Background(), urls) 58 if err != nil { 59 t.Fatalf("failed to add member %v", err) 60 } 61 62 if !reflect.DeepEqual(resp.Member.PeerURLs, urls) { 63 t.Errorf("urls = %v, want %v", urls, resp.Member.PeerURLs) 64 } 65 } 66 67 func TestMemberAddWithExistingURLs(t *testing.T) { 68 integration2.BeforeTest(t) 69 70 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) 71 defer clus.Terminate(t) 72 73 capi := clus.RandClient() 74 75 resp, err := capi.MemberList(context.Background()) 76 if err != nil { 77 t.Fatalf("failed to list member %v", err) 78 } 79 80 existingURL := resp.Members[0].PeerURLs[0] 81 _, err = capi.MemberAdd(context.Background(), []string{existingURL}) 82 expectedErrKeywords := "Peer URLs already exists" 83 if err == nil { 84 t.Fatalf("expecting add member to fail, got no error") 85 } 86 if !strings.Contains(err.Error(), expectedErrKeywords) { 87 t.Errorf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) 88 } 89 } 90 91 func TestMemberRemove(t *testing.T) { 92 integration2.BeforeTest(t) 93 94 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) 95 defer clus.Terminate(t) 96 97 capi := clus.Client(1) 98 resp, err := capi.MemberList(context.Background()) 99 if err != nil { 100 t.Fatalf("failed to list member %v", err) 101 } 102 103 rmvID := resp.Members[0].ID 104 // indexes in capi member list don't necessarily match cluster member list; 105 // find member that is not the client to remove 106 for _, m := range resp.Members { 107 mURLs, _ := types.NewURLs(m.PeerURLs) 108 if !reflect.DeepEqual(mURLs, clus.Members[1].ServerConfig.PeerURLs) { 109 rmvID = m.ID 110 break 111 } 112 } 113 114 _, err = capi.MemberRemove(context.Background(), rmvID) 115 if err != nil { 116 t.Fatalf("failed to remove member %v", err) 117 } 118 119 resp, err = capi.MemberList(context.Background()) 120 if err != nil { 121 t.Fatalf("failed to list member %v", err) 122 } 123 124 if len(resp.Members) != 2 { 125 t.Errorf("number of members = %d, want %d", len(resp.Members), 2) 126 } 127 } 128 129 func TestMemberUpdate(t *testing.T) { 130 integration2.BeforeTest(t) 131 132 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) 133 defer clus.Terminate(t) 134 135 capi := clus.RandClient() 136 resp, err := capi.MemberList(context.Background()) 137 if err != nil { 138 t.Fatalf("failed to list member %v", err) 139 } 140 141 urls := []string{"http://127.0.0.1:1234"} 142 _, err = capi.MemberUpdate(context.Background(), resp.Members[0].ID, urls) 143 if err != nil { 144 t.Fatalf("failed to update member %v", err) 145 } 146 147 resp, err = capi.MemberList(context.Background()) 148 if err != nil { 149 t.Fatalf("failed to list member %v", err) 150 } 151 152 if !reflect.DeepEqual(resp.Members[0].PeerURLs, urls) { 153 t.Errorf("urls = %v, want %v", urls, resp.Members[0].PeerURLs) 154 } 155 } 156 157 func TestMemberAddUpdateWrongURLs(t *testing.T) { 158 integration2.BeforeTest(t) 159 160 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 1}) 161 defer clus.Terminate(t) 162 163 capi := clus.RandClient() 164 tt := [][]string{ 165 // missing protocol scheme 166 {"://127.0.0.1:2379"}, 167 // unsupported scheme 168 {"mailto://127.0.0.1:2379"}, 169 // not conform to host:port 170 {"http://127.0.0.1"}, 171 // contain a path 172 {"http://127.0.0.1:2379/path"}, 173 // first path segment in URL cannot contain colon 174 {"127.0.0.1:1234"}, 175 // URL scheme must be http, https, unix, or unixs 176 {"localhost:1234"}, 177 } 178 for i := range tt { 179 _, err := capi.MemberAdd(context.Background(), tt[i]) 180 if err == nil { 181 t.Errorf("#%d: MemberAdd err = nil, but error", i) 182 } 183 _, err = capi.MemberUpdate(context.Background(), 0, tt[i]) 184 if err == nil { 185 t.Errorf("#%d: MemberUpdate err = nil, but error", i) 186 } 187 } 188 } 189 190 func TestMemberAddForLearner(t *testing.T) { 191 integration2.BeforeTest(t) 192 193 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) 194 defer clus.Terminate(t) 195 196 capi := clus.RandClient() 197 198 urls := []string{"http://127.0.0.1:1234"} 199 resp, err := capi.MemberAddAsLearner(context.Background(), urls) 200 if err != nil { 201 t.Fatalf("failed to add member %v", err) 202 } 203 204 if !resp.Member.IsLearner { 205 t.Errorf("Added a member as learner, got resp.Member.IsLearner = %v", resp.Member.IsLearner) 206 } 207 208 numberOfLearners := 0 209 for _, m := range resp.Members { 210 if m.IsLearner { 211 numberOfLearners++ 212 } 213 } 214 if numberOfLearners != 1 { 215 t.Errorf("Added 1 learner node to cluster, got %d", numberOfLearners) 216 } 217 } 218 219 func TestMemberPromote(t *testing.T) { 220 integration2.BeforeTest(t) 221 222 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, DisableStrictReconfigCheck: true}) 223 defer clus.Terminate(t) 224 225 // member promote request can be sent to any server in cluster, 226 // the request will be auto-forwarded to leader on server-side. 227 // This test explicitly includes the server-side forwarding by 228 // sending the request to follower. 229 leaderIdx := clus.WaitLeader(t) 230 followerIdx := (leaderIdx + 1) % 3 231 capi := clus.Client(followerIdx) 232 233 urls := []string{"http://127.0.0.1:1234"} 234 memberAddResp, err := capi.MemberAddAsLearner(context.Background(), urls) 235 if err != nil { 236 t.Fatalf("failed to add member %v", err) 237 } 238 239 if !memberAddResp.Member.IsLearner { 240 t.Fatalf("Added a member as learner, got resp.Member.IsLearner = %v", memberAddResp.Member.IsLearner) 241 } 242 learnerID := memberAddResp.Member.ID 243 244 numberOfLearners := 0 245 for _, m := range memberAddResp.Members { 246 if m.IsLearner { 247 numberOfLearners++ 248 } 249 } 250 if numberOfLearners != 1 { 251 t.Fatalf("Added 1 learner node to cluster, got %d", numberOfLearners) 252 } 253 254 // learner is not started yet. Expect learner progress check to fail. 255 // As the result, member promote request will fail. 256 _, err = capi.MemberPromote(context.Background(), learnerID) 257 expectedErrKeywords := "can only promote a learner member which is in sync with leader" 258 if err == nil { 259 t.Fatalf("expecting promote not ready learner to fail, got no error") 260 } 261 if !strings.Contains(err.Error(), expectedErrKeywords) { 262 t.Fatalf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) 263 } 264 265 // create and launch learner member based on the response of V3 Member Add API. 266 // (the response has information on peer urls of the existing members in cluster) 267 learnerMember := clus.MustNewMember(t, memberAddResp) 268 269 if err := learnerMember.Launch(); err != nil { 270 t.Fatal(err) 271 } 272 273 // retry until promote succeed or timeout 274 timeout := time.After(5 * time.Second) 275 for { 276 select { 277 case <-time.After(500 * time.Millisecond): 278 case <-timeout: 279 t.Fatalf("failed all attempts to promote learner member, last error: %v", err) 280 } 281 282 _, err = capi.MemberPromote(context.Background(), learnerID) 283 // successfully promoted learner 284 if err == nil { 285 break 286 } 287 // if member promote fails due to learner not ready, retry. 288 // otherwise fails the test. 289 if !strings.Contains(err.Error(), expectedErrKeywords) { 290 t.Fatalf("unexpected error when promoting learner member: %v", err) 291 } 292 } 293 } 294 295 // TestMemberPromoteMemberNotLearner ensures that promoting a voting member fails. 296 func TestMemberPromoteMemberNotLearner(t *testing.T) { 297 integration2.BeforeTest(t) 298 299 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) 300 defer clus.Terminate(t) 301 302 // member promote request can be sent to any server in cluster, 303 // the request will be auto-forwarded to leader on server-side. 304 // This test explicitly includes the server-side forwarding by 305 // sending the request to follower. 306 leaderIdx := clus.WaitLeader(t) 307 followerIdx := (leaderIdx + 1) % 3 308 cli := clus.Client(followerIdx) 309 310 resp, err := cli.MemberList(context.Background()) 311 if err != nil { 312 t.Fatalf("failed to list member %v", err) 313 } 314 if len(resp.Members) != 3 { 315 t.Fatalf("number of members = %d, want %d", len(resp.Members), 3) 316 } 317 318 // promoting any of the voting members in cluster should fail 319 expectedErrKeywords := "can only promote a learner member" 320 for _, m := range resp.Members { 321 _, err = cli.MemberPromote(context.Background(), m.ID) 322 if err == nil { 323 t.Fatalf("expect promoting voting member to fail, got no error") 324 } 325 if !strings.Contains(err.Error(), expectedErrKeywords) { 326 t.Fatalf("expect error to contain %s, got %s", expectedErrKeywords, err.Error()) 327 } 328 } 329 } 330 331 // TestMemberPromoteMemberNotExist ensures that promoting a member that does not exist in cluster fails. 332 func TestMemberPromoteMemberNotExist(t *testing.T) { 333 integration2.BeforeTest(t) 334 335 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3}) 336 defer clus.Terminate(t) 337 338 // member promote request can be sent to any server in cluster, 339 // the request will be auto-forwarded to leader on server-side. 340 // This test explicitly includes the server-side forwarding by 341 // sending the request to follower. 342 leaderIdx := clus.WaitLeader(t) 343 followerIdx := (leaderIdx + 1) % 3 344 cli := clus.Client(followerIdx) 345 346 resp, err := cli.MemberList(context.Background()) 347 if err != nil { 348 t.Fatalf("failed to list member %v", err) 349 } 350 if len(resp.Members) != 3 { 351 t.Fatalf("number of members = %d, want %d", len(resp.Members), 3) 352 } 353 354 // generate an random ID that does not exist in cluster 355 var randID uint64 356 for { 357 randID = rand.Uint64() 358 notExist := true 359 for _, m := range resp.Members { 360 if m.ID == randID { 361 notExist = false 362 break 363 } 364 } 365 if notExist { 366 break 367 } 368 } 369 370 expectedErrKeywords := "member not found" 371 _, err = cli.MemberPromote(context.Background(), randID) 372 if err == nil { 373 t.Fatalf("expect promoting voting member to fail, got no error") 374 } 375 if !strings.Contains(err.Error(), expectedErrKeywords) { 376 t.Errorf("expect error to contain %s, got %s", expectedErrKeywords, err.Error()) 377 } 378 } 379 380 // TestMaxLearnerInCluster verifies that the maximum number of learners allowed in a cluster 381 func TestMaxLearnerInCluster(t *testing.T) { 382 integration2.BeforeTest(t) 383 384 // 1. start with a cluster with 3 voting member and max learner 2 385 clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, ExperimentalMaxLearners: 2, DisableStrictReconfigCheck: true}) 386 defer clus.Terminate(t) 387 388 // 2. adding 2 learner members should succeed 389 for i := 0; i < 2; i++ { 390 _, err := clus.Client(0).MemberAddAsLearner(context.Background(), []string{fmt.Sprintf("http://127.0.0.1:123%d", i)}) 391 if err != nil { 392 t.Fatalf("failed to add learner member %v", err) 393 } 394 } 395 396 // ensure client endpoint is voting member 397 leaderIdx := clus.WaitLeader(t) 398 capi := clus.Client(leaderIdx) 399 resp1, err := capi.MemberList(context.Background()) 400 if err != nil { 401 t.Fatalf("failed to get member list") 402 } 403 numberOfLearners := 0 404 for _, m := range resp1.Members { 405 if m.IsLearner { 406 numberOfLearners++ 407 } 408 } 409 if numberOfLearners != 2 { 410 t.Fatalf("added 2 learner node to cluster, got %d", numberOfLearners) 411 } 412 413 // 3. cluster has 3 voting member and 2 learner, adding another learner should fail 414 _, err = clus.Client(0).MemberAddAsLearner(context.Background(), []string{"http://127.0.0.1:2342"}) 415 if err == nil { 416 t.Fatalf("expect member add to fail, got no error") 417 } 418 expectedErrKeywords := "too many learner members in cluster" 419 if !strings.Contains(err.Error(), expectedErrKeywords) { 420 t.Fatalf("expecting error to contain %s, got %s", expectedErrKeywords, err.Error()) 421 } 422 423 // 4. cluster has 3 voting member and 1 learner, adding a voting member should succeed 424 _, err = clus.Client(0).MemberAdd(context.Background(), []string{"http://127.0.0.1:3453"}) 425 if err != nil { 426 t.Errorf("failed to add member %v", err) 427 } 428 }