vitess.io/vitess@v0.16.2/go/vt/topo/etcd2topo/server_test.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package etcd2topo 18 19 import ( 20 "context" 21 "fmt" 22 "os" 23 "os/exec" 24 "path" 25 "testing" 26 "time" 27 28 "vitess.io/vitess/go/testfiles" 29 "vitess.io/vitess/go/vt/log" 30 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 31 "vitess.io/vitess/go/vt/tlstest" 32 "vitess.io/vitess/go/vt/topo" 33 "vitess.io/vitess/go/vt/topo/test" 34 35 clientv3 "go.etcd.io/etcd/client/v3" 36 ) 37 38 // startEtcd starts an etcd subprocess, and waits for it to be ready. 39 func startEtcd(t *testing.T) string { 40 // Create a temporary directory. 41 dataDir := t.TempDir() 42 43 // Get our two ports to listen to. 44 port := testfiles.GoVtTopoEtcd2topoPort 45 name := "vitess_unit_test" 46 clientAddr := fmt.Sprintf("http://localhost:%v", port) 47 peerAddr := fmt.Sprintf("http://localhost:%v", port+1) 48 initialCluster := fmt.Sprintf("%v=%v", name, peerAddr) 49 50 cmd := exec.Command("etcd", 51 "-name", name, 52 "-advertise-client-urls", clientAddr, 53 "-initial-advertise-peer-urls", peerAddr, 54 "-listen-client-urls", clientAddr, 55 "-listen-peer-urls", peerAddr, 56 "-initial-cluster", initialCluster, 57 "-data-dir", dataDir) 58 err := cmd.Start() 59 if err != nil { 60 t.Fatalf("failed to start etcd: %v", err) 61 } 62 63 // Create a client to connect to the created etcd. 64 cli, err := clientv3.New(clientv3.Config{ 65 Endpoints: []string{clientAddr}, 66 DialTimeout: 5 * time.Second, 67 }) 68 if err != nil { 69 t.Fatalf("newCellClient(%v) failed: %v", clientAddr, err) 70 } 71 defer cli.Close() 72 73 // Wait until we can list "/", or timeout. 74 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 75 defer cancel() 76 start := time.Now() 77 for { 78 if _, err := cli.Get(ctx, "/"); err == nil { 79 break 80 } 81 if time.Since(start) > 10*time.Second { 82 t.Fatalf("Failed to start etcd daemon in time") 83 } 84 time.Sleep(10 * time.Millisecond) 85 } 86 t.Cleanup(func() { 87 // log error 88 if err := cmd.Process.Kill(); err != nil { 89 log.Errorf("cmd.Process.Kill() failed : %v", err) 90 } 91 // log error 92 if err := cmd.Wait(); err != nil { 93 log.Errorf("cmd.wait() failed : %v", err) 94 } 95 }) 96 97 return clientAddr 98 } 99 100 // startEtcdWithTLS starts an etcd subprocess with TLS setup, and waits for it to be ready. 101 func startEtcdWithTLS(t *testing.T) (string, *tlstest.ClientServerKeyPairs) { 102 // Create a temporary directory. 103 dataDir := t.TempDir() 104 105 // Get our two ports to listen to. 106 port := testfiles.GoVtTopoEtcd2topoPort 107 name := "vitess_unit_test" 108 clientAddr := fmt.Sprintf("https://localhost:%v", port+2) 109 peerAddr := fmt.Sprintf("https://localhost:%v", port+3) 110 initialCluster := fmt.Sprintf("%v=%v", name, peerAddr) 111 112 certs := tlstest.CreateClientServerCertPairs(dataDir) 113 114 cmd := exec.Command("etcd", 115 "-name", name, 116 "-advertise-client-urls", clientAddr, 117 "-initial-advertise-peer-urls", peerAddr, 118 "-listen-client-urls", clientAddr, 119 "-listen-peer-urls", peerAddr, 120 "-initial-cluster", initialCluster, 121 "-cert-file", certs.ServerCert, 122 "-key-file", certs.ServerKey, 123 "-trusted-ca-file", certs.ClientCA, 124 "-peer-trusted-ca-file", certs.ClientCA, 125 "-peer-cert-file", certs.ServerCert, 126 "-peer-key-file", certs.ServerKey, 127 "-client-cert-auth", 128 "-data-dir", dataDir) 129 130 cmd.Stderr = os.Stderr 131 cmd.Stdout = os.Stdout 132 err := cmd.Start() 133 if err != nil { 134 t.Fatalf("failed to start etcd: %v", err) 135 } 136 137 tlsConfig, err := newTLSConfig(certs.ClientCert, certs.ClientKey, certs.ServerCA) 138 if err != nil { 139 t.Fatalf("failed to get tls.Config: %v", err) 140 } 141 142 var cli *clientv3.Client 143 // Create client 144 start := time.Now() 145 for { 146 // Create a client to connect to the created etcd. 147 cli, err = clientv3.New(clientv3.Config{ 148 Endpoints: []string{clientAddr}, 149 TLS: tlsConfig, 150 DialTimeout: 5 * time.Second, 151 }) 152 if err == nil { 153 break 154 } 155 t.Logf("error establishing client for etcd tls test: %v", err) 156 if time.Since(start) > 60*time.Second { 157 t.Fatalf("failed to start client for etcd tls test in time") 158 } 159 time.Sleep(100 * time.Millisecond) 160 } 161 defer cli.Close() 162 163 // Wait until we can list "/", or timeout. 164 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 165 defer cancel() 166 start = time.Now() 167 for { 168 if _, err := cli.Get(ctx, "/"); err == nil { 169 break 170 } 171 if time.Since(start) > 60*time.Second { 172 t.Fatalf("failed to start etcd daemon in time") 173 } 174 time.Sleep(100 * time.Millisecond) 175 } 176 t.Cleanup(func() { 177 // log error 178 if err := cmd.Process.Kill(); err != nil { 179 log.Errorf("cmd.Process.Kill() failed : %v", err) 180 } 181 // log error 182 if err := cmd.Wait(); err != nil { 183 log.Errorf("cmd.wait() failed : %v", err) 184 } 185 }) 186 187 return clientAddr, &certs 188 } 189 190 func TestEtcd2TLS(t *testing.T) { 191 // Start a single etcd in the background. 192 clientAddr, certs := startEtcdWithTLS(t) 193 194 testIndex := 0 195 testRoot := fmt.Sprintf("/test-%v", testIndex) 196 197 // Create the server on the new root. 198 server, err := NewServerWithOpts(clientAddr, testRoot, certs.ClientCert, certs.ClientKey, certs.ServerCA) 199 if err != nil { 200 t.Fatalf("NewServerWithOpts failed: %v", err) 201 } 202 defer server.Close() 203 204 testCtx := context.Background() 205 testKey := "testkey" 206 testVal := "testval" 207 _, err = server.Create(testCtx, testKey, []byte(testVal)) 208 if err != nil { 209 t.Fatalf("Failed to set key value pair: %v", err) 210 } 211 val, _, err := server.Get(testCtx, testKey) 212 if err != nil { 213 t.Fatalf("Failed to retrieve value at key we just set: %v", err) 214 } 215 if string(val) != testVal { 216 t.Fatalf("Value returned doesn't match %s, err: %v", testVal, err) 217 } 218 } 219 220 func TestEtcd2Topo(t *testing.T) { 221 // Start a single etcd in the background. 222 clientAddr := startEtcd(t) 223 224 testIndex := 0 225 newServer := func() *topo.Server { 226 // Each test will use its own sub-directories. 227 testRoot := fmt.Sprintf("/test-%v", testIndex) 228 testIndex++ 229 230 // Create the server on the new root. 231 ts, err := topo.OpenServer("etcd2", clientAddr, path.Join(testRoot, topo.GlobalCell)) 232 if err != nil { 233 t.Fatalf("OpenServer() failed: %v", err) 234 } 235 236 // Create the CellInfo. 237 if err := ts.CreateCellInfo(context.Background(), test.LocalCellName, &topodatapb.CellInfo{ 238 ServerAddress: clientAddr, 239 Root: path.Join(testRoot, test.LocalCellName), 240 }); err != nil { 241 t.Fatalf("CreateCellInfo() failed: %v", err) 242 } 243 244 return ts 245 } 246 247 // Run the TopoServerTestSuite tests. 248 test.TopoServerTestSuite(t, func() *topo.Server { 249 return newServer() 250 }, []string{}) 251 252 // Run etcd-specific tests. 253 ts := newServer() 254 testKeyspaceLock(t, ts) 255 ts.Close() 256 } 257 258 // testKeyspaceLock tests etcd-specific heartbeat (TTL). 259 // Note TTL granularity is in seconds, even though the API uses time.Duration. 260 // So we have to wait a long time in these tests. 261 func testKeyspaceLock(t *testing.T, ts *topo.Server) { 262 ctx := context.Background() 263 keyspacePath := path.Join(topo.KeyspacesPath, "test_keyspace") 264 if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil { 265 t.Fatalf("CreateKeyspace: %v", err) 266 } 267 268 conn, err := ts.ConnForCell(ctx, topo.GlobalCell) 269 if err != nil { 270 t.Fatalf("ConnForCell failed: %v", err) 271 } 272 273 // Long TTL, unlock before lease runs out. 274 leaseTTL = 1000 275 lockDescriptor, err := conn.Lock(ctx, keyspacePath, "ttl") 276 if err != nil { 277 t.Fatalf("Lock failed: %v", err) 278 } 279 if err := lockDescriptor.Unlock(ctx); err != nil { 280 t.Fatalf("Unlock failed: %v", err) 281 } 282 283 // Short TTL, make sure it doesn't expire. 284 leaseTTL = 1 285 lockDescriptor, err = conn.Lock(ctx, keyspacePath, "short ttl") 286 if err != nil { 287 t.Fatalf("Lock failed: %v", err) 288 } 289 time.Sleep(2 * time.Second) 290 if err := lockDescriptor.Unlock(ctx); err != nil { 291 t.Fatalf("Unlock failed: %v", err) 292 } 293 }