vitess.io/vitess@v0.16.2/go/vt/topo/etcd2topo/server_test.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package etcd2topo
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"os/exec"
    24  	"path"
    25  	"testing"
    26  	"time"
    27  
    28  	"vitess.io/vitess/go/testfiles"
    29  	"vitess.io/vitess/go/vt/log"
    30  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    31  	"vitess.io/vitess/go/vt/tlstest"
    32  	"vitess.io/vitess/go/vt/topo"
    33  	"vitess.io/vitess/go/vt/topo/test"
    34  
    35  	clientv3 "go.etcd.io/etcd/client/v3"
    36  )
    37  
    38  // startEtcd starts an etcd subprocess, and waits for it to be ready.
    39  func startEtcd(t *testing.T) string {
    40  	// Create a temporary directory.
    41  	dataDir := t.TempDir()
    42  
    43  	// Get our two ports to listen to.
    44  	port := testfiles.GoVtTopoEtcd2topoPort
    45  	name := "vitess_unit_test"
    46  	clientAddr := fmt.Sprintf("http://localhost:%v", port)
    47  	peerAddr := fmt.Sprintf("http://localhost:%v", port+1)
    48  	initialCluster := fmt.Sprintf("%v=%v", name, peerAddr)
    49  
    50  	cmd := exec.Command("etcd",
    51  		"-name", name,
    52  		"-advertise-client-urls", clientAddr,
    53  		"-initial-advertise-peer-urls", peerAddr,
    54  		"-listen-client-urls", clientAddr,
    55  		"-listen-peer-urls", peerAddr,
    56  		"-initial-cluster", initialCluster,
    57  		"-data-dir", dataDir)
    58  	err := cmd.Start()
    59  	if err != nil {
    60  		t.Fatalf("failed to start etcd: %v", err)
    61  	}
    62  
    63  	// Create a client to connect to the created etcd.
    64  	cli, err := clientv3.New(clientv3.Config{
    65  		Endpoints:   []string{clientAddr},
    66  		DialTimeout: 5 * time.Second,
    67  	})
    68  	if err != nil {
    69  		t.Fatalf("newCellClient(%v) failed: %v", clientAddr, err)
    70  	}
    71  	defer cli.Close()
    72  
    73  	// Wait until we can list "/", or timeout.
    74  	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
    75  	defer cancel()
    76  	start := time.Now()
    77  	for {
    78  		if _, err := cli.Get(ctx, "/"); err == nil {
    79  			break
    80  		}
    81  		if time.Since(start) > 10*time.Second {
    82  			t.Fatalf("Failed to start etcd daemon in time")
    83  		}
    84  		time.Sleep(10 * time.Millisecond)
    85  	}
    86  	t.Cleanup(func() {
    87  		// log error
    88  		if err := cmd.Process.Kill(); err != nil {
    89  			log.Errorf("cmd.Process.Kill() failed : %v", err)
    90  		}
    91  		// log error
    92  		if err := cmd.Wait(); err != nil {
    93  			log.Errorf("cmd.wait() failed : %v", err)
    94  		}
    95  	})
    96  
    97  	return clientAddr
    98  }
    99  
   100  // startEtcdWithTLS starts an etcd subprocess with TLS setup, and waits for it to be ready.
   101  func startEtcdWithTLS(t *testing.T) (string, *tlstest.ClientServerKeyPairs) {
   102  	// Create a temporary directory.
   103  	dataDir := t.TempDir()
   104  
   105  	// Get our two ports to listen to.
   106  	port := testfiles.GoVtTopoEtcd2topoPort
   107  	name := "vitess_unit_test"
   108  	clientAddr := fmt.Sprintf("https://localhost:%v", port+2)
   109  	peerAddr := fmt.Sprintf("https://localhost:%v", port+3)
   110  	initialCluster := fmt.Sprintf("%v=%v", name, peerAddr)
   111  
   112  	certs := tlstest.CreateClientServerCertPairs(dataDir)
   113  
   114  	cmd := exec.Command("etcd",
   115  		"-name", name,
   116  		"-advertise-client-urls", clientAddr,
   117  		"-initial-advertise-peer-urls", peerAddr,
   118  		"-listen-client-urls", clientAddr,
   119  		"-listen-peer-urls", peerAddr,
   120  		"-initial-cluster", initialCluster,
   121  		"-cert-file", certs.ServerCert,
   122  		"-key-file", certs.ServerKey,
   123  		"-trusted-ca-file", certs.ClientCA,
   124  		"-peer-trusted-ca-file", certs.ClientCA,
   125  		"-peer-cert-file", certs.ServerCert,
   126  		"-peer-key-file", certs.ServerKey,
   127  		"-client-cert-auth",
   128  		"-data-dir", dataDir)
   129  
   130  	cmd.Stderr = os.Stderr
   131  	cmd.Stdout = os.Stdout
   132  	err := cmd.Start()
   133  	if err != nil {
   134  		t.Fatalf("failed to start etcd: %v", err)
   135  	}
   136  
   137  	tlsConfig, err := newTLSConfig(certs.ClientCert, certs.ClientKey, certs.ServerCA)
   138  	if err != nil {
   139  		t.Fatalf("failed to get tls.Config: %v", err)
   140  	}
   141  
   142  	var cli *clientv3.Client
   143  	// Create client
   144  	start := time.Now()
   145  	for {
   146  		// Create a client to connect to the created etcd.
   147  		cli, err = clientv3.New(clientv3.Config{
   148  			Endpoints:   []string{clientAddr},
   149  			TLS:         tlsConfig,
   150  			DialTimeout: 5 * time.Second,
   151  		})
   152  		if err == nil {
   153  			break
   154  		}
   155  		t.Logf("error establishing client for etcd tls test: %v", err)
   156  		if time.Since(start) > 60*time.Second {
   157  			t.Fatalf("failed to start client for etcd tls test in time")
   158  		}
   159  		time.Sleep(100 * time.Millisecond)
   160  	}
   161  	defer cli.Close()
   162  
   163  	// Wait until we can list "/", or timeout.
   164  	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
   165  	defer cancel()
   166  	start = time.Now()
   167  	for {
   168  		if _, err := cli.Get(ctx, "/"); err == nil {
   169  			break
   170  		}
   171  		if time.Since(start) > 60*time.Second {
   172  			t.Fatalf("failed to start etcd daemon in time")
   173  		}
   174  		time.Sleep(100 * time.Millisecond)
   175  	}
   176  	t.Cleanup(func() {
   177  		// log error
   178  		if err := cmd.Process.Kill(); err != nil {
   179  			log.Errorf("cmd.Process.Kill() failed : %v", err)
   180  		}
   181  		// log error
   182  		if err := cmd.Wait(); err != nil {
   183  			log.Errorf("cmd.wait() failed : %v", err)
   184  		}
   185  	})
   186  
   187  	return clientAddr, &certs
   188  }
   189  
   190  func TestEtcd2TLS(t *testing.T) {
   191  	// Start a single etcd in the background.
   192  	clientAddr, certs := startEtcdWithTLS(t)
   193  
   194  	testIndex := 0
   195  	testRoot := fmt.Sprintf("/test-%v", testIndex)
   196  
   197  	// Create the server on the new root.
   198  	server, err := NewServerWithOpts(clientAddr, testRoot, certs.ClientCert, certs.ClientKey, certs.ServerCA)
   199  	if err != nil {
   200  		t.Fatalf("NewServerWithOpts failed: %v", err)
   201  	}
   202  	defer server.Close()
   203  
   204  	testCtx := context.Background()
   205  	testKey := "testkey"
   206  	testVal := "testval"
   207  	_, err = server.Create(testCtx, testKey, []byte(testVal))
   208  	if err != nil {
   209  		t.Fatalf("Failed to set key value pair: %v", err)
   210  	}
   211  	val, _, err := server.Get(testCtx, testKey)
   212  	if err != nil {
   213  		t.Fatalf("Failed to retrieve value at key we just set: %v", err)
   214  	}
   215  	if string(val) != testVal {
   216  		t.Fatalf("Value returned doesn't match %s, err: %v", testVal, err)
   217  	}
   218  }
   219  
   220  func TestEtcd2Topo(t *testing.T) {
   221  	// Start a single etcd in the background.
   222  	clientAddr := startEtcd(t)
   223  
   224  	testIndex := 0
   225  	newServer := func() *topo.Server {
   226  		// Each test will use its own sub-directories.
   227  		testRoot := fmt.Sprintf("/test-%v", testIndex)
   228  		testIndex++
   229  
   230  		// Create the server on the new root.
   231  		ts, err := topo.OpenServer("etcd2", clientAddr, path.Join(testRoot, topo.GlobalCell))
   232  		if err != nil {
   233  			t.Fatalf("OpenServer() failed: %v", err)
   234  		}
   235  
   236  		// Create the CellInfo.
   237  		if err := ts.CreateCellInfo(context.Background(), test.LocalCellName, &topodatapb.CellInfo{
   238  			ServerAddress: clientAddr,
   239  			Root:          path.Join(testRoot, test.LocalCellName),
   240  		}); err != nil {
   241  			t.Fatalf("CreateCellInfo() failed: %v", err)
   242  		}
   243  
   244  		return ts
   245  	}
   246  
   247  	// Run the TopoServerTestSuite tests.
   248  	test.TopoServerTestSuite(t, func() *topo.Server {
   249  		return newServer()
   250  	}, []string{})
   251  
   252  	// Run etcd-specific tests.
   253  	ts := newServer()
   254  	testKeyspaceLock(t, ts)
   255  	ts.Close()
   256  }
   257  
   258  // testKeyspaceLock tests etcd-specific heartbeat (TTL).
   259  // Note TTL granularity is in seconds, even though the API uses time.Duration.
   260  // So we have to wait a long time in these tests.
   261  func testKeyspaceLock(t *testing.T, ts *topo.Server) {
   262  	ctx := context.Background()
   263  	keyspacePath := path.Join(topo.KeyspacesPath, "test_keyspace")
   264  	if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil {
   265  		t.Fatalf("CreateKeyspace: %v", err)
   266  	}
   267  
   268  	conn, err := ts.ConnForCell(ctx, topo.GlobalCell)
   269  	if err != nil {
   270  		t.Fatalf("ConnForCell failed: %v", err)
   271  	}
   272  
   273  	// Long TTL, unlock before lease runs out.
   274  	leaseTTL = 1000
   275  	lockDescriptor, err := conn.Lock(ctx, keyspacePath, "ttl")
   276  	if err != nil {
   277  		t.Fatalf("Lock failed: %v", err)
   278  	}
   279  	if err := lockDescriptor.Unlock(ctx); err != nil {
   280  		t.Fatalf("Unlock failed: %v", err)
   281  	}
   282  
   283  	// Short TTL, make sure it doesn't expire.
   284  	leaseTTL = 1
   285  	lockDescriptor, err = conn.Lock(ctx, keyspacePath, "short ttl")
   286  	if err != nil {
   287  		t.Fatalf("Lock failed: %v", err)
   288  	}
   289  	time.Sleep(2 * time.Second)
   290  	if err := lockDescriptor.Unlock(ctx); err != nil {
   291  		t.Fatalf("Unlock failed: %v", err)
   292  	}
   293  }