github.com/m3db/m3@v1.5.0/src/cluster/integration/etcd/etcd.go (about)

     1  // Copyright (c) 2017 Uber Technologies, Inc.
     2  //
     3  // Permission is hereby granted, free of charge, to any person obtaining a copy
     4  // of this software and associated documentation files (the "Software"), to deal
     5  // in the Software without restriction, including without limitation the rights
     6  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     7  // copies of the Software, and to permit persons to whom the Software is
     8  // furnished to do so, subject to the following conditions:
     9  //
    10  // The above copyright notice and this permission notice shall be included in
    11  // all copies or substantial portions of the Software.
    12  //
    13  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    14  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    15  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    16  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    17  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    18  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    19  // THE SOFTWARE.
    20  
    21  package etcd
    22  
    23  import (
    24  	"encoding/json"
    25  	"fmt"
    26  	"io/ioutil"
    27  	"net/http"
    28  	"net/url"
    29  	"os"
    30  	"strings"
    31  	"time"
    32  
    33  	"github.com/m3db/m3/src/cluster/client"
    34  	etcdclient "github.com/m3db/m3/src/cluster/client/etcd"
    35  	"github.com/m3db/m3/src/cluster/services"
    36  	xclock "github.com/m3db/m3/src/x/clock"
    37  	"github.com/m3db/m3/src/x/errors"
    38  
    39  	"go.etcd.io/etcd/server/v3/embed"
    40  )
    41  
    42  type embeddedKV struct {
    43  	etcd *embed.Etcd
    44  	opts Options
    45  	dir  string
    46  }
    47  
    48  // New creates a new EmbeddedKV
    49  func New(opts Options) (EmbeddedKV, error) {
    50  	dir, err := ioutil.TempDir("", opts.Dir())
    51  	if err != nil {
    52  		return nil, err
    53  	}
    54  	cfg := embed.NewConfig()
    55  	cfg.Dir = dir
    56  	cfg.Logger = "zap"
    57  
    58  	setRandomPorts(cfg)
    59  	e, err := embed.StartEtcd(cfg)
    60  	if err != nil {
    61  		return nil, fmt.Errorf("unable to start etcd, err: %v", err)
    62  	}
    63  	return &embeddedKV{
    64  		etcd: e,
    65  		opts: opts,
    66  		dir:  dir,
    67  	}, nil
    68  }
    69  
    70  func setRandomPorts(cfg *embed.Config) {
    71  	randomPortURL, err := url.Parse("http://localhost:0")
    72  	if err != nil {
    73  		panic(err.Error())
    74  	}
    75  
    76  	cfg.LPUrls = []url.URL{*randomPortURL}
    77  	cfg.APUrls = []url.URL{*randomPortURL}
    78  	cfg.LCUrls = []url.URL{*randomPortURL}
    79  	cfg.ACUrls = []url.URL{*randomPortURL}
    80  
    81  	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
    82  }
    83  
    84  func (e *embeddedKV) Close() error {
    85  	var multi errors.MultiError
    86  
    87  	// see if there's any errors
    88  	select {
    89  	case err := <-e.etcd.Err():
    90  		multi = multi.Add(err)
    91  	default:
    92  	}
    93  
    94  	// shutdown and release
    95  	e.etcd.Server.Stop()
    96  	e.etcd.Close()
    97  
    98  	multi = multi.Add(os.RemoveAll(e.dir))
    99  	return multi.FinalError()
   100  }
   101  
   102  func (e *embeddedKV) Start() error {
   103  	timeout := e.opts.InitTimeout()
   104  	select {
   105  	case <-e.etcd.Server.ReadyNotify():
   106  		break
   107  	case <-time.After(timeout):
   108  		return fmt.Errorf("etcd server took too long to start")
   109  	}
   110  
   111  	// ensure v3 api endpoints are available, https://github.com/coreos/etcd/pull/7075
   112  	apiVersionEndpoint := fmt.Sprintf("http://%s/version", e.etcd.Clients[0].Addr().String())
   113  	fn := func() bool { return version3Available(apiVersionEndpoint) }
   114  	ok := xclock.WaitUntil(fn, timeout)
   115  	if !ok {
   116  		return fmt.Errorf("api version 3 not available")
   117  	}
   118  
   119  	return nil
   120  }
   121  
   122  type versionResponse struct {
   123  	Version string `json:"etcdcluster"`
   124  }
   125  
   126  func version3Available(endpoint string) bool {
   127  	resp, err := http.Get(endpoint)
   128  	if err != nil {
   129  		return false
   130  	}
   131  	if resp.StatusCode != 200 {
   132  		return false
   133  	}
   134  	defer resp.Body.Close()
   135  
   136  	decoder := json.NewDecoder(resp.Body)
   137  	var data versionResponse
   138  	err = decoder.Decode(&data)
   139  	if err != nil {
   140  		return false
   141  	}
   142  
   143  	return strings.Index(data.Version, "3.") == 0
   144  }
   145  
   146  func (e *embeddedKV) Endpoints() []string {
   147  	addresses := make([]string, 0, len(e.etcd.Clients))
   148  	for _, c := range e.etcd.Clients {
   149  		addresses = append(addresses, c.Addr().String())
   150  	}
   151  	return addresses
   152  }
   153  
   154  func (e *embeddedKV) ConfigServiceClient(fns ...ClientOptFn) (client.Client, error) {
   155  	eopts := etcdclient.NewOptions().
   156  		SetInstrumentOptions(e.opts.InstrumentOptions()).
   157  		SetServicesOptions(services.NewOptions().SetInitTimeout(e.opts.InitTimeout())).
   158  		SetClusters([]etcdclient.Cluster{
   159  			etcdclient.NewCluster().SetZone(e.opts.Zone()).SetEndpoints(e.Endpoints()),
   160  		}).
   161  		SetService(e.opts.ServiceID()).
   162  		SetEnv(e.opts.Environment()).
   163  		SetZone(e.opts.Zone())
   164  	for _, fn := range fns {
   165  		eopts = fn(eopts)
   166  	}
   167  	return etcdclient.NewConfigServiceClient(eopts)
   168  }