github.com/lfch/etcd-io/tests/v3@v3.0.0-20221004140520-eac99acd3e9d/functional/rpcpb/member.go (about)

     1  // Copyright 2018 The etcd Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package rpcpb
    16  
    17  import (
    18  	"context"
    19  	"crypto/tls"
    20  	"fmt"
    21  	"net/url"
    22  	"os"
    23  	"time"
    24  
    25  	pb "github.com/lfch/etcd-io/api/v3/etcdserverpb"
    26  	"github.com/lfch/etcd-io/client/pkg/v3/logutil"
    27  	"github.com/lfch/etcd-io/client/pkg/v3/transport"
    28  	"github.com/lfch/etcd-io/client/v3"
    29  	"github.com/lfch/etcd-io/etcdutl/v3/snapshot"
    30  
    31  	"github.com/dustin/go-humanize"
    32  	"go.uber.org/zap"
    33  	grpc "google.golang.org/grpc"
    34  	"google.golang.org/grpc/credentials"
    35  )
    36  
    37  // ElectionTimeout returns an election timeout duration.
    38  func (m *Member) ElectionTimeout() time.Duration {
    39  	return time.Duration(m.Etcd.ElectionTimeoutMs) * time.Millisecond
    40  }
    41  
    42  // DialEtcdGRPCServer creates a raw gRPC connection to an etcd member.
    43  func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) {
    44  	dialOpts := []grpc.DialOption{
    45  		grpc.WithTimeout(5 * time.Second),
    46  		grpc.WithBlock(),
    47  	}
    48  
    49  	secure := false
    50  	for _, cu := range m.Etcd.AdvertiseClientURLs {
    51  		u, err := url.Parse(cu)
    52  		if err != nil {
    53  			return nil, err
    54  		}
    55  		if u.Scheme == "https" { // TODO: handle unix
    56  			secure = true
    57  		}
    58  	}
    59  
    60  	if secure {
    61  		// assume save TLS assets are already stord on disk
    62  		tlsInfo := transport.TLSInfo{
    63  			CertFile:      m.ClientCertPath,
    64  			KeyFile:       m.ClientKeyPath,
    65  			TrustedCAFile: m.ClientTrustedCAPath,
    66  
    67  			// TODO: remove this with generated certs
    68  			// only need it for auto TLS
    69  			InsecureSkipVerify: true,
    70  		}
    71  		tlsConfig, err := tlsInfo.ClientConfig()
    72  		if err != nil {
    73  			return nil, err
    74  		}
    75  		creds := credentials.NewTLS(tlsConfig)
    76  		dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
    77  	} else {
    78  		dialOpts = append(dialOpts, grpc.WithInsecure())
    79  	}
    80  	dialOpts = append(dialOpts, opts...)
    81  	return grpc.Dial(m.EtcdClientEndpoint, dialOpts...)
    82  }
    83  
    84  // CreateEtcdClientConfig creates a client configuration from member.
    85  func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3.Config, err error) {
    86  	secure := false
    87  	for _, cu := range m.Etcd.AdvertiseClientURLs {
    88  		var u *url.URL
    89  		u, err = url.Parse(cu)
    90  		if err != nil {
    91  			return nil, err
    92  		}
    93  		if u.Scheme == "https" { // TODO: handle unix
    94  			secure = true
    95  		}
    96  	}
    97  
    98  	// TODO: make this configurable
    99  	level := "error"
   100  	if os.Getenv("ETCD_CLIENT_DEBUG") != "" {
   101  		level = "debug"
   102  	}
   103  	lcfg := logutil.DefaultZapLoggerConfig
   104  	lcfg.Level = zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(level))
   105  
   106  	cfg = &clientv3.Config{
   107  		Endpoints:   []string{m.EtcdClientEndpoint},
   108  		DialTimeout: 10 * time.Second,
   109  		DialOptions: opts,
   110  		LogConfig:   &lcfg,
   111  	}
   112  	if secure {
   113  		// assume save TLS assets are already stord on disk
   114  		tlsInfo := transport.TLSInfo{
   115  			CertFile:      m.ClientCertPath,
   116  			KeyFile:       m.ClientKeyPath,
   117  			TrustedCAFile: m.ClientTrustedCAPath,
   118  
   119  			// TODO: remove this with generated certs
   120  			// only need it for auto TLS
   121  			InsecureSkipVerify: true,
   122  		}
   123  		var tlsConfig *tls.Config
   124  		tlsConfig, err = tlsInfo.ClientConfig()
   125  		if err != nil {
   126  			return nil, err
   127  		}
   128  		cfg.TLS = tlsConfig
   129  	}
   130  	return cfg, err
   131  }
   132  
   133  // CreateEtcdClient creates a client from member.
   134  func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) {
   135  	cfg, err := m.CreateEtcdClientConfig(opts...)
   136  	if err != nil {
   137  		return nil, err
   138  	}
   139  	return clientv3.New(*cfg)
   140  }
   141  
   142  // CheckCompact ensures that historical data before given revision has been compacted.
   143  func (m *Member) CheckCompact(rev int64) error {
   144  	cli, err := m.CreateEtcdClient()
   145  	if err != nil {
   146  		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   147  	}
   148  	defer cli.Close()
   149  
   150  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   151  	wch := cli.Watch(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithRev(rev-1))
   152  	wr, ok := <-wch
   153  	cancel()
   154  
   155  	if !ok {
   156  		return fmt.Errorf("watch channel terminated (endpoint %q)", m.EtcdClientEndpoint)
   157  	}
   158  	if wr.CompactRevision != rev {
   159  		return fmt.Errorf("got compact revision %v, wanted %v (endpoint %q)", wr.CompactRevision, rev, m.EtcdClientEndpoint)
   160  	}
   161  
   162  	return nil
   163  }
   164  
   165  // Defrag runs defragmentation on this member.
   166  func (m *Member) Defrag() error {
   167  	cli, err := m.CreateEtcdClient()
   168  	if err != nil {
   169  		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   170  	}
   171  	defer cli.Close()
   172  
   173  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
   174  	_, err = cli.Defragment(ctx, m.EtcdClientEndpoint)
   175  	cancel()
   176  	return err
   177  }
   178  
   179  // RevHash fetches current revision and hash on this member.
   180  func (m *Member) RevHash() (int64, int64, error) {
   181  	conn, err := m.DialEtcdGRPCServer()
   182  	if err != nil {
   183  		return 0, 0, err
   184  	}
   185  	defer conn.Close()
   186  
   187  	mt := pb.NewMaintenanceClient(conn)
   188  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   189  	resp, err := mt.Hash(ctx, &pb.HashRequest{}, grpc.WaitForReady(true))
   190  	cancel()
   191  
   192  	if err != nil {
   193  		return 0, 0, err
   194  	}
   195  
   196  	return resp.Header.Revision, int64(resp.Hash), nil
   197  }
   198  
   199  // Rev fetches current revision on this member.
   200  func (m *Member) Rev(ctx context.Context) (int64, error) {
   201  	cli, err := m.CreateEtcdClient()
   202  	if err != nil {
   203  		return 0, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   204  	}
   205  	defer cli.Close()
   206  
   207  	resp, err := cli.Status(ctx, m.EtcdClientEndpoint)
   208  	if err != nil {
   209  		return 0, err
   210  	}
   211  	return resp.Header.Revision, nil
   212  }
   213  
   214  // Compact compacts member storage with given revision.
   215  // It blocks until it's physically done.
   216  func (m *Member) Compact(rev int64, timeout time.Duration) error {
   217  	cli, err := m.CreateEtcdClient()
   218  	if err != nil {
   219  		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   220  	}
   221  	defer cli.Close()
   222  
   223  	ctx, cancel := context.WithTimeout(context.Background(), timeout)
   224  	_, err = cli.Compact(ctx, rev, clientv3.WithCompactPhysical())
   225  	cancel()
   226  	return err
   227  }
   228  
   229  // IsLeader returns true if this member is the current cluster leader.
   230  func (m *Member) IsLeader() (bool, error) {
   231  	cli, err := m.CreateEtcdClient()
   232  	if err != nil {
   233  		return false, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   234  	}
   235  	defer cli.Close()
   236  
   237  	resp, err := cli.Status(context.Background(), m.EtcdClientEndpoint)
   238  	if err != nil {
   239  		return false, err
   240  	}
   241  	return resp.Header.MemberId == resp.Leader, nil
   242  }
   243  
   244  // WriteHealthKey writes a health key to this member.
   245  func (m *Member) WriteHealthKey() error {
   246  	cli, err := m.CreateEtcdClient()
   247  	if err != nil {
   248  		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   249  	}
   250  	defer cli.Close()
   251  
   252  	// give enough time-out in case expensive requests (range/delete) are pending
   253  	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
   254  	_, err = cli.Put(ctx, "health", "good")
   255  	cancel()
   256  	if err != nil {
   257  		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   258  	}
   259  	return nil
   260  }
   261  
   262  // SaveSnapshot downloads a snapshot file from this member, locally.
   263  // It's meant to requested remotely, so that local member can store
   264  // snapshot file on local disk.
   265  func (m *Member) SaveSnapshot(lg *zap.Logger) (err error) {
   266  	// remove existing snapshot first
   267  	if err = os.RemoveAll(m.SnapshotPath); err != nil {
   268  		return err
   269  	}
   270  
   271  	var ccfg *clientv3.Config
   272  	ccfg, err = m.CreateEtcdClientConfig()
   273  	if err != nil {
   274  		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
   275  	}
   276  
   277  	lg.Info(
   278  		"snapshot save START",
   279  		zap.String("member-name", m.Etcd.Name),
   280  		zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
   281  		zap.String("snapshot-path", m.SnapshotPath),
   282  	)
   283  	now := time.Now()
   284  	mgr := snapshot.NewV3(lg)
   285  	version, err := mgr.Save(context.Background(), *ccfg, m.SnapshotPath)
   286  	if err != nil {
   287  		return err
   288  	}
   289  	took := time.Since(now)
   290  
   291  	var fi os.FileInfo
   292  	fi, err = os.Stat(m.SnapshotPath)
   293  	if err != nil {
   294  		return err
   295  	}
   296  	var st snapshot.Status
   297  	st, err = mgr.Status(m.SnapshotPath)
   298  	if err != nil {
   299  		return err
   300  	}
   301  	m.SnapshotInfo = &SnapshotInfo{
   302  		MemberName:        m.Etcd.Name,
   303  		MemberClientURLs:  m.Etcd.AdvertiseClientURLs,
   304  		SnapshotPath:      m.SnapshotPath,
   305  		SnapshotFileSize:  humanize.Bytes(uint64(fi.Size())),
   306  		SnapshotTotalSize: humanize.Bytes(uint64(st.TotalSize)),
   307  		SnapshotTotalKey:  int64(st.TotalKey),
   308  		SnapshotHash:      int64(st.Hash),
   309  		SnapshotRevision:  st.Revision,
   310  		Took:              fmt.Sprintf("%v", took),
   311  		Version:           version,
   312  	}
   313  	lg.Info(
   314  		"snapshot save END",
   315  		zap.String("member-name", m.SnapshotInfo.MemberName),
   316  		zap.String("member-version", m.SnapshotInfo.Version),
   317  		zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
   318  		zap.String("snapshot-path", m.SnapshotPath),
   319  		zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
   320  		zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
   321  		zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
   322  		zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
   323  		zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
   324  		zap.String("took", m.SnapshotInfo.Took),
   325  	)
   326  	return nil
   327  }
   328  
   329  // RestoreSnapshot restores a cluster from a given snapshot file on disk.
   330  // It's meant to requested remotely, so that local member can load the
   331  // snapshot file from local disk.
   332  func (m *Member) RestoreSnapshot(lg *zap.Logger) (err error) {
   333  	if err = os.RemoveAll(m.EtcdOnSnapshotRestore.DataDir); err != nil {
   334  		return err
   335  	}
   336  	if err = os.RemoveAll(m.EtcdOnSnapshotRestore.WALDir); err != nil {
   337  		return err
   338  	}
   339  
   340  	lg.Info(
   341  		"snapshot restore START",
   342  		zap.String("member-name", m.Etcd.Name),
   343  		zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
   344  		zap.String("snapshot-path", m.SnapshotPath),
   345  	)
   346  	now := time.Now()
   347  	mgr := snapshot.NewV3(lg)
   348  	err = mgr.Restore(snapshot.RestoreConfig{
   349  		SnapshotPath:        m.SnapshotInfo.SnapshotPath,
   350  		Name:                m.EtcdOnSnapshotRestore.Name,
   351  		OutputDataDir:       m.EtcdOnSnapshotRestore.DataDir,
   352  		OutputWALDir:        m.EtcdOnSnapshotRestore.WALDir,
   353  		PeerURLs:            m.EtcdOnSnapshotRestore.AdvertisePeerURLs,
   354  		InitialCluster:      m.EtcdOnSnapshotRestore.InitialCluster,
   355  		InitialClusterToken: m.EtcdOnSnapshotRestore.InitialClusterToken,
   356  		SkipHashCheck:       false,
   357  		// TODO: set SkipHashCheck it true, to recover from existing db file
   358  	})
   359  	took := time.Since(now)
   360  	lg.Info(
   361  		"snapshot restore END",
   362  		zap.String("member-name", m.SnapshotInfo.MemberName),
   363  		zap.String("member-version", m.SnapshotInfo.Version),
   364  		zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
   365  		zap.String("snapshot-path", m.SnapshotPath),
   366  		zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
   367  		zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
   368  		zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
   369  		zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
   370  		zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
   371  		zap.String("took", took.String()),
   372  		zap.Error(err),
   373  	)
   374  	return err
   375  }