go.etcd.io/etcd@v3.3.27+incompatible/functional/rpcpb/member.go (about) 1 // Copyright 2018 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package rpcpb 16 17 import ( 18 "context" 19 "crypto/tls" 20 "fmt" 21 "net/url" 22 "os" 23 "time" 24 25 "github.com/coreos/etcd/clientv3" 26 pb "github.com/coreos/etcd/etcdserver/etcdserverpb" 27 "github.com/coreos/etcd/pkg/transport" 28 "github.com/coreos/etcd/snapshot" 29 30 "github.com/dustin/go-humanize" 31 "go.uber.org/zap" 32 grpc "google.golang.org/grpc" 33 "google.golang.org/grpc/credentials" 34 ) 35 36 // ElectionTimeout returns an election timeout duration. 37 func (m *Member) ElectionTimeout() time.Duration { 38 return time.Duration(m.Etcd.ElectionTimeoutMs) * time.Millisecond 39 } 40 41 // DialEtcdGRPCServer creates a raw gRPC connection to an etcd member. 42 func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) { 43 dialOpts := []grpc.DialOption{ 44 grpc.WithTimeout(5 * time.Second), 45 grpc.WithBlock(), 46 } 47 48 secure := false 49 for _, cu := range m.Etcd.AdvertiseClientURLs { 50 u, err := url.Parse(cu) 51 if err != nil { 52 return nil, err 53 } 54 if u.Scheme == "https" { // TODO: handle unix 55 secure = true 56 } 57 } 58 59 if secure { 60 // assume save TLS assets are already stord on disk 61 tlsInfo := transport.TLSInfo{ 62 CertFile: m.ClientCertPath, 63 KeyFile: m.ClientKeyPath, 64 TrustedCAFile: m.ClientTrustedCAPath, 65 66 // TODO: remove this with generated certs 67 // only need it for auto TLS 68 InsecureSkipVerify: true, 69 } 70 tlsConfig, err := tlsInfo.ClientConfig() 71 if err != nil { 72 return nil, err 73 } 74 creds := credentials.NewTLS(tlsConfig) 75 dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds)) 76 } else { 77 dialOpts = append(dialOpts, grpc.WithInsecure()) 78 } 79 dialOpts = append(dialOpts, opts...) 80 return grpc.Dial(m.EtcdClientEndpoint, dialOpts...) 81 } 82 83 // CreateEtcdClientConfig creates a client configuration from member. 84 func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3.Config, err error) { 85 secure := false 86 for _, cu := range m.Etcd.AdvertiseClientURLs { 87 var u *url.URL 88 u, err = url.Parse(cu) 89 if err != nil { 90 return nil, err 91 } 92 if u.Scheme == "https" { // TODO: handle unix 93 secure = true 94 } 95 } 96 97 cfg = &clientv3.Config{ 98 Endpoints: []string{m.EtcdClientEndpoint}, 99 DialTimeout: 10 * time.Second, 100 DialOptions: opts, 101 } 102 if secure { 103 // assume save TLS assets are already stord on disk 104 tlsInfo := transport.TLSInfo{ 105 CertFile: m.ClientCertPath, 106 KeyFile: m.ClientKeyPath, 107 TrustedCAFile: m.ClientTrustedCAPath, 108 109 // TODO: remove this with generated certs 110 // only need it for auto TLS 111 InsecureSkipVerify: true, 112 } 113 var tlsConfig *tls.Config 114 tlsConfig, err = tlsInfo.ClientConfig() 115 if err != nil { 116 return nil, err 117 } 118 cfg.TLS = tlsConfig 119 } 120 return cfg, err 121 } 122 123 // CreateEtcdClient creates a client from member. 124 func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) { 125 cfg, err := m.CreateEtcdClientConfig(opts...) 126 if err != nil { 127 return nil, err 128 } 129 return clientv3.New(*cfg) 130 } 131 132 // CheckCompact ensures that historical data before given revision has been compacted. 133 func (m *Member) CheckCompact(rev int64) error { 134 cli, err := m.CreateEtcdClient() 135 if err != nil { 136 return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 137 } 138 defer cli.Close() 139 140 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 141 wch := cli.Watch(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithRev(rev-1)) 142 wr, ok := <-wch 143 cancel() 144 145 if !ok { 146 return fmt.Errorf("watch channel terminated (endpoint %q)", m.EtcdClientEndpoint) 147 } 148 if wr.CompactRevision != rev { 149 return fmt.Errorf("got compact revision %v, wanted %v (endpoint %q)", wr.CompactRevision, rev, m.EtcdClientEndpoint) 150 } 151 152 return nil 153 } 154 155 // Defrag runs defragmentation on this member. 156 func (m *Member) Defrag() error { 157 cli, err := m.CreateEtcdClient() 158 if err != nil { 159 return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 160 } 161 defer cli.Close() 162 163 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) 164 _, err = cli.Defragment(ctx, m.EtcdClientEndpoint) 165 cancel() 166 return err 167 } 168 169 // RevHash fetches current revision and hash on this member. 170 func (m *Member) RevHash() (int64, int64, error) { 171 conn, err := m.DialEtcdGRPCServer() 172 if err != nil { 173 return 0, 0, err 174 } 175 defer conn.Close() 176 177 mt := pb.NewMaintenanceClient(conn) 178 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 179 resp, err := mt.Hash(ctx, &pb.HashRequest{}, grpc.FailFast(false)) 180 cancel() 181 182 if err != nil { 183 return 0, 0, err 184 } 185 186 return resp.Header.Revision, int64(resp.Hash), nil 187 } 188 189 // Rev fetches current revision on this member. 190 func (m *Member) Rev(ctx context.Context) (int64, error) { 191 cli, err := m.CreateEtcdClient() 192 if err != nil { 193 return 0, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 194 } 195 defer cli.Close() 196 197 resp, err := cli.Status(ctx, m.EtcdClientEndpoint) 198 if err != nil { 199 return 0, err 200 } 201 return resp.Header.Revision, nil 202 } 203 204 // Compact compacts member storage with given revision. 205 // It blocks until it's physically done. 206 func (m *Member) Compact(rev int64, timeout time.Duration) error { 207 cli, err := m.CreateEtcdClient() 208 if err != nil { 209 return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 210 } 211 defer cli.Close() 212 213 ctx, cancel := context.WithTimeout(context.Background(), timeout) 214 _, err = cli.Compact(ctx, rev, clientv3.WithCompactPhysical()) 215 cancel() 216 return err 217 } 218 219 // IsLeader returns true if this member is the current cluster leader. 220 func (m *Member) IsLeader() (bool, error) { 221 cli, err := m.CreateEtcdClient() 222 if err != nil { 223 return false, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 224 } 225 defer cli.Close() 226 227 resp, err := cli.Status(context.Background(), m.EtcdClientEndpoint) 228 if err != nil { 229 return false, err 230 } 231 return resp.Header.MemberId == resp.Leader, nil 232 } 233 234 // WriteHealthKey writes a health key to this member. 235 func (m *Member) WriteHealthKey() error { 236 cli, err := m.CreateEtcdClient() 237 if err != nil { 238 return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 239 } 240 defer cli.Close() 241 242 // give enough time-out in case expensive requests (range/delete) are pending 243 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 244 _, err = cli.Put(ctx, "health", "good") 245 cancel() 246 if err != nil { 247 return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 248 } 249 return nil 250 } 251 252 // SaveSnapshot downloads a snapshot file from this member, locally. 253 // It's meant to requested remotely, so that local member can store 254 // snapshot file on local disk. 255 func (m *Member) SaveSnapshot(lg *zap.Logger) (err error) { 256 // remove existing snapshot first 257 if err = os.RemoveAll(m.SnapshotPath); err != nil { 258 return err 259 } 260 261 var ccfg *clientv3.Config 262 ccfg, err = m.CreateEtcdClientConfig() 263 if err != nil { 264 return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint) 265 } 266 267 lg.Info( 268 "snapshot save START", 269 zap.String("member-name", m.Etcd.Name), 270 zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs), 271 zap.String("snapshot-path", m.SnapshotPath), 272 ) 273 now := time.Now() 274 mgr := snapshot.NewV3(lg) 275 if err = mgr.Save(context.Background(), *ccfg, m.SnapshotPath); err != nil { 276 return err 277 } 278 took := time.Since(now) 279 280 var fi os.FileInfo 281 fi, err = os.Stat(m.SnapshotPath) 282 if err != nil { 283 return err 284 } 285 var st snapshot.Status 286 st, err = mgr.Status(m.SnapshotPath) 287 if err != nil { 288 return err 289 } 290 m.SnapshotInfo = &SnapshotInfo{ 291 MemberName: m.Etcd.Name, 292 MemberClientURLs: m.Etcd.AdvertiseClientURLs, 293 SnapshotPath: m.SnapshotPath, 294 SnapshotFileSize: humanize.Bytes(uint64(fi.Size())), 295 SnapshotTotalSize: humanize.Bytes(uint64(st.TotalSize)), 296 SnapshotTotalKey: int64(st.TotalKey), 297 SnapshotHash: int64(st.Hash), 298 SnapshotRevision: st.Revision, 299 Took: fmt.Sprintf("%v", took), 300 } 301 lg.Info( 302 "snapshot save END", 303 zap.String("member-name", m.SnapshotInfo.MemberName), 304 zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs), 305 zap.String("snapshot-path", m.SnapshotPath), 306 zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize), 307 zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize), 308 zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey), 309 zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash), 310 zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision), 311 zap.String("took", m.SnapshotInfo.Took), 312 ) 313 return nil 314 } 315 316 // RestoreSnapshot restores a cluster from a given snapshot file on disk. 317 // It's meant to requested remotely, so that local member can load the 318 // snapshot file from local disk. 319 func (m *Member) RestoreSnapshot(lg *zap.Logger) (err error) { 320 if err = os.RemoveAll(m.EtcdOnSnapshotRestore.DataDir); err != nil { 321 return err 322 } 323 if err = os.RemoveAll(m.EtcdOnSnapshotRestore.WALDir); err != nil { 324 return err 325 } 326 327 lg.Info( 328 "snapshot restore START", 329 zap.String("member-name", m.Etcd.Name), 330 zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs), 331 zap.String("snapshot-path", m.SnapshotPath), 332 ) 333 now := time.Now() 334 mgr := snapshot.NewV3(lg) 335 err = mgr.Restore(snapshot.RestoreConfig{ 336 SnapshotPath: m.SnapshotInfo.SnapshotPath, 337 Name: m.EtcdOnSnapshotRestore.Name, 338 OutputDataDir: m.EtcdOnSnapshotRestore.DataDir, 339 OutputWALDir: m.EtcdOnSnapshotRestore.WALDir, 340 PeerURLs: m.EtcdOnSnapshotRestore.AdvertisePeerURLs, 341 InitialCluster: m.EtcdOnSnapshotRestore.InitialCluster, 342 InitialClusterToken: m.EtcdOnSnapshotRestore.InitialClusterToken, 343 SkipHashCheck: false, 344 // TODO: set SkipHashCheck it true, to recover from existing db file 345 }) 346 took := time.Since(now) 347 lg.Info( 348 "snapshot restore END", 349 zap.String("member-name", m.SnapshotInfo.MemberName), 350 zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs), 351 zap.String("snapshot-path", m.SnapshotPath), 352 zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize), 353 zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize), 354 zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey), 355 zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash), 356 zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision), 357 zap.String("took", took.String()), 358 zap.Error(err), 359 ) 360 return err 361 }