go.etcd.io/etcd@v3.3.27+incompatible/etcdmain/grpc_proxy.go (about) 1 // Copyright 2016 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package etcdmain 16 17 import ( 18 "context" 19 "crypto/tls" 20 "crypto/x509" 21 "fmt" 22 "io/ioutil" 23 "math" 24 "net" 25 "net/http" 26 "net/url" 27 "os" 28 "path/filepath" 29 "time" 30 31 "github.com/coreos/etcd/clientv3" 32 "github.com/coreos/etcd/clientv3/leasing" 33 "github.com/coreos/etcd/clientv3/namespace" 34 "github.com/coreos/etcd/clientv3/ordering" 35 "github.com/coreos/etcd/etcdserver/api/v3election/v3electionpb" 36 "github.com/coreos/etcd/etcdserver/api/v3lock/v3lockpb" 37 pb "github.com/coreos/etcd/etcdserver/etcdserverpb" 38 "github.com/coreos/etcd/pkg/debugutil" 39 "github.com/coreos/etcd/pkg/transport" 40 "github.com/coreos/etcd/proxy/grpcproxy" 41 42 "github.com/coreos/pkg/capnslog" 43 grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" 44 "github.com/soheilhy/cmux" 45 "github.com/spf13/cobra" 46 "google.golang.org/grpc" 47 "google.golang.org/grpc/grpclog" 48 ) 49 50 var ( 51 grpcProxyListenAddr string 52 grpcProxyMetricsListenAddr string 53 grpcProxyEndpoints []string 54 grpcProxyDNSCluster string 55 grpcProxyInsecureDiscovery bool 56 grpcProxyDataDir string 57 grpcMaxCallSendMsgSize int 58 grpcMaxCallRecvMsgSize int 59 60 // tls for connecting to etcd 61 62 grpcProxyCA string 63 grpcProxyCert string 64 grpcProxyKey string 65 grpcProxyInsecureSkipTLSVerify bool 66 67 // tls for clients connecting to proxy 68 69 grpcProxyListenCA string 70 grpcProxyListenCert string 71 grpcProxyListenKey string 72 grpcProxyListenAutoTLS bool 73 grpcProxyListenCRL string 74 75 grpcProxyAdvertiseClientURL string 76 grpcProxyResolverPrefix string 77 grpcProxyResolverTTL int 78 79 grpcProxyNamespace string 80 grpcProxyLeasing string 81 82 grpcProxyEnablePprof bool 83 grpcProxyEnableOrdering bool 84 85 grpcProxyDebug bool 86 ) 87 88 const defaultGRPCMaxCallSendMsgSize = 1.5 * 1024 * 1024 89 90 func init() { 91 rootCmd.AddCommand(newGRPCProxyCommand()) 92 } 93 94 // newGRPCProxyCommand returns the cobra command for "grpc-proxy". 95 func newGRPCProxyCommand() *cobra.Command { 96 lpc := &cobra.Command{ 97 Use: "grpc-proxy <subcommand>", 98 Short: "grpc-proxy related command", 99 } 100 lpc.AddCommand(newGRPCProxyStartCommand()) 101 102 return lpc 103 } 104 105 func newGRPCProxyStartCommand() *cobra.Command { 106 cmd := cobra.Command{ 107 Use: "start", 108 Short: "start the grpc proxy", 109 Run: startGRPCProxy, 110 } 111 112 cmd.Flags().StringVar(&grpcProxyListenAddr, "listen-addr", "127.0.0.1:23790", "listen address") 113 cmd.Flags().StringVar(&grpcProxyDNSCluster, "discovery-srv", "", "DNS domain used to bootstrap initial cluster") 114 cmd.Flags().StringVar(&grpcProxyMetricsListenAddr, "metrics-addr", "", "listen for endpoint /metrics requests on an additional interface") 115 cmd.Flags().BoolVar(&grpcProxyInsecureDiscovery, "insecure-discovery", false, "accept insecure SRV records") 116 cmd.Flags().StringSliceVar(&grpcProxyEndpoints, "endpoints", []string{"127.0.0.1:2379"}, "comma separated etcd cluster endpoints") 117 cmd.Flags().StringVar(&grpcProxyAdvertiseClientURL, "advertise-client-url", "127.0.0.1:23790", "advertise address to register (must be reachable by client)") 118 cmd.Flags().StringVar(&grpcProxyResolverPrefix, "resolver-prefix", "", "prefix to use for registering proxy (must be shared with other grpc-proxy members)") 119 cmd.Flags().IntVar(&grpcProxyResolverTTL, "resolver-ttl", 0, "specify TTL, in seconds, when registering proxy endpoints") 120 cmd.Flags().StringVar(&grpcProxyNamespace, "namespace", "", "string to prefix to all keys for namespacing requests") 121 cmd.Flags().BoolVar(&grpcProxyEnablePprof, "enable-pprof", false, `Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/"`) 122 cmd.Flags().StringVar(&grpcProxyDataDir, "data-dir", "default.proxy", "Data directory for persistent data") 123 cmd.Flags().IntVar(&grpcMaxCallSendMsgSize, "max-send-bytes", defaultGRPCMaxCallSendMsgSize, "message send limits in bytes (default value is 1.5 MiB)") 124 cmd.Flags().IntVar(&grpcMaxCallRecvMsgSize, "max-recv-bytes", math.MaxInt32, "message receive limits in bytes (default value is math.MaxInt32)") 125 126 // client TLS for connecting to server 127 cmd.Flags().StringVar(&grpcProxyCert, "cert", "", "identify secure connections with etcd servers using this TLS certificate file") 128 cmd.Flags().StringVar(&grpcProxyKey, "key", "", "identify secure connections with etcd servers using this TLS key file") 129 cmd.Flags().StringVar(&grpcProxyCA, "cacert", "", "verify certificates of TLS-enabled secure etcd servers using this CA bundle") 130 cmd.Flags().BoolVar(&grpcProxyInsecureSkipTLSVerify, "insecure-skip-tls-verify", false, "skip authentication of etcd server TLS certificates (CAUTION: this option should be enabled only for testing purposes)") 131 132 // client TLS for connecting to proxy 133 cmd.Flags().StringVar(&grpcProxyListenCert, "cert-file", "", "identify secure connections to the proxy using this TLS certificate file") 134 cmd.Flags().StringVar(&grpcProxyListenKey, "key-file", "", "identify secure connections to the proxy using this TLS key file") 135 cmd.Flags().StringVar(&grpcProxyListenCA, "trusted-ca-file", "", "verify certificates of TLS-enabled secure proxy using this CA bundle") 136 cmd.Flags().BoolVar(&grpcProxyListenAutoTLS, "auto-tls", false, "proxy TLS using generated certificates") 137 cmd.Flags().StringVar(&grpcProxyListenCRL, "client-crl-file", "", "proxy client certificate revocation list file.") 138 139 // experimental flags 140 cmd.Flags().BoolVar(&grpcProxyEnableOrdering, "experimental-serializable-ordering", false, "Ensure serializable reads have monotonically increasing store revisions across endpoints.") 141 cmd.Flags().StringVar(&grpcProxyLeasing, "experimental-leasing-prefix", "", "leasing metadata prefix for disconnected linearized reads.") 142 143 cmd.Flags().BoolVar(&grpcProxyDebug, "debug", false, "Enable debug-level logging for grpc-proxy.") 144 145 return &cmd 146 } 147 148 func startGRPCProxy(cmd *cobra.Command, args []string) { 149 checkArgs() 150 151 capnslog.SetGlobalLogLevel(capnslog.INFO) 152 if grpcProxyDebug { 153 capnslog.SetGlobalLogLevel(capnslog.DEBUG) 154 grpc.EnableTracing = true 155 // enable info, warning, error 156 grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr)) 157 } else { 158 // only discard info 159 grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr)) 160 } 161 162 tlsinfo := newTLS(grpcProxyListenCA, grpcProxyListenCert, grpcProxyListenKey) 163 if tlsinfo == nil && grpcProxyListenAutoTLS { 164 host := []string{"https://" + grpcProxyListenAddr} 165 dir := filepath.Join(grpcProxyDataDir, "fixtures", "proxy") 166 autoTLS, err := transport.SelfCert(dir, host) 167 if err != nil { 168 plog.Fatal(err) 169 } 170 tlsinfo = &autoTLS 171 } 172 if tlsinfo != nil { 173 plog.Infof("ServerTLS: %s", tlsinfo) 174 } 175 m := mustListenCMux(tlsinfo) 176 177 grpcl := m.Match(cmux.HTTP2()) 178 defer func() { 179 grpcl.Close() 180 plog.Infof("stopping listening for grpc-proxy client requests on %s", grpcProxyListenAddr) 181 }() 182 183 client := mustNewClient() 184 httpClient := mustNewHTTPClient() 185 186 srvhttp, httpl := mustHTTPListener(m, tlsinfo, client) 187 errc := make(chan error) 188 go func() { errc <- newGRPCProxyServer(client).Serve(grpcl) }() 189 go func() { errc <- srvhttp.Serve(httpl) }() 190 go func() { errc <- m.Serve() }() 191 if len(grpcProxyMetricsListenAddr) > 0 { 192 mhttpl := mustMetricsListener(tlsinfo) 193 go func() { 194 mux := http.NewServeMux() 195 grpcproxy.HandleMetrics(mux, httpClient, client.Endpoints()) 196 grpcproxy.HandleHealth(mux, client) 197 plog.Fatal(http.Serve(mhttpl, mux)) 198 }() 199 } 200 201 // grpc-proxy is initialized, ready to serve 202 notifySystemd() 203 204 fmt.Fprintln(os.Stderr, <-errc) 205 os.Exit(1) 206 } 207 208 func checkArgs() { 209 if grpcProxyResolverPrefix != "" && grpcProxyResolverTTL < 1 { 210 fmt.Fprintln(os.Stderr, fmt.Errorf("invalid resolver-ttl %d", grpcProxyResolverTTL)) 211 os.Exit(1) 212 } 213 if grpcProxyResolverPrefix == "" && grpcProxyResolverTTL > 0 { 214 fmt.Fprintln(os.Stderr, fmt.Errorf("invalid resolver-prefix %q", grpcProxyResolverPrefix)) 215 os.Exit(1) 216 } 217 if grpcProxyResolverPrefix != "" && grpcProxyResolverTTL > 0 && grpcProxyAdvertiseClientURL == "" { 218 fmt.Fprintln(os.Stderr, fmt.Errorf("invalid advertise-client-url %q", grpcProxyAdvertiseClientURL)) 219 os.Exit(1) 220 } 221 } 222 223 func mustNewClient() *clientv3.Client { 224 srvs := discoverEndpoints(grpcProxyDNSCluster, grpcProxyCA, grpcProxyInsecureDiscovery) 225 eps := srvs.Endpoints 226 if len(eps) == 0 { 227 eps = grpcProxyEndpoints 228 } 229 cfg, err := newClientCfg(eps) 230 if err != nil { 231 fmt.Fprintln(os.Stderr, err) 232 os.Exit(1) 233 } 234 cfg.DialOptions = append(cfg.DialOptions, 235 grpc.WithUnaryInterceptor(grpcproxy.AuthUnaryClientInterceptor)) 236 cfg.DialOptions = append(cfg.DialOptions, 237 grpc.WithStreamInterceptor(grpcproxy.AuthStreamClientInterceptor)) 238 client, err := clientv3.New(*cfg) 239 if err != nil { 240 fmt.Fprintln(os.Stderr, err) 241 os.Exit(1) 242 } 243 return client 244 } 245 246 func newClientCfg(eps []string) (*clientv3.Config, error) { 247 // set tls if any one tls option set 248 cfg := clientv3.Config{ 249 Endpoints: eps, 250 DialTimeout: 5 * time.Second, 251 } 252 253 if grpcMaxCallSendMsgSize > 0 { 254 cfg.MaxCallSendMsgSize = grpcMaxCallSendMsgSize 255 } 256 if grpcMaxCallRecvMsgSize > 0 { 257 cfg.MaxCallRecvMsgSize = grpcMaxCallRecvMsgSize 258 } 259 260 tls := newTLS(grpcProxyCA, grpcProxyCert, grpcProxyKey) 261 if tls == nil && grpcProxyInsecureSkipTLSVerify { 262 tls = &transport.TLSInfo{} 263 } 264 if tls != nil { 265 clientTLS, err := tls.ClientConfig() 266 if err != nil { 267 return nil, err 268 } 269 clientTLS.InsecureSkipVerify = grpcProxyInsecureSkipTLSVerify 270 if clientTLS.InsecureSkipVerify { 271 plog.Warningf("--insecure-skip-tls-verify was given, this grpc proxy process skips authentication of etcd server TLS certificates. This option should be enabled only for testing purposes.") 272 } 273 cfg.TLS = clientTLS 274 plog.Infof("ClientTLS: %s", tls) 275 } 276 return &cfg, nil 277 } 278 279 func newTLS(ca, cert, key string) *transport.TLSInfo { 280 if ca == "" && cert == "" && key == "" { 281 return nil 282 } 283 return &transport.TLSInfo{CAFile: ca, CertFile: cert, KeyFile: key} 284 } 285 286 func mustListenCMux(tlsinfo *transport.TLSInfo) cmux.CMux { 287 l, err := net.Listen("tcp", grpcProxyListenAddr) 288 if err != nil { 289 fmt.Fprintln(os.Stderr, err) 290 os.Exit(1) 291 } 292 293 if l, err = transport.NewKeepAliveListener(l, "tcp", nil); err != nil { 294 fmt.Fprintln(os.Stderr, err) 295 os.Exit(1) 296 } 297 if tlsinfo != nil { 298 tlsinfo.CRLFile = grpcProxyListenCRL 299 if l, err = transport.NewTLSListener(l, tlsinfo); err != nil { 300 plog.Fatal(err) 301 } 302 } 303 304 plog.Infof("listening for grpc-proxy client requests on %s", grpcProxyListenAddr) 305 return cmux.New(l) 306 } 307 308 func newGRPCProxyServer(client *clientv3.Client) *grpc.Server { 309 if grpcProxyEnableOrdering { 310 vf := ordering.NewOrderViolationSwitchEndpointClosure(*client) 311 client.KV = ordering.NewKV(client.KV, vf) 312 plog.Infof("waiting for linearized read from cluster to recover ordering") 313 for { 314 _, err := client.KV.Get(context.TODO(), "_", clientv3.WithKeysOnly()) 315 if err == nil { 316 break 317 } 318 plog.Warningf("ordering recovery failed, retrying in 1s (%v)", err) 319 time.Sleep(time.Second) 320 } 321 } 322 323 if len(grpcProxyNamespace) > 0 { 324 client.KV = namespace.NewKV(client.KV, grpcProxyNamespace) 325 client.Watcher = namespace.NewWatcher(client.Watcher, grpcProxyNamespace) 326 client.Lease = namespace.NewLease(client.Lease, grpcProxyNamespace) 327 } 328 329 if len(grpcProxyLeasing) > 0 { 330 client.KV, _, _ = leasing.NewKV(client, grpcProxyLeasing) 331 } 332 333 kvp, _ := grpcproxy.NewKvProxy(client) 334 watchp, _ := grpcproxy.NewWatchProxy(client) 335 if grpcProxyResolverPrefix != "" { 336 grpcproxy.Register(client, grpcProxyResolverPrefix, grpcProxyAdvertiseClientURL, grpcProxyResolverTTL) 337 } 338 clusterp, _ := grpcproxy.NewClusterProxy(client, grpcProxyAdvertiseClientURL, grpcProxyResolverPrefix) 339 leasep, _ := grpcproxy.NewLeaseProxy(client) 340 mainp := grpcproxy.NewMaintenanceProxy(client) 341 authp := grpcproxy.NewAuthProxy(client) 342 electionp := grpcproxy.NewElectionProxy(client) 343 lockp := grpcproxy.NewLockProxy(client) 344 345 server := grpc.NewServer( 346 grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), 347 grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), 348 grpc.MaxConcurrentStreams(math.MaxUint32), 349 ) 350 351 pb.RegisterKVServer(server, kvp) 352 pb.RegisterWatchServer(server, watchp) 353 pb.RegisterClusterServer(server, clusterp) 354 pb.RegisterLeaseServer(server, leasep) 355 pb.RegisterMaintenanceServer(server, mainp) 356 pb.RegisterAuthServer(server, authp) 357 v3electionpb.RegisterElectionServer(server, electionp) 358 v3lockpb.RegisterLockServer(server, lockp) 359 360 return server 361 } 362 363 func mustHTTPListener(m cmux.CMux, tlsinfo *transport.TLSInfo, c *clientv3.Client) (*http.Server, net.Listener) { 364 httpClient := mustNewHTTPClient() 365 httpmux := http.NewServeMux() 366 httpmux.HandleFunc("/", http.NotFound) 367 grpcproxy.HandleMetrics(httpmux, httpClient, c.Endpoints()) 368 grpcproxy.HandleHealth(httpmux, c) 369 if grpcProxyEnablePprof { 370 for p, h := range debugutil.PProfHandlers() { 371 httpmux.Handle(p, h) 372 } 373 plog.Infof("pprof is enabled under %s", debugutil.HTTPPrefixPProf) 374 } 375 srvhttp := &http.Server{Handler: httpmux} 376 377 if tlsinfo == nil { 378 return srvhttp, m.Match(cmux.HTTP1()) 379 } 380 381 srvTLS, err := tlsinfo.ServerConfig() 382 if err != nil { 383 plog.Fatalf("could not setup TLS (%v)", err) 384 } 385 srvhttp.TLSConfig = srvTLS 386 return srvhttp, m.Match(cmux.Any()) 387 } 388 389 func mustNewHTTPClient() *http.Client { 390 transport, err := newHTTPTransport(grpcProxyCA, grpcProxyCert, grpcProxyKey) 391 if err != nil { 392 fmt.Fprintln(os.Stderr, err) 393 os.Exit(1) 394 } 395 return &http.Client{Transport: transport} 396 } 397 398 func newHTTPTransport(ca, cert, key string) (*http.Transport, error) { 399 tr := &http.Transport{} 400 401 if ca != "" && cert != "" && key != "" { 402 caCert, err := ioutil.ReadFile(ca) 403 if err != nil { 404 return nil, err 405 } 406 keyPair, err := tls.LoadX509KeyPair(cert, key) 407 if err != nil { 408 return nil, err 409 } 410 caPool := x509.NewCertPool() 411 caPool.AppendCertsFromPEM(caCert) 412 413 tlsConfig := &tls.Config{ 414 Certificates: []tls.Certificate{keyPair}, 415 RootCAs: caPool, 416 } 417 tlsConfig.BuildNameToCertificate() 418 tr.TLSClientConfig = tlsConfig 419 } else if grpcProxyInsecureSkipTLSVerify { 420 tlsConfig := &tls.Config{InsecureSkipVerify: grpcProxyInsecureSkipTLSVerify} 421 tr.TLSClientConfig = tlsConfig 422 } 423 return tr, nil 424 } 425 426 func mustMetricsListener(tlsinfo *transport.TLSInfo) net.Listener { 427 murl, err := url.Parse(grpcProxyMetricsListenAddr) 428 if err != nil { 429 fmt.Fprintf(os.Stderr, "cannot parse %q", grpcProxyMetricsListenAddr) 430 os.Exit(1) 431 } 432 ml, err := transport.NewListener(murl.Host, murl.Scheme, tlsinfo) 433 if err != nil { 434 fmt.Fprintln(os.Stderr, err) 435 os.Exit(1) 436 } 437 plog.Info("grpc-proxy: listening for metrics on ", murl.String()) 438 return ml 439 }