go.temporal.io/server@v1.23.0/common/membership/grpc_resolver.go (about)

     1  // The MIT License
     2  //
     3  // Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
     4  //
     5  // Copyright (c) 2020 Uber Technologies, Inc.
     6  //
     7  // Permission is hereby granted, free of charge, to any person obtaining a copy
     8  // of this software and associated documentation files (the "Software"), to deal
     9  // in the Software without restriction, including without limitation the rights
    10  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    11  // copies of the Software, and to permit persons to whom the Software is
    12  // furnished to do so, subject to the following conditions:
    13  //
    14  // The above copyright notice and this permission notice shall be included in
    15  // all copies or substantial portions of the Software.
    16  //
    17  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    18  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    19  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    20  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    21  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    22  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    23  // THE SOFTWARE.
    24  
    25  package membership
    26  
    27  import (
    28  	"errors"
    29  	"fmt"
    30  	"sync"
    31  	"sync/atomic"
    32  
    33  	"go.uber.org/fx"
    34  	"google.golang.org/grpc/resolver"
    35  
    36  	"go.temporal.io/server/common/primitives"
    37  )
    38  
    39  const grpcResolverScheme = "membership"
    40  
    41  // GRPCResolver is an empty type used to enforce a dependency using fx so that we're guaranteed to have initialized
    42  // the global builder before we use it.
    43  type GRPCResolver struct{}
    44  
    45  var (
    46  	GRPCResolverModule = fx.Options(
    47  		fx.Provide(initializeBuilder),
    48  	)
    49  
    50  	globalGrpcBuilder grpcBuilder
    51  )
    52  
    53  func init() {
    54  	// This must be called in init to avoid race conditions. We don't have a Monitor yet, so we'll leave it nil and
    55  	// initialize it with fx.
    56  	resolver.Register(&globalGrpcBuilder)
    57  }
    58  
    59  func initializeBuilder(monitor Monitor) GRPCResolver {
    60  	globalGrpcBuilder.monitor.Store(monitor)
    61  	return GRPCResolver{}
    62  }
    63  
    64  func (g *GRPCResolver) MakeURL(service primitives.ServiceName) string {
    65  	return fmt.Sprintf("%s://%s", grpcResolverScheme, string(service))
    66  }
    67  
    68  type grpcBuilder struct {
    69  	monitor atomic.Value // Monitor
    70  }
    71  
    72  func (m *grpcBuilder) Scheme() string {
    73  	return grpcResolverScheme
    74  }
    75  
    76  func (m *grpcBuilder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
    77  	monitor, ok := m.monitor.Load().(Monitor)
    78  	if !ok {
    79  		return nil, errors.New("grpc resolver has not been initialized yet")
    80  	}
    81  	// See MakeURL: the service ends up as the "host" of the parsed URL
    82  	service := target.URL.Host
    83  	serviceResolver, err := monitor.GetResolver(primitives.ServiceName(service))
    84  	if err != nil {
    85  		return nil, err
    86  	}
    87  	grpcResolver := &grpcResolver{
    88  		cc:       cc,
    89  		r:        serviceResolver,
    90  		notifyCh: make(chan *ChangedEvent, 1),
    91  	}
    92  	if err := grpcResolver.start(); err != nil {
    93  		return nil, err
    94  	}
    95  	return grpcResolver, nil
    96  }
    97  
    98  type grpcResolver struct {
    99  	cc       resolver.ClientConn
   100  	r        ServiceResolver
   101  	notifyCh chan *ChangedEvent
   102  	wg       sync.WaitGroup
   103  }
   104  
   105  func (m *grpcResolver) start() error {
   106  	if err := m.r.AddListener(fmt.Sprintf("%p", m), m.notifyCh); err != nil {
   107  		return err
   108  	}
   109  	m.wg.Add(1)
   110  	go m.listen()
   111  
   112  	// Try once to get address synchronously. If this fails, it's okay, we'll listen for
   113  	// changes and update the resolver later.
   114  	m.resolve()
   115  	return nil
   116  }
   117  
   118  func (m *grpcResolver) listen() {
   119  	for range m.notifyCh {
   120  		m.resolve()
   121  	}
   122  	m.wg.Done()
   123  }
   124  
   125  func (m *grpcResolver) resolve() {
   126  	members := m.r.Members()
   127  	if len(members) == 0 {
   128  		// grpc considers it an error if we report no addresses, and fails the connection eagerly.
   129  		// Instead, just poke membership and then wait until it notifies us.
   130  		m.r.RequestRefresh()
   131  		return
   132  	}
   133  	addresses := make([]resolver.Address, 0, len(members))
   134  	for _, hostInfo := range members {
   135  		addresses = append(addresses, resolver.Address{
   136  			Addr: hostInfo.GetAddress(),
   137  		})
   138  	}
   139  	if err := m.cc.UpdateState(resolver.State{Addresses: addresses}); err != nil {
   140  		fmt.Printf("error updating state in gRPC resolver: %v", err)
   141  	}
   142  }
   143  
   144  func (m *grpcResolver) ResolveNow(_ resolver.ResolveNowOptions) {
   145  	select {
   146  	case m.notifyCh <- nil:
   147  	default:
   148  	}
   149  }
   150  
   151  func (m *grpcResolver) Close() {
   152  	if err := m.r.RemoveListener(fmt.Sprintf("%p", m)); err != nil {
   153  		fmt.Printf("error removing listener from gRPC resolver: %v", err)
   154  	}
   155  	close(m.notifyCh)
   156  	m.wg.Wait() // wait until listen() exits
   157  }