github.com/hxx258456/ccgo@v0.0.5-0.20230213014102-48b35f46f66f/grpc/balancer/grpclb/grpclb_util.go (about)

     1  /*
     2   *
     3   * Copyright 2016 gRPC authors.
     4   *
     5   * Licensed under the Apache License, Version 2.0 (the "License");
     6   * you may not use this file except in compliance with the License.
     7   * You may obtain a copy of the License at
     8   *
     9   *     http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   *
    17   */
    18  
    19  package grpclb
    20  
    21  import (
    22  	"fmt"
    23  	"sync"
    24  	"time"
    25  
    26  	"github.com/hxx258456/ccgo/grpc/balancer"
    27  	"github.com/hxx258456/ccgo/grpc/resolver"
    28  )
    29  
    30  // The parent ClientConn should re-resolve when grpclb loses connection to the
    31  // remote balancer. When the ClientConn inside grpclb gets a TransientFailure,
    32  // it calls lbManualResolver.ResolveNow(), which calls parent ClientConn's
    33  // ResolveNow, and eventually results in re-resolve happening in parent
    34  // ClientConn's resolver (DNS for example).
    35  //
    36  //                          parent
    37  //                          ClientConn
    38  //  +-----------------------------------------------------------------+
    39  //  |             parent          +---------------------------------+ |
    40  //  | DNS         ClientConn      |  grpclb                         | |
    41  //  | resolver    balancerWrapper |                                 | |
    42  //  | +              +            |    grpclb          grpclb       | |
    43  //  | |              |            |    ManualResolver  ClientConn   | |
    44  //  | |              |            |     +              +            | |
    45  //  | |              |            |     |              | Transient  | |
    46  //  | |              |            |     |              | Failure    | |
    47  //  | |              |            |     |  <---------  |            | |
    48  //  | |              | <--------------- |  ResolveNow  |            | |
    49  //  | |  <---------  | ResolveNow |     |              |            | |
    50  //  | |  ResolveNow  |            |     |              |            | |
    51  //  | |              |            |     |              |            | |
    52  //  | +              +            |     +              +            | |
    53  //  |                             +---------------------------------+ |
    54  //  +-----------------------------------------------------------------+
    55  
    56  // lbManualResolver is used by the ClientConn inside grpclb. It's a manual
    57  // resolver with a special ResolveNow() function.
    58  //
    59  // When ResolveNow() is called, it calls ResolveNow() on the parent ClientConn,
    60  // so when grpclb client lose contact with remote balancers, the parent
    61  // ClientConn's resolver will re-resolve.
    62  type lbManualResolver struct {
    63  	scheme string
    64  	ccr    resolver.ClientConn
    65  
    66  	ccb balancer.ClientConn
    67  }
    68  
    69  func (r *lbManualResolver) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) {
    70  	r.ccr = cc
    71  	return r, nil
    72  }
    73  
    74  func (r *lbManualResolver) Scheme() string {
    75  	return r.scheme
    76  }
    77  
    78  // ResolveNow calls resolveNow on the parent ClientConn.
    79  func (r *lbManualResolver) ResolveNow(o resolver.ResolveNowOptions) {
    80  	r.ccb.ResolveNow(o)
    81  }
    82  
    83  // Close is a noop for Resolver.
    84  func (*lbManualResolver) Close() {}
    85  
    86  // UpdateState calls cc.UpdateState.
    87  func (r *lbManualResolver) UpdateState(s resolver.State) {
    88  	r.ccr.UpdateState(s)
    89  }
    90  
    91  const subConnCacheTime = time.Second * 10
    92  
    93  // lbCacheClientConn is a wrapper balancer.ClientConn with a SubConn cache.
    94  // SubConns will be kept in cache for subConnCacheTime before being removed.
    95  //
    96  // Its new and remove methods are updated to do cache first.
    97  type lbCacheClientConn struct {
    98  	cc      balancer.ClientConn
    99  	timeout time.Duration
   100  
   101  	mu sync.Mutex
   102  	// subConnCache only keeps subConns that are being deleted.
   103  	subConnCache  map[resolver.Address]*subConnCacheEntry
   104  	subConnToAddr map[balancer.SubConn]resolver.Address
   105  }
   106  
   107  type subConnCacheEntry struct {
   108  	sc balancer.SubConn
   109  
   110  	cancel        func()
   111  	abortDeleting bool
   112  }
   113  
   114  func newLBCacheClientConn(cc balancer.ClientConn) *lbCacheClientConn {
   115  	return &lbCacheClientConn{
   116  		cc:            cc,
   117  		timeout:       subConnCacheTime,
   118  		subConnCache:  make(map[resolver.Address]*subConnCacheEntry),
   119  		subConnToAddr: make(map[balancer.SubConn]resolver.Address),
   120  	}
   121  }
   122  
   123  func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
   124  	if len(addrs) != 1 {
   125  		return nil, fmt.Errorf("grpclb calling NewSubConn with addrs of length %v", len(addrs))
   126  	}
   127  	addrWithoutAttrs := addrs[0]
   128  	addrWithoutAttrs.Attributes = nil
   129  
   130  	ccc.mu.Lock()
   131  	defer ccc.mu.Unlock()
   132  	if entry, ok := ccc.subConnCache[addrWithoutAttrs]; ok {
   133  		// If entry is in subConnCache, the SubConn was being deleted.
   134  		// cancel function will never be nil.
   135  		entry.cancel()
   136  		delete(ccc.subConnCache, addrWithoutAttrs)
   137  		return entry.sc, nil
   138  	}
   139  
   140  	scNew, err := ccc.cc.NewSubConn(addrs, opts)
   141  	if err != nil {
   142  		return nil, err
   143  	}
   144  
   145  	ccc.subConnToAddr[scNew] = addrWithoutAttrs
   146  	return scNew, nil
   147  }
   148  
   149  func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) {
   150  	ccc.mu.Lock()
   151  	defer ccc.mu.Unlock()
   152  	addr, ok := ccc.subConnToAddr[sc]
   153  	if !ok {
   154  		return
   155  	}
   156  
   157  	if entry, ok := ccc.subConnCache[addr]; ok {
   158  		if entry.sc != sc {
   159  			// This could happen if NewSubConn was called multiple times for the
   160  			// same address, and those SubConns are all removed. We remove sc
   161  			// immediately here.
   162  			delete(ccc.subConnToAddr, sc)
   163  			ccc.cc.RemoveSubConn(sc)
   164  		}
   165  		return
   166  	}
   167  
   168  	entry := &subConnCacheEntry{
   169  		sc: sc,
   170  	}
   171  	ccc.subConnCache[addr] = entry
   172  
   173  	timer := time.AfterFunc(ccc.timeout, func() {
   174  		ccc.mu.Lock()
   175  		defer ccc.mu.Unlock()
   176  		if entry.abortDeleting {
   177  			return
   178  		}
   179  		ccc.cc.RemoveSubConn(sc)
   180  		delete(ccc.subConnToAddr, sc)
   181  		delete(ccc.subConnCache, addr)
   182  	})
   183  	entry.cancel = func() {
   184  		if !timer.Stop() {
   185  			// If stop was not successful, the timer has fired (this can only
   186  			// happen in a race). But the deleting function is blocked on ccc.mu
   187  			// because the mutex was held by the caller of this function.
   188  			//
   189  			// Set abortDeleting to true to abort the deleting function. When
   190  			// the lock is released, the deleting function will acquire the
   191  			// lock, check the value of abortDeleting and return.
   192  			entry.abortDeleting = true
   193  		}
   194  	}
   195  }
   196  
   197  func (ccc *lbCacheClientConn) UpdateState(s balancer.State) {
   198  	ccc.cc.UpdateState(s)
   199  }
   200  
   201  func (ccc *lbCacheClientConn) close() {
   202  	ccc.mu.Lock()
   203  	// Only cancel all existing timers. There's no need to remove SubConns.
   204  	for _, entry := range ccc.subConnCache {
   205  		entry.cancel()
   206  	}
   207  	ccc.mu.Unlock()
   208  }