github.com/hxx258456/ccgo@v0.0.5-0.20230213014102-48b35f46f66f/grpc/balancer/grpclb/grpclb_util.go (about) 1 /* 2 * 3 * Copyright 2016 gRPC authors. 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 * 17 */ 18 19 package grpclb 20 21 import ( 22 "fmt" 23 "sync" 24 "time" 25 26 "github.com/hxx258456/ccgo/grpc/balancer" 27 "github.com/hxx258456/ccgo/grpc/resolver" 28 ) 29 30 // The parent ClientConn should re-resolve when grpclb loses connection to the 31 // remote balancer. When the ClientConn inside grpclb gets a TransientFailure, 32 // it calls lbManualResolver.ResolveNow(), which calls parent ClientConn's 33 // ResolveNow, and eventually results in re-resolve happening in parent 34 // ClientConn's resolver (DNS for example). 35 // 36 // parent 37 // ClientConn 38 // +-----------------------------------------------------------------+ 39 // | parent +---------------------------------+ | 40 // | DNS ClientConn | grpclb | | 41 // | resolver balancerWrapper | | | 42 // | + + | grpclb grpclb | | 43 // | | | | ManualResolver ClientConn | | 44 // | | | | + + | | 45 // | | | | | | Transient | | 46 // | | | | | | Failure | | 47 // | | | | | <--------- | | | 48 // | | | <--------------- | ResolveNow | | | 49 // | | <--------- | ResolveNow | | | | | 50 // | | ResolveNow | | | | | | 51 // | | | | | | | | 52 // | + + | + + | | 53 // | +---------------------------------+ | 54 // +-----------------------------------------------------------------+ 55 56 // lbManualResolver is used by the ClientConn inside grpclb. It's a manual 57 // resolver with a special ResolveNow() function. 58 // 59 // When ResolveNow() is called, it calls ResolveNow() on the parent ClientConn, 60 // so when grpclb client lose contact with remote balancers, the parent 61 // ClientConn's resolver will re-resolve. 62 type lbManualResolver struct { 63 scheme string 64 ccr resolver.ClientConn 65 66 ccb balancer.ClientConn 67 } 68 69 func (r *lbManualResolver) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { 70 r.ccr = cc 71 return r, nil 72 } 73 74 func (r *lbManualResolver) Scheme() string { 75 return r.scheme 76 } 77 78 // ResolveNow calls resolveNow on the parent ClientConn. 79 func (r *lbManualResolver) ResolveNow(o resolver.ResolveNowOptions) { 80 r.ccb.ResolveNow(o) 81 } 82 83 // Close is a noop for Resolver. 84 func (*lbManualResolver) Close() {} 85 86 // UpdateState calls cc.UpdateState. 87 func (r *lbManualResolver) UpdateState(s resolver.State) { 88 r.ccr.UpdateState(s) 89 } 90 91 const subConnCacheTime = time.Second * 10 92 93 // lbCacheClientConn is a wrapper balancer.ClientConn with a SubConn cache. 94 // SubConns will be kept in cache for subConnCacheTime before being removed. 95 // 96 // Its new and remove methods are updated to do cache first. 97 type lbCacheClientConn struct { 98 cc balancer.ClientConn 99 timeout time.Duration 100 101 mu sync.Mutex 102 // subConnCache only keeps subConns that are being deleted. 103 subConnCache map[resolver.Address]*subConnCacheEntry 104 subConnToAddr map[balancer.SubConn]resolver.Address 105 } 106 107 type subConnCacheEntry struct { 108 sc balancer.SubConn 109 110 cancel func() 111 abortDeleting bool 112 } 113 114 func newLBCacheClientConn(cc balancer.ClientConn) *lbCacheClientConn { 115 return &lbCacheClientConn{ 116 cc: cc, 117 timeout: subConnCacheTime, 118 subConnCache: make(map[resolver.Address]*subConnCacheEntry), 119 subConnToAddr: make(map[balancer.SubConn]resolver.Address), 120 } 121 } 122 123 func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { 124 if len(addrs) != 1 { 125 return nil, fmt.Errorf("grpclb calling NewSubConn with addrs of length %v", len(addrs)) 126 } 127 addrWithoutAttrs := addrs[0] 128 addrWithoutAttrs.Attributes = nil 129 130 ccc.mu.Lock() 131 defer ccc.mu.Unlock() 132 if entry, ok := ccc.subConnCache[addrWithoutAttrs]; ok { 133 // If entry is in subConnCache, the SubConn was being deleted. 134 // cancel function will never be nil. 135 entry.cancel() 136 delete(ccc.subConnCache, addrWithoutAttrs) 137 return entry.sc, nil 138 } 139 140 scNew, err := ccc.cc.NewSubConn(addrs, opts) 141 if err != nil { 142 return nil, err 143 } 144 145 ccc.subConnToAddr[scNew] = addrWithoutAttrs 146 return scNew, nil 147 } 148 149 func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { 150 ccc.mu.Lock() 151 defer ccc.mu.Unlock() 152 addr, ok := ccc.subConnToAddr[sc] 153 if !ok { 154 return 155 } 156 157 if entry, ok := ccc.subConnCache[addr]; ok { 158 if entry.sc != sc { 159 // This could happen if NewSubConn was called multiple times for the 160 // same address, and those SubConns are all removed. We remove sc 161 // immediately here. 162 delete(ccc.subConnToAddr, sc) 163 ccc.cc.RemoveSubConn(sc) 164 } 165 return 166 } 167 168 entry := &subConnCacheEntry{ 169 sc: sc, 170 } 171 ccc.subConnCache[addr] = entry 172 173 timer := time.AfterFunc(ccc.timeout, func() { 174 ccc.mu.Lock() 175 defer ccc.mu.Unlock() 176 if entry.abortDeleting { 177 return 178 } 179 ccc.cc.RemoveSubConn(sc) 180 delete(ccc.subConnToAddr, sc) 181 delete(ccc.subConnCache, addr) 182 }) 183 entry.cancel = func() { 184 if !timer.Stop() { 185 // If stop was not successful, the timer has fired (this can only 186 // happen in a race). But the deleting function is blocked on ccc.mu 187 // because the mutex was held by the caller of this function. 188 // 189 // Set abortDeleting to true to abort the deleting function. When 190 // the lock is released, the deleting function will acquire the 191 // lock, check the value of abortDeleting and return. 192 entry.abortDeleting = true 193 } 194 } 195 } 196 197 func (ccc *lbCacheClientConn) UpdateState(s balancer.State) { 198 ccc.cc.UpdateState(s) 199 } 200 201 func (ccc *lbCacheClientConn) close() { 202 ccc.mu.Lock() 203 // Only cancel all existing timers. There's no need to remove SubConns. 204 for _, entry := range ccc.subConnCache { 205 entry.cancel() 206 } 207 ccc.mu.Unlock() 208 }