go.etcd.io/etcd@v3.3.27+incompatible/clientv3/balancer/balancer.go (about) 1 // Copyright 2018 The etcd Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package balancer implements client balancer. 16 package balancer 17 18 import ( 19 "strconv" 20 "sync" 21 "time" 22 23 "github.com/coreos/etcd/clientv3/balancer/connectivity" 24 "github.com/coreos/etcd/clientv3/balancer/picker" 25 26 "go.uber.org/zap" 27 "google.golang.org/grpc/balancer" 28 grpcconnectivity "google.golang.org/grpc/connectivity" 29 "google.golang.org/grpc/resolver" 30 _ "google.golang.org/grpc/resolver/dns" // register DNS resolver 31 _ "google.golang.org/grpc/resolver/passthrough" // register passthrough resolver 32 ) 33 34 // Config defines balancer configurations. 35 type Config struct { 36 // Policy configures balancer policy. 37 Policy picker.Policy 38 39 // Picker implements gRPC picker. 40 // Leave empty if "Policy" field is not custom. 41 // TODO: currently custom policy is not supported. 42 // Picker picker.Picker 43 44 // Name defines an additional name for balancer. 45 // Useful for balancer testing to avoid register conflicts. 46 // If empty, defaults to policy name. 47 Name string 48 49 // Logger configures balancer logging. 50 // If nil, logs are discarded. 51 Logger *zap.Logger 52 } 53 54 // RegisterBuilder creates and registers a builder. Since this function calls balancer.Register, it 55 // must be invoked at initialization time. 56 func RegisterBuilder(cfg Config) { 57 bb := &builder{cfg} 58 balancer.Register(bb) 59 60 bb.cfg.Logger.Debug( 61 "registered balancer", 62 zap.String("policy", bb.cfg.Policy.String()), 63 zap.String("name", bb.cfg.Name), 64 ) 65 } 66 67 type builder struct { 68 cfg Config 69 } 70 71 // Build is called initially when creating "ccBalancerWrapper". 72 // "grpc.Dial" is called to this client connection. 73 // Then, resolved addresses will be handled via "HandleResolvedAddrs". 74 func (b *builder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { 75 bb := &baseBalancer{ 76 id: strconv.FormatInt(time.Now().UnixNano(), 36), 77 policy: b.cfg.Policy, 78 name: b.cfg.Name, 79 lg: b.cfg.Logger, 80 81 addrToSc: make(map[resolver.Address]balancer.SubConn), 82 scToAddr: make(map[balancer.SubConn]resolver.Address), 83 scToSt: make(map[balancer.SubConn]grpcconnectivity.State), 84 85 currentConn: nil, 86 connectivityRecorder: connectivity.New(b.cfg.Logger), 87 88 // initialize picker always returns "ErrNoSubConnAvailable" 89 picker: picker.NewErr(balancer.ErrNoSubConnAvailable), 90 } 91 92 // TODO: support multiple connections 93 bb.mu.Lock() 94 bb.currentConn = cc 95 bb.mu.Unlock() 96 97 bb.lg.Info( 98 "built balancer", 99 zap.String("balancer-id", bb.id), 100 zap.String("policy", bb.policy.String()), 101 zap.String("resolver-target", cc.Target()), 102 ) 103 return bb 104 } 105 106 // Name implements "grpc/balancer.Builder" interface. 107 func (b *builder) Name() string { return b.cfg.Name } 108 109 // Balancer defines client balancer interface. 110 type Balancer interface { 111 // Balancer is called on specified client connection. Client initiates gRPC 112 // connection with "grpc.Dial(addr, grpc.WithBalancerName)", and then those resolved 113 // addresses are passed to "grpc/balancer.Balancer.HandleResolvedAddrs". 114 // For each resolved address, balancer calls "balancer.ClientConn.NewSubConn". 115 // "grpc/balancer.Balancer.HandleSubConnStateChange" is called when connectivity state 116 // changes, thus requires failover logic in this method. 117 balancer.Balancer 118 119 // Picker calls "Pick" for every client request. 120 picker.Picker 121 } 122 123 type baseBalancer struct { 124 id string 125 policy picker.Policy 126 name string 127 lg *zap.Logger 128 129 mu sync.RWMutex 130 131 addrToSc map[resolver.Address]balancer.SubConn 132 scToAddr map[balancer.SubConn]resolver.Address 133 scToSt map[balancer.SubConn]grpcconnectivity.State 134 135 currentConn balancer.ClientConn 136 connectivityRecorder connectivity.Recorder 137 138 picker picker.Picker 139 } 140 141 // HandleResolvedAddrs implements "grpc/balancer.Balancer" interface. 142 // gRPC sends initial or updated resolved addresses from "Build". 143 func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { 144 if err != nil { 145 bb.lg.Warn("HandleResolvedAddrs called with error", zap.String("balancer-id", bb.id), zap.Error(err)) 146 return 147 } 148 bb.lg.Info("resolved", 149 zap.String("picker", bb.picker.String()), 150 zap.String("balancer-id", bb.id), 151 zap.Strings("addresses", addrsToStrings(addrs)), 152 ) 153 154 bb.mu.Lock() 155 defer bb.mu.Unlock() 156 157 resolved := make(map[resolver.Address]struct{}) 158 for _, addr := range addrs { 159 resolved[addr] = struct{}{} 160 if _, ok := bb.addrToSc[addr]; !ok { 161 sc, err := bb.currentConn.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{}) 162 if err != nil { 163 bb.lg.Warn("NewSubConn failed", zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr)) 164 continue 165 } 166 bb.lg.Info("created subconn", zap.String("address", addr.Addr)) 167 bb.addrToSc[addr] = sc 168 bb.scToAddr[sc] = addr 169 bb.scToSt[sc] = grpcconnectivity.Idle 170 sc.Connect() 171 } 172 } 173 174 for addr, sc := range bb.addrToSc { 175 if _, ok := resolved[addr]; !ok { 176 // was removed by resolver or failed to create subconn 177 bb.currentConn.RemoveSubConn(sc) 178 delete(bb.addrToSc, addr) 179 180 bb.lg.Info( 181 "removed subconn", 182 zap.String("picker", bb.picker.String()), 183 zap.String("balancer-id", bb.id), 184 zap.String("address", addr.Addr), 185 zap.String("subconn", scToString(sc)), 186 ) 187 188 // Keep the state of this sc in bb.scToSt until sc's state becomes Shutdown. 189 // The entry will be deleted in HandleSubConnStateChange. 190 // (DO NOT) delete(bb.scToAddr, sc) 191 // (DO NOT) delete(bb.scToSt, sc) 192 } 193 } 194 } 195 196 // HandleSubConnStateChange implements "grpc/balancer.Balancer" interface. 197 func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s grpcconnectivity.State) { 198 bb.mu.Lock() 199 defer bb.mu.Unlock() 200 201 old, ok := bb.scToSt[sc] 202 if !ok { 203 bb.lg.Warn( 204 "state change for an unknown subconn", 205 zap.String("picker", bb.picker.String()), 206 zap.String("balancer-id", bb.id), 207 zap.String("subconn", scToString(sc)), 208 zap.Int("subconn-size", len(bb.scToAddr)), 209 zap.String("state", s.String()), 210 ) 211 return 212 } 213 214 bb.lg.Info( 215 "state changed", 216 zap.String("picker", bb.picker.String()), 217 zap.String("balancer-id", bb.id), 218 zap.Bool("connected", s == grpcconnectivity.Ready), 219 zap.String("subconn", scToString(sc)), 220 zap.Int("subconn-size", len(bb.scToAddr)), 221 zap.String("address", bb.scToAddr[sc].Addr), 222 zap.String("old-state", old.String()), 223 zap.String("new-state", s.String()), 224 ) 225 226 bb.scToSt[sc] = s 227 switch s { 228 case grpcconnectivity.Idle: 229 sc.Connect() 230 case grpcconnectivity.Shutdown: 231 // When an address was removed by resolver, b called RemoveSubConn but 232 // kept the sc's state in scToSt. Remove state for this sc here. 233 delete(bb.scToAddr, sc) 234 delete(bb.scToSt, sc) 235 } 236 237 oldAggrState := bb.connectivityRecorder.GetCurrentState() 238 bb.connectivityRecorder.RecordTransition(old, s) 239 240 // Update balancer picker when one of the following happens: 241 // - this sc became ready from not-ready 242 // - this sc became not-ready from ready 243 // - the aggregated state of balancer became TransientFailure from non-TransientFailure 244 // - the aggregated state of balancer became non-TransientFailure from TransientFailure 245 if (s == grpcconnectivity.Ready) != (old == grpcconnectivity.Ready) || 246 (bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure) != (oldAggrState == grpcconnectivity.TransientFailure) { 247 bb.updatePicker() 248 } 249 250 bb.currentConn.UpdateBalancerState(bb.connectivityRecorder.GetCurrentState(), bb.picker) 251 } 252 253 func (bb *baseBalancer) updatePicker() { 254 if bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure { 255 bb.picker = picker.NewErr(balancer.ErrTransientFailure) 256 bb.lg.Info( 257 "updated picker to transient error picker", 258 zap.String("picker", bb.picker.String()), 259 zap.String("balancer-id", bb.id), 260 zap.String("policy", bb.policy.String()), 261 ) 262 return 263 } 264 265 // only pass ready subconns to picker 266 scToAddr := make(map[balancer.SubConn]resolver.Address) 267 for addr, sc := range bb.addrToSc { 268 if st, ok := bb.scToSt[sc]; ok && st == grpcconnectivity.Ready { 269 scToAddr[sc] = addr 270 } 271 } 272 273 bb.picker = picker.New(picker.Config{ 274 Policy: bb.policy, 275 Logger: bb.lg, 276 SubConnToResolverAddress: scToAddr, 277 }) 278 bb.lg.Info( 279 "updated picker", 280 zap.String("picker", bb.picker.String()), 281 zap.String("balancer-id", bb.id), 282 zap.String("policy", bb.policy.String()), 283 zap.Strings("subconn-ready", scsToStrings(scToAddr)), 284 zap.Int("subconn-size", len(scToAddr)), 285 ) 286 } 287 288 // Close implements "grpc/balancer.Balancer" interface. 289 // Close is a nop because base balancer doesn't have internal state to clean up, 290 // and it doesn't need to call RemoveSubConn for the SubConns. 291 func (bb *baseBalancer) Close() { 292 // TODO 293 }