github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/rpc/breaker.go (about) 1 // Copyright 2016 The Cockroach Authors. 2 // 3 // Use of this software is governed by the Business Source License 4 // included in the file licenses/BSL.txt. 5 // 6 // As of the Change Date specified in that file, in accordance with 7 // the Business Source License, use of this software will be governed 8 // by the Apache License, Version 2.0, included in the file 9 // licenses/APL.txt. 10 11 package rpc 12 13 import ( 14 "context" 15 "time" 16 17 "github.com/cenkalti/backoff" 18 circuit "github.com/cockroachdb/circuitbreaker" 19 "github.com/cockroachdb/cockroach/pkg/util/hlc" 20 "github.com/cockroachdb/cockroach/pkg/util/log" 21 "github.com/facebookgo/clock" 22 ) 23 24 const maxBackoff = time.Second 25 26 // breakerClock is an implementation of clock.Clock that internally uses an 27 // hlc.Clock. It is used to bridge the hlc clock to the circuit breaker 28 // clocks. Note that it only implements the After() and Now() methods needed by 29 // circuit breakers and backoffs. 30 type breakerClock struct { 31 clock *hlc.Clock 32 } 33 34 func (c *breakerClock) After(d time.Duration) <-chan time.Time { 35 return time.After(d) 36 } 37 38 func (c *breakerClock) AfterFunc(d time.Duration, f func()) *clock.Timer { 39 panic("unimplemented") 40 } 41 42 func (c *breakerClock) Now() time.Time { 43 return c.clock.PhysicalTime() 44 } 45 46 func (c *breakerClock) Sleep(d time.Duration) { 47 panic("unimplemented") 48 } 49 50 func (c *breakerClock) Tick(d time.Duration) <-chan time.Time { 51 panic("unimplemented") 52 } 53 54 func (c *breakerClock) Ticker(d time.Duration) *clock.Ticker { 55 panic("unimplemented") 56 } 57 58 func (c *breakerClock) Timer(d time.Duration) *clock.Timer { 59 panic("unimplemented") 60 } 61 62 var _ clock.Clock = &breakerClock{} 63 64 // newBackOff creates a new exponential backoff properly configured for RPC 65 // connection backoff. 66 func newBackOff(clock backoff.Clock) backoff.BackOff { 67 // This exponential backoff limits the circuit breaker to 1 second 68 // intervals between successive attempts to resolve a node address 69 // and connect via GRPC. 70 // 71 // NB (nota Ben): MaxInterval should be less than the Raft election timeout 72 // (1.5s) to avoid disruptions. A newly restarted node will be in follower 73 // mode with no knowledge of the Raft leader. If it doesn't hear from a 74 // leader before the election timeout expires, it will start to campaign, 75 // which can be disruptive. Therefore the leader needs to get in touch (via 76 // Raft heartbeats) with such nodes within one election timeout of their 77 // restart, which won't happen if their backoff is too high. 78 b := &backoff.ExponentialBackOff{ 79 InitialInterval: 500 * time.Millisecond, 80 RandomizationFactor: 0.5, 81 Multiplier: 1.5, 82 MaxInterval: maxBackoff, 83 MaxElapsedTime: 0, 84 Clock: clock, 85 } 86 b.Reset() 87 return b 88 } 89 90 func newBreaker(ctx context.Context, name string, clock clock.Clock) *circuit.Breaker { 91 return circuit.NewBreakerWithOptions(&circuit.Options{ 92 Name: name, 93 BackOff: newBackOff(clock), 94 Clock: clock, 95 ShouldTrip: circuit.ThresholdTripFunc(1), 96 Logger: breakerLogger{ctx}, 97 }) 98 } 99 100 // breakerLogger implements circuit.Logger to expose logging from the 101 // circuitbreaker package. Debugf is logged with a vmodule level of 2 so to see 102 // the circuitbreaker debug messages set --vmodule=breaker=2 103 type breakerLogger struct { 104 ctx context.Context 105 } 106 107 func (r breakerLogger) Debugf(format string, v ...interface{}) { 108 if log.V(2) { 109 log.InfofDepth(r.ctx, 1, format, v...) 110 } 111 } 112 113 func (r breakerLogger) Infof(format string, v ...interface{}) { 114 log.InfofDepth(r.ctx, 1, format, v...) 115 }