github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/rpc/breaker.go (about)

     1  // Copyright 2016 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package rpc
    12  
    13  import (
    14  	"context"
    15  	"time"
    16  
    17  	"github.com/cenkalti/backoff"
    18  	circuit "github.com/cockroachdb/circuitbreaker"
    19  	"github.com/cockroachdb/cockroach/pkg/util/hlc"
    20  	"github.com/cockroachdb/cockroach/pkg/util/log"
    21  	"github.com/facebookgo/clock"
    22  )
    23  
    24  const maxBackoff = time.Second
    25  
    26  // breakerClock is an implementation of clock.Clock that internally uses an
    27  // hlc.Clock. It is used to bridge the hlc clock to the circuit breaker
    28  // clocks. Note that it only implements the After() and Now() methods needed by
    29  // circuit breakers and backoffs.
    30  type breakerClock struct {
    31  	clock *hlc.Clock
    32  }
    33  
    34  func (c *breakerClock) After(d time.Duration) <-chan time.Time {
    35  	return time.After(d)
    36  }
    37  
    38  func (c *breakerClock) AfterFunc(d time.Duration, f func()) *clock.Timer {
    39  	panic("unimplemented")
    40  }
    41  
    42  func (c *breakerClock) Now() time.Time {
    43  	return c.clock.PhysicalTime()
    44  }
    45  
    46  func (c *breakerClock) Sleep(d time.Duration) {
    47  	panic("unimplemented")
    48  }
    49  
    50  func (c *breakerClock) Tick(d time.Duration) <-chan time.Time {
    51  	panic("unimplemented")
    52  }
    53  
    54  func (c *breakerClock) Ticker(d time.Duration) *clock.Ticker {
    55  	panic("unimplemented")
    56  }
    57  
    58  func (c *breakerClock) Timer(d time.Duration) *clock.Timer {
    59  	panic("unimplemented")
    60  }
    61  
    62  var _ clock.Clock = &breakerClock{}
    63  
    64  // newBackOff creates a new exponential backoff properly configured for RPC
    65  // connection backoff.
    66  func newBackOff(clock backoff.Clock) backoff.BackOff {
    67  	// This exponential backoff limits the circuit breaker to 1 second
    68  	// intervals between successive attempts to resolve a node address
    69  	// and connect via GRPC.
    70  	//
    71  	// NB (nota Ben): MaxInterval should be less than the Raft election timeout
    72  	// (1.5s) to avoid disruptions. A newly restarted node will be in follower
    73  	// mode with no knowledge of the Raft leader. If it doesn't hear from a
    74  	// leader before the election timeout expires, it will start to campaign,
    75  	// which can be disruptive. Therefore the leader needs to get in touch (via
    76  	// Raft heartbeats) with such nodes within one election timeout of their
    77  	// restart, which won't happen if their backoff is too high.
    78  	b := &backoff.ExponentialBackOff{
    79  		InitialInterval:     500 * time.Millisecond,
    80  		RandomizationFactor: 0.5,
    81  		Multiplier:          1.5,
    82  		MaxInterval:         maxBackoff,
    83  		MaxElapsedTime:      0,
    84  		Clock:               clock,
    85  	}
    86  	b.Reset()
    87  	return b
    88  }
    89  
    90  func newBreaker(ctx context.Context, name string, clock clock.Clock) *circuit.Breaker {
    91  	return circuit.NewBreakerWithOptions(&circuit.Options{
    92  		Name:       name,
    93  		BackOff:    newBackOff(clock),
    94  		Clock:      clock,
    95  		ShouldTrip: circuit.ThresholdTripFunc(1),
    96  		Logger:     breakerLogger{ctx},
    97  	})
    98  }
    99  
   100  // breakerLogger implements circuit.Logger to expose logging from the
   101  // circuitbreaker package. Debugf is logged with a vmodule level of 2 so to see
   102  // the circuitbreaker debug messages set --vmodule=breaker=2
   103  type breakerLogger struct {
   104  	ctx context.Context
   105  }
   106  
   107  func (r breakerLogger) Debugf(format string, v ...interface{}) {
   108  	if log.V(2) {
   109  		log.InfofDepth(r.ctx, 1, format, v...)
   110  	}
   111  }
   112  
   113  func (r breakerLogger) Infof(format string, v ...interface{}) {
   114  	log.InfofDepth(r.ctx, 1, format, v...)
   115  }