github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvserver/split_delay_helper_test.go (about)

     1  // Copyright 2018 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package kvserver
    12  
    13  import (
    14  	"context"
    15  	"testing"
    16  	"time"
    17  
    18  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    19  	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
    20  	"github.com/stretchr/testify/assert"
    21  	"go.etcd.io/etcd/raft"
    22  	"go.etcd.io/etcd/raft/tracker"
    23  )
    24  
    25  type testSplitDelayHelper struct {
    26  	numAttempts int
    27  
    28  	rangeID    roachpb.RangeID
    29  	raftStatus *raft.Status
    30  	sleep      func()
    31  
    32  	slept, emptyProposed int
    33  }
    34  
    35  func (h *testSplitDelayHelper) RaftStatus(context.Context) (roachpb.RangeID, *raft.Status) {
    36  	return h.rangeID, h.raftStatus
    37  }
    38  func (h *testSplitDelayHelper) ProposeEmptyCommand(ctx context.Context) {
    39  	h.emptyProposed++
    40  }
    41  func (h *testSplitDelayHelper) NumAttempts() int {
    42  	return h.numAttempts
    43  }
    44  func (h *testSplitDelayHelper) Sleep(context.Context) time.Duration {
    45  	if h.sleep != nil {
    46  		h.sleep()
    47  	}
    48  	h.slept++
    49  	return time.Second
    50  }
    51  
    52  var _ splitDelayHelperI = (*testSplitDelayHelper)(nil)
    53  
    54  func TestSplitDelayToAvoidSnapshot(t *testing.T) {
    55  	defer leaktest.AfterTest(t)()
    56  
    57  	ctx := context.Background()
    58  
    59  	t.Run("disabled", func(t *testing.T) {
    60  		// Should immediately bail out if told to run zero attempts.
    61  		h := &testSplitDelayHelper{
    62  			numAttempts: 0,
    63  			rangeID:     1,
    64  			raftStatus:  nil,
    65  		}
    66  		s := maybeDelaySplitToAvoidSnapshot(ctx, h)
    67  		assert.Equal(t, "", s)
    68  		assert.Equal(t, 0, h.slept)
    69  	})
    70  
    71  	t.Run("follower", func(t *testing.T) {
    72  		// Should immediately bail out if run on non-leader.
    73  		h := &testSplitDelayHelper{
    74  			numAttempts: 5,
    75  			rangeID:     1,
    76  			raftStatus:  nil,
    77  		}
    78  		s := maybeDelaySplitToAvoidSnapshot(ctx, h)
    79  		assert.Equal(t, "; not Raft leader", s)
    80  		assert.Equal(t, 0, h.slept)
    81  	})
    82  
    83  	t.Run("inactive", func(t *testing.T) {
    84  		h := &testSplitDelayHelper{
    85  			numAttempts: 5,
    86  			rangeID:     1,
    87  			raftStatus: &raft.Status{
    88  				Progress: map[uint64]tracker.Progress{
    89  					2: {State: tracker.StateProbe},
    90  				},
    91  			},
    92  		}
    93  		s := maybeDelaySplitToAvoidSnapshot(ctx, h)
    94  		// We try to wake up the follower once, but then give up on it.
    95  		assert.Equal(t, "; r1/2 inactive; delayed split for 1.0s to avoid Raft snapshot", s)
    96  		assert.Equal(t, 1, h.slept)
    97  		assert.Equal(t, 1, h.emptyProposed)
    98  	})
    99  
   100  	for _, state := range []tracker.StateType{tracker.StateProbe, tracker.StateSnapshot} {
   101  		t.Run(state.String(), func(t *testing.T) {
   102  			h := &testSplitDelayHelper{
   103  				numAttempts: 5,
   104  				rangeID:     1,
   105  				raftStatus: &raft.Status{
   106  					Progress: map[uint64]tracker.Progress{
   107  						2: {
   108  							State:        state,
   109  							RecentActive: true,
   110  							ProbeSent:    true, // Unifies string output below.
   111  							Inflights:    &tracker.Inflights{},
   112  						},
   113  						// Healthy follower just for kicks.
   114  						3: {State: tracker.StateReplicate},
   115  					},
   116  				},
   117  			}
   118  			s := maybeDelaySplitToAvoidSnapshot(ctx, h)
   119  			assert.Equal(t, "; replica r1/2 not caught up: "+state.String()+
   120  				" match=0 next=0 paused; delayed split for 5.0s to avoid Raft snapshot (without success)", s)
   121  			assert.Equal(t, 5, h.slept)
   122  			assert.Equal(t, 5, h.emptyProposed)
   123  		})
   124  	}
   125  
   126  	t.Run("immediately-replicating", func(t *testing.T) {
   127  		h := &testSplitDelayHelper{
   128  			numAttempts: 5,
   129  			rangeID:     1,
   130  			raftStatus: &raft.Status{
   131  				Progress: map[uint64]tracker.Progress{
   132  					2: {State: tracker.StateReplicate}, // intentionally not recently active
   133  				},
   134  			},
   135  		}
   136  		s := maybeDelaySplitToAvoidSnapshot(ctx, h)
   137  		assert.Equal(t, "", s)
   138  		assert.Equal(t, 0, h.slept)
   139  		assert.Equal(t, 0, h.emptyProposed)
   140  	})
   141  
   142  	t.Run("becomes-replicating", func(t *testing.T) {
   143  		h := &testSplitDelayHelper{
   144  			numAttempts: 5,
   145  			rangeID:     1,
   146  			raftStatus: &raft.Status{
   147  				Progress: map[uint64]tracker.Progress{
   148  					2: {State: tracker.StateProbe, RecentActive: true, Inflights: &tracker.Inflights{}},
   149  				},
   150  			},
   151  		}
   152  		// The fourth attempt will see the follower catch up.
   153  		h.sleep = func() {
   154  			if h.slept == 2 {
   155  				pr := h.raftStatus.Progress[2]
   156  				pr.State = tracker.StateReplicate
   157  				h.raftStatus.Progress[2] = pr
   158  			}
   159  		}
   160  		s := maybeDelaySplitToAvoidSnapshot(ctx, h)
   161  		assert.Equal(t, "; delayed split for 3.0s to avoid Raft snapshot", s)
   162  		assert.Equal(t, 3, h.slept)
   163  		assert.Equal(t, 3, h.emptyProposed)
   164  	})
   165  }