github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvserver/client_atomic_membership_change_test.go (about)

     1  // Copyright 2019 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package kvserver_test
    12  
    13  import (
    14  	"context"
    15  	"strings"
    16  	"testing"
    17  
    18  	"github.com/cockroachdb/cockroach/pkg/base"
    19  	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
    20  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    21  	"github.com/cockroachdb/cockroach/pkg/testutils"
    22  	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
    23  	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
    24  	"github.com/cockroachdb/errors"
    25  	"github.com/kr/pretty"
    26  	"github.com/stretchr/testify/assert"
    27  	"github.com/stretchr/testify/require"
    28  	"go.etcd.io/etcd/raft/confchange"
    29  	"go.etcd.io/etcd/raft/tracker"
    30  )
    31  
    32  // TestAtomicReplicationChange is a simple smoke test for atomic membership
    33  // changes.
    34  func TestAtomicReplicationChange(t *testing.T) {
    35  	defer leaktest.AfterTest(t)()
    36  	ctx := context.Background()
    37  
    38  	args := base.TestClusterArgs{
    39  		ServerArgs: base.TestServerArgs{
    40  			Knobs: base.TestingKnobs{
    41  				Store: &kvserver.StoreTestingKnobs{},
    42  			},
    43  		},
    44  		ReplicationMode: base.ReplicationManual,
    45  	}
    46  	tc := testcluster.StartTestCluster(t, 6, args)
    47  	defer tc.Stopper().Stop(ctx)
    48  
    49  	_, err := tc.ServerConn(0).Exec(`SET CLUSTER SETTING kv.atomic_replication_changes.enabled = true`)
    50  	require.NoError(t, err)
    51  
    52  	// Create a range and put it on n1, n2, n3. Intentionally do this one at a
    53  	// time so we're not using atomic replication changes yet.
    54  	k := tc.ScratchRange(t)
    55  	desc, err := tc.AddReplicas(k, tc.Target(1))
    56  	require.NoError(t, err)
    57  	desc, err = tc.AddReplicas(k, tc.Target(2))
    58  	require.NoError(t, err)
    59  
    60  	runChange := func(expDesc roachpb.RangeDescriptor, chgs []roachpb.ReplicationChange) roachpb.RangeDescriptor {
    61  		t.Helper()
    62  		desc, err := tc.Servers[0].DB().AdminChangeReplicas(ctx, k, expDesc, chgs)
    63  		require.NoError(t, err)
    64  
    65  		return *desc
    66  	}
    67  
    68  	checkDesc := func(desc roachpb.RangeDescriptor, expStores ...roachpb.StoreID) {
    69  		testutils.SucceedsSoon(t, func() error {
    70  			var sawStores []roachpb.StoreID
    71  			for _, s := range tc.Servers {
    72  				r, _, _ := s.Stores().GetReplicaForRangeID(desc.RangeID)
    73  				if r == nil {
    74  					continue
    75  				}
    76  				if _, found := desc.GetReplicaDescriptor(r.StoreID()); !found {
    77  					// There's a replica but it's not in the new descriptor, so
    78  					// it should be replicaGC'ed soon.
    79  					return errors.Errorf("%s should have been removed", r)
    80  				}
    81  				sawStores = append(sawStores, r.StoreID())
    82  				// Check that in-mem descriptor of repl is up-to-date.
    83  				if diff := pretty.Diff(&desc, r.Desc()); len(diff) > 0 {
    84  					return errors.Errorf("diff(want, have):\n%s", strings.Join(diff, "\n"))
    85  				}
    86  				// Check that conf state is up to date. This can fail even though
    87  				// the descriptor already matches since the descriptor is updated
    88  				// a hair earlier.
    89  				cfg, _, err := confchange.Restore(confchange.Changer{
    90  					Tracker:   tracker.MakeProgressTracker(1),
    91  					LastIndex: 1,
    92  				}, desc.Replicas().ConfState())
    93  				require.NoError(t, err)
    94  				act := r.RaftStatus().Config.Voters
    95  				if diff := pretty.Diff(cfg.Voters, act); len(diff) > 0 {
    96  					return errors.Errorf("diff(exp,act):\n%s", strings.Join(diff, "\n"))
    97  				}
    98  			}
    99  			assert.Equal(t, expStores, sawStores)
   100  			return nil
   101  		})
   102  	}
   103  
   104  	// Run a fairly general change.
   105  	desc = runChange(desc, []roachpb.ReplicationChange{
   106  		{ChangeType: roachpb.ADD_REPLICA, Target: tc.Target(3)},
   107  		{ChangeType: roachpb.ADD_REPLICA, Target: tc.Target(5)},
   108  		{ChangeType: roachpb.REMOVE_REPLICA, Target: tc.Target(2)},
   109  		{ChangeType: roachpb.ADD_REPLICA, Target: tc.Target(4)},
   110  	})
   111  
   112  	// Replicas should now live on all stores except s3.
   113  	checkDesc(desc, 1, 2, 4, 5, 6)
   114  
   115  	// Transfer the lease to s5.
   116  	require.NoError(t, tc.TransferRangeLease(desc, tc.Target(4)))
   117  
   118  	// Rebalance back down all the way.
   119  	desc = runChange(desc, []roachpb.ReplicationChange{
   120  		{ChangeType: roachpb.REMOVE_REPLICA, Target: tc.Target(0)},
   121  		{ChangeType: roachpb.REMOVE_REPLICA, Target: tc.Target(1)},
   122  		{ChangeType: roachpb.REMOVE_REPLICA, Target: tc.Target(3)},
   123  		{ChangeType: roachpb.REMOVE_REPLICA, Target: tc.Target(5)},
   124  	})
   125  
   126  	// Only a lone voter on s5 should be left over.
   127  	checkDesc(desc, 5)
   128  }