github.com/m3db/m3@v1.5.0/src/dbnode/integration/write_quorum_test.go (about)

     1  // +build integration
     2  
     3  // Copyright (c) 2016 Uber Technologies, Inc.
     4  //
     5  // Permission is hereby granted, free of charge, to any person obtaining a copy
     6  // of this software and associated documentation files (the "Software"), to deal
     7  // in the Software without restriction, including without limitation the rights
     8  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     9  // copies of the Software, and to permit persons to whom the Software is
    10  // furnished to do so, subject to the following conditions:
    11  //
    12  // The above copyright notice and this permission notice shall be included in
    13  // all copies or substantial portions of the Software.
    14  //
    15  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    16  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    17  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    18  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    19  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    20  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    21  // THE SOFTWARE.
    22  
    23  package integration
    24  
    25  import (
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/m3db/m3/src/cluster/services"
    30  	"github.com/m3db/m3/src/cluster/shard"
    31  	"github.com/m3db/m3/src/dbnode/client"
    32  	"github.com/m3db/m3/src/dbnode/integration/generate"
    33  	"github.com/m3db/m3/src/dbnode/namespace"
    34  	"github.com/m3db/m3/src/dbnode/topology"
    35  	"github.com/m3db/m3/src/x/ident"
    36  	xtime "github.com/m3db/m3/src/x/time"
    37  
    38  	"github.com/stretchr/testify/assert"
    39  	"github.com/stretchr/testify/require"
    40  )
    41  
    42  func TestNormalQuorumOnlyOneUp(t *testing.T) {
    43  	if testing.Short() {
    44  		t.SkipNow()
    45  	}
    46  
    47  	numShards := defaultNumShards
    48  	minShard := uint32(0)
    49  	maxShard := uint32(numShards - 1)
    50  
    51  	// nodes = m3db nodes
    52  	nodes, closeFn, testWrite := makeTestWrite(t, numShards, []services.ServiceInstance{
    53  		node(t, 0, newClusterShardsRange(minShard, maxShard, shard.Available)),
    54  		node(t, 1, newClusterShardsRange(minShard, maxShard, shard.Available)),
    55  		node(t, 2, newClusterShardsRange(minShard, maxShard, shard.Available)),
    56  	})
    57  	defer closeFn()
    58  
    59  	require.NoError(t, nodes[0].StartServer())
    60  	defer func() { require.NoError(t, nodes[0].StopServer()) }()
    61  
    62  	// Writes succeed to one node
    63  	assert.NoError(t, testWrite(topology.ConsistencyLevelOne))
    64  	assert.Error(t, testWrite(topology.ConsistencyLevelMajority))
    65  	assert.Error(t, testWrite(topology.ConsistencyLevelAll))
    66  }
    67  
    68  func TestNormalQuorumOnlyTwoUp(t *testing.T) {
    69  	if testing.Short() {
    70  		t.SkipNow()
    71  	}
    72  
    73  	numShards := defaultNumShards
    74  	minShard := uint32(0)
    75  	maxShard := uint32(numShards - 1)
    76  
    77  	// nodes = m3db nodes
    78  	nodes, closeFn, testWrite := makeTestWrite(t, numShards, []services.ServiceInstance{
    79  		node(t, 0, newClusterShardsRange(minShard, maxShard, shard.Available)),
    80  		node(t, 1, newClusterShardsRange(minShard, maxShard, shard.Available)),
    81  		node(t, 2, newClusterShardsRange(minShard, maxShard, shard.Available)),
    82  	})
    83  	defer closeFn()
    84  
    85  	require.NoError(t, nodes[0].StartServer())
    86  	defer func() { require.NoError(t, nodes[0].StopServer()) }()
    87  	require.NoError(t, nodes[1].StartServer())
    88  	defer func() { require.NoError(t, nodes[1].StopServer()) }()
    89  
    90  	// Writes succeed to two nodes
    91  	assert.NoError(t, testWrite(topology.ConsistencyLevelOne))
    92  	assert.NoError(t, testWrite(topology.ConsistencyLevelMajority))
    93  	assert.Error(t, testWrite(topology.ConsistencyLevelAll))
    94  }
    95  
    96  func TestNormalQuorumAllUp(t *testing.T) {
    97  	if testing.Short() {
    98  		t.SkipNow()
    99  	}
   100  
   101  	numShards := defaultNumShards
   102  	minShard := uint32(0)
   103  	maxShard := uint32(numShards - 1)
   104  
   105  	// nodes = m3db nodes
   106  	nodes, closeFn, testWrite := makeTestWrite(t, numShards, []services.ServiceInstance{
   107  		node(t, 0, newClusterShardsRange(minShard, maxShard, shard.Available)),
   108  		node(t, 1, newClusterShardsRange(minShard, maxShard, shard.Available)),
   109  		node(t, 2, newClusterShardsRange(minShard, maxShard, shard.Available)),
   110  	})
   111  	defer closeFn()
   112  
   113  	require.NoError(t, nodes[0].StartServer())
   114  	defer func() { require.NoError(t, nodes[0].StopServer()) }()
   115  	require.NoError(t, nodes[1].StartServer())
   116  	defer func() { require.NoError(t, nodes[1].StopServer()) }()
   117  	require.NoError(t, nodes[2].StartServer())
   118  	defer func() { require.NoError(t, nodes[2].StopServer()) }()
   119  
   120  	// Writes succeed to all nodes
   121  	assert.NoError(t, testWrite(topology.ConsistencyLevelOne))
   122  	assert.NoError(t, testWrite(topology.ConsistencyLevelMajority))
   123  	assert.NoError(t, testWrite(topology.ConsistencyLevelAll))
   124  }
   125  
   126  func TestAddNodeQuorumOnlyLeavingInitializingUp(t *testing.T) {
   127  	if testing.Short() {
   128  		t.SkipNow()
   129  	}
   130  
   131  	numShards := defaultNumShards
   132  	minShard := uint32(0)
   133  	maxShard := uint32(numShards - 1)
   134  
   135  	// nodes = m3db nodes
   136  	nodes, closeFn, testWrite := makeTestWrite(t, numShards, []services.ServiceInstance{
   137  		node(t, 0, newClusterShardsRange(minShard, maxShard, shard.Leaving)),
   138  		node(t, 1, newClusterShardsRange(minShard, maxShard, shard.Available)),
   139  		node(t, 2, newClusterShardsRange(minShard, maxShard, shard.Available)),
   140  		node(t, 3, newClusterShardsRange(minShard, maxShard, shard.Initializing)),
   141  	})
   142  	defer closeFn()
   143  
   144  	require.NoError(t, nodes[0].StartServer())
   145  	defer func() { require.NoError(t, nodes[0].StopServer()) }()
   146  
   147  	require.NoError(t, nodes[3].StartServerDontWaitBootstrap())
   148  	defer func() { require.NoError(t, nodes[3].StopServer()) }()
   149  
   150  	// No writes succeed to available nodes
   151  	assert.Error(t, testWrite(topology.ConsistencyLevelOne))
   152  	assert.Error(t, testWrite(topology.ConsistencyLevelMajority))
   153  	assert.Error(t, testWrite(topology.ConsistencyLevelAll))
   154  }
   155  
   156  func TestAddNodeQuorumOnlyOneNormalAndLeavingInitializingUp(t *testing.T) {
   157  	if testing.Short() {
   158  		t.SkipNow()
   159  	}
   160  
   161  	numShards := defaultNumShards
   162  	minShard := uint32(0)
   163  	maxShard := uint32(numShards - 1)
   164  
   165  	// nodes = m3db nodes
   166  	nodes, closeFn, testWrite := makeTestWrite(t, numShards, []services.ServiceInstance{
   167  		node(t, 0, newClusterShardsRange(minShard, maxShard, shard.Leaving)),
   168  		node(t, 1, newClusterShardsRange(minShard, maxShard, shard.Available)),
   169  		node(t, 2, newClusterShardsRange(minShard, maxShard, shard.Available)),
   170  		node(t, 3, newClusterShardsRange(minShard, maxShard, shard.Initializing)),
   171  	})
   172  	defer closeFn()
   173  
   174  	require.NoError(t, nodes[0].StartServer())
   175  	defer func() { require.NoError(t, nodes[0].StopServer()) }()
   176  	require.NoError(t, nodes[1].StartServer())
   177  	defer func() { require.NoError(t, nodes[1].StopServer()) }()
   178  	require.NoError(t, nodes[3].StartServerDontWaitBootstrap())
   179  	defer func() { require.NoError(t, nodes[3].StopServer()) }()
   180  
   181  	// Writes succeed to one available node
   182  	assert.NoError(t, testWrite(topology.ConsistencyLevelOne))
   183  	assert.Error(t, testWrite(topology.ConsistencyLevelMajority))
   184  	assert.Error(t, testWrite(topology.ConsistencyLevelAll))
   185  }
   186  
   187  func TestAddNodeQuorumAllUp(t *testing.T) {
   188  	if testing.Short() {
   189  		t.SkipNow()
   190  	}
   191  
   192  	numShards := defaultNumShards
   193  	minShard := uint32(0)
   194  	maxShard := uint32(numShards - 1)
   195  
   196  	// nodes = m3db nodes
   197  	nodes, closeFn, testWrite := makeTestWrite(t, numShards, []services.ServiceInstance{
   198  		node(t, 0, newClusterShardsRange(minShard, maxShard, shard.Leaving)),
   199  		node(t, 1, newClusterShardsRange(minShard, maxShard, shard.Available)),
   200  		node(t, 2, newClusterShardsRange(minShard, maxShard, shard.Available)),
   201  		node(t, 3, newClusterShardsRange(minShard, maxShard, shard.Initializing)),
   202  	})
   203  	defer closeFn()
   204  
   205  	require.NoError(t, nodes[0].StartServer())
   206  	defer func() { require.NoError(t, nodes[0].StopServer()) }()
   207  	require.NoError(t, nodes[1].StartServer())
   208  	defer func() { require.NoError(t, nodes[1].StopServer()) }()
   209  	require.NoError(t, nodes[2].StartServer())
   210  	defer func() { require.NoError(t, nodes[2].StopServer()) }()
   211  	require.NoError(t, nodes[3].StartServerDontWaitBootstrap())
   212  	defer func() { require.NoError(t, nodes[3].StopServer()) }()
   213  
   214  	// Writes succeed to two available nodes
   215  	assert.NoError(t, testWrite(topology.ConsistencyLevelOne))
   216  	assert.NoError(t, testWrite(topology.ConsistencyLevelMajority))
   217  	assert.Error(t, testWrite(topology.ConsistencyLevelAll))
   218  }
   219  
   220  type testWriteFn func(topology.ConsistencyLevel) error
   221  
   222  func makeTestWrite(
   223  	t *testing.T,
   224  	numShards int,
   225  	instances []services.ServiceInstance,
   226  ) (testSetups, closeFn, testWriteFn) {
   227  	nsOpts := namespace.NewOptions()
   228  	md, err := namespace.NewMetadata(testNamespaces[0],
   229  		nsOpts.SetRetentionOptions(nsOpts.RetentionOptions().SetRetentionPeriod(6*time.Hour)))
   230  	require.NoError(t, err)
   231  
   232  	nspaces := []namespace.Metadata{md}
   233  	nodes, topoInit, closeFn := newNodes(t, numShards, instances, nspaces, false)
   234  	now := nodes[0].NowFn()()
   235  
   236  	for _, node := range nodes {
   237  		node.SetOpts(node.Opts().SetNumShards(numShards))
   238  		for _, ns := range node.Namespaces() {
   239  			// write empty data files to disk so nodes could bootstrap
   240  			require.NoError(t, writeTestDataToDisk(ns, node, generate.SeriesBlocksByStart{}, 0))
   241  		}
   242  	}
   243  
   244  	clientopts := client.NewOptions().
   245  		SetClusterConnectConsistencyLevel(topology.ConnectConsistencyLevelNone).
   246  		SetClusterConnectTimeout(2 * time.Second).
   247  		SetWriteRequestTimeout(2 * time.Second).
   248  		SetTopologyInitializer(topoInit)
   249  
   250  	testWrite := func(cLevel topology.ConsistencyLevel) error {
   251  		clientopts = clientopts.SetWriteConsistencyLevel(cLevel)
   252  		c, err := client.NewClient(clientopts)
   253  		require.NoError(t, err)
   254  
   255  		s, err := c.NewSession()
   256  		require.NoError(t, err)
   257  
   258  		return s.Write(nspaces[0].ID(), ident.StringID("quorumTest"), now, 42, xtime.Second, nil)
   259  	}
   260  
   261  	return nodes, closeFn, testWrite
   262  }