github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/kv/kvnemesis/kvnemesis_test.go (about)

     1  // Copyright 2020 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package kvnemesis
    12  
    13  import (
    14  	"context"
    15  	gosql "database/sql"
    16  	"fmt"
    17  	"testing"
    18  	"time"
    19  
    20  	"github.com/cockroachdb/cockroach/pkg/base"
    21  	"github.com/cockroachdb/cockroach/pkg/kv"
    22  	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
    23  	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
    24  	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
    25  	"github.com/cockroachdb/cockroach/pkg/util/log"
    26  	"github.com/cockroachdb/cockroach/pkg/util/randutil"
    27  	"github.com/stretchr/testify/require"
    28  )
    29  
    30  func TestKVNemesisSingleNode(t *testing.T) {
    31  	defer leaktest.AfterTest(t)()
    32  
    33  	ctx := context.Background()
    34  	tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{})
    35  	defer tc.Stopper().Stop(ctx)
    36  	db := tc.Server(0).DB()
    37  	sqlDB := tc.ServerConn(0)
    38  	sqlutils.MakeSQLRunner(sqlDB).Exec(t, `SET CLUSTER SETTING kv.rangefeed.enabled = true`)
    39  
    40  	config := NewDefaultConfig()
    41  	config.NumNodes, config.NumReplicas = 1, 1
    42  	rng, _ := randutil.NewPseudoRand()
    43  	ct := sqlClosedTimestampTargetInterval{sqlDBs: []*gosql.DB{sqlDB}}
    44  	failures, err := RunNemesis(ctx, rng, ct, config, db)
    45  	require.NoError(t, err, `%+v`, err)
    46  
    47  	for _, failure := range failures {
    48  		t.Errorf("failure:\n%+v", failure)
    49  	}
    50  }
    51  
    52  func TestKVNemesisMultiNode(t *testing.T) {
    53  	defer leaktest.AfterTest(t)()
    54  
    55  	// 4 nodes so we have somewhere to move 3x replicated ranges to.
    56  	const numNodes = 4
    57  	ctx := context.Background()
    58  	tc := testcluster.StartTestCluster(t, numNodes, base.TestClusterArgs{})
    59  	defer tc.Stopper().Stop(ctx)
    60  	dbs, sqlDBs := make([]*kv.DB, numNodes), make([]*gosql.DB, numNodes)
    61  	for i := 0; i < numNodes; i++ {
    62  		dbs[i] = tc.Server(i).DB()
    63  		sqlDBs[i] = tc.ServerConn(i)
    64  	}
    65  	sqlutils.MakeSQLRunner(sqlDBs[0]).Exec(t, `SET CLUSTER SETTING kv.rangefeed.enabled = true`)
    66  
    67  	config := NewDefaultConfig()
    68  	config.NumNodes, config.NumReplicas = numNodes, 3
    69  	// kvnemesis found a rare bug with closed timestamps when merges happen
    70  	// on a multinode cluster. Disable the combo for now to keep the test
    71  	// from flaking. See #44878.
    72  	config.Ops.Merge = MergeConfig{}
    73  	rng, _ := randutil.NewPseudoRand()
    74  	ct := sqlClosedTimestampTargetInterval{sqlDBs: sqlDBs}
    75  	failures, err := RunNemesis(ctx, rng, ct, config, dbs...)
    76  	require.NoError(t, err, `%+v`, err)
    77  
    78  	for _, failure := range failures {
    79  		t.Errorf("failure:\n%+v", failure)
    80  	}
    81  }
    82  
    83  type sqlClosedTimestampTargetInterval struct {
    84  	sqlDBs []*gosql.DB
    85  }
    86  
    87  func (x sqlClosedTimestampTargetInterval) Set(ctx context.Context, d time.Duration) error {
    88  	var err error
    89  	for i, sqlDB := range x.sqlDBs {
    90  		q := fmt.Sprintf(`SET CLUSTER SETTING kv.closed_timestamp.target_duration = '%s'`, d)
    91  		if _, err = sqlDB.Exec(q); err == nil {
    92  			return nil
    93  		}
    94  		log.Infof(ctx, "node %d could not set target duration: %+v", i, err)
    95  	}
    96  	log.Infof(ctx, "all nodes could not set target duration: %+v", err)
    97  	return err
    98  }
    99  
   100  func (x sqlClosedTimestampTargetInterval) ResetToDefault(ctx context.Context) error {
   101  	var err error
   102  	for i, sqlDB := range x.sqlDBs {
   103  		q := fmt.Sprintf(`SET CLUSTER SETTING kv.closed_timestamp.target_duration TO DEFAULT`)
   104  		if _, err = sqlDB.Exec(q); err == nil {
   105  			return nil
   106  		}
   107  		log.Infof(ctx, "node %d could not reset target duration: %+v", i, err)
   108  	}
   109  	log.Infof(ctx, "all nodes could not reset target duration: %+v", err)
   110  	return err
   111  }