vitess.io/vitess@v0.16.2/go/test/endtoend/tabletgateway/buffer/reshard/sharded_buffer_test.go (about)

     1  /*
     2  Copyright 2021 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package reshard
    18  
    19  import (
    20  	"fmt"
    21  	"testing"
    22  	"time"
    23  
    24  	"github.com/buger/jsonparser"
    25  
    26  	"vitess.io/vitess/go/vt/log"
    27  
    28  	"github.com/stretchr/testify/assert"
    29  	"github.com/stretchr/testify/require"
    30  
    31  	"vitess.io/vitess/go/test/endtoend/tabletgateway/buffer"
    32  
    33  	"vitess.io/vitess/go/test/endtoend/cluster"
    34  )
    35  
    36  const (
    37  	maxWait              = 10 * time.Second
    38  	acceptableLagSeconds = 5
    39  )
    40  
    41  func waitForLowLag(t *testing.T, clusterInstance *cluster.LocalProcessCluster, keyspace, workflow string) {
    42  	var lagSeconds int64
    43  	waitDuration := 500 * time.Millisecond
    44  	duration := maxWait
    45  	for duration > 0 {
    46  		output, err := clusterInstance.VtctlclientProcess.ExecuteCommandWithOutput("Workflow", fmt.Sprintf("%s.%s", keyspace, workflow), "Show")
    47  		require.NoError(t, err)
    48  		lagSeconds, err = jsonparser.GetInt([]byte(output), "MaxVReplicationTransactionLag")
    49  
    50  		require.NoError(t, err)
    51  		if lagSeconds <= acceptableLagSeconds {
    52  			log.Infof("waitForLowLag acceptable for workflow %s, keyspace %s, current lag is %d", workflow, keyspace, lagSeconds)
    53  			break
    54  		} else {
    55  			log.Infof("waitForLowLag too high for workflow %s, keyspace %s, current lag is %d", workflow, keyspace, lagSeconds)
    56  		}
    57  		time.Sleep(waitDuration)
    58  		duration -= waitDuration
    59  	}
    60  
    61  	if duration <= 0 {
    62  		t.Fatalf("waitForLowLag timed out for workflow %s, keyspace %s, current lag is %d", workflow, keyspace, lagSeconds)
    63  	}
    64  }
    65  
    66  func reshard02(t *testing.T, clusterInstance *cluster.LocalProcessCluster, keyspaceName string, reads, writes buffer.QueryEngine) {
    67  	keyspace := &cluster.Keyspace{Name: keyspaceName}
    68  	err := clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, false)
    69  	require.NoError(t, err)
    70  	workflowName := "buf2buf"
    71  	workflow := fmt.Sprintf("%s.%s", keyspaceName, "buf2buf")
    72  
    73  	err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "--", "--source_shards", "0", "--target_shards", "-80,80-", "Create", workflow)
    74  	require.NoError(t, err)
    75  
    76  	// Execute the resharding operation
    77  	reads.ExpectQueries(25)
    78  	writes.ExpectQueries(25)
    79  
    80  	waitForLowLag(t, clusterInstance, keyspaceName, workflowName)
    81  	err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "--", "--tablet_types=rdonly,replica", "SwitchTraffic", workflow)
    82  	require.NoError(t, err)
    83  
    84  	err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "--", "--tablet_types=primary", "SwitchTraffic", workflow)
    85  	require.NoError(t, err)
    86  
    87  	err = clusterInstance.VtctlclientProcess.ExecuteCommand("Reshard", "Complete", workflow)
    88  	require.NoError(t, err)
    89  }
    90  
    91  const vschema = `{
    92     "sharded": true,
    93     "vindexes": {
    94       "hash_index": {
    95         "type": "hash"
    96       }
    97     },
    98     "tables": {
    99       "buffer": {
   100          "column_vindexes": [
   101           {
   102             "column": "id",
   103             "name": "hash_index"
   104           }
   105         ]
   106       }
   107     }
   108   }`
   109  
   110  func assertResharding(t *testing.T, shard string, stats *buffer.VTGateBufferingStats) {
   111  	stopLabel := fmt.Sprintf("%s.%s", shard, "ReshardingComplete")
   112  
   113  	assert.Greater(t, stats.BufferFailoverDurationSumMs[shard], 0)
   114  	assert.Greater(t, stats.BufferRequestsBuffered[shard], 0)
   115  	assert.Greater(t, stats.BufferStops[stopLabel], 0)
   116  }
   117  
   118  func TestBufferResharding(t *testing.T) {
   119  	t.Run("slow queries", func(t *testing.T) {
   120  		bt := &buffer.BufferingTest{
   121  			Assert:      assertResharding,
   122  			Failover:    reshard02,
   123  			SlowQueries: true,
   124  			VSchema:     vschema,
   125  		}
   126  		bt.Test(t)
   127  	})
   128  
   129  	t.Run("fast queries", func(t *testing.T) {
   130  		bt := &buffer.BufferingTest{
   131  			Assert:      assertResharding,
   132  			Failover:    reshard02,
   133  			SlowQueries: false,
   134  			VSchema:     vschema,
   135  		}
   136  		bt.Test(t)
   137  	})
   138  }