vitess.io/vitess@v0.16.2/go/test/endtoend/tabletgateway/vtgate_test.go (about)

     1  /*
     2  Copyright 2020 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  
    16  This tests select/insert using the unshared keyspace added in main_test
    17  */
    18  package healthcheck
    19  
    20  import (
    21  	"context"
    22  	"encoding/json"
    23  	"fmt"
    24  	"io"
    25  	"net/http"
    26  	"reflect"
    27  	"strings"
    28  	"testing"
    29  	"time"
    30  
    31  	"vitess.io/vitess/go/test/endtoend/utils"
    32  
    33  	"github.com/stretchr/testify/assert"
    34  	"github.com/stretchr/testify/require"
    35  
    36  	"vitess.io/vitess/go/mysql"
    37  
    38  	"vitess.io/vitess/go/test/endtoend/cluster"
    39  )
    40  
    41  func TestVtgateHealthCheck(t *testing.T) {
    42  	defer cluster.PanicHandler(t)
    43  	// Healthcheck interval on tablet is set to 1s, so sleep for 2s
    44  	time.Sleep(2 * time.Second)
    45  	verifyVtgateVariables(t, clusterInstance.VtgateProcess.VerifyURL)
    46  	ctx := context.Background()
    47  	conn, err := mysql.Connect(ctx, &vtParams)
    48  	require.NoError(t, err)
    49  	defer conn.Close()
    50  
    51  	qr := utils.Exec(t, conn, "show vitess_tablets")
    52  	assert.Equal(t, 3, len(qr.Rows), "wrong number of results from show")
    53  }
    54  
    55  func TestVtgateReplicationStatusCheck(t *testing.T) {
    56  	defer cluster.PanicHandler(t)
    57  	// Healthcheck interval on tablet is set to 1s, so sleep for 2s
    58  	time.Sleep(2 * time.Second)
    59  	verifyVtgateVariables(t, clusterInstance.VtgateProcess.VerifyURL)
    60  	ctx := context.Background()
    61  	conn, err := mysql.Connect(ctx, &vtParams)
    62  	require.NoError(t, err)
    63  	defer conn.Close()
    64  
    65  	// Only returns rows for REPLICA and RDONLY tablets -- so should be 2 of them
    66  	qr := utils.Exec(t, conn, "show vitess_replication_status like '%'")
    67  	expectNumRows := 2
    68  	numRows := len(qr.Rows)
    69  	assert.Equal(t, expectNumRows, numRows, fmt.Sprintf("wrong number of results from show vitess_replication_status. Expected %d, got %d", expectNumRows, numRows))
    70  }
    71  
    72  func verifyVtgateVariables(t *testing.T, url string) {
    73  	resp, err := http.Get(url)
    74  	require.NoError(t, err)
    75  	defer resp.Body.Close()
    76  	require.Equal(t, 200, resp.StatusCode, "Vtgate api url response not found")
    77  
    78  	resultMap := make(map[string]any)
    79  	respByte, err := io.ReadAll(resp.Body)
    80  	require.NoError(t, err)
    81  	err = json.Unmarshal(respByte, &resultMap)
    82  	require.NoError(t, err)
    83  	assert.Contains(t, resultMap, "VtgateVSchemaCounts", "Vschema count should be present in variables")
    84  
    85  	vschemaCountMap := getMapFromJSON(resultMap, "VtgateVSchemaCounts")
    86  	assert.Contains(t, vschemaCountMap, "Reload", "Reload count should be present in vschemacount")
    87  
    88  	object := reflect.ValueOf(vschemaCountMap["Reload"])
    89  	assert.Greater(t, object.NumField(), 0, "Reload count should be greater than 0")
    90  	assert.NotContains(t, vschemaCountMap, "WatchError", "There should not be any WatchError in VschemaCount")
    91  	assert.NotContains(t, vschemaCountMap, "Parsing", "There should not be any Parsing in VschemaCount")
    92  	assert.Contains(t, resultMap, "HealthcheckConnections", "HealthcheckConnections count should be present in variables")
    93  
    94  	healthCheckConnection := getMapFromJSON(resultMap, "HealthcheckConnections")
    95  	assert.NotEmpty(t, healthCheckConnection, "Atleast one healthy tablet needs to be present")
    96  	assert.True(t, isPrimaryTabletPresent(healthCheckConnection), "Atleast one primary tablet needs to be present")
    97  }
    98  
    99  func retryNTimes(t *testing.T, maxRetries int, f func() bool) {
   100  	i := 0
   101  	for {
   102  		res := f()
   103  		if res {
   104  			return
   105  		}
   106  		if i > maxRetries {
   107  			t.Fatalf("retried %d times and failed", maxRetries)
   108  			return
   109  		}
   110  		i++
   111  	}
   112  }
   113  
   114  func TestReplicaTransactions(t *testing.T) {
   115  	// TODO(deepthi): this test seems to depend on previous test. Fix tearDown so that tests are independent
   116  	defer cluster.PanicHandler(t)
   117  	// Healthcheck interval on tablet is set to 1s, so sleep for 2s
   118  	time.Sleep(2 * time.Second)
   119  	ctx := context.Background()
   120  	writeConn, err := mysql.Connect(ctx, &vtParams)
   121  	require.NoError(t, err)
   122  	defer writeConn.Close()
   123  
   124  	readConn, err := mysql.Connect(ctx, &vtParams)
   125  	require.NoError(t, err)
   126  	defer readConn.Close()
   127  
   128  	readConn2, err := mysql.Connect(ctx, &vtParams)
   129  	require.NoError(t, err)
   130  	defer readConn2.Close()
   131  
   132  	fetchAllCustomers := "select id, email from customer"
   133  	checkCustomerRows := func(expectedRows int) func() bool {
   134  		return func() bool {
   135  			result := utils.Exec(t, readConn2, fetchAllCustomers)
   136  			return len(result.Rows) == expectedRows
   137  		}
   138  	}
   139  
   140  	// point the replica connections to the replica target
   141  	utils.Exec(t, readConn, "use @replica")
   142  	utils.Exec(t, readConn2, "use @replica")
   143  
   144  	// insert a row using primary
   145  	utils.Exec(t, writeConn, "insert into customer(id, email) values(1,'email1')")
   146  
   147  	// we'll run this query a number of times, and then give up if the row count never reaches this value
   148  	retryNTimes(t, 10 /*maxRetries*/, checkCustomerRows(1))
   149  
   150  	// after a short pause, SELECT the data inside a tx on a replica
   151  	// begin transaction on replica
   152  	utils.Exec(t, readConn, "begin")
   153  	qr := utils.Exec(t, readConn, fetchAllCustomers)
   154  	assert.Equal(t, `[[INT64(1) VARCHAR("email1")]]`, fmt.Sprintf("%v", qr.Rows), "select returned wrong result")
   155  
   156  	// insert more data on primary using a transaction
   157  	utils.Exec(t, writeConn, "begin")
   158  	utils.Exec(t, writeConn, "insert into customer(id, email) values(2,'email2')")
   159  	utils.Exec(t, writeConn, "commit")
   160  
   161  	retryNTimes(t, 10 /*maxRetries*/, checkCustomerRows(2))
   162  
   163  	// replica doesn't see new row because it is in a transaction
   164  	qr2 := utils.Exec(t, readConn, fetchAllCustomers)
   165  	assert.Equal(t, qr.Rows, qr2.Rows)
   166  
   167  	// replica should see new row after closing the transaction
   168  	utils.Exec(t, readConn, "commit")
   169  
   170  	qr3 := utils.Exec(t, readConn, fetchAllCustomers)
   171  	assert.Equal(t, `[[INT64(1) VARCHAR("email1")] [INT64(2) VARCHAR("email2")]]`, fmt.Sprintf("%v", qr3.Rows), "we are not seeing the updates after closing the replica transaction")
   172  
   173  	// begin transaction on replica
   174  	utils.Exec(t, readConn, "begin")
   175  	// try to delete a row, should fail
   176  	utils.AssertContainsError(t, readConn, "delete from customer where id=1", "supported only for primary tablet type, current type: replica")
   177  	utils.Exec(t, readConn, "commit")
   178  
   179  	// begin transaction on replica
   180  	utils.Exec(t, readConn, "begin")
   181  	// try to update a row, should fail
   182  	utils.AssertContainsError(t, readConn, "update customer set email='emailn' where id=1", "supported only for primary tablet type, current type: replica")
   183  	utils.Exec(t, readConn, "commit")
   184  
   185  	// begin transaction on replica
   186  	utils.Exec(t, readConn, "begin")
   187  	// try to insert a row, should fail
   188  	utils.AssertContainsError(t, readConn, "insert into customer(id, email) values(1,'email1')", "supported only for primary tablet type, current type: replica")
   189  	// call rollback just for fun
   190  	utils.Exec(t, readConn, "rollback")
   191  
   192  	// start another transaction
   193  	utils.Exec(t, readConn, "begin")
   194  	utils.Exec(t, readConn, fetchAllCustomers)
   195  	// bring down the tablet and try to select again
   196  	replicaTablet := clusterInstance.Keyspaces[0].Shards[0].Replica()
   197  	// this gives us a "signal: killed" error, ignore it
   198  	_ = replicaTablet.VttabletProcess.TearDown()
   199  	// Healthcheck interval on tablet is set to 1s, so sleep for 2s
   200  	time.Sleep(2 * time.Second)
   201  	utils.AssertContainsError(t, readConn, fetchAllCustomers, "is either down or nonexistent")
   202  
   203  	// bring up the tablet again
   204  	// trying to use the same session/transaction should fail as the vtgate has
   205  	// been restarted and the session lost
   206  	replicaTablet.VttabletProcess.ServingStatus = "SERVING"
   207  	err = replicaTablet.VttabletProcess.Setup()
   208  	require.NoError(t, err)
   209  	serving := replicaTablet.VttabletProcess.WaitForStatus("SERVING", 60*time.Second)
   210  	assert.Equal(t, serving, true, "Tablet did not become ready within a reasonable time")
   211  	utils.AssertContainsError(t, readConn, fetchAllCustomers, "not found")
   212  
   213  	// create a new connection, should be able to query again
   214  	readConn, err = mysql.Connect(ctx, &vtParams)
   215  	require.NoError(t, err)
   216  	utils.Exec(t, readConn, "begin")
   217  	qr4 := utils.Exec(t, readConn, fetchAllCustomers)
   218  	assert.Equal(t, `[[INT64(1) VARCHAR("email1")] [INT64(2) VARCHAR("email2")]]`, fmt.Sprintf("%v", qr4.Rows), "we are not able to reconnect after restart")
   219  }
   220  
   221  func getMapFromJSON(JSON map[string]any, key string) map[string]any {
   222  	result := make(map[string]any)
   223  	object := reflect.ValueOf(JSON[key])
   224  	if object.Kind() == reflect.Map {
   225  		for _, key := range object.MapKeys() {
   226  			value := object.MapIndex(key)
   227  			result[key.String()] = value
   228  		}
   229  	}
   230  	return result
   231  }
   232  
   233  func isPrimaryTabletPresent(tablets map[string]any) bool {
   234  	for key := range tablets {
   235  		if strings.Contains(key, "primary") {
   236  			return true
   237  		}
   238  	}
   239  	return false
   240  }