vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/vreplication/controller_test.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package vreplication
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"testing"
    24  	"time"
    25  
    26  	querypb "vitess.io/vitess/go/vt/proto/query"
    27  
    28  	"vitess.io/vitess/go/sqltypes"
    29  	"vitess.io/vitess/go/sync2"
    30  	"vitess.io/vitess/go/vt/binlog/binlogplayer"
    31  	"vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon"
    32  	"vitess.io/vitess/go/vt/mysqlctl/tmutils"
    33  
    34  	tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
    35  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    36  )
    37  
    38  var (
    39  	testSettingsResponse = &sqltypes.Result{
    40  		Fields: []*querypb.Field{
    41  			{Name: "pos", Type: sqltypes.VarBinary},
    42  			{Name: "stop_pos", Type: sqltypes.VarBinary},
    43  			{Name: "max_tps", Type: sqltypes.Int64},
    44  			{Name: "max_replication_lag", Type: sqltypes.Int64},
    45  			{Name: "state", Type: sqltypes.VarBinary},
    46  			{Name: "workflow_type", Type: sqltypes.Int64},
    47  			{Name: "workflow", Type: sqltypes.VarChar},
    48  			{Name: "workflow_sub_type", Type: sqltypes.Int64},
    49  			{Name: "defer_secondary_keys", Type: sqltypes.Int64},
    50  		},
    51  		InsertID: 0,
    52  		Rows: [][]sqltypes.Value{
    53  			{
    54  				sqltypes.NewVarBinary("MariaDB/0-1-1083"), // pos
    55  				sqltypes.NULL,                          // stop_pos
    56  				sqltypes.NewInt64(9223372036854775807), // max_tps
    57  				sqltypes.NewInt64(9223372036854775807), // max_replication_lag
    58  				sqltypes.NewVarBinary("Running"),       // state
    59  				sqltypes.NewInt64(1),                   // workflow_type
    60  				sqltypes.NewVarChar("wf"),              // workflow
    61  				sqltypes.NewInt64(0),                   // workflow_sub_type
    62  				sqltypes.NewInt64(0),                   // defer_secondary_keys
    63  			},
    64  		},
    65  	}
    66  	testSelectorResponse1 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(1)}}}
    67  	testSelectorResponse2 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(1)}, {sqltypes.NewInt64(2)}}}
    68  	testDMLResponse       = &sqltypes.Result{RowsAffected: 1}
    69  	testPos               = "MariaDB/0-1-1083"
    70  )
    71  
    72  func TestControllerKeyRange(t *testing.T) {
    73  	resetBinlogClient()
    74  	wantTablet := addTablet(100)
    75  	defer deleteTablet(wantTablet)
    76  	params := map[string]string{
    77  		"id":     "1",
    78  		"state":  binlogplayer.BlpRunning,
    79  		"source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName),
    80  	}
    81  
    82  	dbClient := binlogplayer.NewMockDBClient(t)
    83  	dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil)
    84  	dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil)
    85  	dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil)
    86  	dbClient.ExpectRequest("begin", nil, nil)
    87  	dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil)
    88  	dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil)
    89  	dbClient.ExpectRequest("commit", nil, nil)
    90  
    91  	dbClientFactory := func() binlogplayer.DBClient { return dbClient }
    92  	mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)}
    93  
    94  	ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil)
    95  	if err != nil {
    96  		t.Fatal(err)
    97  	}
    98  	defer func() {
    99  		dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil)
   100  		ct.Stop()
   101  	}()
   102  
   103  	dbClient.Wait()
   104  	expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}})
   105  }
   106  
   107  func TestControllerTables(t *testing.T) {
   108  	wantTablet := addTablet(100)
   109  	defer deleteTablet(wantTablet)
   110  	resetBinlogClient()
   111  
   112  	params := map[string]string{
   113  		"id":     "1",
   114  		"state":  binlogplayer.BlpRunning,
   115  		"source": fmt.Sprintf(`keyspace:"%s" shard:"0" tables:"table1" tables:"/funtables_/" `, env.KeyspaceName),
   116  	}
   117  
   118  	dbClient := binlogplayer.NewMockDBClient(t)
   119  	dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil)
   120  	dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil)
   121  	dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil)
   122  	dbClient.ExpectRequest("begin", nil, nil)
   123  	dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil)
   124  	dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil)
   125  	dbClient.ExpectRequest("commit", nil, nil)
   126  
   127  	dbClientFactory := func() binlogplayer.DBClient { return dbClient }
   128  	mysqld := &fakemysqldaemon.FakeMysqlDaemon{
   129  		MysqlPort: sync2.NewAtomicInt32(3306),
   130  		Schema: &tabletmanagerdatapb.SchemaDefinition{
   131  			DatabaseSchema: "",
   132  			TableDefinitions: []*tabletmanagerdatapb.TableDefinition{
   133  				{
   134  					Name:              "table1",
   135  					Columns:           []string{"id", "msg", "keyspace_id"},
   136  					PrimaryKeyColumns: []string{"id"},
   137  					Type:              tmutils.TableBaseTable,
   138  				},
   139  				{
   140  					Name:              "funtables_one",
   141  					Columns:           []string{"id", "msg", "keyspace_id"},
   142  					PrimaryKeyColumns: []string{"id"},
   143  					Type:              tmutils.TableBaseTable,
   144  				},
   145  				{
   146  					Name:              "excluded_table",
   147  					Columns:           []string{"id", "msg", "keyspace_id"},
   148  					PrimaryKeyColumns: []string{"id"},
   149  					Type:              tmutils.TableBaseTable,
   150  				},
   151  			},
   152  		},
   153  	}
   154  
   155  	ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil)
   156  	if err != nil {
   157  		t.Fatal(err)
   158  	}
   159  	defer func() {
   160  		dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil)
   161  		ct.Stop()
   162  	}()
   163  
   164  	dbClient.Wait()
   165  	expectFBCRequest(t, wantTablet, testPos, []string{"table1", "funtables_one"}, nil)
   166  }
   167  
   168  func TestControllerBadID(t *testing.T) {
   169  	params := map[string]string{
   170  		"id": "bad",
   171  	}
   172  	_, err := newController(context.Background(), params, nil, nil, nil, "", "", nil, nil)
   173  	want := `strconv.Atoi: parsing "bad": invalid syntax`
   174  	if err == nil || err.Error() != want {
   175  		t.Errorf("newController err: %v, want %v", err, want)
   176  	}
   177  }
   178  
   179  func TestControllerStopped(t *testing.T) {
   180  	params := map[string]string{
   181  		"id":    "1",
   182  		"state": binlogplayer.BlpStopped,
   183  	}
   184  
   185  	ct, err := newController(context.Background(), params, nil, nil, nil, "", "", nil, nil)
   186  	if err != nil {
   187  		t.Fatal(err)
   188  	}
   189  	defer ct.Stop()
   190  
   191  	select {
   192  	case <-ct.done:
   193  	default:
   194  		t.Errorf("context should be closed, but is not: %v", ct)
   195  	}
   196  }
   197  
   198  func TestControllerOverrides(t *testing.T) {
   199  	resetBinlogClient()
   200  	wantTablet := addTablet(100)
   201  	defer deleteTablet(wantTablet)
   202  
   203  	params := map[string]string{
   204  		"id":           "1",
   205  		"state":        binlogplayer.BlpRunning,
   206  		"source":       fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName),
   207  		"cell":         env.Cells[0],
   208  		"tablet_types": "replica",
   209  	}
   210  
   211  	dbClient := binlogplayer.NewMockDBClient(t)
   212  	dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil)
   213  	dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil)
   214  	dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil)
   215  	dbClient.ExpectRequest("begin", nil, nil)
   216  	dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil)
   217  	dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil)
   218  	dbClient.ExpectRequest("commit", nil, nil)
   219  
   220  	dbClientFactory := func() binlogplayer.DBClient { return dbClient }
   221  	mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)}
   222  
   223  	ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil)
   224  	if err != nil {
   225  		t.Fatal(err)
   226  	}
   227  	defer func() {
   228  		dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil)
   229  		ct.Stop()
   230  	}()
   231  
   232  	dbClient.Wait()
   233  	expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}})
   234  }
   235  
   236  func TestControllerCanceledContext(t *testing.T) {
   237  	defer deleteTablet(addTablet(100))
   238  
   239  	params := map[string]string{
   240  		"id":     "1",
   241  		"state":  binlogplayer.BlpRunning,
   242  		"source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName),
   243  	}
   244  
   245  	ctx, cancel := context.WithCancel(context.Background())
   246  	cancel()
   247  	ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil, nil)
   248  	if err != nil {
   249  		t.Fatal(err)
   250  	}
   251  	defer ct.Stop()
   252  
   253  	select {
   254  	case <-ct.done:
   255  	case <-time.After(1 * time.Second):
   256  		t.Errorf("context should be closed, but is not: %v", ct)
   257  	}
   258  }
   259  
   260  func TestControllerRetry(t *testing.T) {
   261  	savedDelay := retryDelay
   262  	defer func() { retryDelay = savedDelay }()
   263  	retryDelay = 10 * time.Millisecond
   264  
   265  	resetBinlogClient()
   266  	defer deleteTablet(addTablet(100))
   267  
   268  	params := map[string]string{
   269  		"id":           "1",
   270  		"state":        binlogplayer.BlpRunning,
   271  		"source":       fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName),
   272  		"cell":         env.Cells[0],
   273  		"tablet_types": "replica",
   274  	}
   275  
   276  	dbClient := binlogplayer.NewMockDBClient(t)
   277  	dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil)
   278  	dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil)
   279  	dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", nil, errors.New("(expected error)"))
   280  	dbClient.ExpectRequest("update _vt.vreplication set state='Error', message='error (expected error) in selecting vreplication settings select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1' where id=1", testDMLResponse, nil)
   281  	dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil)
   282  	dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil)
   283  	dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil)
   284  	dbClient.ExpectRequest("begin", nil, nil)
   285  	dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil)
   286  	dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil)
   287  	dbClient.ExpectRequest("commit", nil, nil)
   288  	dbClientFactory := func() binlogplayer.DBClient { return dbClient }
   289  	mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)}
   290  
   291  	ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil)
   292  	if err != nil {
   293  		t.Fatal(err)
   294  	}
   295  	defer ct.Stop()
   296  
   297  	dbClient.Wait()
   298  }
   299  
   300  func TestControllerStopPosition(t *testing.T) {
   301  	resetBinlogClient()
   302  	wantTablet := addTablet(100)
   303  	defer deleteTablet(wantTablet)
   304  
   305  	params := map[string]string{
   306  		"id":     "1",
   307  		"state":  binlogplayer.BlpRunning,
   308  		"source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName),
   309  	}
   310  
   311  	dbClient := binlogplayer.NewMockDBClient(t)
   312  	dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil)
   313  	dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil)
   314  	withStop := &sqltypes.Result{
   315  		Fields: []*querypb.Field{
   316  			{Name: "pos", Type: sqltypes.VarBinary},
   317  			{Name: "stop_pos", Type: sqltypes.VarBinary},
   318  			{Name: "max_tps", Type: sqltypes.Int64},
   319  			{Name: "max_replication_lag", Type: sqltypes.Int64},
   320  			{Name: "state", Type: sqltypes.VarBinary},
   321  			{Name: "workflow_type", Type: sqltypes.Int64},
   322  			{Name: "workflow", Type: sqltypes.VarChar},
   323  			{Name: "workflow_sub_type", Type: sqltypes.Int64},
   324  			{Name: "defer_secondary_keys", Type: sqltypes.Int64},
   325  		},
   326  		InsertID: 0,
   327  		Rows: [][]sqltypes.Value{
   328  			{
   329  				sqltypes.NewVarBinary("MariaDB/0-1-1083"), // pos
   330  				sqltypes.NewVarBinary("MariaDB/0-1-1235"), // stop_pos
   331  				sqltypes.NewInt64(9223372036854775807),    // max_tps
   332  				sqltypes.NewInt64(9223372036854775807),    // max_replication_lag
   333  				sqltypes.NewVarBinary("Running"),          // state
   334  				sqltypes.NewInt64(1),                      // workflow_type
   335  				sqltypes.NewVarChar("wf"),                 // workflow
   336  				sqltypes.NewInt64(1),                      // workflow_sub_type
   337  				sqltypes.NewInt64(1),                      // defer_secondary_keys
   338  			},
   339  		},
   340  	}
   341  	dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", withStop, nil)
   342  	dbClient.ExpectRequest("begin", nil, nil)
   343  	dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil)
   344  	dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil)
   345  	dbClient.ExpectRequest("commit", nil, nil)
   346  	dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='Reached stopping position, done playing logs' where id=1", testDMLResponse, nil)
   347  
   348  	dbClientFactory := func() binlogplayer.DBClient { return dbClient }
   349  	mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)}
   350  
   351  	ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil)
   352  	if err != nil {
   353  		t.Fatal(err)
   354  	}
   355  	defer func() {
   356  		dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil)
   357  		ct.Stop()
   358  	}()
   359  
   360  	// Also confirm that replication stopped.
   361  	select {
   362  	case <-ct.done:
   363  	case <-time.After(1 * time.Second):
   364  		t.Errorf("context should be closed, but is not: %v", ct)
   365  	}
   366  
   367  	dbClient.Wait()
   368  	expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}})
   369  }