vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/tm_init_test.go (about)

     1  /*
     2  Copyright 2020 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package tabletmanager
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"testing"
    23  	"time"
    24  
    25  	"github.com/stretchr/testify/assert"
    26  	"github.com/stretchr/testify/require"
    27  
    28  	"vitess.io/vitess/go/mysql"
    29  	"vitess.io/vitess/go/mysql/collations"
    30  	"vitess.io/vitess/go/mysql/fakesqldb"
    31  	"vitess.io/vitess/go/sqltypes"
    32  	"vitess.io/vitess/go/sync2"
    33  	"vitess.io/vitess/go/test/utils"
    34  	"vitess.io/vitess/go/vt/dbconfigs"
    35  	"vitess.io/vitess/go/vt/logutil"
    36  	"vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon"
    37  	"vitess.io/vitess/go/vt/servenv"
    38  	"vitess.io/vitess/go/vt/topo"
    39  	"vitess.io/vitess/go/vt/topo/memorytopo"
    40  	"vitess.io/vitess/go/vt/topotools"
    41  	"vitess.io/vitess/go/vt/vttablet/tabletservermock"
    42  
    43  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    44  	vschemapb "vitess.io/vitess/go/vt/proto/vschema"
    45  )
    46  
    47  var (
    48  	dbServerVersion = "8.0.0"
    49  	charsetName     = "utf8mb4"
    50  	dbsvCollID      = collations.NewEnvironment(dbServerVersion).DefaultCollationForCharset(charsetName).ID()
    51  )
    52  
    53  func TestStartBuildTabletFromInput(t *testing.T) {
    54  	alias := &topodatapb.TabletAlias{
    55  		Cell: "cell",
    56  		Uid:  1,
    57  	}
    58  	port := int32(12)
    59  	grpcport := int32(34)
    60  
    61  	// Hostname should be used as is.
    62  	tabletHostname = "foo"
    63  	initKeyspace = "test_keyspace"
    64  	initShard = "0"
    65  	initTabletType = "replica"
    66  	initDbNameOverride = "aa"
    67  	wantTablet := &topodatapb.Tablet{
    68  		Alias:    alias,
    69  		Hostname: "foo",
    70  		PortMap: map[string]int32{
    71  			"vt":   port,
    72  			"grpc": grpcport,
    73  		},
    74  		Keyspace:             "test_keyspace",
    75  		Shard:                "0",
    76  		KeyRange:             nil,
    77  		Type:                 topodatapb.TabletType_REPLICA,
    78  		Tags:                 map[string]string{},
    79  		DbNameOverride:       "aa",
    80  		DbServerVersion:      dbServerVersion,
    81  		DefaultConnCollation: uint32(dbsvCollID),
    82  	}
    83  
    84  	gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
    85  	require.NoError(t, err)
    86  
    87  	// Hostname should be resolved.
    88  	assert.Equal(t, wantTablet, gotTablet)
    89  	tabletHostname = ""
    90  	gotTablet, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
    91  	require.NoError(t, err)
    92  	assert.NotEqual(t, "", gotTablet.Hostname)
    93  
    94  	// Canonicalize shard name and compute keyrange.
    95  	tabletHostname = "foo"
    96  	initShard = "-C0"
    97  	wantTablet.Shard = "-c0"
    98  	wantTablet.KeyRange = &topodatapb.KeyRange{
    99  		Start: []byte(""),
   100  		End:   []byte("\xc0"),
   101  	}
   102  	gotTablet, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   103  	require.NoError(t, err)
   104  	// KeyRange check is explicit because the next comparison doesn't
   105  	// show the diff well enough.
   106  	assert.Equal(t, wantTablet.KeyRange, gotTablet.KeyRange)
   107  	assert.Equal(t, wantTablet, gotTablet)
   108  
   109  	// Invalid inputs.
   110  	initKeyspace = ""
   111  	initShard = "0"
   112  	_, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   113  	assert.Contains(t, err.Error(), "init_keyspace and init_shard must be specified")
   114  
   115  	initKeyspace = "test_keyspace"
   116  	initShard = ""
   117  	_, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   118  	assert.Contains(t, err.Error(), "init_keyspace and init_shard must be specified")
   119  
   120  	initShard = "x-y"
   121  	_, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   122  	assert.Contains(t, err.Error(), "cannot validate shard name")
   123  
   124  	initShard = "0"
   125  	initTabletType = "bad"
   126  	_, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   127  	assert.Contains(t, err.Error(), "unknown TabletType bad")
   128  
   129  	initTabletType = "primary"
   130  	_, err = BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   131  	assert.Contains(t, err.Error(), "invalid init_tablet_type PRIMARY")
   132  }
   133  
   134  func TestBuildTabletFromInputWithBuildTags(t *testing.T) {
   135  	alias := &topodatapb.TabletAlias{
   136  		Cell: "cell",
   137  		Uid:  1,
   138  	}
   139  	port := int32(12)
   140  	grpcport := int32(34)
   141  
   142  	// Hostname should be used as is.
   143  	tabletHostname = "foo"
   144  	initKeyspace = "test_keyspace"
   145  	initShard = "0"
   146  	initTabletType = "replica"
   147  	initDbNameOverride = "aa"
   148  	skipBuildInfoTags = ""
   149  	defer func() { skipBuildInfoTags = "/.*/" }()
   150  	wantTablet := &topodatapb.Tablet{
   151  		Alias:    alias,
   152  		Hostname: "foo",
   153  		PortMap: map[string]int32{
   154  			"vt":   port,
   155  			"grpc": grpcport,
   156  		},
   157  		Keyspace:             "test_keyspace",
   158  		Shard:                "0",
   159  		KeyRange:             nil,
   160  		Type:                 topodatapb.TabletType_REPLICA,
   161  		Tags:                 servenv.AppVersion.ToStringMap(),
   162  		DbNameOverride:       "aa",
   163  		DbServerVersion:      dbServerVersion,
   164  		DefaultConnCollation: uint32(dbsvCollID),
   165  	}
   166  
   167  	gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil)
   168  	require.NoError(t, err)
   169  	assert.Equal(t, wantTablet, gotTablet)
   170  }
   171  
   172  func TestStartCreateKeyspaceShard(t *testing.T) {
   173  	defer func(saved time.Duration) { rebuildKeyspaceRetryInterval = saved }(rebuildKeyspaceRetryInterval)
   174  	rebuildKeyspaceRetryInterval = 10 * time.Millisecond
   175  
   176  	ctx := context.Background()
   177  	statsTabletTypeCount.ResetAll()
   178  	cell := "cell1"
   179  	ts := memorytopo.NewServer(cell)
   180  	tm := newTestTM(t, ts, 1, "ks", "0")
   181  	defer tm.Stop()
   182  
   183  	assert.Equal(t, "replica", statsTabletType.Get())
   184  	assert.Equal(t, 1, len(statsTabletTypeCount.Counts()))
   185  	assert.Equal(t, int64(1), statsTabletTypeCount.Counts()["replica"])
   186  
   187  	_, err := ts.GetShard(ctx, "ks", "0")
   188  	require.NoError(t, err)
   189  
   190  	ensureSrvKeyspace(t, ts, cell, "ks")
   191  
   192  	srvVSchema, err := ts.GetSrvVSchema(context.Background(), cell)
   193  	require.NoError(t, err)
   194  	wantVSchema := &vschemapb.Keyspace{}
   195  	assert.Equal(t, wantVSchema, srvVSchema.Keyspaces["ks"])
   196  
   197  	// keyspace-shard already created.
   198  	_, err = ts.GetOrCreateShard(ctx, "ks1", "0")
   199  	require.NoError(t, err)
   200  	tm = newTestTM(t, ts, 2, "ks1", "0")
   201  	defer tm.Stop()
   202  	_, err = ts.GetShard(ctx, "ks1", "0")
   203  	require.NoError(t, err)
   204  	ensureSrvKeyspace(t, ts, cell, "ks1")
   205  	srvVSchema, err = ts.GetSrvVSchema(context.Background(), cell)
   206  	require.NoError(t, err)
   207  	assert.Equal(t, wantVSchema, srvVSchema.Keyspaces["ks1"])
   208  
   209  	// srvKeyspace already created
   210  	_, err = ts.GetOrCreateShard(ctx, "ks2", "0")
   211  	require.NoError(t, err)
   212  	err = topotools.RebuildKeyspace(ctx, logutil.NewConsoleLogger(), ts, "ks2", []string{cell}, false)
   213  	require.NoError(t, err)
   214  	tm = newTestTM(t, ts, 3, "ks2", "0")
   215  	defer tm.Stop()
   216  	_, err = ts.GetShard(ctx, "ks2", "0")
   217  	require.NoError(t, err)
   218  	_, err = ts.GetSrvKeyspace(context.Background(), cell, "ks2")
   219  	require.NoError(t, err)
   220  	srvVSchema, err = ts.GetSrvVSchema(context.Background(), cell)
   221  	require.NoError(t, err)
   222  	assert.Equal(t, wantVSchema, srvVSchema.Keyspaces["ks2"])
   223  
   224  	// srvVSchema already created
   225  	_, err = ts.GetOrCreateShard(ctx, "ks3", "0")
   226  	require.NoError(t, err)
   227  	err = topotools.RebuildKeyspace(ctx, logutil.NewConsoleLogger(), ts, "ks3", []string{cell}, false)
   228  	require.NoError(t, err)
   229  	err = ts.RebuildSrvVSchema(ctx, []string{cell})
   230  	require.NoError(t, err)
   231  	tm = newTestTM(t, ts, 4, "ks3", "0")
   232  	defer tm.Stop()
   233  	_, err = ts.GetShard(ctx, "ks3", "0")
   234  	require.NoError(t, err)
   235  	_, err = ts.GetSrvKeyspace(context.Background(), cell, "ks3")
   236  	require.NoError(t, err)
   237  	srvVSchema, err = ts.GetSrvVSchema(context.Background(), cell)
   238  	require.NoError(t, err)
   239  	assert.Equal(t, wantVSchema, srvVSchema.Keyspaces["ks3"])
   240  
   241  	// Multi-shard
   242  	tm1 := newTestTM(t, ts, 5, "ks4", "-80")
   243  	defer tm1.Stop()
   244  
   245  	// Wait a bit and make sure that srvKeyspace is still not created.
   246  	time.Sleep(100 * time.Millisecond)
   247  	_, err = ts.GetSrvKeyspace(context.Background(), cell, "ks4")
   248  	require.True(t, topo.IsErrType(err, topo.NoNode), err)
   249  
   250  	tm2 := newTestTM(t, ts, 6, "ks4", "80-")
   251  	defer tm2.Stop()
   252  	// Now that we've started the tablet for the other shard, srvKeyspace will succeed.
   253  	ensureSrvKeyspace(t, ts, cell, "ks4")
   254  }
   255  
   256  func TestCheckPrimaryShip(t *testing.T) {
   257  	defer func(saved time.Duration) { rebuildKeyspaceRetryInterval = saved }(rebuildKeyspaceRetryInterval)
   258  	rebuildKeyspaceRetryInterval = 10 * time.Millisecond
   259  
   260  	ctx := context.Background()
   261  	cell := "cell1"
   262  	ts := memorytopo.NewServer(cell)
   263  	alias := &topodatapb.TabletAlias{
   264  		Cell: "cell1",
   265  		Uid:  1,
   266  	}
   267  
   268  	// 1. Initialize the tablet as REPLICA.
   269  	// This will create the respective topology records.
   270  	tm := newTestTM(t, ts, 1, "ks", "0")
   271  	tablet := tm.Tablet()
   272  	ensureSrvKeyspace(t, ts, cell, "ks")
   273  	ti, err := ts.GetTablet(ctx, alias)
   274  	require.NoError(t, err)
   275  	assert.Equal(t, topodatapb.TabletType_REPLICA, ti.Type)
   276  	tm.Stop()
   277  
   278  	// 2. Update shard's primary to our alias, then try to init again.
   279  	// (This simulates the case where the PrimaryAlias in the shard record says
   280  	// that we are the primary but the tablet record says otherwise. In that case,
   281  	// we become primary by inheriting the shard record's timestamp.)
   282  	now := time.Now()
   283  	_, err = ts.UpdateShardFields(ctx, "ks", "0", func(si *topo.ShardInfo) error {
   284  		si.PrimaryAlias = alias
   285  		si.PrimaryTermStartTime = logutil.TimeToProto(now)
   286  		// Reassign to now for easier comparison.
   287  		now = si.GetPrimaryTermStartTime()
   288  		return nil
   289  	})
   290  	require.NoError(t, err)
   291  	err = tm.Start(tablet, 0)
   292  	require.NoError(t, err)
   293  	ti, err = ts.GetTablet(ctx, alias)
   294  	require.NoError(t, err)
   295  	assert.Equal(t, topodatapb.TabletType_PRIMARY, ti.Type)
   296  	ter0 := ti.GetPrimaryTermStartTime()
   297  	assert.Equal(t, now, ter0)
   298  	assert.Equal(t, "primary", statsTabletType.Get())
   299  	tm.Stop()
   300  
   301  	// 3. Delete the tablet record. The shard record still says that we are the
   302  	// PRIMARY. Since it is the only source, we assume that its information is
   303  	// correct and start as PRIMARY.
   304  	err = ts.DeleteTablet(ctx, alias)
   305  	require.NoError(t, err)
   306  	err = tm.Start(tablet, 0)
   307  	require.NoError(t, err)
   308  	ti, err = ts.GetTablet(ctx, alias)
   309  	require.NoError(t, err)
   310  	assert.Equal(t, topodatapb.TabletType_PRIMARY, ti.Type)
   311  	ter1 := ti.GetPrimaryTermStartTime()
   312  	tm.Stop()
   313  
   314  	// 4. Fix the tablet record to agree that we're primary.
   315  	// Shard and tablet record are in sync now and we assume that we are actually
   316  	// the PRIMARY.
   317  	ti.Type = topodatapb.TabletType_PRIMARY
   318  	err = ts.UpdateTablet(ctx, ti)
   319  	require.NoError(t, err)
   320  	err = tm.Start(tablet, 0)
   321  	require.NoError(t, err)
   322  	ti, err = ts.GetTablet(ctx, alias)
   323  	require.NoError(t, err)
   324  	assert.Equal(t, topodatapb.TabletType_PRIMARY, ti.Type)
   325  	ter2 := ti.GetPrimaryTermStartTime()
   326  	assert.Equal(t, ter1, ter2)
   327  	tm.Stop()
   328  
   329  	// 5. Subsequent inits will still start the vttablet as PRIMARY.
   330  	err = tm.Start(tablet, 0)
   331  	require.NoError(t, err)
   332  	ti, err = ts.GetTablet(ctx, alias)
   333  	require.NoError(t, err)
   334  	assert.Equal(t, topodatapb.TabletType_PRIMARY, ti.Type)
   335  	ter3 := ti.GetPrimaryTermStartTime()
   336  	assert.Equal(t, ter1, ter3)
   337  	tm.Stop()
   338  
   339  	// 6. If the shard record shows a different primary with an older
   340  	// timestamp, we take over primaryship.
   341  	otherAlias := &topodatapb.TabletAlias{
   342  		Cell: "cell1",
   343  		Uid:  2,
   344  	}
   345  	otherTablet := &topodatapb.Tablet{
   346  		Alias:         otherAlias,
   347  		Keyspace:      "ks",
   348  		Shard:         "0",
   349  		Type:          topodatapb.TabletType_PRIMARY,
   350  		MysqlHostname: "localhost",
   351  		MysqlPort:     1234,
   352  	}
   353  	// Create the tablet record for the primary
   354  	err = ts.CreateTablet(ctx, otherTablet)
   355  	require.NoError(t, err)
   356  	_, err = ts.UpdateShardFields(ctx, "ks", "0", func(si *topo.ShardInfo) error {
   357  		si.PrimaryAlias = otherAlias
   358  		si.PrimaryTermStartTime = logutil.TimeToProto(ter1.Add(-10 * time.Second))
   359  		return nil
   360  	})
   361  	require.NoError(t, err)
   362  	err = tm.Start(tablet, 0)
   363  	require.NoError(t, err)
   364  	ti, err = ts.GetTablet(ctx, alias)
   365  	require.NoError(t, err)
   366  	assert.Equal(t, topodatapb.TabletType_PRIMARY, ti.Type)
   367  	ter4 := ti.GetPrimaryTermStartTime()
   368  	assert.Equal(t, ter1, ter4)
   369  	tm.Stop()
   370  
   371  	// 7. If the shard record shows a different primary with a newer
   372  	// timestamp, we remain replica.
   373  	_, err = ts.UpdateShardFields(ctx, "ks", "0", func(si *topo.ShardInfo) error {
   374  		si.PrimaryAlias = otherAlias
   375  		si.PrimaryTermStartTime = logutil.TimeToProto(ter4.Add(10 * time.Second))
   376  		return nil
   377  	})
   378  	require.NoError(t, err)
   379  	tablet.Type = topodatapb.TabletType_REPLICA
   380  	tablet.PrimaryTermStartTime = nil
   381  	// Get the fakeMySQL and set it up to expect a set replication source command
   382  	fakeMysql := tm.MysqlDaemon.(*fakemysqldaemon.FakeMysqlDaemon)
   383  	fakeMysql.SetReplicationSourceInputs = append(fakeMysql.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", otherTablet.MysqlHostname, otherTablet.MysqlPort))
   384  	fakeMysql.ExpectedExecuteSuperQueryList = []string{
   385  		"STOP SLAVE",
   386  		"RESET SLAVE ALL",
   387  		"FAKE SET MASTER",
   388  		"START SLAVE",
   389  	}
   390  	err = tm.Start(tablet, 0)
   391  	require.NoError(t, err)
   392  	ti, err = ts.GetTablet(ctx, alias)
   393  	require.NoError(t, err)
   394  	assert.Equal(t, topodatapb.TabletType_REPLICA, ti.Type)
   395  	ter5 := ti.GetPrimaryTermStartTime()
   396  	assert.True(t, ter5.IsZero())
   397  	tm.Stop()
   398  }
   399  
   400  func TestStartCheckMysql(t *testing.T) {
   401  	ctx := context.Background()
   402  	cell := "cell1"
   403  	ts := memorytopo.NewServer(cell)
   404  	tablet := newTestTablet(t, 1, "ks", "0")
   405  	cp := mysql.ConnParams{
   406  		Host: "foo",
   407  		Port: 1,
   408  	}
   409  	tm := &TabletManager{
   410  		BatchCtx:            context.Background(),
   411  		TopoServer:          ts,
   412  		MysqlDaemon:         newTestMysqlDaemon(t, 1),
   413  		DBConfigs:           dbconfigs.NewTestDBConfigs(cp, cp, ""),
   414  		QueryServiceControl: tabletservermock.NewController(),
   415  	}
   416  	err := tm.Start(tablet, 0)
   417  	require.NoError(t, err)
   418  	defer tm.Stop()
   419  
   420  	ti, err := ts.GetTablet(ctx, tm.tabletAlias)
   421  	require.NoError(t, err)
   422  	assert.Equal(t, int32(1), ti.MysqlPort)
   423  	assert.Equal(t, "foo", ti.MysqlHostname)
   424  }
   425  
   426  func TestStartFindMysqlPort(t *testing.T) {
   427  	defer func(saved time.Duration) { mysqlPortRetryInterval = saved }(mysqlPortRetryInterval)
   428  	mysqlPortRetryInterval = 1 * time.Millisecond
   429  
   430  	ctx := context.Background()
   431  	cell := "cell1"
   432  	ts := memorytopo.NewServer(cell)
   433  	tablet := newTestTablet(t, 1, "ks", "0")
   434  	fmd := newTestMysqlDaemon(t, -1)
   435  	tm := &TabletManager{
   436  		BatchCtx:            context.Background(),
   437  		TopoServer:          ts,
   438  		MysqlDaemon:         fmd,
   439  		DBConfigs:           &dbconfigs.DBConfigs{},
   440  		QueryServiceControl: tabletservermock.NewController(),
   441  	}
   442  	err := tm.Start(tablet, 0)
   443  	require.NoError(t, err)
   444  	defer tm.Stop()
   445  
   446  	ti, err := ts.GetTablet(ctx, tm.tabletAlias)
   447  	require.NoError(t, err)
   448  	assert.Equal(t, int32(0), ti.MysqlPort)
   449  
   450  	fmd.MysqlPort.Set(3306)
   451  	for i := 0; i < 10; i++ {
   452  		ti, err := ts.GetTablet(ctx, tm.tabletAlias)
   453  		require.NoError(t, err)
   454  		if ti.MysqlPort == 3306 {
   455  			return
   456  		}
   457  		time.Sleep(5 * time.Millisecond)
   458  	}
   459  	assert.Fail(t, "mysql port was not updated")
   460  }
   461  
   462  // Init tablet fixes replication data when safe
   463  func TestStartFixesReplicationData(t *testing.T) {
   464  	ctx := context.Background()
   465  	cell := "cell1"
   466  	ts := memorytopo.NewServer(cell, "cell2")
   467  	tm := newTestTM(t, ts, 1, "ks", "0")
   468  	defer tm.Stop()
   469  	tabletAlias := tm.tabletAlias
   470  
   471  	sri, err := ts.GetShardReplication(ctx, cell, "ks", "0")
   472  	require.NoError(t, err)
   473  	utils.MustMatch(t, tabletAlias, sri.Nodes[0].TabletAlias)
   474  
   475  	// Remove the ShardReplication record, try to create the
   476  	// tablets again, make sure it's fixed.
   477  	err = topo.RemoveShardReplicationRecord(ctx, ts, cell, "ks", "0", tabletAlias)
   478  	require.NoError(t, err)
   479  	sri, err = ts.GetShardReplication(ctx, cell, "ks", "0")
   480  	require.NoError(t, err)
   481  	assert.Equal(t, 0, len(sri.Nodes))
   482  
   483  	// An initTablet will recreate the shard replication data.
   484  	err = tm.initTablet(context.Background())
   485  	require.NoError(t, err)
   486  
   487  	sri, err = ts.GetShardReplication(ctx, cell, "ks", "0")
   488  	require.NoError(t, err)
   489  	utils.MustMatch(t, tabletAlias, sri.Nodes[0].TabletAlias)
   490  }
   491  
   492  // This is a test to make sure a regression does not happen in the future.
   493  // There is code in Start that updates replication data if tablet fails
   494  // to be created due to a NodeExists error. During this particular error we were not doing
   495  // the sanity checks that the provided tablet was the same in the topo.
   496  func TestStartDoesNotUpdateReplicationDataForTabletInWrongShard(t *testing.T) {
   497  	ctx := context.Background()
   498  	ts := memorytopo.NewServer("cell1", "cell2")
   499  	tm := newTestTM(t, ts, 1, "ks", "0")
   500  	tm.Stop()
   501  
   502  	tabletAliases, err := ts.FindAllTabletAliasesInShard(ctx, "ks", "0")
   503  	require.NoError(t, err)
   504  	assert.Equal(t, uint32(1), tabletAliases[0].Uid)
   505  
   506  	tablet := newTestTablet(t, 1, "ks", "-d0")
   507  	require.NoError(t, err)
   508  	err = tm.Start(tablet, 0)
   509  	assert.Contains(t, err.Error(), "existing tablet keyspace and shard ks/0 differ")
   510  
   511  	tablets, err := ts.FindAllTabletAliasesInShard(ctx, "ks", "-d0")
   512  	require.NoError(t, err)
   513  	assert.Equal(t, 0, len(tablets))
   514  }
   515  
   516  func TestCheckTabletTypeResets(t *testing.T) {
   517  	defer func(saved time.Duration) { rebuildKeyspaceRetryInterval = saved }(rebuildKeyspaceRetryInterval)
   518  	rebuildKeyspaceRetryInterval = 10 * time.Millisecond
   519  
   520  	ctx := context.Background()
   521  	cell := "cell1"
   522  	ts := memorytopo.NewServer(cell)
   523  	alias := &topodatapb.TabletAlias{
   524  		Cell: "cell1",
   525  		Uid:  1,
   526  	}
   527  
   528  	// 1. Initialize the tablet as REPLICA.
   529  	// This will create the respective topology records.
   530  	tm := newTestTM(t, ts, 1, "ks", "0")
   531  	tablet := tm.Tablet()
   532  	ensureSrvKeyspace(t, ts, cell, "ks")
   533  	ti, err := ts.GetTablet(ctx, alias)
   534  	require.NoError(t, err)
   535  	assert.Equal(t, topodatapb.TabletType_REPLICA, ti.Type)
   536  	tm.Stop()
   537  
   538  	// 2. Update tablet record with tabletType RESTORE
   539  	_, err = ts.UpdateTabletFields(ctx, alias, func(t *topodatapb.Tablet) error {
   540  		t.Type = topodatapb.TabletType_RESTORE
   541  		return nil
   542  	})
   543  	require.NoError(t, err)
   544  	err = tm.Start(tablet, 0)
   545  	require.NoError(t, err)
   546  	assert.Equal(t, tm.tmState.tablet.Type, tm.tmState.displayState.tablet.Type)
   547  	ti, err = ts.GetTablet(ctx, alias)
   548  	require.NoError(t, err)
   549  	// Verify that it changes back to initTabletType
   550  	assert.Equal(t, topodatapb.TabletType_REPLICA, ti.Type)
   551  
   552  	// 3. Update shard's primary to our alias, then try to init again.
   553  	// (This simulates the case where the PrimaryAlias in the shard record says
   554  	// that we are the primary but the tablet record says otherwise. In that case,
   555  	// we become primary by inheriting the shard record's timestamp.)
   556  	now := time.Now()
   557  	_, err = ts.UpdateShardFields(ctx, "ks", "0", func(si *topo.ShardInfo) error {
   558  		si.PrimaryAlias = alias
   559  		si.PrimaryTermStartTime = logutil.TimeToProto(now)
   560  		// Reassign to now for easier comparison.
   561  		now = si.GetPrimaryTermStartTime()
   562  		return nil
   563  	})
   564  	require.NoError(t, err)
   565  	si, err := tm.createKeyspaceShard(ctx)
   566  	require.NoError(t, err)
   567  	err = tm.checkPrimaryShip(ctx, si)
   568  	require.NoError(t, err)
   569  	assert.Equal(t, tm.tmState.tablet.Type, tm.tmState.displayState.tablet.Type)
   570  	err = tm.initTablet(ctx)
   571  	require.NoError(t, err)
   572  	assert.Equal(t, tm.tmState.tablet.Type, tm.tmState.displayState.tablet.Type)
   573  	ti, err = ts.GetTablet(ctx, alias)
   574  	require.NoError(t, err)
   575  	assert.Equal(t, topodatapb.TabletType_PRIMARY, ti.Type)
   576  	ter0 := ti.GetPrimaryTermStartTime()
   577  	assert.Equal(t, now, ter0)
   578  	tm.Stop()
   579  }
   580  
   581  func TestGetBuildTags(t *testing.T) {
   582  	t.Parallel()
   583  
   584  	tests := []struct {
   585  		in      map[string]string
   586  		skipCSV string
   587  		want    map[string]string
   588  		wantErr bool
   589  	}{
   590  		{
   591  			in: map[string]string{
   592  				"a": "a",
   593  				"b": "b",
   594  				"c": "c",
   595  			},
   596  			skipCSV: "a,c",
   597  			want: map[string]string{
   598  				"b": "b",
   599  			},
   600  		},
   601  		{
   602  			in: map[string]string{
   603  				"hello": "world",
   604  				"help":  "me",
   605  				"good":  "bye",
   606  				"a":     "b",
   607  			},
   608  			skipCSV: "a,/hel.*/",
   609  			want: map[string]string{
   610  				"good": "bye",
   611  			},
   612  		},
   613  		{
   614  			in: map[string]string{
   615  				"a":      "a",
   616  				"/hello": "/hello",
   617  			},
   618  			skipCSV: "/,a", // len(skipTag) <= 1, so not a regexp
   619  			want: map[string]string{
   620  				"/hello": "/hello",
   621  			},
   622  		},
   623  	}
   624  
   625  	for _, tt := range tests {
   626  		tt := tt
   627  		t.Run(tt.skipCSV, func(t *testing.T) {
   628  			t.Parallel()
   629  
   630  			out, err := getBuildTags(tt.in, tt.skipCSV)
   631  			if tt.wantErr {
   632  				assert.Error(t, err)
   633  				return
   634  			}
   635  
   636  			require.NoError(t, err)
   637  			assert.Equal(t, tt.want, out)
   638  		})
   639  	}
   640  }
   641  
   642  func newTestMysqlDaemon(t *testing.T, port int32) *fakemysqldaemon.FakeMysqlDaemon {
   643  	t.Helper()
   644  
   645  	db := fakesqldb.New(t)
   646  	db.AddQueryPattern("SET @@.*", &sqltypes.Result{})
   647  	db.AddQueryPattern("BEGIN", &sqltypes.Result{})
   648  	db.AddQueryPattern("COMMIT", &sqltypes.Result{})
   649  
   650  	mysqld := fakemysqldaemon.NewFakeMysqlDaemon(db)
   651  	mysqld.MysqlPort = sync2.NewAtomicInt32(port)
   652  
   653  	return mysqld
   654  }
   655  
   656  func newTestTM(t *testing.T, ts *topo.Server, uid int, keyspace, shard string) *TabletManager {
   657  	t.Helper()
   658  	ctx := context.Background()
   659  	tablet := newTestTablet(t, uid, keyspace, shard)
   660  	tm := &TabletManager{
   661  		BatchCtx:            ctx,
   662  		TopoServer:          ts,
   663  		MysqlDaemon:         newTestMysqlDaemon(t, 1),
   664  		DBConfigs:           &dbconfigs.DBConfigs{},
   665  		QueryServiceControl: tabletservermock.NewController(),
   666  	}
   667  	err := tm.Start(tablet, 0)
   668  	require.NoError(t, err)
   669  
   670  	// Wait for SrvKeyspace to be rebuilt. We know that it has been built
   671  	// when isShardServing or tabletControls maps is non-empty.
   672  	timeout := time.After(1 * time.Second)
   673  	for {
   674  		select {
   675  		case <-timeout:
   676  			t.Logf("servingKeyspace not initialized for tablet uid - %d", uid)
   677  			return tm
   678  		default:
   679  			isNonEmpty := false
   680  			func() {
   681  				tm.tmState.mu.Lock()
   682  				defer tm.tmState.mu.Unlock()
   683  				if tm.tmState.isShardServing != nil || tm.tmState.tabletControls != nil {
   684  					isNonEmpty = true
   685  				}
   686  			}()
   687  			if isNonEmpty {
   688  				return tm
   689  			}
   690  			time.Sleep(100 * time.Millisecond)
   691  		}
   692  	}
   693  }
   694  
   695  func newTestTablet(t *testing.T, uid int, keyspace, shard string) *topodatapb.Tablet {
   696  	shard, keyRange, err := topo.ValidateShardName(shard)
   697  	require.NoError(t, err)
   698  	return &topodatapb.Tablet{
   699  		Alias: &topodatapb.TabletAlias{
   700  			Cell: "cell1",
   701  			Uid:  uint32(uid),
   702  		},
   703  		Hostname: "localhost",
   704  		PortMap: map[string]int32{
   705  			"vt":   int32(1234),
   706  			"grpc": int32(3456),
   707  		},
   708  		Keyspace: keyspace,
   709  		Shard:    shard,
   710  		KeyRange: keyRange,
   711  		Type:     topodatapb.TabletType_REPLICA,
   712  	}
   713  }
   714  
   715  func ensureSrvKeyspace(t *testing.T, ts *topo.Server, cell, keyspace string) {
   716  	t.Helper()
   717  	found := false
   718  	for i := 0; i < 10; i++ {
   719  		_, err := ts.GetSrvKeyspace(context.Background(), cell, "ks")
   720  		if err == nil {
   721  			found = true
   722  			break
   723  		}
   724  		require.True(t, topo.IsErrType(err, topo.NoNode), err)
   725  		time.Sleep(rebuildKeyspaceRetryInterval)
   726  	}
   727  	assert.True(t, found)
   728  }