vitess.io/vitess@v0.16.2/go/vt/vtctl/reparentutil/planned_reparenter_flaky_test.go (about)

     1  /*
     2  Copyright 2021 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package reparentutil
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"testing"
    24  	"time"
    25  
    26  	"vitess.io/vitess/go/mysql"
    27  
    28  	"vitess.io/vitess/go/test/utils"
    29  
    30  	"github.com/stretchr/testify/assert"
    31  	"github.com/stretchr/testify/require"
    32  
    33  	"vitess.io/vitess/go/vt/logutil"
    34  	"vitess.io/vitess/go/vt/topo"
    35  	"vitess.io/vitess/go/vt/topo/memorytopo"
    36  	"vitess.io/vitess/go/vt/topotools/events"
    37  	"vitess.io/vitess/go/vt/vtctl/grpcvtctldserver/testutil"
    38  	"vitess.io/vitess/go/vt/vttablet/tmclient"
    39  
    40  	replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata"
    41  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    42  	vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
    43  	"vitess.io/vitess/go/vt/proto/vttime"
    44  )
    45  
    46  func TestNewPlannedReparenter(t *testing.T) {
    47  	t.Parallel()
    48  
    49  	tests := []struct {
    50  		name   string
    51  		logger logutil.Logger
    52  	}{
    53  		{
    54  			name:   "default case",
    55  			logger: logutil.NewMemoryLogger(),
    56  		},
    57  		{
    58  			name:   "overrides nil logger with no-op",
    59  			logger: nil,
    60  		},
    61  	}
    62  
    63  	for _, tt := range tests {
    64  		tt := tt
    65  
    66  		t.Run(tt.name, func(t *testing.T) {
    67  			t.Parallel()
    68  
    69  			er := NewPlannedReparenter(nil, nil, tt.logger)
    70  			assert.NotNil(t, er.logger, "NewPlannedReparenter should never result in a nil logger instance on the EmergencyReparenter")
    71  		})
    72  	}
    73  }
    74  
    75  func TestPlannedReparenter_ReparentShard(t *testing.T) {
    76  	t.Parallel()
    77  
    78  	tests := []struct {
    79  		name                string
    80  		ts                  *topo.Server
    81  		tmc                 tmclient.TabletManagerClient
    82  		tablets             []*topodatapb.Tablet
    83  		lockShardBeforeTest bool
    84  
    85  		keyspace string
    86  		shard    string
    87  		opts     PlannedReparentOptions
    88  
    89  		expectedEvent *events.Reparent
    90  		shouldErr     bool
    91  	}{
    92  		{
    93  			name: "success",
    94  			ts:   memorytopo.NewServer("zone1"),
    95  			tmc: &testutil.TabletManagerClient{
    96  				PrimaryPositionResults: map[string]struct {
    97  					Position string
    98  					Error    error
    99  				}{
   100  					"zone1-0000000100": {
   101  						Position: "position1",
   102  						Error:    nil,
   103  					},
   104  				},
   105  				PopulateReparentJournalResults: map[string]error{
   106  					"zone1-0000000100": nil,
   107  				},
   108  				SetReplicationSourceResults: map[string]error{
   109  					"zone1-0000000200": nil,
   110  				},
   111  				SetReadWriteResults: map[string]error{
   112  					"zone1-0000000100": nil,
   113  				},
   114  			},
   115  			tablets: []*topodatapb.Tablet{
   116  				{
   117  					Alias: &topodatapb.TabletAlias{
   118  						Cell: "zone1",
   119  						Uid:  100,
   120  					},
   121  					Type:     topodatapb.TabletType_PRIMARY,
   122  					Keyspace: "testkeyspace",
   123  					Shard:    "-",
   124  				},
   125  				{
   126  					Alias: &topodatapb.TabletAlias{
   127  						Cell: "zone1",
   128  						Uid:  200,
   129  					},
   130  					Type:     topodatapb.TabletType_REPLICA,
   131  					Keyspace: "testkeyspace",
   132  					Shard:    "-",
   133  				},
   134  			},
   135  
   136  			keyspace: "testkeyspace",
   137  			shard:    "-",
   138  			opts: PlannedReparentOptions{
   139  				NewPrimaryAlias: &topodatapb.TabletAlias{
   140  					Cell: "zone1",
   141  					Uid:  100,
   142  				},
   143  			},
   144  
   145  			shouldErr: false,
   146  			expectedEvent: &events.Reparent{
   147  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   148  					PrimaryAlias: &topodatapb.TabletAlias{
   149  						Cell: "zone1",
   150  						Uid:  100,
   151  					},
   152  					KeyRange:         &topodatapb.KeyRange{},
   153  					IsPrimaryServing: true,
   154  				}, nil),
   155  				NewPrimary: &topodatapb.Tablet{
   156  					Alias: &topodatapb.TabletAlias{
   157  						Cell: "zone1",
   158  						Uid:  100,
   159  					},
   160  					Type:     topodatapb.TabletType_PRIMARY,
   161  					Keyspace: "testkeyspace",
   162  					Shard:    "-",
   163  				},
   164  			},
   165  		},
   166  		{
   167  			name: "success - new primary not provided",
   168  			ts:   memorytopo.NewServer("zone1"),
   169  			tmc: &testutil.TabletManagerClient{
   170  				ReplicationStatusResults: map[string]struct {
   171  					Position *replicationdatapb.Status
   172  					Error    error
   173  				}{
   174  					"zone1-0000000200": {
   175  						Position: &replicationdatapb.Status{
   176  							Position: "",
   177  						},
   178  					},
   179  				},
   180  				DemotePrimaryResults: map[string]struct {
   181  					Status *replicationdatapb.PrimaryStatus
   182  					Error  error
   183  				}{
   184  					"zone1-0000000100": {
   185  						Status: &replicationdatapb.PrimaryStatus{
   186  							// value of Position doesn't strictly matter for
   187  							// this test case, as long as it matches the inner
   188  							// key of the WaitForPositionResults map for the
   189  							// primary-elect.
   190  							Position: "position1",
   191  						},
   192  						Error: nil,
   193  					},
   194  				},
   195  				PrimaryPositionResults: map[string]struct {
   196  					Position string
   197  					Error    error
   198  				}{
   199  					"zone1-0000000100": {
   200  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
   201  					},
   202  				},
   203  				PromoteReplicaResults: map[string]struct {
   204  					Result string
   205  					Error  error
   206  				}{
   207  					"zone1-0000000200": {
   208  						Result: "successful reparent journal position",
   209  						Error:  nil,
   210  					},
   211  				},
   212  				SetReplicationSourceResults: map[string]error{
   213  					"zone1-0000000100": nil,
   214  					"zone1-0000000200": nil,
   215  				},
   216  				WaitForPositionResults: map[string]map[string]error{
   217  					"zone1-0000000200": {
   218  						"position1": nil,
   219  					},
   220  				},
   221  				PopulateReparentJournalResults: map[string]error{
   222  					"zone1-0000000200": nil,
   223  				},
   224  			},
   225  			tablets: []*topodatapb.Tablet{
   226  				{
   227  					Alias: &topodatapb.TabletAlias{
   228  						Cell: "zone1",
   229  						Uid:  100,
   230  					},
   231  					Type:     topodatapb.TabletType_PRIMARY,
   232  					Keyspace: "testkeyspace",
   233  					Shard:    "-",
   234  				},
   235  				{
   236  					Alias: &topodatapb.TabletAlias{
   237  						Cell: "zone1",
   238  						Uid:  200,
   239  					},
   240  					Type:     topodatapb.TabletType_REPLICA,
   241  					Keyspace: "testkeyspace",
   242  					Shard:    "-",
   243  				},
   244  			},
   245  
   246  			keyspace: "testkeyspace",
   247  			shard:    "-",
   248  			opts:     PlannedReparentOptions{},
   249  
   250  			shouldErr: false,
   251  			expectedEvent: &events.Reparent{
   252  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   253  					PrimaryAlias: &topodatapb.TabletAlias{
   254  						Cell: "zone1",
   255  						Uid:  100,
   256  					},
   257  					KeyRange:         &topodatapb.KeyRange{},
   258  					IsPrimaryServing: true,
   259  				}, nil),
   260  				OldPrimary: &topodatapb.Tablet{
   261  					Alias: &topodatapb.TabletAlias{
   262  						Cell: "zone1",
   263  						Uid:  100,
   264  					},
   265  					Type:     topodatapb.TabletType_PRIMARY,
   266  					Keyspace: "testkeyspace",
   267  					Shard:    "-",
   268  				},
   269  				NewPrimary: &topodatapb.Tablet{
   270  					Alias: &topodatapb.TabletAlias{
   271  						Cell: "zone1",
   272  						Uid:  200,
   273  					},
   274  					Type:     topodatapb.TabletType_REPLICA,
   275  					Keyspace: "testkeyspace",
   276  					Shard:    "-",
   277  				},
   278  			},
   279  		},
   280  		{
   281  			name: "already locked shard",
   282  			ts:   memorytopo.NewServer("zone1"),
   283  			tmc: &testutil.TabletManagerClient{
   284  				PrimaryPositionResults: map[string]struct {
   285  					Position string
   286  					Error    error
   287  				}{
   288  					"zone1-0000000100": {
   289  						Position: "position1",
   290  						Error:    nil,
   291  					},
   292  				},
   293  				PopulateReparentJournalResults: map[string]error{
   294  					"zone1-0000000100": nil,
   295  				},
   296  				SetReplicationSourceResults: map[string]error{
   297  					"zone1-0000000200": nil,
   298  				},
   299  				SetReadWriteResults: map[string]error{
   300  					"zone1-0000000100": nil,
   301  				},
   302  			},
   303  			tablets: []*topodatapb.Tablet{
   304  				{
   305  					Alias: &topodatapb.TabletAlias{
   306  						Cell: "zone1",
   307  						Uid:  100,
   308  					},
   309  					Type:     topodatapb.TabletType_PRIMARY,
   310  					Keyspace: "testkeyspace",
   311  					Shard:    "-",
   312  				},
   313  				{
   314  					Alias: &topodatapb.TabletAlias{
   315  						Cell: "zone1",
   316  						Uid:  200,
   317  					},
   318  					Type:     topodatapb.TabletType_REPLICA,
   319  					Keyspace: "testkeyspace",
   320  					Shard:    "-",
   321  				},
   322  			},
   323  			lockShardBeforeTest: true,
   324  
   325  			keyspace: "testkeyspace",
   326  			shard:    "-",
   327  			opts: PlannedReparentOptions{
   328  				NewPrimaryAlias: &topodatapb.TabletAlias{
   329  					Cell: "zone1",
   330  					Uid:  100,
   331  				},
   332  			},
   333  
   334  			shouldErr: false,
   335  			expectedEvent: &events.Reparent{
   336  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   337  					PrimaryAlias: &topodatapb.TabletAlias{
   338  						Cell: "zone1",
   339  						Uid:  100,
   340  					},
   341  					KeyRange:         &topodatapb.KeyRange{},
   342  					IsPrimaryServing: true,
   343  				}, nil),
   344  				NewPrimary: &topodatapb.Tablet{
   345  					Alias: &topodatapb.TabletAlias{
   346  						Cell: "zone1",
   347  						Uid:  100,
   348  					},
   349  					Type:     topodatapb.TabletType_PRIMARY,
   350  					Keyspace: "testkeyspace",
   351  					Shard:    "-",
   352  				},
   353  			},
   354  		},
   355  		{
   356  			// The simplest setup required to make an overall ReparentShard call
   357  			// fail is to set NewPrimaryAlias = AvoidPrimaryAlias, which will
   358  			// fail the preflight checks. Other functions are unit-tested
   359  			// thoroughly to cover all the cases.
   360  			name: "reparent fails",
   361  			ts:   memorytopo.NewServer("zone1"),
   362  			tmc:  nil,
   363  			tablets: []*topodatapb.Tablet{
   364  				{
   365  					Alias: &topodatapb.TabletAlias{
   366  						Cell: "zone1",
   367  						Uid:  100,
   368  					},
   369  					Keyspace: "testkeyspace",
   370  					Shard:    "-",
   371  					Type:     topodatapb.TabletType_PRIMARY,
   372  				},
   373  			},
   374  
   375  			keyspace: "testkeyspace",
   376  			shard:    "-",
   377  			opts: PlannedReparentOptions{
   378  				NewPrimaryAlias: &topodatapb.TabletAlias{
   379  					Cell: "zone1",
   380  					Uid:  100,
   381  				},
   382  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   383  					Cell: "zone1",
   384  					Uid:  100,
   385  				},
   386  			},
   387  
   388  			expectedEvent: &events.Reparent{
   389  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   390  					PrimaryAlias: &topodatapb.TabletAlias{
   391  						Cell: "zone1",
   392  						Uid:  100,
   393  					},
   394  					IsPrimaryServing: true,
   395  					KeyRange:         &topodatapb.KeyRange{},
   396  				}, nil),
   397  			},
   398  			shouldErr: true,
   399  		},
   400  	}
   401  
   402  	ctx := context.Background()
   403  	logger := logutil.NewMemoryLogger()
   404  
   405  	for _, tt := range tests {
   406  		tt := tt
   407  
   408  		t.Run(tt.name, func(t *testing.T) {
   409  			t.Parallel()
   410  
   411  			ctx := ctx
   412  
   413  			testutil.AddTablets(ctx, t, tt.ts, &testutil.AddTabletOptions{
   414  				AlsoSetShardPrimary: true,
   415  				SkipShardCreation:   false,
   416  			}, tt.tablets...)
   417  
   418  			if tt.lockShardBeforeTest {
   419  				lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "locking for test")
   420  				require.NoError(t, err, "could not lock %s/%s for test case", tt.keyspace, tt.shard)
   421  
   422  				defer func() {
   423  					unlock(&err)
   424  					require.NoError(t, err, "could not unlock %s/%s after test case", tt.keyspace, tt.shard)
   425  				}()
   426  
   427  				ctx = lctx
   428  			}
   429  
   430  			pr := NewPlannedReparenter(tt.ts, tt.tmc, logger)
   431  			ev, err := pr.ReparentShard(ctx, tt.keyspace, tt.shard, tt.opts)
   432  			if tt.shouldErr {
   433  				assert.Error(t, err)
   434  				AssertReparentEventsEqual(t, tt.expectedEvent, ev)
   435  
   436  				if ev != nil {
   437  					assert.Contains(t, ev.Status, "failed PlannedReparentShard", "expected event status to indicate failed PRS")
   438  				}
   439  
   440  				return
   441  			}
   442  
   443  			assert.NoError(t, err)
   444  			AssertReparentEventsEqual(t, tt.expectedEvent, ev)
   445  			assert.Contains(t, ev.Status, "finished PlannedReparentShard", "expected event status to indicate successful PRS")
   446  		})
   447  	}
   448  }
   449  
   450  func TestPlannedReparenter_getLockAction(t *testing.T) {
   451  	t.Parallel()
   452  
   453  	pr := &PlannedReparenter{}
   454  	tests := []struct {
   455  		name     string
   456  		opts     PlannedReparentOptions
   457  		expected string
   458  	}{
   459  		{
   460  			name:     "no options",
   461  			opts:     PlannedReparentOptions{},
   462  			expected: "PlannedReparentShard(<nil>, AvoidPrimary = <nil>)",
   463  		},
   464  		{
   465  			name: "desired primary only",
   466  			opts: PlannedReparentOptions{
   467  				NewPrimaryAlias: &topodatapb.TabletAlias{
   468  					Cell: "zone1",
   469  					Uid:  100,
   470  				},
   471  			},
   472  			expected: "PlannedReparentShard(zone1-0000000100, AvoidPrimary = <nil>)",
   473  		},
   474  		{
   475  			name: "avoid-primary only",
   476  			opts: PlannedReparentOptions{
   477  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   478  					Cell: "zone1",
   479  					Uid:  500,
   480  				},
   481  			},
   482  			expected: "PlannedReparentShard(<nil>, AvoidPrimary = zone1-0000000500)",
   483  		},
   484  		{
   485  			name: "all options specified",
   486  			opts: PlannedReparentOptions{
   487  				NewPrimaryAlias: &topodatapb.TabletAlias{
   488  					Cell: "zone1",
   489  					Uid:  100,
   490  				},
   491  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   492  					Cell: "zone1",
   493  					Uid:  500,
   494  				},
   495  			},
   496  			expected: "PlannedReparentShard(zone1-0000000100, AvoidPrimary = zone1-0000000500)",
   497  		},
   498  	}
   499  
   500  	for _, tt := range tests {
   501  		tt := tt
   502  
   503  		t.Run(tt.name, func(t *testing.T) {
   504  			t.Parallel()
   505  
   506  			actual := pr.getLockAction(tt.opts)
   507  			assert.Equal(t, tt.expected, actual)
   508  		})
   509  	}
   510  }
   511  
   512  func TestPlannedReparenter_preflightChecks(t *testing.T) {
   513  	t.Parallel()
   514  
   515  	tests := []struct {
   516  		name string
   517  
   518  		ts      *topo.Server
   519  		tmc     tmclient.TabletManagerClient
   520  		tablets []*topodatapb.Tablet
   521  
   522  		ev        *events.Reparent
   523  		keyspace  string
   524  		shard     string
   525  		tabletMap map[string]*topo.TabletInfo
   526  		opts      *PlannedReparentOptions
   527  
   528  		expectedIsNoop bool
   529  		expectedEvent  *events.Reparent
   530  		expectedOpts   *PlannedReparentOptions
   531  		shouldErr      bool
   532  	}{
   533  		{
   534  			name: "invariants hold",
   535  			ev: &events.Reparent{
   536  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   537  					PrimaryAlias: &topodatapb.TabletAlias{
   538  						Cell: "zone1",
   539  						Uid:  500,
   540  					},
   541  				}, nil),
   542  			},
   543  			tabletMap: map[string]*topo.TabletInfo{
   544  				"zone1-0000000100": {
   545  					Tablet: &topodatapb.Tablet{
   546  						Alias: &topodatapb.TabletAlias{
   547  							Cell: "zone1",
   548  							Uid:  100,
   549  						},
   550  					},
   551  				},
   552  			},
   553  			opts: &PlannedReparentOptions{
   554  				NewPrimaryAlias: &topodatapb.TabletAlias{
   555  					Cell: "zone1",
   556  					Uid:  100,
   557  				},
   558  			},
   559  			expectedIsNoop: false,
   560  			expectedEvent: &events.Reparent{
   561  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   562  					PrimaryAlias: &topodatapb.TabletAlias{
   563  						Cell: "zone1",
   564  						Uid:  500,
   565  					},
   566  				}, nil),
   567  				NewPrimary: &topodatapb.Tablet{
   568  					Alias: &topodatapb.TabletAlias{
   569  						Cell: "zone1",
   570  						Uid:  100,
   571  					},
   572  				},
   573  			},
   574  			shouldErr: false,
   575  		},
   576  		{
   577  			name: "invariants hold with primary selection",
   578  			tmc: &testutil.TabletManagerClient{
   579  				ReplicationStatusResults: map[string]struct {
   580  					Position *replicationdatapb.Status
   581  					Error    error
   582  				}{
   583  					"zone1-0000000100": { // most advanced position
   584  						Position: &replicationdatapb.Status{
   585  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
   586  						},
   587  					},
   588  					"zone1-0000000101": {
   589  						Position: &replicationdatapb.Status{
   590  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
   591  						},
   592  					},
   593  				},
   594  			},
   595  			ev: &events.Reparent{
   596  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   597  					PrimaryAlias: &topodatapb.TabletAlias{
   598  						Cell: "zone1",
   599  						Uid:  500,
   600  					},
   601  				}, nil),
   602  			},
   603  			tabletMap: map[string]*topo.TabletInfo{
   604  				"zone1-0000000100": {
   605  					Tablet: &topodatapb.Tablet{
   606  						Alias: &topodatapb.TabletAlias{
   607  							Cell: "zone1",
   608  							Uid:  100,
   609  						},
   610  						Type: topodatapb.TabletType_REPLICA,
   611  					},
   612  				},
   613  				"zone1-0000000101": {
   614  					Tablet: &topodatapb.Tablet{
   615  						Alias: &topodatapb.TabletAlias{
   616  							Cell: "zone1",
   617  							Uid:  101,
   618  						},
   619  						Type: topodatapb.TabletType_REPLICA,
   620  					},
   621  				},
   622  				"zone1-0000000500": {
   623  					Tablet: &topodatapb.Tablet{
   624  						Alias: &topodatapb.TabletAlias{
   625  							Cell: "zone1",
   626  							Uid:  500,
   627  						},
   628  						Type: topodatapb.TabletType_PRIMARY,
   629  					},
   630  				},
   631  			},
   632  			opts: &PlannedReparentOptions{
   633  				// Avoid the current primary.
   634  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   635  					Cell: "zone1",
   636  					Uid:  500,
   637  				},
   638  				durability: &durabilityNone{},
   639  			},
   640  			expectedIsNoop: false,
   641  			expectedEvent: &events.Reparent{
   642  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   643  					PrimaryAlias: &topodatapb.TabletAlias{
   644  						Cell: "zone1",
   645  						Uid:  500,
   646  					},
   647  				}, nil),
   648  				NewPrimary: &topodatapb.Tablet{
   649  					Alias: &topodatapb.TabletAlias{
   650  						Cell: "zone1",
   651  						Uid:  100,
   652  					},
   653  					Type: topodatapb.TabletType_REPLICA,
   654  				},
   655  			},
   656  			expectedOpts: &PlannedReparentOptions{
   657  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   658  					Cell: "zone1",
   659  					Uid:  500,
   660  				},
   661  				// NewPrimaryAlias gets populated by the preflightCheck code
   662  				NewPrimaryAlias: &topodatapb.TabletAlias{
   663  					Cell: "zone1",
   664  					Uid:  100,
   665  				},
   666  				durability: &durabilityNone{},
   667  			},
   668  			shouldErr: false,
   669  		},
   670  		{
   671  			name: "new-primary and avoid-primary match",
   672  			opts: &PlannedReparentOptions{
   673  				NewPrimaryAlias: &topodatapb.TabletAlias{
   674  					Cell: "zone1",
   675  					Uid:  100,
   676  				},
   677  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   678  					Cell: "zone1",
   679  					Uid:  100,
   680  				},
   681  			},
   682  			expectedIsNoop: true,
   683  			shouldErr:      true,
   684  		},
   685  		{
   686  			name: "current shard primary is not avoid-primary",
   687  			ev: &events.Reparent{
   688  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   689  					PrimaryAlias: &topodatapb.TabletAlias{
   690  						Cell: "zone1",
   691  						Uid:  100,
   692  					},
   693  				}, nil),
   694  			},
   695  			opts: &PlannedReparentOptions{
   696  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   697  					Cell: "zone1",
   698  					Uid:  200,
   699  				},
   700  			},
   701  			expectedIsNoop: true, // nothing to do, but not an error!
   702  			shouldErr:      false,
   703  		},
   704  		{
   705  			// this doesn't cause an actual error from ChooseNewPrimary, because
   706  			// there is no way to do that other than something going horribly wrong
   707  			// in go runtime, however we do check that we
   708  			// get a non-nil result from ChooseNewPrimary in preflightChecks and
   709  			// bail out if we don't, so we're forcing that case here.
   710  			name: "cannot choose new primary-elect",
   711  			ev: &events.Reparent{
   712  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   713  					PrimaryAlias: &topodatapb.TabletAlias{
   714  						Cell: "zone1",
   715  						Uid:  100,
   716  					},
   717  				}, nil),
   718  			},
   719  			tabletMap: map[string]*topo.TabletInfo{
   720  				"zone1-0000000100": {
   721  					Tablet: &topodatapb.Tablet{
   722  						Alias: &topodatapb.TabletAlias{
   723  							Cell: "zone1",
   724  							Uid:  100,
   725  						},
   726  					},
   727  				},
   728  			},
   729  			opts: &PlannedReparentOptions{
   730  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   731  					Cell: "zone1",
   732  					Uid:  100,
   733  				},
   734  			},
   735  			expectedIsNoop: true,
   736  			shouldErr:      true,
   737  		},
   738  		{
   739  			name:      "primary-elect is not in tablet map",
   740  			ev:        &events.Reparent{},
   741  			tabletMap: map[string]*topo.TabletInfo{},
   742  			opts: &PlannedReparentOptions{
   743  				NewPrimaryAlias: &topodatapb.TabletAlias{
   744  					Cell: "zone1",
   745  					Uid:  100,
   746  				},
   747  			},
   748  			expectedIsNoop: true,
   749  			shouldErr:      true,
   750  		},
   751  		{
   752  			name: "shard has no current primary",
   753  			ev: &events.Reparent{
   754  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   755  					PrimaryAlias: nil,
   756  				}, nil),
   757  			},
   758  			tabletMap: map[string]*topo.TabletInfo{
   759  				"zone1-0000000100": {
   760  					Tablet: &topodatapb.Tablet{
   761  						Alias: &topodatapb.TabletAlias{
   762  							Cell: "zone1",
   763  							Uid:  100,
   764  						},
   765  					},
   766  				},
   767  			},
   768  			opts: &PlannedReparentOptions{
   769  				NewPrimaryAlias: &topodatapb.TabletAlias{
   770  					Cell: "zone1",
   771  					Uid:  100,
   772  				},
   773  			},
   774  			expectedIsNoop: false,
   775  			expectedEvent: &events.Reparent{
   776  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   777  					PrimaryAlias: nil,
   778  				}, nil),
   779  				NewPrimary: &topodatapb.Tablet{
   780  					Alias: &topodatapb.TabletAlias{
   781  						Cell: "zone1",
   782  						Uid:  100,
   783  					},
   784  				},
   785  			},
   786  			shouldErr: false,
   787  		},
   788  		{
   789  			name: "shard has no current primary and new primary not provided - initialisation test",
   790  			ev: &events.Reparent{
   791  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   792  					PrimaryAlias: nil,
   793  				}, nil),
   794  			},
   795  			tmc: &testutil.TabletManagerClient{
   796  				ReplicationStatusResults: map[string]struct {
   797  					Position *replicationdatapb.Status
   798  					Error    error
   799  				}{
   800  					"zone1-0000000100": { // most advanced position
   801  						Error: mysql.ErrNotReplica,
   802  					},
   803  				},
   804  			},
   805  			tabletMap: map[string]*topo.TabletInfo{
   806  				"zone1-0000000100": {
   807  					Tablet: &topodatapb.Tablet{
   808  						Alias: &topodatapb.TabletAlias{
   809  							Cell: "zone1",
   810  							Uid:  100,
   811  						},
   812  						Type: topodatapb.TabletType_REPLICA,
   813  					},
   814  				},
   815  			},
   816  			opts:           &PlannedReparentOptions{},
   817  			expectedIsNoop: false,
   818  			expectedEvent: &events.Reparent{
   819  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   820  					PrimaryAlias: nil,
   821  				}, nil),
   822  				NewPrimary: &topodatapb.Tablet{
   823  					Alias: &topodatapb.TabletAlias{
   824  						Cell: "zone1",
   825  						Uid:  100,
   826  					},
   827  					Type: topodatapb.TabletType_REPLICA,
   828  				},
   829  			},
   830  			shouldErr: false,
   831  		},
   832  		{
   833  			name: "primary elect can't make forward progress",
   834  			tmc: &testutil.TabletManagerClient{
   835  				ReplicationStatusResults: map[string]struct {
   836  					Position *replicationdatapb.Status
   837  					Error    error
   838  				}{
   839  					"zone1-0000000100": { // most advanced position
   840  						Position: &replicationdatapb.Status{
   841  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
   842  						},
   843  					},
   844  					"zone1-0000000101": {
   845  						Position: &replicationdatapb.Status{
   846  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
   847  						},
   848  					},
   849  				},
   850  			},
   851  			ev: &events.Reparent{
   852  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   853  					PrimaryAlias: &topodatapb.TabletAlias{
   854  						Cell: "zone1",
   855  						Uid:  500,
   856  					},
   857  				}, nil),
   858  			},
   859  			tabletMap: map[string]*topo.TabletInfo{
   860  				"zone1-0000000100": {
   861  					Tablet: &topodatapb.Tablet{
   862  						Alias: &topodatapb.TabletAlias{
   863  							Cell: "zone1",
   864  							Uid:  100,
   865  						},
   866  						Type: topodatapb.TabletType_REPLICA,
   867  					},
   868  				},
   869  				"zone1-0000000101": {
   870  					Tablet: &topodatapb.Tablet{
   871  						Alias: &topodatapb.TabletAlias{
   872  							Cell: "zone1",
   873  							Uid:  101,
   874  						},
   875  						Type: topodatapb.TabletType_REPLICA,
   876  					},
   877  				},
   878  				"zone1-0000000500": {
   879  					Tablet: &topodatapb.Tablet{
   880  						Alias: &topodatapb.TabletAlias{
   881  							Cell: "zone1",
   882  							Uid:  500,
   883  						},
   884  						Type: topodatapb.TabletType_PRIMARY,
   885  					},
   886  				},
   887  			},
   888  			opts: &PlannedReparentOptions{
   889  				// Avoid the current primary.
   890  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
   891  					Cell: "zone1",
   892  					Uid:  500,
   893  				},
   894  				durability: &durabilityCrossCell{},
   895  			},
   896  			expectedIsNoop: true,
   897  			expectedEvent: &events.Reparent{
   898  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
   899  					PrimaryAlias: &topodatapb.TabletAlias{
   900  						Cell: "zone1",
   901  						Uid:  500,
   902  					},
   903  				}, nil),
   904  			},
   905  			shouldErr: true,
   906  		},
   907  	}
   908  
   909  	ctx := context.Background()
   910  	logger := logutil.NewMemoryLogger()
   911  
   912  	for _, tt := range tests {
   913  		tt := tt
   914  
   915  		t.Run(tt.name, func(t *testing.T) {
   916  			t.Parallel()
   917  
   918  			defer func() {
   919  				if tt.expectedEvent != nil {
   920  					AssertReparentEventsEqualWithMessage(t, tt.expectedEvent, tt.ev, "expected preflightChecks to mutate the passed-in event")
   921  				}
   922  
   923  				if tt.expectedOpts != nil {
   924  					utils.MustMatch(t, tt.expectedOpts, tt.opts, "expected preflightChecks to mutate the passed in PlannedReparentOptions")
   925  				}
   926  			}()
   927  
   928  			pr := NewPlannedReparenter(tt.ts, tt.tmc, logger)
   929  			if tt.opts.durability == nil {
   930  				durability, err := GetDurabilityPolicy("none")
   931  				require.NoError(t, err)
   932  				tt.opts.durability = durability
   933  			}
   934  			isNoop, err := pr.preflightChecks(ctx, tt.ev, tt.keyspace, tt.shard, tt.tabletMap, tt.opts)
   935  			if tt.shouldErr {
   936  				assert.Error(t, err)
   937  				assert.Equal(t, tt.expectedIsNoop, isNoop, "preflightChecks returned wrong isNoop signal")
   938  
   939  				return
   940  			}
   941  
   942  			assert.NoError(t, err)
   943  			assert.Equal(t, tt.expectedIsNoop, isNoop, "preflightChecks returned wrong isNoop signal")
   944  		})
   945  	}
   946  }
   947  
   948  func TestPlannedReparenter_performGracefulPromotion(t *testing.T) {
   949  	t.Parallel()
   950  
   951  	tests := []struct {
   952  		name       string
   953  		ts         *topo.Server
   954  		tmc        tmclient.TabletManagerClient
   955  		unlockTopo bool
   956  		ctxTimeout time.Duration
   957  
   958  		ev             *events.Reparent
   959  		keyspace       string
   960  		shard          string
   961  		currentPrimary *topo.TabletInfo
   962  		primaryElect   *topodatapb.Tablet
   963  		tabletMap      map[string]*topo.TabletInfo
   964  		opts           PlannedReparentOptions
   965  
   966  		expectedPos   string
   967  		expectedEvent *events.Reparent
   968  		shouldErr     bool
   969  		// Optional function to run some additional post-test assertions. Will
   970  		// be run in the main test body before the common assertions are run,
   971  		// regardless of the value of tt.shouldErr for that test case.
   972  		extraAssertions func(t *testing.T, pos string, err error)
   973  	}{
   974  		{
   975  			name: "successful promotion",
   976  			ts:   memorytopo.NewServer("zone1"),
   977  			tmc: &testutil.TabletManagerClient{
   978  				DemotePrimaryResults: map[string]struct {
   979  					Status *replicationdatapb.PrimaryStatus
   980  					Error  error
   981  				}{
   982  					"zone1-0000000100": {
   983  						Status: &replicationdatapb.PrimaryStatus{
   984  							// value of Position doesn't strictly matter for
   985  							// this test case, as long as it matches the inner
   986  							// key of the WaitForPositionResults map for the
   987  							// primary-elect.
   988  							Position: "position1",
   989  						},
   990  						Error: nil,
   991  					},
   992  				},
   993  				PrimaryPositionResults: map[string]struct {
   994  					Position string
   995  					Error    error
   996  				}{
   997  					"zone1-0000000100": {
   998  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
   999  					},
  1000  				},
  1001  				PromoteReplicaResults: map[string]struct {
  1002  					Result string
  1003  					Error  error
  1004  				}{
  1005  					"zone1-0000000200": {
  1006  						Result: "successful reparent journal position",
  1007  						Error:  nil,
  1008  					},
  1009  				},
  1010  				SetReplicationSourceResults: map[string]error{
  1011  					"zone1-0000000200": nil,
  1012  				},
  1013  				WaitForPositionResults: map[string]map[string]error{
  1014  					"zone1-0000000200": {
  1015  						"position1": nil,
  1016  					},
  1017  				},
  1018  			},
  1019  			ev:       &events.Reparent{},
  1020  			keyspace: "testkeyspace",
  1021  			shard:    "-",
  1022  			currentPrimary: &topo.TabletInfo{
  1023  				Tablet: &topodatapb.Tablet{
  1024  					Alias: &topodatapb.TabletAlias{
  1025  						Cell: "zone1",
  1026  						Uid:  100,
  1027  					},
  1028  				},
  1029  			},
  1030  			primaryElect: &topodatapb.Tablet{
  1031  				Alias: &topodatapb.TabletAlias{
  1032  					Cell: "zone1",
  1033  					Uid:  200,
  1034  				},
  1035  			},
  1036  			tabletMap:   map[string]*topo.TabletInfo{},
  1037  			opts:        PlannedReparentOptions{},
  1038  			expectedPos: "successful reparent journal position",
  1039  			shouldErr:   false,
  1040  		},
  1041  		{
  1042  			name: "cannot get snapshot of current primary",
  1043  			ts:   memorytopo.NewServer("zone1"),
  1044  			tmc: &testutil.TabletManagerClient{
  1045  				PrimaryPositionResults: map[string]struct {
  1046  					Position string
  1047  					Error    error
  1048  				}{
  1049  					"zone1-0000000100": {
  1050  						Error: assert.AnError,
  1051  					},
  1052  				},
  1053  			},
  1054  			ev:       &events.Reparent{},
  1055  			keyspace: "testkeyspace",
  1056  			shard:    "-",
  1057  			currentPrimary: &topo.TabletInfo{
  1058  				Tablet: &topodatapb.Tablet{
  1059  					Alias: &topodatapb.TabletAlias{
  1060  						Cell: "zone1",
  1061  						Uid:  100,
  1062  					},
  1063  				},
  1064  			},
  1065  			primaryElect: &topodatapb.Tablet{
  1066  				Alias: &topodatapb.TabletAlias{
  1067  					Cell: "zone1",
  1068  					Uid:  200,
  1069  				},
  1070  			},
  1071  			tabletMap: map[string]*topo.TabletInfo{},
  1072  			opts:      PlannedReparentOptions{},
  1073  			shouldErr: true,
  1074  		},
  1075  		{
  1076  			name: "primary-elect fails to catch up to current primary snapshot position",
  1077  			ts:   memorytopo.NewServer("zone1"),
  1078  			tmc: &testutil.TabletManagerClient{
  1079  				PrimaryPositionResults: map[string]struct {
  1080  					Position string
  1081  					Error    error
  1082  				}{
  1083  					"zone1-0000000100": {
  1084  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1085  					},
  1086  				},
  1087  				SetReplicationSourceResults: map[string]error{
  1088  					"zone1-0000000200": assert.AnError,
  1089  				},
  1090  			},
  1091  			ev:       &events.Reparent{},
  1092  			keyspace: "testkeyspace",
  1093  			shard:    "-",
  1094  			currentPrimary: &topo.TabletInfo{
  1095  				Tablet: &topodatapb.Tablet{
  1096  					Alias: &topodatapb.TabletAlias{
  1097  						Cell: "zone1",
  1098  						Uid:  100,
  1099  					},
  1100  				},
  1101  			},
  1102  			primaryElect: &topodatapb.Tablet{
  1103  				Alias: &topodatapb.TabletAlias{
  1104  					Cell: "zone1",
  1105  					Uid:  200,
  1106  				},
  1107  			},
  1108  			tabletMap: map[string]*topo.TabletInfo{},
  1109  			opts:      PlannedReparentOptions{},
  1110  			shouldErr: true,
  1111  		},
  1112  		{
  1113  			name: "primary-elect times out catching up to current primary snapshot position",
  1114  			ts:   memorytopo.NewServer("zone1"),
  1115  			tmc: &testutil.TabletManagerClient{
  1116  				PrimaryPositionResults: map[string]struct {
  1117  					Position string
  1118  					Error    error
  1119  				}{
  1120  					"zone1-0000000100": {
  1121  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1122  					},
  1123  				},
  1124  				SetReplicationSourceDelays: map[string]time.Duration{
  1125  					"zone1-0000000200": time.Millisecond * 100,
  1126  				},
  1127  				SetReplicationSourceResults: map[string]error{
  1128  					"zone1-0000000200": nil,
  1129  				},
  1130  			},
  1131  			ev:       &events.Reparent{},
  1132  			keyspace: "testkeyspace",
  1133  			shard:    "-",
  1134  			currentPrimary: &topo.TabletInfo{
  1135  				Tablet: &topodatapb.Tablet{
  1136  					Alias: &topodatapb.TabletAlias{
  1137  						Cell: "zone1",
  1138  						Uid:  100,
  1139  					},
  1140  				},
  1141  			},
  1142  			primaryElect: &topodatapb.Tablet{
  1143  				Alias: &topodatapb.TabletAlias{
  1144  					Cell: "zone1",
  1145  					Uid:  200,
  1146  				},
  1147  			},
  1148  			tabletMap: map[string]*topo.TabletInfo{},
  1149  			opts: PlannedReparentOptions{
  1150  				WaitReplicasTimeout: time.Millisecond * 10,
  1151  			},
  1152  			shouldErr: true,
  1153  		},
  1154  		{
  1155  			name: "lost topology lock",
  1156  			ts:   memorytopo.NewServer("zone1"),
  1157  			tmc: &testutil.TabletManagerClient{
  1158  				PrimaryPositionResults: map[string]struct {
  1159  					Position string
  1160  					Error    error
  1161  				}{
  1162  					"zone1-0000000100": {
  1163  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1164  					},
  1165  				},
  1166  				SetReplicationSourceResults: map[string]error{
  1167  					"zone1-0000000200": nil,
  1168  				},
  1169  			},
  1170  			unlockTopo: true,
  1171  			ev:         &events.Reparent{},
  1172  			keyspace:   "testkeyspace",
  1173  			shard:      "-",
  1174  			currentPrimary: &topo.TabletInfo{
  1175  				Tablet: &topodatapb.Tablet{
  1176  					Alias: &topodatapb.TabletAlias{
  1177  						Cell: "zone1",
  1178  						Uid:  100,
  1179  					},
  1180  				},
  1181  			},
  1182  			primaryElect: &topodatapb.Tablet{
  1183  				Alias: &topodatapb.TabletAlias{
  1184  					Cell: "zone1",
  1185  					Uid:  200,
  1186  				},
  1187  			},
  1188  			tabletMap: map[string]*topo.TabletInfo{},
  1189  			opts:      PlannedReparentOptions{},
  1190  			shouldErr: true,
  1191  		},
  1192  		{
  1193  			name: "failed to demote current primary",
  1194  			ts:   memorytopo.NewServer("zone1"),
  1195  			tmc: &testutil.TabletManagerClient{
  1196  				DemotePrimaryResults: map[string]struct {
  1197  					Status *replicationdatapb.PrimaryStatus
  1198  					Error  error
  1199  				}{
  1200  					"zone1-0000000100": {
  1201  						Error: assert.AnError,
  1202  					},
  1203  				},
  1204  				PrimaryPositionResults: map[string]struct {
  1205  					Position string
  1206  					Error    error
  1207  				}{
  1208  					"zone1-0000000100": {
  1209  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1210  					},
  1211  				},
  1212  				SetReplicationSourceResults: map[string]error{
  1213  					"zone1-0000000200": nil,
  1214  				},
  1215  			},
  1216  			ev:       &events.Reparent{},
  1217  			keyspace: "testkeyspace",
  1218  			shard:    "-",
  1219  			currentPrimary: &topo.TabletInfo{
  1220  				Tablet: &topodatapb.Tablet{
  1221  					Alias: &topodatapb.TabletAlias{
  1222  						Cell: "zone1",
  1223  						Uid:  100,
  1224  					},
  1225  				},
  1226  			},
  1227  			primaryElect: &topodatapb.Tablet{
  1228  				Alias: &topodatapb.TabletAlias{
  1229  					Cell: "zone1",
  1230  					Uid:  200,
  1231  				},
  1232  			},
  1233  			tabletMap: map[string]*topo.TabletInfo{},
  1234  			opts:      PlannedReparentOptions{},
  1235  			shouldErr: true,
  1236  		},
  1237  		{
  1238  			name: "primary-elect fails to catch up to current primary demotion position",
  1239  			ts:   memorytopo.NewServer("zone1"),
  1240  			tmc: &testutil.TabletManagerClient{
  1241  				DemotePrimaryResults: map[string]struct {
  1242  					Status *replicationdatapb.PrimaryStatus
  1243  					Error  error
  1244  				}{
  1245  					"zone1-0000000100": {
  1246  						Status: &replicationdatapb.PrimaryStatus{
  1247  							// value of Position doesn't strictly matter for
  1248  							// this test case, as long as it matches the inner
  1249  							// key of the WaitForPositionResults map for the
  1250  							// primary-elect.
  1251  							Position: "position1",
  1252  						},
  1253  						Error: nil,
  1254  					},
  1255  				},
  1256  				PrimaryPositionResults: map[string]struct {
  1257  					Position string
  1258  					Error    error
  1259  				}{
  1260  					"zone1-0000000100": {
  1261  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1262  					},
  1263  				},
  1264  				SetReplicationSourceResults: map[string]error{
  1265  					"zone1-0000000200": nil,
  1266  				},
  1267  				WaitForPositionResults: map[string]map[string]error{
  1268  					"zone1-0000000200": {
  1269  						"position1": assert.AnError,
  1270  					},
  1271  				},
  1272  			},
  1273  			ev:       &events.Reparent{},
  1274  			keyspace: "testkeyspace",
  1275  			shard:    "-",
  1276  			currentPrimary: &topo.TabletInfo{
  1277  				Tablet: &topodatapb.Tablet{
  1278  					Alias: &topodatapb.TabletAlias{
  1279  						Cell: "zone1",
  1280  						Uid:  100,
  1281  					},
  1282  				},
  1283  			},
  1284  			primaryElect: &topodatapb.Tablet{
  1285  				Alias: &topodatapb.TabletAlias{
  1286  					Cell: "zone1",
  1287  					Uid:  200,
  1288  				},
  1289  			},
  1290  			tabletMap: map[string]*topo.TabletInfo{},
  1291  			opts:      PlannedReparentOptions{},
  1292  			shouldErr: true,
  1293  		},
  1294  		{
  1295  			name: "primary-elect times out catching up to current primary demotion position",
  1296  			ts:   memorytopo.NewServer("zone1"),
  1297  			tmc: &testutil.TabletManagerClient{
  1298  				DemotePrimaryResults: map[string]struct {
  1299  					Status *replicationdatapb.PrimaryStatus
  1300  					Error  error
  1301  				}{
  1302  					"zone1-0000000100": {
  1303  						Status: &replicationdatapb.PrimaryStatus{
  1304  							// value of Position doesn't strictly matter for
  1305  							// this test case, as long as it matches the inner
  1306  							// key of the WaitForPositionResults map for the
  1307  							// primary-elect.
  1308  							Position: "position1",
  1309  						},
  1310  						Error: nil,
  1311  					},
  1312  				},
  1313  				PrimaryPositionResults: map[string]struct {
  1314  					Position string
  1315  					Error    error
  1316  				}{
  1317  					"zone1-0000000100": {
  1318  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1319  					},
  1320  				},
  1321  				SetReplicationSourceResults: map[string]error{
  1322  					"zone1-0000000200": nil,
  1323  				},
  1324  				WaitForPositionDelays: map[string]time.Duration{
  1325  					"zone1-0000000200": time.Millisecond * 100,
  1326  				},
  1327  				WaitForPositionResults: map[string]map[string]error{
  1328  					"zone1-0000000200": {
  1329  						"position1": nil,
  1330  					},
  1331  				},
  1332  			},
  1333  			ev:       &events.Reparent{},
  1334  			keyspace: "testkeyspace",
  1335  			shard:    "-",
  1336  			currentPrimary: &topo.TabletInfo{
  1337  				Tablet: &topodatapb.Tablet{
  1338  					Alias: &topodatapb.TabletAlias{
  1339  						Cell: "zone1",
  1340  						Uid:  100,
  1341  					},
  1342  				},
  1343  			},
  1344  			primaryElect: &topodatapb.Tablet{
  1345  				Alias: &topodatapb.TabletAlias{
  1346  					Cell: "zone1",
  1347  					Uid:  200,
  1348  				},
  1349  			},
  1350  			tabletMap: map[string]*topo.TabletInfo{},
  1351  			opts: PlannedReparentOptions{
  1352  				WaitReplicasTimeout: time.Millisecond * 10,
  1353  			},
  1354  			shouldErr: true,
  1355  		},
  1356  		{
  1357  			name: "demotion succeeds but parent context times out",
  1358  			ts:   memorytopo.NewServer("zone1"),
  1359  			tmc: &testutil.TabletManagerClient{
  1360  				DemotePrimaryResults: map[string]struct {
  1361  					Status *replicationdatapb.PrimaryStatus
  1362  					Error  error
  1363  				}{
  1364  					"zone1-0000000100": {
  1365  						Status: &replicationdatapb.PrimaryStatus{
  1366  							Position: "position1",
  1367  						},
  1368  						Error: nil,
  1369  					},
  1370  				},
  1371  				PrimaryPositionResults: map[string]struct {
  1372  					Position string
  1373  					Error    error
  1374  				}{
  1375  					"zone1-0000000100": {
  1376  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1377  					},
  1378  				},
  1379  				PromoteReplicaResults: map[string]struct {
  1380  					Result string
  1381  					Error  error
  1382  				}{
  1383  					// This being present means that if we don't encounter a
  1384  					// a case where either WaitForPosition errors, or the parent
  1385  					// context times out, then we will fail the test, since it
  1386  					// will cause the overall function under test to return no
  1387  					// error.
  1388  					"zone1-0000000200": {
  1389  						Result: "success!",
  1390  						Error:  nil,
  1391  					},
  1392  				},
  1393  				SetReplicationSourceResults: map[string]error{
  1394  					"zone1-0000000200": nil,
  1395  				},
  1396  				WaitForPositionPostDelays: map[string]time.Duration{
  1397  					"zone1-0000000200": time.Second * 1,
  1398  				},
  1399  				WaitForPositionResults: map[string]map[string]error{
  1400  					"zone1-0000000200": {
  1401  						"position1": nil,
  1402  					},
  1403  				},
  1404  			},
  1405  			ctxTimeout: time.Millisecond * 4, // WaitForPosition won't return error, but will timeout the parent context
  1406  			ev:         &events.Reparent{},
  1407  			keyspace:   "testkeyspace",
  1408  			shard:      "-",
  1409  			currentPrimary: &topo.TabletInfo{
  1410  				Tablet: &topodatapb.Tablet{
  1411  					Alias: &topodatapb.TabletAlias{
  1412  						Cell: "zone1",
  1413  						Uid:  100,
  1414  					},
  1415  				},
  1416  			},
  1417  			primaryElect: &topodatapb.Tablet{
  1418  				Alias: &topodatapb.TabletAlias{
  1419  					Cell: "zone1",
  1420  					Uid:  200,
  1421  				},
  1422  			},
  1423  			tabletMap: map[string]*topo.TabletInfo{},
  1424  			opts:      PlannedReparentOptions{},
  1425  			shouldErr: true,
  1426  		},
  1427  		{
  1428  			name: "rollback fails",
  1429  			ts:   memorytopo.NewServer("zone1"),
  1430  			tmc: &testutil.TabletManagerClient{
  1431  				DemotePrimaryResults: map[string]struct {
  1432  					Status *replicationdatapb.PrimaryStatus
  1433  					Error  error
  1434  				}{
  1435  					"zone1-0000000100": {
  1436  						Status: &replicationdatapb.PrimaryStatus{
  1437  							// value of Position doesn't strictly matter for
  1438  							// this test case, as long as it matches the inner
  1439  							// key of the WaitForPositionResults map for the
  1440  							// primary-elect.
  1441  							Position: "position1",
  1442  						},
  1443  						Error: nil,
  1444  					},
  1445  				},
  1446  				PrimaryPositionResults: map[string]struct {
  1447  					Position string
  1448  					Error    error
  1449  				}{
  1450  					"zone1-0000000100": {
  1451  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1452  					},
  1453  				},
  1454  				SetReplicationSourceResults: map[string]error{
  1455  					"zone1-0000000200": nil,
  1456  				},
  1457  				WaitForPositionResults: map[string]map[string]error{
  1458  					"zone1-0000000200": {
  1459  						"position1": assert.AnError,
  1460  					},
  1461  				},
  1462  				UndoDemotePrimaryResults: map[string]error{
  1463  					"zone1-0000000100": assert.AnError,
  1464  				},
  1465  			},
  1466  			ev:       &events.Reparent{},
  1467  			keyspace: "testkeyspace",
  1468  			shard:    "-",
  1469  			currentPrimary: &topo.TabletInfo{
  1470  				Tablet: &topodatapb.Tablet{
  1471  					Alias: &topodatapb.TabletAlias{
  1472  						Cell: "zone1",
  1473  						Uid:  100,
  1474  					},
  1475  				},
  1476  			},
  1477  			primaryElect: &topodatapb.Tablet{
  1478  				Alias: &topodatapb.TabletAlias{
  1479  					Cell: "zone1",
  1480  					Uid:  200,
  1481  				},
  1482  			},
  1483  			tabletMap: map[string]*topo.TabletInfo{},
  1484  			opts:      PlannedReparentOptions{},
  1485  			shouldErr: true,
  1486  			extraAssertions: func(t *testing.T, pos string, err error) {
  1487  				assert.Contains(t, err.Error(), "UndoDemotePrimary", "expected error to include information about failed demotion rollback")
  1488  			},
  1489  		},
  1490  		{
  1491  			name: "rollback succeeds",
  1492  			ts:   memorytopo.NewServer("zone1"),
  1493  			tmc: &testutil.TabletManagerClient{
  1494  				DemotePrimaryResults: map[string]struct {
  1495  					Status *replicationdatapb.PrimaryStatus
  1496  					Error  error
  1497  				}{
  1498  					"zone1-0000000100": {
  1499  						Status: &replicationdatapb.PrimaryStatus{
  1500  							// value of Position doesn't strictly matter for
  1501  							// this test case, as long as it matches the inner
  1502  							// key of the WaitForPositionResults map for the
  1503  							// primary-elect.
  1504  							Position: "position1",
  1505  						},
  1506  						Error: nil,
  1507  					},
  1508  				},
  1509  				PrimaryPositionResults: map[string]struct {
  1510  					Position string
  1511  					Error    error
  1512  				}{
  1513  					"zone1-0000000100": {
  1514  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1515  					},
  1516  				},
  1517  				SetReplicationSourceResults: map[string]error{
  1518  					"zone1-0000000200": nil,
  1519  				},
  1520  				WaitForPositionResults: map[string]map[string]error{
  1521  					"zone1-0000000200": {
  1522  						"position1": assert.AnError,
  1523  					},
  1524  				},
  1525  				UndoDemotePrimaryResults: map[string]error{
  1526  					"zone1-0000000100": nil,
  1527  				},
  1528  			},
  1529  			ev:       &events.Reparent{},
  1530  			keyspace: "testkeyspace",
  1531  			shard:    "-",
  1532  			currentPrimary: &topo.TabletInfo{
  1533  				Tablet: &topodatapb.Tablet{
  1534  					Alias: &topodatapb.TabletAlias{
  1535  						Cell: "zone1",
  1536  						Uid:  100,
  1537  					},
  1538  				},
  1539  			},
  1540  			primaryElect: &topodatapb.Tablet{
  1541  				Alias: &topodatapb.TabletAlias{
  1542  					Cell: "zone1",
  1543  					Uid:  200,
  1544  				},
  1545  			},
  1546  			tabletMap: map[string]*topo.TabletInfo{},
  1547  			opts:      PlannedReparentOptions{},
  1548  			shouldErr: true,
  1549  			extraAssertions: func(t *testing.T, pos string, err error) {
  1550  				assert.NotContains(t, err.Error(), "UndoDemotePrimary", "expected error to not include information about failed demotion rollback")
  1551  			},
  1552  		},
  1553  		{
  1554  			name: "primary-elect fails to promote",
  1555  			ts:   memorytopo.NewServer("zone1"),
  1556  			tmc: &testutil.TabletManagerClient{
  1557  				DemotePrimaryResults: map[string]struct {
  1558  					Status *replicationdatapb.PrimaryStatus
  1559  					Error  error
  1560  				}{
  1561  					"zone1-0000000100": {
  1562  						Status: &replicationdatapb.PrimaryStatus{
  1563  							// value of Position doesn't strictly matter for
  1564  							// this test case, as long as it matches the inner
  1565  							// key of the WaitForPositionResults map for the
  1566  							// primary-elect.
  1567  							Position: "position1",
  1568  						},
  1569  						Error: nil,
  1570  					},
  1571  				},
  1572  				PrimaryPositionResults: map[string]struct {
  1573  					Position string
  1574  					Error    error
  1575  				}{
  1576  					"zone1-0000000100": {
  1577  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1578  					},
  1579  				},
  1580  				PromoteReplicaResults: map[string]struct {
  1581  					Result string
  1582  					Error  error
  1583  				}{
  1584  					"zone1-0000000200": {
  1585  						Error: assert.AnError,
  1586  					},
  1587  				},
  1588  				SetReplicationSourceResults: map[string]error{
  1589  					"zone1-0000000200": nil,
  1590  				},
  1591  				WaitForPositionResults: map[string]map[string]error{
  1592  					"zone1-0000000200": {
  1593  						"position1": nil,
  1594  					},
  1595  				},
  1596  			},
  1597  			ev:       &events.Reparent{},
  1598  			keyspace: "testkeyspace",
  1599  			shard:    "-",
  1600  			currentPrimary: &topo.TabletInfo{
  1601  				Tablet: &topodatapb.Tablet{
  1602  					Alias: &topodatapb.TabletAlias{
  1603  						Cell: "zone1",
  1604  						Uid:  100,
  1605  					},
  1606  				},
  1607  			},
  1608  			primaryElect: &topodatapb.Tablet{
  1609  				Alias: &topodatapb.TabletAlias{
  1610  					Cell: "zone1",
  1611  					Uid:  200,
  1612  				},
  1613  			},
  1614  			tabletMap: map[string]*topo.TabletInfo{},
  1615  			opts:      PlannedReparentOptions{},
  1616  			shouldErr: true,
  1617  		},
  1618  		{
  1619  			name: "promotion succeeds but parent context times out",
  1620  			ts:   memorytopo.NewServer("zone1"),
  1621  			tmc: &testutil.TabletManagerClient{
  1622  				DemotePrimaryResults: map[string]struct {
  1623  					Status *replicationdatapb.PrimaryStatus
  1624  					Error  error
  1625  				}{
  1626  					"zone1-0000000100": {
  1627  						Status: &replicationdatapb.PrimaryStatus{
  1628  							// value of Position doesn't strictly matter for
  1629  							// this test case, as long as it matches the inner
  1630  							// key of the WaitForPositionResults map for the
  1631  							// primary-elect.
  1632  							Position: "position1",
  1633  						},
  1634  						Error: nil,
  1635  					},
  1636  				},
  1637  				PrimaryPositionResults: map[string]struct {
  1638  					Position string
  1639  					Error    error
  1640  				}{
  1641  					"zone1-0000000100": {
  1642  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  1643  					},
  1644  				},
  1645  				PromoteReplicaPostDelays: map[string]time.Duration{
  1646  					"zone1-0000000200": time.Millisecond * 100, // 10x the parent context timeout
  1647  				},
  1648  				PromoteReplicaResults: map[string]struct {
  1649  					Result string
  1650  					Error  error
  1651  				}{
  1652  					"zone1-0000000200": {
  1653  						Error: nil,
  1654  					},
  1655  				},
  1656  				SetReplicationSourceResults: map[string]error{
  1657  					"zone1-0000000200": nil,
  1658  				},
  1659  				WaitForPositionResults: map[string]map[string]error{
  1660  					"zone1-0000000200": {
  1661  						"position1": nil,
  1662  					},
  1663  				},
  1664  			},
  1665  			ctxTimeout: time.Millisecond * 10,
  1666  			ev:         &events.Reparent{},
  1667  			keyspace:   "testkeyspace",
  1668  			shard:      "-",
  1669  			currentPrimary: &topo.TabletInfo{
  1670  				Tablet: &topodatapb.Tablet{
  1671  					Alias: &topodatapb.TabletAlias{
  1672  						Cell: "zone1",
  1673  						Uid:  100,
  1674  					},
  1675  				},
  1676  			},
  1677  			primaryElect: &topodatapb.Tablet{
  1678  				Alias: &topodatapb.TabletAlias{
  1679  					Cell: "zone1",
  1680  					Uid:  200,
  1681  				},
  1682  			},
  1683  			tabletMap: map[string]*topo.TabletInfo{},
  1684  			opts:      PlannedReparentOptions{},
  1685  			shouldErr: true,
  1686  		},
  1687  	}
  1688  
  1689  	ctx := context.Background()
  1690  	logger := logutil.NewMemoryLogger()
  1691  
  1692  	for _, tt := range tests {
  1693  		tt := tt
  1694  
  1695  		t.Run(tt.name, func(t *testing.T) {
  1696  			t.Parallel()
  1697  
  1698  			ctx := ctx
  1699  
  1700  			testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{
  1701  				Keyspace: tt.keyspace,
  1702  				Name:     tt.shard,
  1703  			})
  1704  
  1705  			if !tt.unlockTopo {
  1706  				lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock")
  1707  				require.NoError(t, err, "could not lock %s/%s for testing", tt.keyspace, tt.shard)
  1708  
  1709  				defer func() {
  1710  					unlock(&err)
  1711  					require.NoError(t, err, "could not unlock %s/%s during testing", tt.keyspace, tt.shard)
  1712  				}()
  1713  
  1714  				ctx = lctx
  1715  			}
  1716  
  1717  			pr := NewPlannedReparenter(tt.ts, tt.tmc, logger)
  1718  
  1719  			if tt.ctxTimeout > 0 {
  1720  				_ctx, cancel := context.WithTimeout(ctx, tt.ctxTimeout)
  1721  				defer cancel()
  1722  
  1723  				ctx = _ctx
  1724  			}
  1725  
  1726  			durability, err := GetDurabilityPolicy("none")
  1727  			require.NoError(t, err)
  1728  			tt.opts.durability = durability
  1729  
  1730  			pos, err := pr.performGracefulPromotion(
  1731  				ctx,
  1732  				tt.ev,
  1733  				tt.keyspace,
  1734  				tt.shard,
  1735  				tt.currentPrimary,
  1736  				tt.primaryElect,
  1737  				tt.tabletMap,
  1738  				tt.opts,
  1739  			)
  1740  
  1741  			if tt.extraAssertions != nil {
  1742  				tt.extraAssertions(t, pos, err)
  1743  			}
  1744  
  1745  			if tt.shouldErr {
  1746  				assert.Error(t, err)
  1747  
  1748  				return
  1749  			}
  1750  
  1751  			assert.NoError(t, err)
  1752  			assert.Equal(t, tt.expectedPos, pos)
  1753  		})
  1754  	}
  1755  }
  1756  
  1757  func TestPlannedReparenter_performInitialPromotion(t *testing.T) {
  1758  	t.Parallel()
  1759  
  1760  	tests := []struct {
  1761  		name       string
  1762  		ts         *topo.Server
  1763  		tmc        tmclient.TabletManagerClient
  1764  		ctxTimeout time.Duration
  1765  
  1766  		ev           *events.Reparent
  1767  		keyspace     string
  1768  		shard        string
  1769  		primaryElect *topodatapb.Tablet
  1770  
  1771  		expectedPos string
  1772  		shouldErr   bool
  1773  	}{
  1774  		{
  1775  			name: "successful promotion",
  1776  			ts:   memorytopo.NewServer("zone1"),
  1777  			tmc: &testutil.TabletManagerClient{
  1778  				InitPrimaryResults: map[string]struct {
  1779  					Result string
  1780  					Error  error
  1781  				}{
  1782  					"zone1-0000000200": {
  1783  						Result: "successful reparent journal position",
  1784  						Error:  nil,
  1785  					},
  1786  				},
  1787  			},
  1788  			ev:       &events.Reparent{},
  1789  			keyspace: "testkeyspace",
  1790  			shard:    "-",
  1791  			primaryElect: &topodatapb.Tablet{
  1792  				Alias: &topodatapb.TabletAlias{
  1793  					Cell: "zone1",
  1794  					Uid:  200,
  1795  				},
  1796  			},
  1797  			expectedPos: "successful reparent journal position",
  1798  			shouldErr:   false,
  1799  		},
  1800  		{
  1801  			name: "primary-elect fails to promote",
  1802  			ts:   memorytopo.NewServer("zone1"),
  1803  			tmc: &testutil.TabletManagerClient{
  1804  				InitPrimaryResults: map[string]struct {
  1805  					Result string
  1806  					Error  error
  1807  				}{
  1808  					"zone1-0000000200": {
  1809  						Error: assert.AnError,
  1810  					},
  1811  				},
  1812  			},
  1813  			ev:       &events.Reparent{},
  1814  			keyspace: "testkeyspace",
  1815  			shard:    "-",
  1816  			primaryElect: &topodatapb.Tablet{
  1817  				Alias: &topodatapb.TabletAlias{
  1818  					Cell: "zone1",
  1819  					Uid:  200,
  1820  				},
  1821  			},
  1822  			shouldErr: true,
  1823  		},
  1824  		{
  1825  			name: "promotion succeeds but parent context times out",
  1826  			ts:   memorytopo.NewServer("zone1"),
  1827  			tmc: &testutil.TabletManagerClient{
  1828  				InitPrimaryPostDelays: map[string]time.Duration{
  1829  					"zone1-0000000200": time.Millisecond * 100, // 10x the parent context timeout
  1830  				},
  1831  				InitPrimaryResults: map[string]struct {
  1832  					Result string
  1833  					Error  error
  1834  				}{
  1835  					"zone1-0000000200": {
  1836  						Error: nil,
  1837  					},
  1838  				},
  1839  			},
  1840  			ctxTimeout: time.Millisecond * 10,
  1841  			ev:         &events.Reparent{},
  1842  			keyspace:   "testkeyspace",
  1843  			shard:      "-",
  1844  			primaryElect: &topodatapb.Tablet{
  1845  				Alias: &topodatapb.TabletAlias{
  1846  					Cell: "zone1",
  1847  					Uid:  200,
  1848  				},
  1849  			},
  1850  			shouldErr: true,
  1851  		},
  1852  	}
  1853  
  1854  	ctx := context.Background()
  1855  	logger := logutil.NewMemoryLogger()
  1856  
  1857  	for _, tt := range tests {
  1858  		tt := tt
  1859  
  1860  		t.Run(tt.name, func(t *testing.T) {
  1861  			t.Parallel()
  1862  
  1863  			ctx := ctx
  1864  
  1865  			testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{
  1866  				Keyspace: tt.keyspace,
  1867  				Name:     tt.shard,
  1868  			})
  1869  
  1870  			pr := NewPlannedReparenter(tt.ts, tt.tmc, logger)
  1871  
  1872  			if tt.ctxTimeout > 0 {
  1873  				_ctx, cancel := context.WithTimeout(ctx, tt.ctxTimeout)
  1874  				defer cancel()
  1875  
  1876  				ctx = _ctx
  1877  			}
  1878  
  1879  			durability, err := GetDurabilityPolicy("none")
  1880  			require.NoError(t, err)
  1881  			pos, err := pr.performInitialPromotion(
  1882  				ctx,
  1883  				tt.primaryElect,
  1884  				PlannedReparentOptions{durability: durability},
  1885  			)
  1886  
  1887  			if tt.shouldErr {
  1888  				assert.Error(t, err)
  1889  				return
  1890  			}
  1891  
  1892  			assert.NoError(t, err)
  1893  			assert.Equal(t, tt.expectedPos, pos)
  1894  		})
  1895  	}
  1896  }
  1897  
  1898  func TestPlannedReparenter_performPartialPromotionRecovery(t *testing.T) {
  1899  	t.Parallel()
  1900  
  1901  	tests := []struct {
  1902  		name         string
  1903  		tmc          tmclient.TabletManagerClient
  1904  		timeout      time.Duration
  1905  		primaryElect *topodatapb.Tablet
  1906  		expectedPos  string
  1907  		shouldErr    bool
  1908  	}{
  1909  		{
  1910  			name: "successful recovery",
  1911  			tmc: &testutil.TabletManagerClient{
  1912  				PrimaryPositionResults: map[string]struct {
  1913  					Position string
  1914  					Error    error
  1915  				}{
  1916  					"zone1-0000000100": {
  1917  						Position: "position1",
  1918  						Error:    nil,
  1919  					},
  1920  				},
  1921  				SetReadWriteResults: map[string]error{
  1922  					"zone1-0000000100": nil,
  1923  				},
  1924  			},
  1925  			primaryElect: &topodatapb.Tablet{
  1926  				Alias: &topodatapb.TabletAlias{
  1927  					Cell: "zone1",
  1928  					Uid:  100,
  1929  				},
  1930  			},
  1931  			expectedPos: "position1",
  1932  			shouldErr:   false,
  1933  		},
  1934  		{
  1935  			name: "failed to SetReadWrite",
  1936  			tmc: &testutil.TabletManagerClient{
  1937  				SetReadWriteResults: map[string]error{
  1938  					"zone1-0000000100": assert.AnError,
  1939  				},
  1940  			},
  1941  			primaryElect: &topodatapb.Tablet{
  1942  				Alias: &topodatapb.TabletAlias{
  1943  					Cell: "zone1",
  1944  					Uid:  100,
  1945  				},
  1946  			},
  1947  			shouldErr: true,
  1948  		},
  1949  		{
  1950  			name: "SetReadWrite timed out",
  1951  			tmc: &testutil.TabletManagerClient{
  1952  				SetReadWriteDelays: map[string]time.Duration{
  1953  					"zone1-0000000100": time.Millisecond * 50,
  1954  				},
  1955  				SetReadWriteResults: map[string]error{
  1956  					"zone1-0000000100": nil,
  1957  				},
  1958  			},
  1959  			timeout: time.Millisecond * 10,
  1960  			primaryElect: &topodatapb.Tablet{
  1961  				Alias: &topodatapb.TabletAlias{
  1962  					Cell: "zone1",
  1963  					Uid:  100,
  1964  				},
  1965  			},
  1966  			shouldErr: true,
  1967  		},
  1968  		{
  1969  			name: "failed to get PrimaryPosition from refreshed primary",
  1970  			tmc: &testutil.TabletManagerClient{
  1971  				PrimaryPositionResults: map[string]struct {
  1972  					Position string
  1973  					Error    error
  1974  				}{
  1975  					"zone1-0000000100": {
  1976  						Position: "",
  1977  						Error:    assert.AnError,
  1978  					},
  1979  				},
  1980  				SetReadWriteResults: map[string]error{
  1981  					"zone1-0000000100": nil,
  1982  				},
  1983  			},
  1984  			primaryElect: &topodatapb.Tablet{
  1985  				Alias: &topodatapb.TabletAlias{
  1986  					Cell: "zone1",
  1987  					Uid:  100,
  1988  				},
  1989  			},
  1990  			shouldErr: true,
  1991  		},
  1992  		{
  1993  			name: "PrimaryPosition timed out",
  1994  			tmc: &testutil.TabletManagerClient{
  1995  				PrimaryPositionDelays: map[string]time.Duration{
  1996  					"zone1-0000000100": time.Millisecond * 50,
  1997  				},
  1998  				PrimaryPositionResults: map[string]struct {
  1999  					Position string
  2000  					Error    error
  2001  				}{
  2002  					"zone1-0000000100": {
  2003  						Position: "position1",
  2004  						Error:    nil,
  2005  					},
  2006  				},
  2007  				SetReadWriteResults: map[string]error{
  2008  					"zone1-0000000100": nil,
  2009  				},
  2010  			},
  2011  			timeout: time.Millisecond * 10,
  2012  			primaryElect: &topodatapb.Tablet{
  2013  				Alias: &topodatapb.TabletAlias{
  2014  					Cell: "zone1",
  2015  					Uid:  100,
  2016  				},
  2017  			},
  2018  			shouldErr: true,
  2019  		},
  2020  	}
  2021  
  2022  	ctx := context.Background()
  2023  	logger := logutil.NewMemoryLogger()
  2024  
  2025  	for _, tt := range tests {
  2026  		tt := tt
  2027  
  2028  		t.Run(tt.name, func(t *testing.T) {
  2029  			t.Parallel()
  2030  
  2031  			ctx := ctx
  2032  			pr := NewPlannedReparenter(nil, tt.tmc, logger)
  2033  
  2034  			if tt.timeout > 0 {
  2035  				_ctx, cancel := context.WithTimeout(ctx, tt.timeout)
  2036  				defer cancel()
  2037  
  2038  				ctx = _ctx
  2039  			}
  2040  
  2041  			rp, err := pr.performPartialPromotionRecovery(ctx, tt.primaryElect)
  2042  			if tt.shouldErr {
  2043  				assert.Error(t, err)
  2044  
  2045  				return
  2046  			}
  2047  
  2048  			assert.NoError(t, err)
  2049  			assert.Equal(t, tt.expectedPos, rp, "performPartialPromotionRecovery gave unexpected reparent journal position")
  2050  		})
  2051  	}
  2052  }
  2053  
  2054  func TestPlannedReparenter_performPotentialPromotion(t *testing.T) {
  2055  	t.Parallel()
  2056  
  2057  	tests := []struct {
  2058  		name       string
  2059  		ts         *topo.Server
  2060  		tmc        tmclient.TabletManagerClient
  2061  		timeout    time.Duration
  2062  		unlockTopo bool
  2063  
  2064  		keyspace     string
  2065  		shard        string
  2066  		primaryElect *topodatapb.Tablet
  2067  		tabletMap    map[string]*topo.TabletInfo
  2068  
  2069  		expectedPos string
  2070  		shouldErr   bool
  2071  	}{
  2072  		{
  2073  			name: "success",
  2074  			ts:   memorytopo.NewServer("zone1"),
  2075  			tmc: &testutil.TabletManagerClient{
  2076  				DemotePrimaryResults: map[string]struct {
  2077  					Status *replicationdatapb.PrimaryStatus
  2078  					Error  error
  2079  				}{
  2080  					"zone1-0000000100": {
  2081  						Status: &replicationdatapb.PrimaryStatus{
  2082  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2083  						},
  2084  						Error: nil,
  2085  					},
  2086  					"zone1-0000000101": {
  2087  						Status: &replicationdatapb.PrimaryStatus{
  2088  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2089  						},
  2090  						Error: nil,
  2091  					},
  2092  					"zone1-0000000102": {
  2093  						Status: &replicationdatapb.PrimaryStatus{
  2094  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
  2095  						},
  2096  						Error: nil,
  2097  					},
  2098  				},
  2099  				PromoteReplicaResults: map[string]struct {
  2100  					Result string
  2101  					Error  error
  2102  				}{
  2103  					"zone1-0000000100": {
  2104  						Result: "reparent journal position",
  2105  						Error:  nil,
  2106  					},
  2107  				},
  2108  			},
  2109  			unlockTopo: false,
  2110  			keyspace:   "testkeyspace",
  2111  			shard:      "-",
  2112  			primaryElect: &topodatapb.Tablet{
  2113  				Alias: &topodatapb.TabletAlias{
  2114  					Cell: "zone1",
  2115  					Uid:  100,
  2116  				},
  2117  			},
  2118  			tabletMap: map[string]*topo.TabletInfo{
  2119  				"zone1-0000000100": {
  2120  					Tablet: &topodatapb.Tablet{
  2121  						Alias: &topodatapb.TabletAlias{
  2122  							Cell: "zone1",
  2123  							Uid:  100,
  2124  						},
  2125  					},
  2126  				},
  2127  				"zone1-0000000101": {
  2128  					Tablet: &topodatapb.Tablet{
  2129  						Alias: &topodatapb.TabletAlias{
  2130  							Cell: "zone1",
  2131  							Uid:  101,
  2132  						},
  2133  					},
  2134  				},
  2135  				"zone1-0000000102": {
  2136  					Tablet: &topodatapb.Tablet{
  2137  						Alias: &topodatapb.TabletAlias{
  2138  							Cell: "zone1",
  2139  							Uid:  102,
  2140  						},
  2141  					},
  2142  				},
  2143  			},
  2144  			expectedPos: "reparent journal position",
  2145  			shouldErr:   false,
  2146  		},
  2147  		{
  2148  			name: "failed to DemotePrimary on a tablet",
  2149  			ts:   memorytopo.NewServer("zone1"),
  2150  			tmc: &testutil.TabletManagerClient{
  2151  				DemotePrimaryResults: map[string]struct {
  2152  					Status *replicationdatapb.PrimaryStatus
  2153  					Error  error
  2154  				}{
  2155  					"zone1-0000000100": {
  2156  						Status: nil,
  2157  						Error:  assert.AnError,
  2158  					},
  2159  				},
  2160  			},
  2161  			unlockTopo: false,
  2162  			keyspace:   "testkeyspace",
  2163  			shard:      "-",
  2164  			primaryElect: &topodatapb.Tablet{
  2165  				Alias: &topodatapb.TabletAlias{
  2166  					Cell: "zone1",
  2167  					Uid:  100,
  2168  				},
  2169  			},
  2170  			tabletMap: map[string]*topo.TabletInfo{
  2171  				"zone1-0000000100": {
  2172  					Tablet: &topodatapb.Tablet{
  2173  						Alias: &topodatapb.TabletAlias{
  2174  							Cell: "zone1",
  2175  							Uid:  100,
  2176  						},
  2177  					},
  2178  				},
  2179  			},
  2180  			shouldErr: true,
  2181  		},
  2182  		{
  2183  			name: "timed out during DemotePrimary on a tablet",
  2184  			ts:   memorytopo.NewServer("zone1"),
  2185  			tmc: &testutil.TabletManagerClient{
  2186  				DemotePrimaryDelays: map[string]time.Duration{
  2187  					"zone1-0000000100": time.Millisecond * 50,
  2188  				},
  2189  				DemotePrimaryResults: map[string]struct {
  2190  					Status *replicationdatapb.PrimaryStatus
  2191  					Error  error
  2192  				}{
  2193  					"zone1-0000000100": {
  2194  						Status: &replicationdatapb.PrimaryStatus{
  2195  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2196  						},
  2197  						Error: nil,
  2198  					},
  2199  				},
  2200  			},
  2201  			timeout:    time.Millisecond * 10,
  2202  			unlockTopo: false,
  2203  			keyspace:   "testkeyspace",
  2204  			shard:      "-",
  2205  			primaryElect: &topodatapb.Tablet{
  2206  				Alias: &topodatapb.TabletAlias{
  2207  					Cell: "zone1",
  2208  					Uid:  100,
  2209  				},
  2210  			},
  2211  			tabletMap: map[string]*topo.TabletInfo{
  2212  				"zone1-0000000100": {
  2213  					Tablet: &topodatapb.Tablet{
  2214  						Alias: &topodatapb.TabletAlias{
  2215  							Cell: "zone1",
  2216  							Uid:  100,
  2217  						},
  2218  					},
  2219  				},
  2220  			},
  2221  			shouldErr: true,
  2222  		},
  2223  		{
  2224  			name: "failed to DecodePosition on a tablet's demote position",
  2225  			ts:   memorytopo.NewServer("zone1"),
  2226  			tmc: &testutil.TabletManagerClient{
  2227  				DemotePrimaryResults: map[string]struct {
  2228  					Status *replicationdatapb.PrimaryStatus
  2229  					Error  error
  2230  				}{
  2231  					"zone1-0000000100": {
  2232  						Status: &replicationdatapb.PrimaryStatus{
  2233  							Position: "MySQL56/this-is-nonsense",
  2234  						},
  2235  						Error: nil,
  2236  					},
  2237  				},
  2238  			},
  2239  			unlockTopo: false,
  2240  			keyspace:   "testkeyspace",
  2241  			shard:      "-",
  2242  			primaryElect: &topodatapb.Tablet{
  2243  				Alias: &topodatapb.TabletAlias{
  2244  					Cell: "zone1",
  2245  					Uid:  100,
  2246  				},
  2247  			},
  2248  			tabletMap: map[string]*topo.TabletInfo{
  2249  				"zone1-0000000100": {
  2250  					Tablet: &topodatapb.Tablet{
  2251  						Alias: &topodatapb.TabletAlias{
  2252  							Cell: "zone1",
  2253  							Uid:  100,
  2254  						},
  2255  					},
  2256  				},
  2257  			},
  2258  			shouldErr: true,
  2259  		},
  2260  		{
  2261  			name:       "primary-elect not in tablet map",
  2262  			ts:         memorytopo.NewServer("zone1"),
  2263  			tmc:        &testutil.TabletManagerClient{},
  2264  			unlockTopo: false,
  2265  			keyspace:   "testkeyspace",
  2266  			shard:      "-",
  2267  			primaryElect: &topodatapb.Tablet{
  2268  				Alias: &topodatapb.TabletAlias{
  2269  					Cell: "zone1",
  2270  					Uid:  100,
  2271  				},
  2272  			},
  2273  			tabletMap: map[string]*topo.TabletInfo{},
  2274  			shouldErr: true,
  2275  		},
  2276  		{
  2277  			name: "primary-elect not most at most advanced position",
  2278  			ts:   memorytopo.NewServer("zone1"),
  2279  			tmc: &testutil.TabletManagerClient{
  2280  				DemotePrimaryResults: map[string]struct {
  2281  					Status *replicationdatapb.PrimaryStatus
  2282  					Error  error
  2283  				}{
  2284  					"zone1-0000000100": {
  2285  						Status: &replicationdatapb.PrimaryStatus{
  2286  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2287  						},
  2288  						Error: nil,
  2289  					},
  2290  					"zone1-0000000101": {
  2291  						Status: &replicationdatapb.PrimaryStatus{
  2292  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2293  						},
  2294  						Error: nil,
  2295  					},
  2296  					"zone1-0000000102": {
  2297  						Status: &replicationdatapb.PrimaryStatus{
  2298  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10000",
  2299  						},
  2300  						Error: nil,
  2301  					},
  2302  				},
  2303  			},
  2304  			unlockTopo: false,
  2305  			keyspace:   "testkeyspace",
  2306  			shard:      "-",
  2307  			primaryElect: &topodatapb.Tablet{
  2308  				Alias: &topodatapb.TabletAlias{
  2309  					Cell: "zone1",
  2310  					Uid:  100,
  2311  				},
  2312  			},
  2313  			tabletMap: map[string]*topo.TabletInfo{
  2314  				"zone1-0000000100": {
  2315  					Tablet: &topodatapb.Tablet{
  2316  						Alias: &topodatapb.TabletAlias{
  2317  							Cell: "zone1",
  2318  							Uid:  100,
  2319  						},
  2320  					},
  2321  				},
  2322  				"zone1-0000000101": {
  2323  					Tablet: &topodatapb.Tablet{
  2324  						Alias: &topodatapb.TabletAlias{
  2325  							Cell: "zone1",
  2326  							Uid:  101,
  2327  						},
  2328  					},
  2329  				},
  2330  				"zone1-0000000102": {
  2331  					Tablet: &topodatapb.Tablet{
  2332  						Alias: &topodatapb.TabletAlias{
  2333  							Cell: "zone1",
  2334  							Uid:  102,
  2335  						},
  2336  					},
  2337  				},
  2338  			},
  2339  			shouldErr: true,
  2340  		},
  2341  		{
  2342  			name: "lost topology lock",
  2343  			ts:   memorytopo.NewServer("zone1"),
  2344  			tmc: &testutil.TabletManagerClient{
  2345  				DemotePrimaryResults: map[string]struct {
  2346  					Status *replicationdatapb.PrimaryStatus
  2347  					Error  error
  2348  				}{
  2349  					"zone1-0000000100": {
  2350  						Status: &replicationdatapb.PrimaryStatus{
  2351  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2352  						},
  2353  						Error: nil,
  2354  					},
  2355  					"zone1-0000000101": {
  2356  						Status: &replicationdatapb.PrimaryStatus{
  2357  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2358  						},
  2359  						Error: nil,
  2360  					},
  2361  					"zone1-0000000102": {
  2362  						Status: &replicationdatapb.PrimaryStatus{
  2363  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2364  						},
  2365  						Error: nil,
  2366  					},
  2367  				},
  2368  			},
  2369  			unlockTopo: true,
  2370  			keyspace:   "testkeyspace",
  2371  			shard:      "-",
  2372  			primaryElect: &topodatapb.Tablet{
  2373  				Alias: &topodatapb.TabletAlias{
  2374  					Cell: "zone1",
  2375  					Uid:  100,
  2376  				},
  2377  			},
  2378  			tabletMap: map[string]*topo.TabletInfo{
  2379  				"zone1-0000000100": {
  2380  					Tablet: &topodatapb.Tablet{
  2381  						Alias: &topodatapb.TabletAlias{
  2382  							Cell: "zone1",
  2383  							Uid:  100,
  2384  						},
  2385  					},
  2386  				},
  2387  				"zone1-0000000101": {
  2388  					Tablet: &topodatapb.Tablet{
  2389  						Alias: &topodatapb.TabletAlias{
  2390  							Cell: "zone1",
  2391  							Uid:  101,
  2392  						},
  2393  					},
  2394  				},
  2395  				"zone1-0000000102": {
  2396  					Tablet: &topodatapb.Tablet{
  2397  						Alias: &topodatapb.TabletAlias{
  2398  							Cell: "zone1",
  2399  							Uid:  102,
  2400  						},
  2401  					},
  2402  				},
  2403  			},
  2404  			shouldErr: true,
  2405  		},
  2406  		{
  2407  			name: "failed to promote primary-elect",
  2408  			ts:   memorytopo.NewServer("zone1"),
  2409  			tmc: &testutil.TabletManagerClient{
  2410  				DemotePrimaryResults: map[string]struct {
  2411  					Status *replicationdatapb.PrimaryStatus
  2412  					Error  error
  2413  				}{
  2414  					"zone1-0000000100": {
  2415  						Status: &replicationdatapb.PrimaryStatus{
  2416  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2417  						},
  2418  						Error: nil,
  2419  					},
  2420  					"zone1-0000000101": {
  2421  						Status: &replicationdatapb.PrimaryStatus{
  2422  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2423  						},
  2424  						Error: nil,
  2425  					},
  2426  					"zone1-0000000102": {
  2427  						Status: &replicationdatapb.PrimaryStatus{
  2428  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
  2429  						},
  2430  						Error: nil,
  2431  					},
  2432  				},
  2433  				PromoteReplicaResults: map[string]struct {
  2434  					Result string
  2435  					Error  error
  2436  				}{
  2437  					"zone1-0000000100": {
  2438  						Result: "",
  2439  						Error:  assert.AnError,
  2440  					},
  2441  				},
  2442  			},
  2443  			unlockTopo: false,
  2444  			keyspace:   "testkeyspace",
  2445  			shard:      "-",
  2446  			primaryElect: &topodatapb.Tablet{
  2447  				Alias: &topodatapb.TabletAlias{
  2448  					Cell: "zone1",
  2449  					Uid:  100,
  2450  				},
  2451  			},
  2452  			tabletMap: map[string]*topo.TabletInfo{
  2453  				"zone1-0000000100": {
  2454  					Tablet: &topodatapb.Tablet{
  2455  						Alias: &topodatapb.TabletAlias{
  2456  							Cell: "zone1",
  2457  							Uid:  100,
  2458  						},
  2459  					},
  2460  				},
  2461  				"zone1-0000000101": {
  2462  					Tablet: &topodatapb.Tablet{
  2463  						Alias: &topodatapb.TabletAlias{
  2464  							Cell: "zone1",
  2465  							Uid:  101,
  2466  						},
  2467  					},
  2468  				},
  2469  				"zone1-0000000102": {
  2470  					Tablet: &topodatapb.Tablet{
  2471  						Alias: &topodatapb.TabletAlias{
  2472  							Cell: "zone1",
  2473  							Uid:  102,
  2474  						},
  2475  					},
  2476  				},
  2477  			},
  2478  			shouldErr: true,
  2479  		},
  2480  		{
  2481  			name: "timed out while promoting primary-elect",
  2482  			ts:   memorytopo.NewServer("zone1"),
  2483  			tmc: &testutil.TabletManagerClient{
  2484  				DemotePrimaryResults: map[string]struct {
  2485  					Status *replicationdatapb.PrimaryStatus
  2486  					Error  error
  2487  				}{
  2488  					"zone1-0000000100": {
  2489  						Status: &replicationdatapb.PrimaryStatus{
  2490  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2491  						},
  2492  						Error: nil,
  2493  					},
  2494  					"zone1-0000000101": {
  2495  						Status: &replicationdatapb.PrimaryStatus{
  2496  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2497  						},
  2498  						Error: nil,
  2499  					},
  2500  					"zone1-0000000102": {
  2501  						Status: &replicationdatapb.PrimaryStatus{
  2502  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-5",
  2503  						},
  2504  						Error: nil,
  2505  					},
  2506  				},
  2507  				PromoteReplicaDelays: map[string]time.Duration{
  2508  					"zone1-0000000100": time.Millisecond * 100,
  2509  				},
  2510  				PromoteReplicaResults: map[string]struct {
  2511  					Result string
  2512  					Error  error
  2513  				}{
  2514  					"zone1-0000000100": {
  2515  						Result: "reparent journal position",
  2516  						Error:  nil,
  2517  					},
  2518  				},
  2519  			},
  2520  			timeout:    time.Millisecond * 50,
  2521  			unlockTopo: false,
  2522  			keyspace:   "testkeyspace",
  2523  			shard:      "-",
  2524  			primaryElect: &topodatapb.Tablet{
  2525  				Alias: &topodatapb.TabletAlias{
  2526  					Cell: "zone1",
  2527  					Uid:  100,
  2528  				},
  2529  			},
  2530  			tabletMap: map[string]*topo.TabletInfo{
  2531  				"zone1-0000000100": {
  2532  					Tablet: &topodatapb.Tablet{
  2533  						Alias: &topodatapb.TabletAlias{
  2534  							Cell: "zone1",
  2535  							Uid:  100,
  2536  						},
  2537  					},
  2538  				},
  2539  				"zone1-0000000101": {
  2540  					Tablet: &topodatapb.Tablet{
  2541  						Alias: &topodatapb.TabletAlias{
  2542  							Cell: "zone1",
  2543  							Uid:  101,
  2544  						},
  2545  					},
  2546  				},
  2547  				"zone1-0000000102": {
  2548  					Tablet: &topodatapb.Tablet{
  2549  						Alias: &topodatapb.TabletAlias{
  2550  							Cell: "zone1",
  2551  							Uid:  102,
  2552  						},
  2553  					},
  2554  				},
  2555  			},
  2556  			shouldErr: true,
  2557  		},
  2558  	}
  2559  
  2560  	ctx := context.Background()
  2561  	logger := logutil.NewMemoryLogger()
  2562  
  2563  	for _, tt := range tests {
  2564  		tt := tt
  2565  
  2566  		t.Run(tt.name, func(t *testing.T) {
  2567  			t.Parallel()
  2568  
  2569  			ctx := ctx
  2570  			pr := NewPlannedReparenter(nil, tt.tmc, logger)
  2571  
  2572  			testutil.AddShards(ctx, t, tt.ts, &vtctldatapb.Shard{
  2573  				Keyspace: tt.keyspace,
  2574  				Name:     tt.shard,
  2575  			})
  2576  
  2577  			if !tt.unlockTopo {
  2578  				lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "test lock")
  2579  				require.NoError(t, err, "could not lock %s/%s for testing", tt.keyspace, tt.shard)
  2580  
  2581  				defer func() {
  2582  					unlock(&err)
  2583  					require.NoError(t, err, "could not unlock %s/%s during testing", tt.keyspace, tt.shard)
  2584  				}()
  2585  
  2586  				ctx = lctx
  2587  			}
  2588  
  2589  			if tt.timeout > 0 {
  2590  				_ctx, cancel := context.WithTimeout(ctx, tt.timeout)
  2591  				defer cancel()
  2592  
  2593  				ctx = _ctx
  2594  			}
  2595  
  2596  			durability, err := GetDurabilityPolicy("none")
  2597  			require.NoError(t, err)
  2598  			rp, err := pr.performPotentialPromotion(ctx, tt.keyspace, tt.shard, tt.primaryElect, tt.tabletMap, PlannedReparentOptions{durability: durability})
  2599  			if tt.shouldErr {
  2600  				assert.Error(t, err)
  2601  
  2602  				return
  2603  			}
  2604  
  2605  			assert.NoError(t, err)
  2606  			assert.Equal(t, tt.expectedPos, rp)
  2607  		})
  2608  	}
  2609  }
  2610  
  2611  func TestPlannedReparenter_reparentShardLocked(t *testing.T) {
  2612  	t.Parallel()
  2613  
  2614  	tests := []struct {
  2615  		name       string
  2616  		ts         *topo.Server
  2617  		tmc        tmclient.TabletManagerClient
  2618  		tablets    []*topodatapb.Tablet
  2619  		unlockTopo bool
  2620  
  2621  		ev       *events.Reparent
  2622  		keyspace string
  2623  		shard    string
  2624  		opts     PlannedReparentOptions
  2625  
  2626  		shouldErr     bool
  2627  		expectedEvent *events.Reparent
  2628  	}{
  2629  		{
  2630  			name: "success: current primary cannot be determined", // "Case (1)"
  2631  			ts:   memorytopo.NewServer("zone1"),
  2632  			tmc: &testutil.TabletManagerClient{
  2633  				DemotePrimaryResults: map[string]struct {
  2634  					Status *replicationdatapb.PrimaryStatus
  2635  					Error  error
  2636  				}{
  2637  					"zone1-0000000100": {
  2638  						Status: &replicationdatapb.PrimaryStatus{
  2639  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2640  						},
  2641  						Error: nil,
  2642  					},
  2643  					"zone1-0000000200": {
  2644  						Status: &replicationdatapb.PrimaryStatus{
  2645  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2646  						},
  2647  						Error: nil,
  2648  					},
  2649  				},
  2650  				PopulateReparentJournalResults: map[string]error{
  2651  					"zone1-0000000200": nil, // zone1-200 gets promoted
  2652  				},
  2653  				PromoteReplicaResults: map[string]struct {
  2654  					Result string
  2655  					Error  error
  2656  				}{
  2657  					"zone1-0000000200": {
  2658  						Result: "reparent journal position",
  2659  						Error:  nil,
  2660  					},
  2661  				},
  2662  				SetReplicationSourceResults: map[string]error{
  2663  					"zone1-0000000100": nil, // zone1-100 gets reparented under zone1-200
  2664  				},
  2665  			},
  2666  			tablets: []*topodatapb.Tablet{
  2667  				{
  2668  					Alias: &topodatapb.TabletAlias{
  2669  						Cell: "zone1",
  2670  						Uid:  100,
  2671  					},
  2672  					Type: topodatapb.TabletType_PRIMARY,
  2673  					PrimaryTermStartTime: &vttime.Time{
  2674  						Seconds:     1000,
  2675  						Nanoseconds: 500,
  2676  					},
  2677  					Hostname: "primary1", // claims to be PRIMARY with same term as primary2
  2678  					Keyspace: "testkeyspace",
  2679  					Shard:    "-",
  2680  				},
  2681  				{
  2682  					Alias: &topodatapb.TabletAlias{
  2683  						Cell: "zone1",
  2684  						Uid:  200,
  2685  					},
  2686  					Type: topodatapb.TabletType_PRIMARY,
  2687  					PrimaryTermStartTime: &vttime.Time{
  2688  						Seconds:     1000,
  2689  						Nanoseconds: 500,
  2690  					},
  2691  					Hostname: "primary2", // claims to be PRIMARY with same term as primary1
  2692  					Keyspace: "testkeyspace",
  2693  					Shard:    "-",
  2694  				},
  2695  			},
  2696  
  2697  			ev:       &events.Reparent{},
  2698  			keyspace: "testkeyspace",
  2699  			shard:    "-",
  2700  			opts: PlannedReparentOptions{
  2701  				NewPrimaryAlias: &topodatapb.TabletAlias{ // We want primary2 to be the true primary.
  2702  					Cell: "zone1",
  2703  					Uid:  200,
  2704  				},
  2705  			},
  2706  
  2707  			shouldErr: false,
  2708  		},
  2709  		{
  2710  			name: "success: current primary is desired primary", // "Case (2)"
  2711  			ts:   memorytopo.NewServer("zone1"),
  2712  			tmc: &testutil.TabletManagerClient{
  2713  				PrimaryPositionResults: map[string]struct {
  2714  					Position string
  2715  					Error    error
  2716  				}{
  2717  					"zone1-0000000100": {
  2718  						Position: "position1",
  2719  						Error:    nil,
  2720  					},
  2721  				},
  2722  				PopulateReparentJournalResults: map[string]error{
  2723  					"zone1-0000000100": nil,
  2724  				},
  2725  				SetReplicationSourceResults: map[string]error{
  2726  					"zone1-0000000200": nil,
  2727  				},
  2728  				SetReadWriteResults: map[string]error{
  2729  					"zone1-0000000100": nil,
  2730  				},
  2731  			},
  2732  			tablets: []*topodatapb.Tablet{
  2733  				{
  2734  					Alias: &topodatapb.TabletAlias{
  2735  						Cell: "zone1",
  2736  						Uid:  100,
  2737  					},
  2738  					Type:     topodatapb.TabletType_PRIMARY,
  2739  					Keyspace: "testkeyspace",
  2740  					Shard:    "-",
  2741  				},
  2742  				{
  2743  					Alias: &topodatapb.TabletAlias{
  2744  						Cell: "zone1",
  2745  						Uid:  200,
  2746  					},
  2747  					Type:     topodatapb.TabletType_REPLICA,
  2748  					Keyspace: "testkeyspace",
  2749  					Shard:    "-",
  2750  				},
  2751  			},
  2752  
  2753  			ev:       &events.Reparent{},
  2754  			keyspace: "testkeyspace",
  2755  			shard:    "-",
  2756  			opts: PlannedReparentOptions{
  2757  				NewPrimaryAlias: &topodatapb.TabletAlias{
  2758  					Cell: "zone1",
  2759  					Uid:  100,
  2760  				},
  2761  			},
  2762  
  2763  			shouldErr: false,
  2764  			expectedEvent: &events.Reparent{
  2765  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  2766  					PrimaryAlias: &topodatapb.TabletAlias{
  2767  						Cell: "zone1",
  2768  						Uid:  100,
  2769  					},
  2770  					KeyRange:         &topodatapb.KeyRange{},
  2771  					IsPrimaryServing: true,
  2772  				}, nil),
  2773  				NewPrimary: &topodatapb.Tablet{
  2774  					Alias: &topodatapb.TabletAlias{
  2775  						Cell: "zone1",
  2776  						Uid:  100,
  2777  					},
  2778  					Type:     topodatapb.TabletType_PRIMARY,
  2779  					Keyspace: "testkeyspace",
  2780  					Shard:    "-",
  2781  				},
  2782  			},
  2783  		},
  2784  		{
  2785  			name: "success: graceful promotion", // "Case (3)"
  2786  			ts:   memorytopo.NewServer("zone1"),
  2787  			tmc: &testutil.TabletManagerClient{
  2788  				DemotePrimaryResults: map[string]struct {
  2789  					Status *replicationdatapb.PrimaryStatus
  2790  					Error  error
  2791  				}{
  2792  					"zone1-0000000100": {
  2793  						Status: &replicationdatapb.PrimaryStatus{
  2794  							// a few more transactions happen after waiting for replication
  2795  							Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10",
  2796  						},
  2797  						Error: nil,
  2798  					},
  2799  				},
  2800  				PrimaryPositionResults: map[string]struct {
  2801  					Position string
  2802  					Error    error
  2803  				}{
  2804  					"zone1-0000000100": {
  2805  						Position: "MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-8",
  2806  						Error:    nil,
  2807  					},
  2808  				},
  2809  				PopulateReparentJournalResults: map[string]error{
  2810  					"zone1-0000000200": nil,
  2811  				},
  2812  				PromoteReplicaResults: map[string]struct {
  2813  					Result string
  2814  					Error  error
  2815  				}{
  2816  					"zone1-0000000200": {
  2817  						Result: "reparent journal position",
  2818  						Error:  nil,
  2819  					},
  2820  				},
  2821  				SetReplicationSourceResults: map[string]error{
  2822  					"zone1-0000000100": nil, // called during reparentTablets to make oldPrimary a replica of newPrimary
  2823  					"zone1-0000000200": nil, // called during performGracefulPromotion to ensure newPrimary is caught up
  2824  				},
  2825  				WaitForPositionResults: map[string]map[string]error{
  2826  					"zone1-0000000200": {
  2827  						"MySQL56/3E11FA47-71CA-11E1-9E33-C80AA9429562:1-10": nil,
  2828  					},
  2829  				},
  2830  			},
  2831  			tablets: []*topodatapb.Tablet{
  2832  				{
  2833  					Alias: &topodatapb.TabletAlias{
  2834  						Cell: "zone1",
  2835  						Uid:  100,
  2836  					},
  2837  					Type: topodatapb.TabletType_PRIMARY,
  2838  					PrimaryTermStartTime: &vttime.Time{
  2839  						Seconds:     1000,
  2840  						Nanoseconds: 500,
  2841  					},
  2842  					Keyspace: "testkeyspace",
  2843  					Shard:    "-",
  2844  				},
  2845  				{
  2846  					Alias: &topodatapb.TabletAlias{
  2847  						Cell: "zone1",
  2848  						Uid:  200,
  2849  					},
  2850  					Type:     topodatapb.TabletType_REPLICA,
  2851  					Keyspace: "testkeyspace",
  2852  					Shard:    "-",
  2853  				},
  2854  			},
  2855  
  2856  			ev:       &events.Reparent{},
  2857  			keyspace: "testkeyspace",
  2858  			shard:    "-",
  2859  			opts: PlannedReparentOptions{
  2860  				NewPrimaryAlias: &topodatapb.TabletAlias{
  2861  					Cell: "zone1",
  2862  					Uid:  200,
  2863  				},
  2864  			},
  2865  
  2866  			shouldErr: false,
  2867  		},
  2868  		{
  2869  			name:       "shard not found",
  2870  			ts:         memorytopo.NewServer("zone1"),
  2871  			tmc:        nil,
  2872  			tablets:    nil,
  2873  			unlockTopo: true,
  2874  
  2875  			ev:       &events.Reparent{},
  2876  			keyspace: "testkeyspace",
  2877  			shard:    "-",
  2878  			opts:     PlannedReparentOptions{},
  2879  
  2880  			shouldErr:     true,
  2881  			expectedEvent: &events.Reparent{},
  2882  		},
  2883  		{
  2884  			name: "shard initialization",
  2885  			ts:   memorytopo.NewServer("zone1"),
  2886  			tmc: &testutil.TabletManagerClient{
  2887  				PopulateReparentJournalResults: map[string]error{
  2888  					"zone1-0000000200": nil,
  2889  				},
  2890  				InitPrimaryResults: map[string]struct {
  2891  					Result string
  2892  					Error  error
  2893  				}{
  2894  					"zone1-0000000200": {
  2895  						Result: "reparent journal position",
  2896  						Error:  nil,
  2897  					},
  2898  				},
  2899  				SetReplicationSourceResults: map[string]error{
  2900  					"zone1-0000000100": nil, // called during reparentTablets to make this tablet a replica of newPrimary
  2901  				},
  2902  			},
  2903  			tablets: []*topodatapb.Tablet{
  2904  				// Shard has no current primary in the beginning.
  2905  				{
  2906  					Alias: &topodatapb.TabletAlias{
  2907  						Cell: "zone1",
  2908  						Uid:  100,
  2909  					},
  2910  					Type:     topodatapb.TabletType_REPLICA,
  2911  					Keyspace: "testkeyspace",
  2912  					Shard:    "-",
  2913  				},
  2914  				{
  2915  					Alias: &topodatapb.TabletAlias{
  2916  						Cell: "zone1",
  2917  						Uid:  200,
  2918  					},
  2919  					Type:     topodatapb.TabletType_REPLICA,
  2920  					Keyspace: "testkeyspace",
  2921  					Shard:    "-",
  2922  				},
  2923  			},
  2924  
  2925  			ev:       &events.Reparent{},
  2926  			keyspace: "testkeyspace",
  2927  			shard:    "-",
  2928  			opts: PlannedReparentOptions{
  2929  				NewPrimaryAlias: &topodatapb.TabletAlias{
  2930  					Cell: "zone1",
  2931  					Uid:  200,
  2932  				},
  2933  			},
  2934  
  2935  			shouldErr: false,
  2936  			expectedEvent: &events.Reparent{
  2937  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  2938  					KeyRange:         &topodatapb.KeyRange{},
  2939  					IsPrimaryServing: true,
  2940  				}, nil),
  2941  				NewPrimary: &topodatapb.Tablet{
  2942  					Alias: &topodatapb.TabletAlias{
  2943  						Cell: "zone1",
  2944  						Uid:  200,
  2945  					},
  2946  					Type:     topodatapb.TabletType_REPLICA,
  2947  					Keyspace: "testkeyspace",
  2948  					Shard:    "-",
  2949  				},
  2950  			},
  2951  		},
  2952  		{
  2953  			name: "shard initialization with no new primary provided",
  2954  			ts:   memorytopo.NewServer("zone1"),
  2955  			tmc: &testutil.TabletManagerClient{
  2956  				PopulateReparentJournalResults: map[string]error{
  2957  					"zone1-0000000200": nil,
  2958  				},
  2959  				InitPrimaryResults: map[string]struct {
  2960  					Result string
  2961  					Error  error
  2962  				}{
  2963  					"zone1-0000000200": {
  2964  						Result: "reparent journal position",
  2965  						Error:  nil,
  2966  					},
  2967  				},
  2968  				ReplicationStatusResults: map[string]struct {
  2969  					Position *replicationdatapb.Status
  2970  					Error    error
  2971  				}{
  2972  					"zone1-0000000200": {
  2973  						Error: mysql.ErrNotReplica,
  2974  					},
  2975  					"zone1-0000000100": {
  2976  						Error: fmt.Errorf("not providing replication status, so that 200 wins"),
  2977  					},
  2978  				},
  2979  				SetReplicationSourceResults: map[string]error{
  2980  					"zone1-0000000100": nil, // called during reparentTablets to make this tablet a replica of newPrimary
  2981  				},
  2982  			},
  2983  			tablets: []*topodatapb.Tablet{
  2984  				// Shard has no current primary in the beginning.
  2985  				{
  2986  					Alias: &topodatapb.TabletAlias{
  2987  						Cell: "zone1",
  2988  						Uid:  100,
  2989  					},
  2990  					Type:     topodatapb.TabletType_REPLICA,
  2991  					Keyspace: "testkeyspace",
  2992  					Shard:    "-",
  2993  				},
  2994  				{
  2995  					Alias: &topodatapb.TabletAlias{
  2996  						Cell: "zone1",
  2997  						Uid:  200,
  2998  					},
  2999  					Type:     topodatapb.TabletType_REPLICA,
  3000  					Keyspace: "testkeyspace",
  3001  					Shard:    "-",
  3002  				},
  3003  			},
  3004  
  3005  			ev:        &events.Reparent{},
  3006  			keyspace:  "testkeyspace",
  3007  			shard:     "-",
  3008  			opts:      PlannedReparentOptions{},
  3009  			shouldErr: false,
  3010  			expectedEvent: &events.Reparent{
  3011  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  3012  					KeyRange:         &topodatapb.KeyRange{},
  3013  					IsPrimaryServing: true,
  3014  				}, nil),
  3015  				NewPrimary: &topodatapb.Tablet{
  3016  					Alias: &topodatapb.TabletAlias{
  3017  						Cell: "zone1",
  3018  						Uid:  200,
  3019  					},
  3020  					Type:     topodatapb.TabletType_REPLICA,
  3021  					Keyspace: "testkeyspace",
  3022  					Shard:    "-",
  3023  				},
  3024  			},
  3025  		},
  3026  		{
  3027  			name: "preflight checks determine PRS is no-op",
  3028  			ts:   memorytopo.NewServer("zone1"),
  3029  			tmc:  nil,
  3030  			tablets: []*topodatapb.Tablet{
  3031  				{
  3032  					Alias: &topodatapb.TabletAlias{
  3033  						Cell: "zone1",
  3034  						Uid:  100,
  3035  					},
  3036  					Type:     topodatapb.TabletType_PRIMARY,
  3037  					Keyspace: "testkeyspace",
  3038  					Shard:    "-",
  3039  				},
  3040  				{
  3041  					Alias: &topodatapb.TabletAlias{
  3042  						Cell: "zone1",
  3043  						Uid:  200,
  3044  					},
  3045  					Type:     topodatapb.TabletType_REPLICA,
  3046  					Keyspace: "testkeyspace",
  3047  					Shard:    "-",
  3048  				},
  3049  			},
  3050  
  3051  			ev:       &events.Reparent{},
  3052  			keyspace: "testkeyspace",
  3053  			shard:    "-",
  3054  			opts: PlannedReparentOptions{
  3055  				// This is not the shard primary, so nothing to do.
  3056  				AvoidPrimaryAlias: &topodatapb.TabletAlias{
  3057  					Cell: "zone1",
  3058  					Uid:  200,
  3059  				},
  3060  			},
  3061  
  3062  			shouldErr: false,
  3063  			expectedEvent: &events.Reparent{
  3064  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  3065  					PrimaryAlias: &topodatapb.TabletAlias{
  3066  						Cell: "zone1",
  3067  						Uid:  100,
  3068  					},
  3069  					KeyRange:         &topodatapb.KeyRange{},
  3070  					IsPrimaryServing: true,
  3071  				}, nil),
  3072  			},
  3073  		},
  3074  		{
  3075  			name: "promotion step fails",
  3076  			ts:   memorytopo.NewServer("zone1"),
  3077  			tmc: &testutil.TabletManagerClient{
  3078  				SetReadWriteResults: map[string]error{
  3079  					"zone1-0000000100": assert.AnError,
  3080  				},
  3081  			},
  3082  			tablets: []*topodatapb.Tablet{
  3083  				{
  3084  					Alias: &topodatapb.TabletAlias{
  3085  						Cell: "zone1",
  3086  						Uid:  100,
  3087  					},
  3088  					Type:     topodatapb.TabletType_PRIMARY,
  3089  					Keyspace: "testkeyspace",
  3090  					Shard:    "-",
  3091  				},
  3092  				{
  3093  					Alias: &topodatapb.TabletAlias{
  3094  						Cell: "zone1",
  3095  						Uid:  200,
  3096  					},
  3097  					Type:     topodatapb.TabletType_REPLICA,
  3098  					Keyspace: "testkeyspace",
  3099  					Shard:    "-",
  3100  				},
  3101  			},
  3102  
  3103  			ev:       &events.Reparent{},
  3104  			keyspace: "testkeyspace",
  3105  			shard:    "-",
  3106  			opts: PlannedReparentOptions{
  3107  				NewPrimaryAlias: &topodatapb.TabletAlias{
  3108  					Cell: "zone1",
  3109  					Uid:  100,
  3110  				},
  3111  			},
  3112  
  3113  			shouldErr: true,
  3114  			expectedEvent: &events.Reparent{
  3115  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  3116  					PrimaryAlias: &topodatapb.TabletAlias{
  3117  						Cell: "zone1",
  3118  						Uid:  100,
  3119  					},
  3120  					KeyRange:         &topodatapb.KeyRange{},
  3121  					IsPrimaryServing: true,
  3122  				}, nil),
  3123  				NewPrimary: &topodatapb.Tablet{
  3124  					Alias: &topodatapb.TabletAlias{
  3125  						Cell: "zone1",
  3126  						Uid:  100,
  3127  					},
  3128  					Type:     topodatapb.TabletType_PRIMARY,
  3129  					Keyspace: "testkeyspace",
  3130  					Shard:    "-",
  3131  				},
  3132  			},
  3133  		},
  3134  		{
  3135  			name: "lost topology lock",
  3136  			ts:   memorytopo.NewServer("zone1"),
  3137  			tmc: &testutil.TabletManagerClient{
  3138  				PrimaryPositionResults: map[string]struct {
  3139  					Position string
  3140  					Error    error
  3141  				}{
  3142  					"zone1-0000000100": {
  3143  						Position: "position1",
  3144  						Error:    nil,
  3145  					},
  3146  				},
  3147  				SetReadWriteResults: map[string]error{
  3148  					"zone1-0000000100": nil,
  3149  				},
  3150  			},
  3151  			tablets: []*topodatapb.Tablet{
  3152  				{
  3153  					Alias: &topodatapb.TabletAlias{
  3154  						Cell: "zone1",
  3155  						Uid:  100,
  3156  					},
  3157  					Type:     topodatapb.TabletType_PRIMARY,
  3158  					Keyspace: "testkeyspace",
  3159  					Shard:    "-",
  3160  				},
  3161  				{
  3162  					Alias: &topodatapb.TabletAlias{
  3163  						Cell: "zone1",
  3164  						Uid:  200,
  3165  					},
  3166  					Type:     topodatapb.TabletType_REPLICA,
  3167  					Keyspace: "testkeyspace",
  3168  					Shard:    "-",
  3169  				},
  3170  			},
  3171  			unlockTopo: true,
  3172  
  3173  			ev:       &events.Reparent{},
  3174  			keyspace: "testkeyspace",
  3175  			shard:    "-",
  3176  			opts: PlannedReparentOptions{
  3177  				// This is not the shard primary, so nothing to do.
  3178  				NewPrimaryAlias: &topodatapb.TabletAlias{
  3179  					Cell: "zone1",
  3180  					Uid:  100,
  3181  				},
  3182  			},
  3183  
  3184  			shouldErr: true,
  3185  			expectedEvent: &events.Reparent{
  3186  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  3187  					PrimaryAlias: &topodatapb.TabletAlias{
  3188  						Cell: "zone1",
  3189  						Uid:  100,
  3190  					},
  3191  					KeyRange:         &topodatapb.KeyRange{},
  3192  					IsPrimaryServing: true,
  3193  				}, nil),
  3194  				NewPrimary: &topodatapb.Tablet{
  3195  					Alias: &topodatapb.TabletAlias{
  3196  						Cell: "zone1",
  3197  						Uid:  100,
  3198  					},
  3199  					Type:     topodatapb.TabletType_PRIMARY,
  3200  					Keyspace: "testkeyspace",
  3201  					Shard:    "-",
  3202  				},
  3203  			},
  3204  		},
  3205  		{
  3206  			name: "failed to reparent tablets",
  3207  			ts:   memorytopo.NewServer("zone1"),
  3208  			tmc: &testutil.TabletManagerClient{
  3209  				PrimaryPositionResults: map[string]struct {
  3210  					Position string
  3211  					Error    error
  3212  				}{
  3213  					"zone1-0000000100": {
  3214  						Position: "position1",
  3215  						Error:    nil,
  3216  					},
  3217  				},
  3218  				PopulateReparentJournalResults: map[string]error{
  3219  					"zone1-0000000100": assert.AnError,
  3220  				},
  3221  				SetReadWriteResults: map[string]error{
  3222  					"zone1-0000000100": nil,
  3223  				},
  3224  			},
  3225  			tablets: []*topodatapb.Tablet{
  3226  				{
  3227  					Alias: &topodatapb.TabletAlias{
  3228  						Cell: "zone1",
  3229  						Uid:  100,
  3230  					},
  3231  					Type:     topodatapb.TabletType_PRIMARY,
  3232  					Keyspace: "testkeyspace",
  3233  					Shard:    "-",
  3234  				},
  3235  				{
  3236  					Alias: &topodatapb.TabletAlias{
  3237  						Cell: "zone1",
  3238  						Uid:  200,
  3239  					},
  3240  					Type:     topodatapb.TabletType_REPLICA,
  3241  					Keyspace: "testkeyspace",
  3242  					Shard:    "-",
  3243  				},
  3244  			},
  3245  
  3246  			ev:       &events.Reparent{},
  3247  			keyspace: "testkeyspace",
  3248  			shard:    "-",
  3249  			opts: PlannedReparentOptions{
  3250  				NewPrimaryAlias: &topodatapb.TabletAlias{
  3251  					Cell: "zone1",
  3252  					Uid:  100,
  3253  				},
  3254  			},
  3255  
  3256  			shouldErr: true,
  3257  			expectedEvent: &events.Reparent{
  3258  				ShardInfo: *topo.NewShardInfo("testkeyspace", "-", &topodatapb.Shard{
  3259  					PrimaryAlias: &topodatapb.TabletAlias{
  3260  						Cell: "zone1",
  3261  						Uid:  100,
  3262  					},
  3263  					KeyRange:         &topodatapb.KeyRange{},
  3264  					IsPrimaryServing: true,
  3265  				}, nil),
  3266  				NewPrimary: &topodatapb.Tablet{
  3267  					Alias: &topodatapb.TabletAlias{
  3268  						Cell: "zone1",
  3269  						Uid:  100,
  3270  					},
  3271  					Type:     topodatapb.TabletType_PRIMARY,
  3272  					Keyspace: "testkeyspace",
  3273  					Shard:    "-",
  3274  				},
  3275  			},
  3276  		},
  3277  	}
  3278  
  3279  	ctx := context.Background()
  3280  	logger := logutil.NewMemoryLogger()
  3281  
  3282  	for _, tt := range tests {
  3283  		tt := tt
  3284  
  3285  		t.Run(tt.name, func(t *testing.T) {
  3286  			t.Parallel()
  3287  
  3288  			ctx := ctx
  3289  
  3290  			testutil.AddTablets(ctx, t, tt.ts, &testutil.AddTabletOptions{
  3291  				AlsoSetShardPrimary:  true,
  3292  				ForceSetShardPrimary: true, // Some of our test cases count on having multiple primaries, so let the last one "win".
  3293  				SkipShardCreation:    false,
  3294  			}, tt.tablets...)
  3295  
  3296  			if !tt.unlockTopo {
  3297  				lctx, unlock, err := tt.ts.LockShard(ctx, tt.keyspace, tt.shard, "locking for testing")
  3298  				require.NoError(t, err, "could not lock %s/%s for testing", tt.keyspace, tt.shard)
  3299  
  3300  				defer func() {
  3301  					unlock(&err)
  3302  					require.NoError(t, err, "error while unlocking %s/%s after test case", tt.keyspace, tt.shard)
  3303  				}()
  3304  
  3305  				ctx = lctx
  3306  			}
  3307  
  3308  			if tt.expectedEvent != nil {
  3309  				defer func() {
  3310  					AssertReparentEventsEqualWithMessage(t, tt.expectedEvent, tt.ev, "expected reparentShardLocked to mutate the passed-in event")
  3311  				}()
  3312  			}
  3313  
  3314  			pr := NewPlannedReparenter(tt.ts, tt.tmc, logger)
  3315  
  3316  			err := pr.reparentShardLocked(ctx, tt.ev, tt.keyspace, tt.shard, tt.opts)
  3317  			if tt.shouldErr {
  3318  				assert.Error(t, err)
  3319  
  3320  				return
  3321  			}
  3322  
  3323  			assert.NoError(t, err)
  3324  		})
  3325  	}
  3326  }
  3327  
  3328  func TestPlannedReparenter_reparentTablets(t *testing.T) {
  3329  	t.Parallel()
  3330  
  3331  	tests := []struct {
  3332  		name string
  3333  		tmc  tmclient.TabletManagerClient
  3334  
  3335  		durability              string
  3336  		ev                      *events.Reparent
  3337  		reparentJournalPosition string
  3338  		tabletMap               map[string]*topo.TabletInfo
  3339  		opts                    PlannedReparentOptions
  3340  
  3341  		shouldErr bool
  3342  	}{
  3343  		{
  3344  			name:       "success - durability = none",
  3345  			durability: "none",
  3346  			tmc: &testutil.TabletManagerClient{
  3347  				PopulateReparentJournalResults: map[string]error{
  3348  					"zone1-0000000100": nil,
  3349  				},
  3350  				SetReplicationSourceResults: map[string]error{
  3351  					"zone1-0000000200": nil,
  3352  					"zone1-0000000201": nil,
  3353  					"zone1-0000000202": nil,
  3354  				},
  3355  				SetReplicationSourceSemiSync: map[string]bool{
  3356  					"zone1-0000000200": false,
  3357  					"zone1-0000000201": false,
  3358  					"zone1-0000000202": false,
  3359  				},
  3360  			},
  3361  			ev: &events.Reparent{
  3362  				NewPrimary: &topodatapb.Tablet{
  3363  					Alias: &topodatapb.TabletAlias{
  3364  						Cell: "zone1",
  3365  						Uid:  100,
  3366  					},
  3367  					Type: topodatapb.TabletType_PRIMARY,
  3368  				},
  3369  			},
  3370  			tabletMap: map[string]*topo.TabletInfo{
  3371  				"zone1-0000000100": {
  3372  					Tablet: &topodatapb.Tablet{
  3373  						Alias: &topodatapb.TabletAlias{
  3374  							Cell: "zone1",
  3375  							Uid:  100,
  3376  						},
  3377  						Type: topodatapb.TabletType_PRIMARY,
  3378  					},
  3379  				},
  3380  				"zone1-0000000200": {
  3381  					Tablet: &topodatapb.Tablet{
  3382  						Alias: &topodatapb.TabletAlias{
  3383  							Cell: "zone1",
  3384  							Uid:  200,
  3385  						},
  3386  						Type: topodatapb.TabletType_REPLICA,
  3387  					},
  3388  				},
  3389  				"zone1-0000000201": {
  3390  					Tablet: &topodatapb.Tablet{
  3391  						Alias: &topodatapb.TabletAlias{
  3392  							Cell: "zone1",
  3393  							Uid:  201,
  3394  						},
  3395  						Type: topodatapb.TabletType_REPLICA,
  3396  					},
  3397  				},
  3398  				"zone1-0000000202": {
  3399  					Tablet: &topodatapb.Tablet{
  3400  						Alias: &topodatapb.TabletAlias{
  3401  							Cell: "zone1",
  3402  							Uid:  202,
  3403  						},
  3404  						Type: topodatapb.TabletType_REPLICA,
  3405  					},
  3406  				},
  3407  			},
  3408  			shouldErr: false,
  3409  		},
  3410  		{
  3411  			name:       "success - durability = semi_sync",
  3412  			durability: "semi_sync",
  3413  			tmc: &testutil.TabletManagerClient{
  3414  				PopulateReparentJournalResults: map[string]error{
  3415  					"zone1-0000000100": nil,
  3416  				},
  3417  				SetReplicationSourceResults: map[string]error{
  3418  					"zone1-0000000200": nil,
  3419  					"zone1-0000000201": nil,
  3420  					"zone1-0000000202": nil,
  3421  				},
  3422  				SetReplicationSourceSemiSync: map[string]bool{
  3423  					"zone1-0000000200": true,
  3424  					"zone1-0000000201": true,
  3425  					"zone1-0000000202": false,
  3426  				},
  3427  			},
  3428  			ev: &events.Reparent{
  3429  				NewPrimary: &topodatapb.Tablet{
  3430  					Alias: &topodatapb.TabletAlias{
  3431  						Cell: "zone1",
  3432  						Uid:  100,
  3433  					},
  3434  					Type: topodatapb.TabletType_PRIMARY,
  3435  				},
  3436  			},
  3437  			tabletMap: map[string]*topo.TabletInfo{
  3438  				"zone1-0000000100": {
  3439  					Tablet: &topodatapb.Tablet{
  3440  						Alias: &topodatapb.TabletAlias{
  3441  							Cell: "zone1",
  3442  							Uid:  100,
  3443  						},
  3444  						Type: topodatapb.TabletType_PRIMARY,
  3445  					},
  3446  				},
  3447  				"zone1-0000000200": {
  3448  					Tablet: &topodatapb.Tablet{
  3449  						Alias: &topodatapb.TabletAlias{
  3450  							Cell: "zone1",
  3451  							Uid:  200,
  3452  						},
  3453  						Type: topodatapb.TabletType_REPLICA,
  3454  					},
  3455  				},
  3456  				"zone1-0000000201": {
  3457  					Tablet: &topodatapb.Tablet{
  3458  						Alias: &topodatapb.TabletAlias{
  3459  							Cell: "zone1",
  3460  							Uid:  201,
  3461  						},
  3462  						Type: topodatapb.TabletType_REPLICA,
  3463  					},
  3464  				},
  3465  				"zone1-0000000202": {
  3466  					Tablet: &topodatapb.Tablet{
  3467  						Alias: &topodatapb.TabletAlias{
  3468  							Cell: "zone1",
  3469  							Uid:  202,
  3470  						},
  3471  						Type: topodatapb.TabletType_RDONLY,
  3472  					},
  3473  				},
  3474  			},
  3475  			shouldErr: false,
  3476  		},
  3477  		{
  3478  			name: "SetReplicationSource failed on replica",
  3479  			tmc: &testutil.TabletManagerClient{
  3480  				PopulateReparentJournalResults: map[string]error{
  3481  					"zone1-0000000100": nil,
  3482  				},
  3483  				SetReplicationSourceResults: map[string]error{
  3484  					"zone1-0000000200": nil,
  3485  					"zone1-0000000201": assert.AnError,
  3486  					"zone1-0000000202": nil,
  3487  				},
  3488  			},
  3489  			ev: &events.Reparent{
  3490  				NewPrimary: &topodatapb.Tablet{
  3491  					Alias: &topodatapb.TabletAlias{
  3492  						Cell: "zone1",
  3493  						Uid:  100,
  3494  					},
  3495  					Type: topodatapb.TabletType_PRIMARY,
  3496  				},
  3497  			},
  3498  			tabletMap: map[string]*topo.TabletInfo{
  3499  				"zone1-0000000100": {
  3500  					Tablet: &topodatapb.Tablet{
  3501  						Alias: &topodatapb.TabletAlias{
  3502  							Cell: "zone1",
  3503  							Uid:  100,
  3504  						},
  3505  						Type: topodatapb.TabletType_PRIMARY,
  3506  					},
  3507  				},
  3508  				"zone1-0000000200": {
  3509  					Tablet: &topodatapb.Tablet{
  3510  						Alias: &topodatapb.TabletAlias{
  3511  							Cell: "zone1",
  3512  							Uid:  200,
  3513  						},
  3514  						Type: topodatapb.TabletType_REPLICA,
  3515  					},
  3516  				},
  3517  				"zone1-0000000201": {
  3518  					Tablet: &topodatapb.Tablet{
  3519  						Alias: &topodatapb.TabletAlias{
  3520  							Cell: "zone1",
  3521  							Uid:  201,
  3522  						},
  3523  						Type: topodatapb.TabletType_REPLICA,
  3524  					},
  3525  				},
  3526  				"zone1-0000000202": {
  3527  					Tablet: &topodatapb.Tablet{
  3528  						Alias: &topodatapb.TabletAlias{
  3529  							Cell: "zone1",
  3530  							Uid:  202,
  3531  						},
  3532  						Type: topodatapb.TabletType_REPLICA,
  3533  					},
  3534  				},
  3535  			},
  3536  			shouldErr: true,
  3537  		},
  3538  		{
  3539  			name: "SetReplicationSource timed out on replica",
  3540  			tmc: &testutil.TabletManagerClient{
  3541  				PopulateReparentJournalResults: map[string]error{
  3542  					"zone1-0000000100": nil,
  3543  				},
  3544  				SetReplicationSourceDelays: map[string]time.Duration{
  3545  					"zone1-0000000201": time.Millisecond * 50,
  3546  				},
  3547  				SetReplicationSourceResults: map[string]error{
  3548  					"zone1-0000000200": nil,
  3549  					"zone1-0000000201": nil,
  3550  					"zone1-0000000202": nil,
  3551  				},
  3552  			},
  3553  			ev: &events.Reparent{
  3554  				NewPrimary: &topodatapb.Tablet{
  3555  					Alias: &topodatapb.TabletAlias{
  3556  						Cell: "zone1",
  3557  						Uid:  100,
  3558  					},
  3559  					Type: topodatapb.TabletType_PRIMARY,
  3560  				},
  3561  			},
  3562  			tabletMap: map[string]*topo.TabletInfo{
  3563  				"zone1-0000000100": {
  3564  					Tablet: &topodatapb.Tablet{
  3565  						Alias: &topodatapb.TabletAlias{
  3566  							Cell: "zone1",
  3567  							Uid:  100,
  3568  						},
  3569  						Type: topodatapb.TabletType_PRIMARY,
  3570  					},
  3571  				},
  3572  				"zone1-0000000200": {
  3573  					Tablet: &topodatapb.Tablet{
  3574  						Alias: &topodatapb.TabletAlias{
  3575  							Cell: "zone1",
  3576  							Uid:  200,
  3577  						},
  3578  						Type: topodatapb.TabletType_REPLICA,
  3579  					},
  3580  				},
  3581  				"zone1-0000000201": {
  3582  					Tablet: &topodatapb.Tablet{
  3583  						Alias: &topodatapb.TabletAlias{
  3584  							Cell: "zone1",
  3585  							Uid:  201,
  3586  						},
  3587  						Type: topodatapb.TabletType_REPLICA,
  3588  					},
  3589  				},
  3590  				"zone1-0000000202": {
  3591  					Tablet: &topodatapb.Tablet{
  3592  						Alias: &topodatapb.TabletAlias{
  3593  							Cell: "zone1",
  3594  							Uid:  202,
  3595  						},
  3596  						Type: topodatapb.TabletType_REPLICA,
  3597  					},
  3598  				},
  3599  			},
  3600  			opts: PlannedReparentOptions{
  3601  				WaitReplicasTimeout: time.Millisecond * 10,
  3602  			},
  3603  			shouldErr: true,
  3604  		},
  3605  		{
  3606  			name: "PopulateReparentJournal failed out on new primary",
  3607  			tmc: &testutil.TabletManagerClient{
  3608  				PopulateReparentJournalResults: map[string]error{
  3609  					"zone1-0000000100": assert.AnError,
  3610  				},
  3611  				SetReplicationSourceResults: map[string]error{
  3612  					"zone1-0000000200": nil,
  3613  					"zone1-0000000201": nil,
  3614  					"zone1-0000000202": nil,
  3615  				},
  3616  			},
  3617  			ev: &events.Reparent{
  3618  				NewPrimary: &topodatapb.Tablet{
  3619  					Alias: &topodatapb.TabletAlias{
  3620  						Cell: "zone1",
  3621  						Uid:  100,
  3622  					},
  3623  					Type: topodatapb.TabletType_PRIMARY,
  3624  				},
  3625  			},
  3626  			tabletMap: map[string]*topo.TabletInfo{
  3627  				"zone1-0000000100": {
  3628  					Tablet: &topodatapb.Tablet{
  3629  						Alias: &topodatapb.TabletAlias{
  3630  							Cell: "zone1",
  3631  							Uid:  100,
  3632  						},
  3633  						Type: topodatapb.TabletType_PRIMARY,
  3634  					},
  3635  				},
  3636  				"zone1-0000000200": {
  3637  					Tablet: &topodatapb.Tablet{
  3638  						Alias: &topodatapb.TabletAlias{
  3639  							Cell: "zone1",
  3640  							Uid:  200,
  3641  						},
  3642  						Type: topodatapb.TabletType_REPLICA,
  3643  					},
  3644  				},
  3645  				"zone1-0000000201": {
  3646  					Tablet: &topodatapb.Tablet{
  3647  						Alias: &topodatapb.TabletAlias{
  3648  							Cell: "zone1",
  3649  							Uid:  201,
  3650  						},
  3651  						Type: topodatapb.TabletType_REPLICA,
  3652  					},
  3653  				},
  3654  				"zone1-0000000202": {
  3655  					Tablet: &topodatapb.Tablet{
  3656  						Alias: &topodatapb.TabletAlias{
  3657  							Cell: "zone1",
  3658  							Uid:  202,
  3659  						},
  3660  						Type: topodatapb.TabletType_REPLICA,
  3661  					},
  3662  				},
  3663  			},
  3664  			shouldErr: true,
  3665  		},
  3666  		{
  3667  			name: "PopulateReparentJournal timed out on new primary",
  3668  			tmc: &testutil.TabletManagerClient{
  3669  				PopulateReparentJournalDelays: map[string]time.Duration{
  3670  					"zone1-0000000100": time.Millisecond * 50,
  3671  				},
  3672  				PopulateReparentJournalResults: map[string]error{
  3673  					"zone1-0000000100": nil,
  3674  				},
  3675  				SetReplicationSourceResults: map[string]error{
  3676  					"zone1-0000000200": nil,
  3677  					"zone1-0000000201": nil,
  3678  					"zone1-0000000202": nil,
  3679  				},
  3680  			},
  3681  			ev: &events.Reparent{
  3682  				NewPrimary: &topodatapb.Tablet{
  3683  					Alias: &topodatapb.TabletAlias{
  3684  						Cell: "zone1",
  3685  						Uid:  100,
  3686  					},
  3687  					Type: topodatapb.TabletType_PRIMARY,
  3688  				},
  3689  			},
  3690  			tabletMap: map[string]*topo.TabletInfo{
  3691  				"zone1-0000000100": {
  3692  					Tablet: &topodatapb.Tablet{
  3693  						Alias: &topodatapb.TabletAlias{
  3694  							Cell: "zone1",
  3695  							Uid:  100,
  3696  						},
  3697  						Type: topodatapb.TabletType_PRIMARY,
  3698  					},
  3699  				},
  3700  				"zone1-0000000200": {
  3701  					Tablet: &topodatapb.Tablet{
  3702  						Alias: &topodatapb.TabletAlias{
  3703  							Cell: "zone1",
  3704  							Uid:  200,
  3705  						},
  3706  						Type: topodatapb.TabletType_REPLICA,
  3707  					},
  3708  				},
  3709  				"zone1-0000000201": {
  3710  					Tablet: &topodatapb.Tablet{
  3711  						Alias: &topodatapb.TabletAlias{
  3712  							Cell: "zone1",
  3713  							Uid:  201,
  3714  						},
  3715  						Type: topodatapb.TabletType_REPLICA,
  3716  					},
  3717  				},
  3718  				"zone1-0000000202": {
  3719  					Tablet: &topodatapb.Tablet{
  3720  						Alias: &topodatapb.TabletAlias{
  3721  							Cell: "zone1",
  3722  							Uid:  202,
  3723  						},
  3724  						Type: topodatapb.TabletType_REPLICA,
  3725  					},
  3726  				},
  3727  			},
  3728  			opts: PlannedReparentOptions{
  3729  				WaitReplicasTimeout: time.Millisecond * 10,
  3730  			},
  3731  			shouldErr: true,
  3732  		},
  3733  	}
  3734  
  3735  	ctx := context.Background()
  3736  	logger := logutil.NewMemoryLogger()
  3737  
  3738  	for _, tt := range tests {
  3739  		tt := tt
  3740  
  3741  		t.Run(tt.name, func(t *testing.T) {
  3742  			t.Parallel()
  3743  
  3744  			pr := NewPlannedReparenter(nil, tt.tmc, logger)
  3745  			durabilityPolicy := "none"
  3746  			if tt.durability != "" {
  3747  				durabilityPolicy = tt.durability
  3748  			}
  3749  			durability, err := GetDurabilityPolicy(durabilityPolicy)
  3750  			require.NoError(t, err)
  3751  			tt.opts.durability = durability
  3752  			err = pr.reparentTablets(ctx, tt.ev, tt.reparentJournalPosition, tt.tabletMap, tt.opts)
  3753  			if tt.shouldErr {
  3754  				assert.Error(t, err)
  3755  
  3756  				return
  3757  			}
  3758  
  3759  			assert.NoError(t, err)
  3760  		})
  3761  	}
  3762  }
  3763  
  3764  // (TODO:@ajm88) when unifying all the mock TMClient implementations (which will
  3765  // most likely end up in go/vt/vtctl/testutil), move these to the same testutil
  3766  // package.
  3767  func AssertReparentEventsEqualWithMessage(t *testing.T, expected *events.Reparent, actual *events.Reparent, msg string) {
  3768  	t.Helper()
  3769  
  3770  	if msg != "" && !strings.HasSuffix(msg, " ") {
  3771  		msg = msg + ": "
  3772  	}
  3773  
  3774  	if expected == nil {
  3775  		assert.Nil(t, actual, "%sexpected nil Reparent event", msg)
  3776  		return
  3777  	}
  3778  
  3779  	if actual == nil {
  3780  		// Note: the reason we don't use require.NotNil here is because it would
  3781  		// fail the entire test, rather than just this one helper, which is
  3782  		// intended to be an atomic assertion. However, we also don't want to
  3783  		// have to add a bunch of nil-guards below, as it would complicate the
  3784  		// code, so we're going to duplicate the nil check to force a failure
  3785  		// and bail early.
  3786  		assert.NotNil(t, actual, "%sexpected non-nil Reparent event", msg)
  3787  		return
  3788  	}
  3789  
  3790  	removeVersion := func(si *topo.ShardInfo) *topo.ShardInfo {
  3791  		return topo.NewShardInfo(si.Keyspace(), si.ShardName(), si.Shard, nil)
  3792  	}
  3793  
  3794  	utils.MustMatch(t, removeVersion(&expected.ShardInfo), removeVersion(&actual.ShardInfo), msg+"Reparent.ShardInfo mismatch")
  3795  	utils.MustMatch(t, &expected.NewPrimary, &actual.NewPrimary, msg+"Reparent.NewPrimary mismatch")
  3796  	utils.MustMatch(t, &expected.OldPrimary, &actual.OldPrimary, msg+"Reparent.OldPrimary mismatch")
  3797  }
  3798  
  3799  func AssertReparentEventsEqual(t *testing.T, expected *events.Reparent, actual *events.Reparent) {
  3800  	t.Helper()
  3801  
  3802  	AssertReparentEventsEqualWithMessage(t, expected, actual, "")
  3803  }