vitess.io/vitess@v0.16.2/go/vt/wrangler/traffic_switcher_test.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package wrangler 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "reflect" 24 "strings" 25 "testing" 26 "time" 27 28 "github.com/google/go-cmp/cmp" 29 "github.com/stretchr/testify/require" 30 31 "vitess.io/vitess/go/sqltypes" 32 "vitess.io/vitess/go/vt/topo" 33 "vitess.io/vitess/go/vt/topotools" 34 "vitess.io/vitess/go/vt/vtctl/workflow" 35 36 binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" 37 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 38 ) 39 40 var ( 41 resultid1 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(1)}}} 42 resultid2 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(2)}}} 43 resultid3 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(3)}}} 44 resultid12 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(1)}, {sqltypes.NewInt64(2)}}} 45 resultid1234 = &sqltypes.Result{ 46 Rows: [][]sqltypes.Value{{ 47 sqltypes.NewInt64(1), 48 }, { 49 sqltypes.NewInt64(2), 50 }, { 51 sqltypes.NewInt64(3), 52 }, { 53 sqltypes.NewInt64(4), 54 }}, 55 } 56 resultid34 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(3)}, {sqltypes.NewInt64(4)}}} 57 resultid3456 = &sqltypes.Result{ 58 Rows: [][]sqltypes.Value{{ 59 sqltypes.NewInt64(3), 60 }, { 61 sqltypes.NewInt64(4), 62 }, { 63 sqltypes.NewInt64(5), 64 }, { 65 sqltypes.NewInt64(6), 66 }}, 67 } 68 ) 69 70 const ( 71 tsCheckJournals = "select val from _vt.resharding_journal where id=.*" 72 ) 73 74 // TestTableMigrate tests table mode migrations. 75 // This has to be kept in sync with TestShardMigrate. 76 func TestTableMigrateMainflow(t *testing.T) { 77 ctx := context.Background() 78 tme := newTestTableMigrater(ctx, t) 79 defer tme.close(t) 80 81 checkCellRouting(t, tme.wr, "cell1", map[string][]string{ 82 "t1": {"ks1.t1"}, 83 "ks2.t1": {"ks1.t1"}, 84 "t2": {"ks1.t2"}, 85 "ks2.t2": {"ks1.t2"}, 86 }) 87 88 tme.expectNoPreviousJournals() 89 //------------------------------------------------------------------------------------------------------------------- 90 // Single cell RDONLY migration. 91 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, []string{"cell1"}, workflow.DirectionForward, false) 92 if err != nil { 93 t.Fatal(err) 94 } 95 checkCellRouting(t, tme.wr, "cell1", map[string][]string{ 96 "t1": {"ks1.t1"}, 97 "ks2.t1": {"ks1.t1"}, 98 "t2": {"ks1.t2"}, 99 "ks2.t2": {"ks1.t2"}, 100 "t1@rdonly": {"ks2.t1"}, 101 "ks2.t1@rdonly": {"ks2.t1"}, 102 "ks1.t1@rdonly": {"ks2.t1"}, 103 "t2@rdonly": {"ks2.t2"}, 104 "ks2.t2@rdonly": {"ks2.t2"}, 105 "ks1.t2@rdonly": {"ks2.t2"}, 106 }) 107 checkCellRouting(t, tme.wr, "cell2", map[string][]string{ 108 "t1": {"ks1.t1"}, 109 "ks2.t1": {"ks1.t1"}, 110 "t2": {"ks1.t2"}, 111 "ks2.t2": {"ks1.t2"}, 112 }) 113 verifyQueries(t, tme.allDBClients) 114 115 tme.expectNoPreviousJournals() 116 //------------------------------------------------------------------------------------------------------------------- 117 // Other cell REPLICA migration. 118 // The global routing already contains redirections for rdonly. 119 // So, adding routes for replica and deploying to cell2 will also cause 120 // cell2 to switch rdonly. This is a quirk that can be fixed later if necessary. 121 // TODO(sougou): check if it's worth fixing, or clearly document the quirk. 122 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionForward, false) 123 if err != nil { 124 t.Fatal(err) 125 } 126 checkCellRouting(t, tme.wr, "cell1", map[string][]string{ 127 "t1": {"ks1.t1"}, 128 "ks2.t1": {"ks1.t1"}, 129 "t2": {"ks1.t2"}, 130 "ks2.t2": {"ks1.t2"}, 131 "t1@rdonly": {"ks2.t1"}, 132 "ks2.t1@rdonly": {"ks2.t1"}, 133 "ks1.t1@rdonly": {"ks2.t1"}, 134 "t2@rdonly": {"ks2.t2"}, 135 "ks2.t2@rdonly": {"ks2.t2"}, 136 "ks1.t2@rdonly": {"ks2.t2"}, 137 }) 138 checkCellRouting(t, tme.wr, "cell2", map[string][]string{ 139 "t1": {"ks1.t1"}, 140 "ks2.t1": {"ks1.t1"}, 141 "t2": {"ks1.t2"}, 142 "ks2.t2": {"ks1.t2"}, 143 "t1@rdonly": {"ks2.t1"}, 144 "ks2.t1@rdonly": {"ks2.t1"}, 145 "ks1.t1@rdonly": {"ks2.t1"}, 146 "t2@rdonly": {"ks2.t2"}, 147 "ks2.t2@rdonly": {"ks2.t2"}, 148 "ks1.t2@rdonly": {"ks2.t2"}, 149 "t1@replica": {"ks2.t1"}, 150 "ks2.t1@replica": {"ks2.t1"}, 151 "ks1.t1@replica": {"ks2.t1"}, 152 "t2@replica": {"ks2.t2"}, 153 "ks2.t2@replica": {"ks2.t2"}, 154 "ks1.t2@replica": {"ks2.t2"}, 155 }) 156 verifyQueries(t, tme.allDBClients) 157 tme.expectNoPreviousJournals() 158 //------------------------------------------------------------------------------------------------------------------- 159 // Single cell backward REPLICA migration. 160 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionBackward, false) 161 if err != nil { 162 t.Fatal(err) 163 } 164 checkCellRouting(t, tme.wr, "cell1", map[string][]string{ 165 "t1": {"ks1.t1"}, 166 "ks2.t1": {"ks1.t1"}, 167 "t2": {"ks1.t2"}, 168 "ks2.t2": {"ks1.t2"}, 169 "t1@rdonly": {"ks2.t1"}, 170 "ks2.t1@rdonly": {"ks2.t1"}, 171 "ks1.t1@rdonly": {"ks2.t1"}, 172 "t2@rdonly": {"ks2.t2"}, 173 "ks2.t2@rdonly": {"ks2.t2"}, 174 "ks1.t2@rdonly": {"ks2.t2"}, 175 }) 176 checkCellRouting(t, tme.wr, "cell2", map[string][]string{ 177 "t1": {"ks1.t1"}, 178 "ks2.t1": {"ks1.t1"}, 179 "t2": {"ks1.t2"}, 180 "ks2.t2": {"ks1.t2"}, 181 "t1@rdonly": {"ks1.t1"}, 182 "ks2.t1@rdonly": {"ks1.t1"}, 183 "ks1.t1@rdonly": {"ks1.t1"}, 184 "t2@rdonly": {"ks1.t2"}, 185 "ks2.t2@rdonly": {"ks1.t2"}, 186 "ks1.t2@rdonly": {"ks1.t2"}, 187 "t1@replica": {"ks1.t1"}, 188 "ks2.t1@replica": {"ks1.t1"}, 189 "ks1.t1@replica": {"ks1.t1"}, 190 "t2@replica": {"ks1.t2"}, 191 "ks2.t2@replica": {"ks1.t2"}, 192 "ks1.t2@replica": {"ks1.t2"}, 193 }) 194 verifyQueries(t, tme.allDBClients) 195 196 tme.expectNoPreviousJournals() 197 //------------------------------------------------------------------------------------------------------------------- 198 // Switch all REPLICA. 199 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 200 if err != nil { 201 t.Fatal(err) 202 } 203 checkRouting(t, tme.wr, map[string][]string{ 204 "t1": {"ks1.t1"}, 205 "ks2.t1": {"ks1.t1"}, 206 "t2": {"ks1.t2"}, 207 "ks2.t2": {"ks1.t2"}, 208 "t1@rdonly": {"ks2.t1"}, 209 "ks2.t1@rdonly": {"ks2.t1"}, 210 "ks1.t1@rdonly": {"ks2.t1"}, 211 "t2@rdonly": {"ks2.t2"}, 212 "ks2.t2@rdonly": {"ks2.t2"}, 213 "ks1.t2@rdonly": {"ks2.t2"}, 214 "t1@replica": {"ks2.t1"}, 215 "ks2.t1@replica": {"ks2.t1"}, 216 "ks1.t1@replica": {"ks2.t1"}, 217 "t2@replica": {"ks2.t2"}, 218 "ks2.t2@replica": {"ks2.t2"}, 219 "ks1.t2@replica": {"ks2.t2"}, 220 }) 221 verifyQueries(t, tme.allDBClients) 222 223 tme.expectNoPreviousJournals() 224 //------------------------------------------------------------------------------------------------------------------- 225 // All cells RDONLY backward migration. 226 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionBackward, false) 227 if err != nil { 228 t.Fatal(err) 229 } 230 checkRouting(t, tme.wr, map[string][]string{ 231 "t1": {"ks1.t1"}, 232 "ks2.t1": {"ks1.t1"}, 233 "t2": {"ks1.t2"}, 234 "ks2.t2": {"ks1.t2"}, 235 "t1@replica": {"ks2.t1"}, 236 "ks2.t1@replica": {"ks2.t1"}, 237 "ks1.t1@replica": {"ks2.t1"}, 238 "t2@replica": {"ks2.t2"}, 239 "ks2.t2@replica": {"ks2.t2"}, 240 "ks1.t2@replica": {"ks2.t2"}, 241 "t1@rdonly": {"ks1.t1"}, 242 "ks2.t1@rdonly": {"ks1.t1"}, 243 "ks1.t1@rdonly": {"ks1.t1"}, 244 "t2@rdonly": {"ks1.t2"}, 245 "ks2.t2@rdonly": {"ks1.t2"}, 246 "ks1.t2@rdonly": {"ks1.t2"}, 247 }) 248 verifyQueries(t, tme.allDBClients) 249 250 tme.expectNoPreviousJournals() 251 //------------------------------------------------------------------------------------------------------------------- 252 // All cells RDONLY backward migration. 253 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionBackward, false) 254 if err != nil { 255 t.Fatal(err) 256 } 257 checkRouting(t, tme.wr, map[string][]string{ 258 "t1": {"ks1.t1"}, 259 "ks2.t1": {"ks1.t1"}, 260 "t2": {"ks1.t2"}, 261 "ks2.t2": {"ks1.t2"}, 262 "t1@replica": {"ks1.t1"}, 263 "ks2.t1@replica": {"ks1.t1"}, 264 "ks1.t1@replica": {"ks1.t1"}, 265 "t2@replica": {"ks1.t2"}, 266 "ks2.t2@replica": {"ks1.t2"}, 267 "ks1.t2@replica": {"ks1.t2"}, 268 "t1@rdonly": {"ks1.t1"}, 269 "ks2.t1@rdonly": {"ks1.t1"}, 270 "ks1.t1@rdonly": {"ks1.t1"}, 271 "t2@rdonly": {"ks1.t2"}, 272 "ks2.t2@rdonly": {"ks1.t2"}, 273 "ks1.t2@rdonly": {"ks1.t2"}, 274 }) 275 verifyQueries(t, tme.allDBClients) 276 277 //------------------------------------------------------------------------------------------------------------------- 278 // Can't switch primary with SwitchReads. 279 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_PRIMARY}, nil, workflow.DirectionForward, false) 280 want := "tablet type must be REPLICA or RDONLY: PRIMARY" 281 if err == nil || err.Error() != want { 282 t.Errorf("SwitchReads(primary) err: %v, want %v", err, want) 283 } 284 verifyQueries(t, tme.allDBClients) 285 286 //------------------------------------------------------------------------------------------------------------------- 287 // Test SwitchWrites cancelation on failure. 288 289 tme.expectNoPreviousJournals() 290 // Switch all the reads first. 291 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 292 if err != nil { 293 t.Fatal(err) 294 } 295 tme.expectNoPreviousJournals() 296 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 297 if err != nil { 298 t.Fatal(err) 299 } 300 checkRouting(t, tme.wr, map[string][]string{ 301 "t1": {"ks1.t1"}, 302 "ks2.t1": {"ks1.t1"}, 303 "t2": {"ks1.t2"}, 304 "ks2.t2": {"ks1.t2"}, 305 "t1@replica": {"ks2.t1"}, 306 "ks2.t1@replica": {"ks2.t1"}, 307 "ks1.t1@replica": {"ks2.t1"}, 308 "t2@replica": {"ks2.t2"}, 309 "ks2.t2@replica": {"ks2.t2"}, 310 "ks1.t2@replica": {"ks2.t2"}, 311 "t1@rdonly": {"ks2.t1"}, 312 "ks2.t1@rdonly": {"ks2.t1"}, 313 "ks1.t1@rdonly": {"ks2.t1"}, 314 "t2@rdonly": {"ks2.t2"}, 315 "ks2.t2@rdonly": {"ks2.t2"}, 316 "ks1.t2@rdonly": {"ks2.t2"}, 317 }) 318 319 checkJournals := func() { 320 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 321 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 322 } 323 checkJournals() 324 325 deleteReverseReplicaion := func() { 326 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 327 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 328 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 329 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 330 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 331 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 332 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 333 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 334 } 335 cancelMigration := func() { 336 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 337 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 338 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 339 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 340 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 341 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 342 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 343 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 344 345 deleteReverseReplicaion() 346 } 347 cancelMigration() 348 349 switchWrites(tme) 350 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, false, true, false) 351 want = "DeadlineExceeded" 352 if err == nil || !strings.Contains(err.Error(), want) { 353 t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) 354 } 355 verifyQueries(t, tme.allDBClients) 356 checkRouting(t, tme.wr, map[string][]string{ 357 "t1": {"ks1.t1"}, 358 "ks2.t1": {"ks1.t1"}, 359 "t2": {"ks1.t2"}, 360 "ks2.t2": {"ks1.t2"}, 361 "t1@replica": {"ks2.t1"}, 362 "ks2.t1@replica": {"ks2.t1"}, 363 "ks1.t1@replica": {"ks2.t1"}, 364 "t2@replica": {"ks2.t2"}, 365 "ks2.t2@replica": {"ks2.t2"}, 366 "ks1.t2@replica": {"ks2.t2"}, 367 "t1@rdonly": {"ks2.t1"}, 368 "ks2.t1@rdonly": {"ks2.t1"}, 369 "ks1.t1@rdonly": {"ks2.t1"}, 370 "t2@rdonly": {"ks2.t2"}, 371 "ks2.t2@rdonly": {"ks2.t2"}, 372 "ks1.t2@rdonly": {"ks2.t2"}, 373 }) 374 checkDenyList(t, tme.ts, "ks1:-40", nil) 375 checkDenyList(t, tme.ts, "ks1:40-", nil) 376 checkDenyList(t, tme.ts, "ks2:-80", nil) 377 checkDenyList(t, tme.ts, "ks2:80-", nil) 378 379 //------------------------------------------------------------------------------------------------------------------- 380 // Test successful SwitchWrites. 381 382 checkJournals() 383 384 waitForCatchup := func() { 385 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 386 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 387 "pos|state|message", 388 "varchar|varchar|varchar"), 389 "MariaDB/5-456-892|Running", 390 ) 391 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 392 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 393 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 394 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 395 396 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('Stopped') 397 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 398 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 399 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 400 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 401 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 402 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 403 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 404 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 405 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 406 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 407 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 408 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 409 } 410 waitForCatchup() 411 412 createReverseVReplication := func() { 413 deleteReverseReplicaion() 414 415 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*in_keyrange.*c1.*hash.*-40.*t2.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 416 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*in_keyrange.*c1.*hash.*-40.*t2.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 417 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*in_keyrange.*c1.*hash.*40-.*t2.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 418 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*in_keyrange.*c1.*hash.*40-.*t2.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 419 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 420 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 421 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 422 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 423 } 424 createReverseVReplication() 425 426 createJournals := func() { 427 journal1 := "insert into _vt.resharding_journal.*7672494164556733923,.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*-80.*MariaDB/5-456-893.*participants.*40.*40" 428 tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) 429 journal2 := "insert into _vt.resharding_journal.*7672494164556733923,.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*80.*participants.*40.*40" 430 tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) 431 } 432 createJournals() 433 434 startReverseVReplication := func() { 435 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1'", resultid34, nil) 436 tme.dbSourceClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 437 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 438 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 439 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1'", resultid34, nil) 440 tme.dbSourceClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 441 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 442 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 443 } 444 startReverseVReplication() 445 446 deleteTargetVReplication := func() { 447 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 448 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 449 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 450 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 451 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 452 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 453 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 454 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 455 } 456 deleteTargetVReplication() 457 458 journalID, _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 459 if err != nil { 460 t.Fatal(err) 461 } 462 if journalID != 7672494164556733923 { 463 t.Errorf("journal id: %d, want 7672494164556733923", journalID) 464 } 465 466 checkRouting(t, tme.wr, map[string][]string{ 467 "t1": {"ks2.t1"}, 468 "ks1.t1": {"ks2.t1"}, 469 "t2": {"ks2.t2"}, 470 "ks1.t2": {"ks2.t2"}, 471 "t1@replica": {"ks2.t1"}, 472 "ks2.t1@replica": {"ks2.t1"}, 473 "ks1.t1@replica": {"ks2.t1"}, 474 "t2@replica": {"ks2.t2"}, 475 "ks2.t2@replica": {"ks2.t2"}, 476 "ks1.t2@replica": {"ks2.t2"}, 477 "t1@rdonly": {"ks2.t1"}, 478 "ks2.t1@rdonly": {"ks2.t1"}, 479 "ks1.t1@rdonly": {"ks2.t1"}, 480 "t2@rdonly": {"ks2.t2"}, 481 "ks2.t2@rdonly": {"ks2.t2"}, 482 "ks1.t2@rdonly": {"ks2.t2"}, 483 }) 484 checkDenyList(t, tme.ts, "ks1:-40", []string{"t1", "t2"}) 485 checkDenyList(t, tme.ts, "ks1:40-", []string{"t1", "t2"}) 486 checkDenyList(t, tme.ts, "ks2:-80", nil) 487 checkDenyList(t, tme.ts, "ks2:80-", nil) 488 489 verifyQueries(t, tme.allDBClients) 490 } 491 492 // TestShardMigrate tests table mode migrations. 493 // This has to be kept in sync with TestTableMigrate. 494 func TestShardMigrateMainflow(t *testing.T) { 495 ctx := context.Background() 496 tme := newTestShardMigrater(ctx, t, []string{"-40", "40-"}, []string{"-80", "80-"}) 497 defer tme.close(t) 498 499 // Initial check 500 checkServedTypes(t, tme.ts, "ks:-40", 3) 501 checkServedTypes(t, tme.ts, "ks:40-", 3) 502 checkServedTypes(t, tme.ts, "ks:-80", 0) 503 checkServedTypes(t, tme.ts, "ks:80-", 0) 504 505 tme.expectNoPreviousJournals() 506 //------------------------------------------------------------------------------------------------------------------- 507 // Single cell RDONLY migration. 508 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, []string{"cell1"}, workflow.DirectionForward, false) 509 if err != nil { 510 t.Fatal(err) 511 } 512 checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) 513 checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) 514 checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) 515 checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) 516 checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 3) 517 checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 3) 518 checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 0) 519 checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 0) 520 verifyQueries(t, tme.allDBClients) 521 522 tme.expectNoPreviousJournals() 523 //------------------------------------------------------------------------------------------------------------------- 524 // Other cell REPLICA migration. 525 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionForward, false) 526 if err != nil { 527 t.Fatal(err) 528 } 529 checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) 530 checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) 531 checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) 532 checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) 533 checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 1) 534 checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 1) 535 checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 2) 536 checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 2) 537 verifyQueries(t, tme.allDBClients) 538 539 tme.expectNoPreviousJournals() 540 //------------------------------------------------------------------------------------------------------------------- 541 // Single cell backward REPLICA migration. 542 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionBackward, false) 543 if err != nil { 544 t.Fatal(err) 545 } 546 checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) 547 checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) 548 checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) 549 checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) 550 checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 3) 551 checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 3) 552 checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 0) 553 checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 0) 554 verifyQueries(t, tme.allDBClients) 555 556 tme.expectNoPreviousJournals() 557 //------------------------------------------------------------------------------------------------------------------- 558 // Switch all RDONLY. 559 // This is an extra step that does not exist in the tables test. 560 // The per-cell migration mechanism is different for tables. So, this 561 // extra step is needed to bring things in sync. 562 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 563 if err != nil { 564 t.Fatal(err) 565 } 566 checkServedTypes(t, tme.ts, "ks:-40", 2) 567 checkServedTypes(t, tme.ts, "ks:40-", 2) 568 checkServedTypes(t, tme.ts, "ks:-80", 1) 569 checkServedTypes(t, tme.ts, "ks:80-", 1) 570 verifyQueries(t, tme.allDBClients) 571 572 tme.expectNoPreviousJournals() 573 //------------------------------------------------------------------------------------------------------------------- 574 // Switch all REPLICA. 575 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 576 if err != nil { 577 t.Fatal(err) 578 } 579 checkServedTypes(t, tme.ts, "ks:-40", 1) 580 checkServedTypes(t, tme.ts, "ks:40-", 1) 581 checkServedTypes(t, tme.ts, "ks:-80", 2) 582 checkServedTypes(t, tme.ts, "ks:80-", 2) 583 verifyQueries(t, tme.allDBClients) 584 585 tme.expectNoPreviousJournals() 586 //------------------------------------------------------------------------------------------------------------------- 587 // All cells RDONLY backward migration. 588 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionBackward, false) 589 if err != nil { 590 t.Fatal(err) 591 } 592 checkServedTypes(t, tme.ts, "ks:-40", 2) 593 checkServedTypes(t, tme.ts, "ks:40-", 2) 594 checkServedTypes(t, tme.ts, "ks:-80", 1) 595 checkServedTypes(t, tme.ts, "ks:80-", 1) 596 verifyQueries(t, tme.allDBClients) 597 598 //------------------------------------------------------------------------------------------------------------------- 599 // Can't switch primary with SwitchReads. 600 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_PRIMARY}, nil, workflow.DirectionForward, false) 601 want := "tablet type must be REPLICA or RDONLY: PRIMARY" 602 if err == nil || err.Error() != want { 603 t.Errorf("SwitchReads(primary) err: %v, want %v", err, want) 604 } 605 verifyQueries(t, tme.allDBClients) 606 607 //------------------------------------------------------------------------------------------------------------------- 608 // Test SwitchWrites cancelation on failure. 609 610 tme.expectNoPreviousJournals() 611 // Switch all the reads first. 612 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 613 if err != nil { 614 t.Fatal(err) 615 } 616 checkServedTypes(t, tme.ts, "ks:-40", 1) 617 checkServedTypes(t, tme.ts, "ks:40-", 1) 618 checkServedTypes(t, tme.ts, "ks:-80", 2) 619 checkServedTypes(t, tme.ts, "ks:80-", 2) 620 checkIfPrimaryServing(t, tme.ts, "ks:-40", true) 621 checkIfPrimaryServing(t, tme.ts, "ks:40-", true) 622 checkIfPrimaryServing(t, tme.ts, "ks:-80", false) 623 checkIfPrimaryServing(t, tme.ts, "ks:80-", false) 624 625 checkJournals := func() { 626 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) 627 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) 628 } 629 checkJournals() 630 631 stopStreams := func() { 632 tme.dbSourceClients[0].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse' and state = 'Stopped' and message != 'FROZEN'", &sqltypes.Result{}, nil) 633 tme.dbSourceClients[1].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse' and state = 'Stopped' and message != 'FROZEN'", &sqltypes.Result{}, nil) 634 tme.dbSourceClients[0].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 635 tme.dbSourceClients[1].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 636 } 637 stopStreams() 638 639 deleteReverseReplicaion := func() { 640 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test_reverse'", resultid3, nil) 641 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test_reverse'", resultid34, nil) 642 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3)", &sqltypes.Result{}, nil) 643 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 644 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3)", &sqltypes.Result{}, nil) 645 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3)", &sqltypes.Result{}, nil) 646 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 647 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 648 } 649 cancelMigration := func() { 650 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 651 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 652 653 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) 654 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) 655 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 656 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (2)", &sqltypes.Result{}, nil) 657 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 658 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 659 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 660 661 deleteReverseReplicaion() 662 } 663 cancelMigration() 664 665 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, false, true, false) 666 want = "DeadlineExceeded" 667 if err == nil || !strings.Contains(err.Error(), want) { 668 t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) 669 } 670 671 verifyQueries(t, tme.allDBClients) 672 checkServedTypes(t, tme.ts, "ks:-40", 1) 673 checkServedTypes(t, tme.ts, "ks:40-", 1) 674 checkServedTypes(t, tme.ts, "ks:-80", 2) 675 checkServedTypes(t, tme.ts, "ks:80-", 2) 676 checkIfPrimaryServing(t, tme.ts, "ks:-40", true) 677 checkIfPrimaryServing(t, tme.ts, "ks:40-", true) 678 checkIfPrimaryServing(t, tme.ts, "ks:-80", false) 679 checkIfPrimaryServing(t, tme.ts, "ks:80-", false) 680 681 //------------------------------------------------------------------------------------------------------------------- 682 // Test successful SwitchWrites. 683 684 checkJournals() 685 stopStreams() 686 687 waitForCatchup := func() { 688 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 689 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 690 "pos|state|message", 691 "varchar|varchar|varchar"), 692 "MariaDB/5-456-892|Running", 693 ) 694 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 695 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 696 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 697 698 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('stopped for cutover') 699 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 700 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 701 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 702 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 703 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 704 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 705 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 706 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 707 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 708 } 709 waitForCatchup() 710 711 createReverseVReplication := func() { 712 deleteReverseReplicaion() 713 714 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*-80.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 715 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*-80.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 716 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*80-.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 717 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 718 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 719 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 720 } 721 createReverseVReplication() 722 723 createJournals := func() { 724 journal1 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*-80.*MariaDB/5-456-893.*participants.*40.*40" 725 tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) 726 journal2 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*shard_gtids.*80.*MariaDB/5-456-893.*participants.*40.*40" 727 tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) 728 } 729 createJournals() 730 731 startReverseVReplication := func() { 732 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) 733 tme.dbSourceClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 734 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 735 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 736 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) 737 tme.dbSourceClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 738 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 739 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 740 } 741 startReverseVReplication() 742 743 freezeTargetVReplication := func() { 744 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) 745 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 746 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 747 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 748 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) 749 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (2)", &sqltypes.Result{}, nil) 750 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 751 } 752 freezeTargetVReplication() 753 754 journalID, _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 755 if err != nil { 756 t.Fatal(err) 757 } 758 if journalID != 6432976123657117097 { 759 t.Errorf("journal id: %d, want 6432976123657117097", journalID) 760 } 761 762 verifyQueries(t, tme.allDBClients) 763 764 checkServedTypes(t, tme.ts, "ks:-40", 0) 765 checkServedTypes(t, tme.ts, "ks:40-", 0) 766 checkServedTypes(t, tme.ts, "ks:-80", 3) 767 checkServedTypes(t, tme.ts, "ks:80-", 3) 768 769 checkIfPrimaryServing(t, tme.ts, "ks:-40", false) 770 checkIfPrimaryServing(t, tme.ts, "ks:40-", false) 771 checkIfPrimaryServing(t, tme.ts, "ks:-80", true) 772 checkIfPrimaryServing(t, tme.ts, "ks:80-", true) 773 774 verifyQueries(t, tme.allDBClients) 775 } 776 777 func TestTableMigrateOneToManyKeepNoArtifacts(t *testing.T) { 778 testTableMigrateOneToMany(t, false, false) 779 } 780 781 func TestTableMigrateOneToManyKeepDataArtifacts(t *testing.T) { 782 testTableMigrateOneToMany(t, true, false) 783 } 784 785 func TestTableMigrateOneToManyKeepRoutingArtifacts(t *testing.T) { 786 testTableMigrateOneToMany(t, false, true) 787 } 788 789 func TestTableMigrateOneToManyKeepAllArtifacts(t *testing.T) { 790 testTableMigrateOneToMany(t, true, true) 791 } 792 793 func testTableMigrateOneToMany(t *testing.T, keepData, keepRoutingRules bool) { 794 ctx := context.Background() 795 tme := newTestTableMigraterCustom(ctx, t, []string{"0"}, []string{"-80", "80-"}, "select * %s") 796 defer tme.close(t) 797 798 tme.expectNoPreviousJournals() 799 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 800 if err != nil { 801 t.Fatal(err) 802 } 803 tme.expectNoPreviousJournals() 804 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 805 if err != nil { 806 t.Fatal(err) 807 } 808 809 waitForCatchup := func() { 810 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 811 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 812 "pos|state|message", 813 "varchar|varchar|varchar"), 814 "MariaDB/5-456-892|Running", 815 ) 816 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 817 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 818 819 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('Stopped') 820 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 821 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 822 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 823 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 824 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 825 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 826 } 827 waitForCatchup() 828 829 deleteReverseReplication := func() { 830 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 831 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 832 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 833 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 834 } 835 836 createReverseVReplication := func() { 837 deleteReverseReplication() 838 839 tme.dbSourceClients[0].addQueryRE(`insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*from `+"`"+"t1`"+`\\".*t2.*from `+"`"+"t2`"+`\\"`, &sqltypes.Result{InsertID: 1}, nil) 840 tme.dbSourceClients[0].addQueryRE(`insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*from `+"`"+"t1`"+`\\".*t2.*from `+"`"+"t2`"+`\\"`, &sqltypes.Result{InsertID: 2}, nil) 841 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 842 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 843 } 844 createReverseVReplication() 845 846 createJournals := func() { 847 journal1 := "insert into _vt.resharding_journal.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*80.*MariaDB/5-456-893.*participants.*0" 848 tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) 849 } 850 createJournals() 851 852 freezeTargetVReplication := func() { 853 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid1, nil) 854 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid1, nil) 855 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1)", &sqltypes.Result{}, nil) 856 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 857 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1)", &sqltypes.Result{}, nil) 858 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 859 } 860 freezeTargetVReplication() 861 862 dropSourcesInvalid := func() { 863 tme.dbTargetClients[0].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 864 tme.dbTargetClients[1].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 865 } 866 dropSourcesInvalid() 867 _, err = tme.wr.DropSources(ctx, tme.targetKeyspace, "test", workflow.DropTable, keepData, keepRoutingRules, false, false) 868 require.Error(t, err, "Workflow has not completed, cannot DropSources") 869 870 tme.dbSourceClients[0].addQueryRE(tsCheckJournals, &sqltypes.Result{}, nil) 871 872 switchWrites(tme) 873 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, false, false) 874 if err != nil { 875 t.Fatal(err) 876 } 877 878 dropSourcesDryRun := func() { 879 tme.dbTargetClients[0].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 880 tme.dbTargetClients[1].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 881 } 882 dropSourcesDryRun() 883 wantdryRunDropSources := []string{ 884 "Lock keyspace ks1", 885 "Lock keyspace ks2", 886 } 887 if !keepData { 888 wantdryRunDropSources = append(wantdryRunDropSources, "Dropping these tables from the database and removing them from the vschema for keyspace ks1:", 889 " Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t1", 890 " Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t2", 891 "Denied tables [t1,t2] will be removed from:", 892 " Keyspace ks1 Shard 0 Tablet 10") 893 } 894 wantdryRunDropSources = append(wantdryRunDropSources, "Delete reverse vreplication streams on source:", 895 " Keyspace ks1 Shard 0 Workflow test_reverse DbName vt_ks1 Tablet 10", 896 "Delete vreplication streams on target:", 897 " Keyspace ks2 Shard -80 Workflow test DbName vt_ks2 Tablet 20", 898 " Keyspace ks2 Shard 80- Workflow test DbName vt_ks2 Tablet 30") 899 if !keepRoutingRules { 900 wantdryRunDropSources = append(wantdryRunDropSources, "Routing rules for participating tables will be deleted") 901 } 902 wantdryRunDropSources = append(wantdryRunDropSources, "Unlock keyspace ks2", "Unlock keyspace ks1") 903 results, err := tme.wr.DropSources(ctx, tme.targetKeyspace, "test", workflow.DropTable, keepData, keepRoutingRules, false, true) 904 require.NoError(t, err) 905 require.Empty(t, cmp.Diff(wantdryRunDropSources, *results)) 906 checkDenyList(t, tme.ts, fmt.Sprintf("%s:%s", "ks1", "0"), []string{"t1", "t2"}) 907 908 dropSourcesDryRunRename := func() { 909 tme.dbTargetClients[0].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 910 tme.dbTargetClients[1].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 911 } 912 dropSourcesDryRunRename() 913 wantdryRunRenameSources := []string{ 914 "Lock keyspace ks1", 915 "Lock keyspace ks2", 916 } 917 if !keepData { 918 wantdryRunRenameSources = append(wantdryRunRenameSources, "Renaming these tables from the database and removing them from the vschema for keyspace ks1:", " "+ 919 "Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t1", 920 " Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t2", 921 "Denied tables [t1,t2] will be removed from:", 922 " Keyspace ks1 Shard 0 Tablet 10") 923 } 924 wantdryRunRenameSources = append(wantdryRunRenameSources, "Delete reverse vreplication streams on source:", 925 " Keyspace ks1 Shard 0 Workflow test_reverse DbName vt_ks1 Tablet 10", 926 "Delete vreplication streams on target:", 927 " Keyspace ks2 Shard -80 Workflow test DbName vt_ks2 Tablet 20", 928 " Keyspace ks2 Shard 80- Workflow test DbName vt_ks2 Tablet 30") 929 if !keepRoutingRules { 930 wantdryRunRenameSources = append(wantdryRunRenameSources, "Routing rules for participating tables will be deleted") 931 } 932 wantdryRunRenameSources = append(wantdryRunRenameSources, "Unlock keyspace ks2", "Unlock keyspace ks1") 933 results, err = tme.wr.DropSources(ctx, tme.targetKeyspace, "test", workflow.RenameTable, keepData, keepRoutingRules, false, true) 934 require.NoError(t, err) 935 require.Empty(t, cmp.Diff(wantdryRunRenameSources, *results)) 936 checkDenyList(t, tme.ts, fmt.Sprintf("%s:%s", "ks1", "0"), []string{"t1", "t2"}) 937 938 dropSources := func() { 939 tme.dbTargetClients[0].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 940 tme.dbTargetClients[1].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil) 941 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", &sqltypes.Result{}, nil) 942 tme.tmeDB.AddQuery(fmt.Sprintf("rename table `vt_ks1`.`t1` TO `vt_ks1`.`%s`", getRenameFileName("t1")), &sqltypes.Result{}) 943 tme.tmeDB.AddQuery(fmt.Sprintf("rename table `vt_ks1`.`t2` TO `vt_ks1`.`%s`", getRenameFileName("t2")), &sqltypes.Result{}) 944 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", &sqltypes.Result{}, nil) // 945 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", &sqltypes.Result{}, nil) 946 } 947 dropSources() 948 949 wantRouting := map[string][]string{ 950 "t1": {"ks2.t1"}, 951 "ks1.t1": {"ks2.t1"}, 952 "t2": {"ks2.t2"}, 953 "ks1.t2": {"ks2.t2"}, 954 "t1@replica": {"ks2.t1"}, 955 "ks2.t1@replica": {"ks2.t1"}, 956 "ks1.t1@replica": {"ks2.t1"}, 957 "t2@replica": {"ks2.t2"}, 958 "ks2.t2@replica": {"ks2.t2"}, 959 "ks1.t2@replica": {"ks2.t2"}, 960 "t1@rdonly": {"ks2.t1"}, 961 "ks2.t1@rdonly": {"ks2.t1"}, 962 "ks1.t1@rdonly": {"ks2.t1"}, 963 "t2@rdonly": {"ks2.t2"}, 964 "ks2.t2@rdonly": {"ks2.t2"}, 965 "ks1.t2@rdonly": {"ks2.t2"}, 966 } 967 checkRouting(t, tme.wr, wantRouting) 968 _, err = tme.wr.DropSources(ctx, tme.targetKeyspace, "test", workflow.RenameTable, keepData, keepRoutingRules, false, false) 969 require.NoError(t, err) 970 var wantDenyList []string 971 if keepData { 972 wantDenyList = []string{"t1", "t2"} 973 } 974 checkDenyList(t, tme.ts, fmt.Sprintf("%s:%s", "ks1", "0"), wantDenyList) 975 if !keepRoutingRules { 976 wantRouting = map[string][]string{} 977 } 978 checkRouting(t, tme.wr, wantRouting) 979 980 verifyQueries(t, tme.allDBClients) 981 } 982 983 func TestTableMigrateOneToManyDryRun(t *testing.T) { 984 var err error 985 ctx := context.Background() 986 tme := newTestTableMigraterCustom(ctx, t, []string{"0"}, []string{"-80", "80-"}, "select * %s") 987 defer tme.close(t) 988 989 wantdryRunReads := []string{ 990 "Lock keyspace ks1", 991 "Switch reads for tables [t1,t2] to keyspace ks2 for tablet types [RDONLY]", 992 "Routing rules for tables [t1,t2] will be updated", 993 "Unlock keyspace ks1", 994 } 995 wantdryRunWrites := []string{ 996 "Lock keyspace ks1", 997 "Lock keyspace ks2", 998 "Stop writes on keyspace ks1, tables [t1,t2]:", 999 "\tKeyspace ks1, Shard 0 at Position MariaDB/5-456-892", 1000 "Wait for VReplication on stopped streams to catchup for up to 1s", 1001 "Create reverse replication workflow test_reverse", 1002 "Create journal entries on source databases", 1003 "Enable writes on keyspace ks2 tables [t1,t2]", 1004 "Switch routing from keyspace ks1 to keyspace ks2", 1005 "Routing rules for tables [t1,t2] will be updated", 1006 "Switch writes completed, freeze and delete vreplication streams on:", 1007 " tablet 20", 1008 " tablet 30", 1009 "Mark vreplication streams frozen on:", 1010 " Keyspace ks2, Shard -80, Tablet 20, Workflow test, DbName vt_ks2", 1011 " Keyspace ks2, Shard 80-, Tablet 30, Workflow test, DbName vt_ks2", 1012 "Unlock keyspace ks2", 1013 "Unlock keyspace ks1", 1014 } 1015 tme.expectNoPreviousJournals() 1016 dryRunResults, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, true) 1017 require.NoError(t, err) 1018 require.Empty(t, cmp.Diff(wantdryRunReads, *dryRunResults)) 1019 1020 tme.expectNoPreviousJournals() 1021 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1022 require.NoError(t, err) 1023 tme.expectNoPreviousJournals() 1024 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1025 require.NoError(t, err) 1026 1027 verifyQueries(t, tme.allDBClients) 1028 1029 // checkJournals 1030 tme.dbSourceClients[0].addQueryRE(tsCheckJournals, &sqltypes.Result{}, nil) 1031 1032 waitForCatchup := func() { 1033 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 1034 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1035 "pos|state|message", 1036 "varchar|varchar|varchar"), 1037 "MariaDB/5-456-892|Running", 1038 ) 1039 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1040 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1041 1042 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('Stopped') 1043 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 1044 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 1045 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 1046 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 1047 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1048 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1049 } 1050 waitForCatchup() 1051 1052 deleteReverseReplicaion := func() { 1053 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1054 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1055 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1056 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1057 } 1058 1059 createReverseVReplication := func() { 1060 deleteReverseReplicaion() 1061 1062 tme.dbSourceClients[0].addQueryRE(`insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*from t1\\".*t2.*from t2\\"`, &sqltypes.Result{InsertID: 1}, nil) 1063 tme.dbSourceClients[0].addQueryRE(`insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*from t1\\".*t2.*from t2\\"`, &sqltypes.Result{InsertID: 2}, nil) 1064 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1065 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1066 } 1067 createReverseVReplication() 1068 1069 createJournals := func() { 1070 journal1 := "insert into _vt.resharding_journal.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*80.*MariaDB/5-456-893.*participants.*0" 1071 tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) 1072 } 1073 createJournals() 1074 1075 deleteTargetVReplication := func() { 1076 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid1, nil) 1077 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid1, nil) 1078 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1)", &sqltypes.Result{}, nil) 1079 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1080 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1)", &sqltypes.Result{}, nil) 1081 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1082 1083 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid1, nil) 1084 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid1, nil) 1085 tme.dbTargetClients[0].addQuery("delete from _vt.vreplication where id in (1)", &sqltypes.Result{}, nil) 1086 tme.dbTargetClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (1)", &sqltypes.Result{}, nil) 1087 tme.dbTargetClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (1)", &sqltypes.Result{}, nil) 1088 tme.dbTargetClients[1].addQuery("delete from _vt.vreplication where id in (1)", &sqltypes.Result{}, nil) 1089 tme.dbTargetClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (1)", &sqltypes.Result{}, nil) 1090 tme.dbTargetClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (1)", &sqltypes.Result{}, nil) 1091 } 1092 deleteTargetVReplication() 1093 1094 switchWrites(tme) 1095 _, results, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, false, true) 1096 require.NoError(t, err) 1097 require.Empty(t, cmp.Diff(wantdryRunWrites, *results)) 1098 } 1099 1100 // TestMigrateFailJournal tests that cancel doesn't get called after point of no return. 1101 // No need to test this for shard migrate because code paths are the same. 1102 func TestMigrateFailJournal(t *testing.T) { 1103 ctx := context.Background() 1104 tme := newTestTableMigrater(ctx, t) 1105 defer tme.close(t) 1106 1107 tme.expectNoPreviousJournals() 1108 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1109 if err != nil { 1110 t.Fatal(err) 1111 } 1112 tme.expectNoPreviousJournals() 1113 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1114 require.NoError(t, err) 1115 1116 // mi.checkJournals 1117 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1118 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1119 1120 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 1121 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1122 "pos|state|message", 1123 "varchar|varchar|varchar"), 1124 "MariaDB/5-456-892|Running", 1125 ) 1126 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1127 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 1128 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1129 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 1130 1131 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('stopped for cutover') 1132 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 1133 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 1134 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 1135 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 1136 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 1137 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 1138 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 1139 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 1140 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1141 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1142 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1143 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1144 1145 // mi.cancelMigration: these must not get called. 1146 cancel1 := "update _vt.vreplication set state = 'Running', stop_pos = null where id in (1)" 1147 cancel2 := "update _vt.vreplication set state = 'Running', stop_pos = null where id in (2)" 1148 tme.dbTargetClients[0].addQuery(cancel1, &sqltypes.Result{}, nil) 1149 tme.dbTargetClients[0].addQuery(cancel2, &sqltypes.Result{}, nil) 1150 tme.dbTargetClients[1].addQuery(cancel1, &sqltypes.Result{}, nil) 1151 tme.dbTargetClients[1].addQuery(cancel2, &sqltypes.Result{}, nil) 1152 1153 deleteReverseReplicaion := func() { 1154 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1155 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1156 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1157 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1158 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1159 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1160 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1161 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1162 } 1163 1164 createReverseVReplication := func() { 1165 deleteReverseReplicaion() 1166 1167 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*in_keyrange.*c1.*hash.*-40.*t2.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 1168 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*in_keyrange.*c1.*hash.*-40.*t2.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 1169 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*in_keyrange.*c1.*hash.*40-.*t2.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 1170 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*in_keyrange.*c1.*hash.*40-.*t2.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 1171 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1172 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1173 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1174 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1175 } 1176 createReverseVReplication() 1177 1178 // Make the journal call fail. 1179 tme.dbSourceClients[0].addQueryRE("insert into _vt.resharding_journal", nil, errors.New("journaling intentionally failed")) 1180 tme.dbSourceClients[1].addQueryRE("insert into _vt.resharding_journal", nil, errors.New("journaling intentionally failed")) 1181 1182 switchWrites(tme) 1183 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 1184 want := "journaling intentionally failed" 1185 if err == nil || !strings.Contains(err.Error(), want) { 1186 t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) 1187 } 1188 1189 // Verify that cancel didn't happen. 1190 if tme.dbTargetClients[0].queries[cancel1].exhausted() { 1191 t.Errorf("tme.dbTargetClients[0].queries[cancel1].exhausted: %v, want false", tme.dbTargetClients[0].queries[cancel1]) 1192 } 1193 if tme.dbTargetClients[1].queries[cancel1].exhausted() { 1194 t.Errorf("tme.dbTargetClients[0].queries[cancel1].exhausted: %v, want false", tme.dbTargetClients[0].queries[cancel1]) 1195 } 1196 if tme.dbTargetClients[0].queries[cancel2].exhausted() { 1197 t.Errorf("tme.dbTargetClients[0].queries[cancel1].exhausted: %v, want false", tme.dbTargetClients[0].queries[cancel1]) 1198 } 1199 } 1200 1201 func TestTableMigrateJournalExists(t *testing.T) { 1202 ctx := context.Background() 1203 tme := newTestTableMigrater(ctx, t) 1204 defer tme.close(t) 1205 1206 tme.expectNoPreviousJournals() 1207 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1208 if err != nil { 1209 t.Fatal(err) 1210 } 1211 tme.expectNoPreviousJournals() 1212 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1213 if err != nil { 1214 t.Fatal(err) 1215 } 1216 // mi.checkJournals: Show one journal as created. 1217 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", sqltypes.MakeTestResult(sqltypes.MakeTestFields("val", "varbinary"), ""), nil) 1218 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1219 1220 // mi.createJournals: Create the missing journal. 1221 journal2 := "insert into _vt.resharding_journal.*7672494164556733923,.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*80.*participants.*40.*40" 1222 tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) 1223 1224 // mi.startReverseVReplication 1225 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1'", resultid34, nil) 1226 tme.dbSourceClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 1227 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 1228 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 1229 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1'", resultid34, nil) 1230 tme.dbSourceClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 1231 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 1232 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 1233 1234 // mi.deleteTargetVReplication 1235 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1236 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1237 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 1238 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1239 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1240 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 1241 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1242 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1243 1244 switchWrites(tme) 1245 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 1246 if err != nil { 1247 t.Fatal(err) 1248 } 1249 1250 // Routes will be redone. 1251 checkRouting(t, tme.wr, map[string][]string{ 1252 "t1": {"ks2.t1"}, 1253 "ks1.t1": {"ks2.t1"}, 1254 "t2": {"ks2.t2"}, 1255 "ks1.t2": {"ks2.t2"}, 1256 "t1@replica": {"ks2.t1"}, 1257 "ks2.t1@replica": {"ks2.t1"}, 1258 "ks1.t1@replica": {"ks2.t1"}, 1259 "t2@replica": {"ks2.t2"}, 1260 "ks2.t2@replica": {"ks2.t2"}, 1261 "ks1.t2@replica": {"ks2.t2"}, 1262 "t1@rdonly": {"ks2.t1"}, 1263 "ks2.t1@rdonly": {"ks2.t1"}, 1264 "ks1.t1@rdonly": {"ks2.t1"}, 1265 "t2@rdonly": {"ks2.t2"}, 1266 "ks2.t2@rdonly": {"ks2.t2"}, 1267 "ks1.t2@rdonly": {"ks2.t2"}, 1268 }) 1269 // We're showing that there are no denied tables. But in real life, 1270 // tables on ks1 should be denied from the previous failed attempt. 1271 checkDenyList(t, tme.ts, "ks1:-40", nil) 1272 checkDenyList(t, tme.ts, "ks1:40-", nil) 1273 checkDenyList(t, tme.ts, "ks2:-80", nil) 1274 checkDenyList(t, tme.ts, "ks2:80-", nil) 1275 1276 verifyQueries(t, tme.allDBClients) 1277 } 1278 1279 func TestShardMigrateJournalExists(t *testing.T) { 1280 ctx := context.Background() 1281 tme := newTestShardMigrater(ctx, t, []string{"-40", "40-"}, []string{"-80", "80-"}) 1282 defer tme.stopTablets(t) 1283 1284 tme.expectNoPreviousJournals() 1285 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1286 if err != nil { 1287 t.Fatal(err) 1288 } 1289 tme.expectNoPreviousJournals() 1290 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1291 if err != nil { 1292 t.Fatal(err) 1293 } 1294 1295 // mi.checkJournals 1296 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", sqltypes.MakeTestResult(sqltypes.MakeTestFields("val", "varbinary"), ""), nil) 1297 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) 1298 1299 // mi.creaetJournals: Create the missing journal. 1300 journal2 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*shard_gtids.*80.*MariaDB/5-456-893.*participants.*40.*40" 1301 tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) 1302 1303 // mi.startReverseVReplication 1304 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) 1305 tme.dbSourceClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 1306 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 1307 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 1308 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) 1309 tme.dbSourceClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 1310 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 1311 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 1312 1313 // mi.deleteTargetVReplication 1314 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) 1315 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 1316 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1317 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1318 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) 1319 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (2)", &sqltypes.Result{}, nil) 1320 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1321 1322 switchWrites(tme) 1323 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 1324 if err != nil { 1325 t.Fatal(err) 1326 } 1327 1328 checkServedTypes(t, tme.ts, "ks:-40", 0) 1329 checkServedTypes(t, tme.ts, "ks:40-", 0) 1330 checkServedTypes(t, tme.ts, "ks:-80", 3) 1331 checkServedTypes(t, tme.ts, "ks:80-", 3) 1332 1333 checkIfPrimaryServing(t, tme.ts, "ks:-40", false) 1334 checkIfPrimaryServing(t, tme.ts, "ks:40-", false) 1335 checkIfPrimaryServing(t, tme.ts, "ks:-80", true) 1336 checkIfPrimaryServing(t, tme.ts, "ks:80-", true) 1337 1338 verifyQueries(t, tme.allDBClients) 1339 } 1340 1341 func TestTableMigrateCancel(t *testing.T) { 1342 ctx := context.Background() 1343 tme := newTestTableMigrater(ctx, t) 1344 defer tme.close(t) 1345 1346 tme.expectNoPreviousJournals() 1347 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1348 if err != nil { 1349 t.Fatal(err) 1350 } 1351 tme.expectNoPreviousJournals() 1352 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1353 if err != nil { 1354 t.Fatal(err) 1355 } 1356 1357 checkJournals := func() { 1358 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1359 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1360 } 1361 checkJournals() 1362 1363 deleteReverseReplicaion := func() { 1364 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1365 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1366 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1367 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1368 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1369 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1370 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1371 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1372 } 1373 cancelMigration := func() { 1374 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1375 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1376 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 1377 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 1378 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 1379 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 1380 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 1381 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 1382 1383 deleteReverseReplicaion() 1384 } 1385 cancelMigration() 1386 1387 switchWrites(tme) 1388 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, true, false, false, false) 1389 if err != nil { 1390 t.Fatal(err) 1391 } 1392 verifyQueries(t, tme.allDBClients) 1393 } 1394 1395 func TestTableMigrateCancelDryRun(t *testing.T) { 1396 ctx := context.Background() 1397 tme := newTestTableMigrater(ctx, t) 1398 defer tme.close(t) 1399 1400 want := []string{ 1401 "Lock keyspace ks1", 1402 "Lock keyspace ks2", 1403 "Cancel stream migrations as requested", 1404 "Unlock keyspace ks2", 1405 "Unlock keyspace ks1", 1406 } 1407 1408 tme.expectNoPreviousJournals() 1409 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1410 if err != nil { 1411 t.Fatal(err) 1412 } 1413 tme.expectNoPreviousJournals() 1414 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1415 if err != nil { 1416 t.Fatal(err) 1417 } 1418 1419 checkJournals := func() { 1420 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1421 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1422 } 1423 checkJournals() 1424 1425 deleteReverseReplicaion := func() { 1426 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1427 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1428 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1429 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1430 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1431 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1432 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1433 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1434 } 1435 cancelMigration := func() { 1436 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1437 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1438 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 1439 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 1440 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 1441 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 1442 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 1443 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 1444 1445 deleteReverseReplicaion() 1446 } 1447 cancelMigration() 1448 1449 switchWrites(tme) 1450 _, dryRunResults, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, true, false, false, true) 1451 require.NoError(t, err) 1452 require.Empty(t, cmp.Diff(want, *dryRunResults)) 1453 } 1454 1455 func TestTableMigrateNoReverse(t *testing.T) { 1456 ctx := context.Background() 1457 tme := newTestTableMigrater(ctx, t) 1458 defer tme.close(t) 1459 1460 tme.expectNoPreviousJournals() 1461 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1462 if err != nil { 1463 t.Fatal(err) 1464 } 1465 tme.expectNoPreviousJournals() 1466 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1467 if err != nil { 1468 t.Fatal(err) 1469 } 1470 1471 checkJournals := func() { 1472 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1473 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=7672494164556733923", &sqltypes.Result{}, nil) 1474 } 1475 checkJournals() 1476 1477 waitForCatchup := func() { 1478 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 1479 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1480 "pos|state|message", 1481 "varchar|varchar|varchar"), 1482 "MariaDB/5-456-892|Running", 1483 ) 1484 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1485 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 1486 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1487 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 1488 1489 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('Stopped') 1490 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 1491 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 1492 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 1493 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 1494 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 1495 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 1496 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 1497 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 1498 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1499 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1500 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1501 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1502 } 1503 waitForCatchup() 1504 1505 deleteReverseReplicaion := func() { 1506 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1507 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", resultid34, nil) 1508 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1509 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1510 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1511 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1512 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1513 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1514 } 1515 1516 createReverseVReplication := func() { 1517 deleteReverseReplicaion() 1518 1519 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*in_keyrange.*c1.*hash.*-40.*t2.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 1520 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*in_keyrange.*c1.*hash.*-40.*t2.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 1521 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*-80.*t1.*in_keyrange.*c1.*hash.*40-.*t2.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 1522 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*test_reverse.*ks2.*80-.*t1.*in_keyrange.*c1.*hash.*40-.*t2.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 1523 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1524 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1525 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1526 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1527 } 1528 createReverseVReplication() 1529 1530 createJournals := func() { 1531 journal1 := "insert into _vt.resharding_journal.*7672494164556733923,.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*-80.*MariaDB/5-456-893.*participants.*40.*40" 1532 tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) 1533 journal2 := "insert into _vt.resharding_journal.*7672494164556733923,.*tables.*t1.*t2.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*80.*participants.*40.*40" 1534 tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) 1535 } 1536 createJournals() 1537 1538 deleteTargetVReplication := func() { 1539 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1540 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", resultid12, nil) 1541 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 1542 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1543 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1544 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 1545 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 1546 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 1547 } 1548 deleteTargetVReplication() 1549 1550 switchWrites(tme) 1551 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, false, false) 1552 if err != nil { 1553 t.Fatal(err) 1554 } 1555 verifyQueries(t, tme.allDBClients) 1556 } 1557 1558 func TestMigrateFrozen(t *testing.T) { 1559 ctx := context.Background() 1560 tme := newTestTableMigrater(ctx, t) 1561 defer tme.close(t) 1562 1563 tme.expectNoPreviousJournals() 1564 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1565 if err != nil { 1566 t.Fatal(err) 1567 } 1568 1569 tme.expectNoPreviousJournals() 1570 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1571 if err != nil { 1572 t.Fatal(err) 1573 } 1574 1575 bls1 := &binlogdatapb.BinlogSource{ 1576 Keyspace: "ks1", 1577 Shard: "-40", 1578 Filter: &binlogdatapb.Filter{ 1579 Rules: []*binlogdatapb.Rule{{ 1580 Match: "t1", 1581 Filter: "", 1582 }}, 1583 }, 1584 } 1585 tme.dbTargetClients[0].addQuery(streamInfoKs2, sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1586 "id|source|message|cell|tablet_types", 1587 "int64|varchar|varchar|varchar|varchar"), 1588 fmt.Sprintf("1|%v|FROZEN||", bls1), 1589 ), nil) 1590 tme.dbTargetClients[1].addQuery(streamInfoKs2, &sqltypes.Result{}, nil) 1591 1592 switchWrites(tme) 1593 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, false, true, false) 1594 if err != nil { 1595 t.Fatal(err) 1596 } 1597 verifyQueries(t, tme.allDBClients) 1598 } 1599 1600 func TestMigrateNoStreamsFound(t *testing.T) { 1601 ctx := context.Background() 1602 tme := newTestTableMigrater(ctx, t) 1603 defer tme.close(t) 1604 1605 tme.dbTargetClients[0].addQuery(streamInfoKs2, &sqltypes.Result{}, nil) 1606 tme.dbTargetClients[1].addQuery(streamInfoKs2, &sqltypes.Result{}, nil) 1607 1608 tme.expectNoPreviousJournals() 1609 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1610 want := "workflow test not found in keyspace ks2" 1611 if err == nil || !strings.Contains(err.Error(), want) { 1612 t.Errorf("SwitchReads: %v, must contain %v", err, want) 1613 } 1614 } 1615 1616 func TestMigrateDistinctSources(t *testing.T) { 1617 ctx := context.Background() 1618 tme := newTestTableMigrater(ctx, t) 1619 defer tme.close(t) 1620 1621 bls := &binlogdatapb.BinlogSource{ 1622 Keyspace: "ks2", 1623 Shard: "-80", 1624 Filter: &binlogdatapb.Filter{ 1625 Rules: []*binlogdatapb.Rule{{ 1626 Match: "t1", 1627 Filter: "select * from t1 where in_keyrange('-80')", 1628 }, { 1629 Match: "t2", 1630 Filter: "select * from t2 where in_keyrange('-80')", 1631 }}, 1632 }, 1633 } 1634 tme.dbTargetClients[0].addQuery(streamInfoKs2, sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1635 "id|source|message|cell|tablet_types", 1636 "int64|varchar|varchar|varchar|varchar"), 1637 fmt.Sprintf("1|%v|||", bls), 1638 ), nil) 1639 1640 tme.expectNoPreviousJournals() 1641 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1642 want := "source keyspaces are mismatched across streams" 1643 if err == nil || !strings.Contains(err.Error(), want) { 1644 t.Errorf("SwitchReads: %v, must contain %v", err, want) 1645 } 1646 } 1647 1648 func TestMigrateMismatchedTables(t *testing.T) { 1649 ctx := context.Background() 1650 tme := newTestTableMigrater(ctx, t) 1651 defer tme.close(t) 1652 1653 bls := &binlogdatapb.BinlogSource{ 1654 Keyspace: "ks1", 1655 Shard: "-40", 1656 Filter: &binlogdatapb.Filter{ 1657 Rules: []*binlogdatapb.Rule{{ 1658 Match: "t1", 1659 Filter: "select * from t1 where in_keyrange('-80')", 1660 }}, 1661 }, 1662 } 1663 tme.dbTargetClients[0].addQuery(streamInfoKs2, sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1664 "id|source|message|cell|tablet_types", 1665 "int64|varchar|varchar|varchar|varchar"), 1666 fmt.Sprintf("1|%v|||", bls)), 1667 nil, 1668 ) 1669 1670 tme.expectNoPreviousJournals() 1671 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1672 want := "table lists are mismatched across streams" 1673 if err == nil || !strings.Contains(err.Error(), want) { 1674 t.Errorf("SwitchReads: %v, must contain %v", err, want) 1675 } 1676 } 1677 1678 func TestTableMigrateAllShardsNotPresent(t *testing.T) { 1679 ctx := context.Background() 1680 tme := newTestTableMigrater(ctx, t) 1681 defer tme.close(t) 1682 1683 tme.dbTargetClients[0].addQuery(streamInfoKs2, &sqltypes.Result{}, nil) 1684 1685 tme.expectNoPreviousJournals() 1686 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1687 want := "mismatched shards for keyspace" 1688 if err == nil || !strings.Contains(err.Error(), want) { 1689 t.Errorf("SwitchReads: %v, must contain %v", err, want) 1690 } 1691 } 1692 1693 func TestMigrateNoTableWildcards(t *testing.T) { 1694 ctx := context.Background() 1695 tme := newTestTableMigrater(ctx, t) 1696 defer tme.close(t) 1697 1698 // validate that no previous journals exist 1699 tme.dbSourceClients[0].addQueryRE(tsCheckJournals, &sqltypes.Result{}, nil) 1700 tme.dbSourceClients[1].addQueryRE(tsCheckJournals, &sqltypes.Result{}, nil) 1701 1702 bls1 := &binlogdatapb.BinlogSource{ 1703 Keyspace: "ks1", 1704 Shard: "-40", 1705 Filter: &binlogdatapb.Filter{ 1706 Rules: []*binlogdatapb.Rule{{ 1707 Match: "/.*", 1708 Filter: "", 1709 }}, 1710 }, 1711 } 1712 bls2 := &binlogdatapb.BinlogSource{ 1713 Keyspace: "ks1", 1714 Shard: "40-", 1715 Filter: &binlogdatapb.Filter{ 1716 Rules: []*binlogdatapb.Rule{{ 1717 Match: "/.*", 1718 Filter: "", 1719 }}, 1720 }, 1721 } 1722 tme.dbTargetClients[0].addQuery(streamInfoKs2, sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1723 "id|source|message|cell|tablet_types", 1724 "int64|varchar|varchar|varchar|varchar"), 1725 fmt.Sprintf("1|%v|||", bls1), 1726 fmt.Sprintf("2|%v|||", bls2), 1727 ), nil) 1728 bls3 := &binlogdatapb.BinlogSource{ 1729 Keyspace: "ks1", 1730 Shard: "40-", 1731 Filter: &binlogdatapb.Filter{ 1732 Rules: []*binlogdatapb.Rule{{ 1733 Match: "/.*", 1734 Filter: "", 1735 }}, 1736 }, 1737 } 1738 tme.dbTargetClients[1].addQuery(streamInfoKs2, sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1739 "id|source|message|cell|tablet_types", 1740 "int64|varchar|varchar|varchar|varchar"), 1741 fmt.Sprintf("1|%v|||", bls3), 1742 ), nil) 1743 tme.expectNoPreviousJournals() 1744 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1745 want := "cannot migrate streams with wild card table names: /.*" 1746 if err == nil || !strings.Contains(err.Error(), want) { 1747 t.Errorf("SwitchReads: %v, must contain %v", err, want) 1748 } 1749 } 1750 1751 func TestReverseVReplicationUpdateQuery(t *testing.T) { 1752 ts := &trafficSwitcher{ 1753 reverseWorkflow: "wf", 1754 } 1755 dbname := "db" 1756 type tCase struct { 1757 optCells string 1758 optTabletTypes string 1759 targetCell string 1760 sourceCell string 1761 want string 1762 } 1763 updateQuery := "update _vt.vreplication set cell = '%s', tablet_types = '%s' where workflow = 'wf' and db_name = 'db'" 1764 tCases := []tCase{ 1765 { 1766 targetCell: "cell1", sourceCell: "cell1", optCells: "cell1", optTabletTypes: "", 1767 want: fmt.Sprintf(updateQuery, "cell1", ""), 1768 }, 1769 { 1770 targetCell: "cell1", sourceCell: "cell2", optCells: "cell1", optTabletTypes: "", 1771 want: fmt.Sprintf(updateQuery, "cell2", ""), 1772 }, 1773 { 1774 targetCell: "cell1", sourceCell: "cell2", optCells: "cell2", optTabletTypes: "", 1775 want: fmt.Sprintf(updateQuery, "cell2", ""), 1776 }, 1777 { 1778 targetCell: "cell1", sourceCell: "cell1", optCells: "cell1,cell2", optTabletTypes: "replica,primary", 1779 want: fmt.Sprintf(updateQuery, "cell1,cell2", "replica,primary"), 1780 }, 1781 { 1782 targetCell: "cell1", sourceCell: "cell1", optCells: "", optTabletTypes: "replica,primary", 1783 want: fmt.Sprintf(updateQuery, "", "replica,primary"), 1784 }, 1785 } 1786 for _, tc := range tCases { 1787 t.Run("", func(t *testing.T) { 1788 ts.optCells = tc.optCells 1789 ts.optTabletTypes = tc.optTabletTypes 1790 got := ts.getReverseVReplicationUpdateQuery(tc.targetCell, tc.sourceCell, dbname) 1791 require.Equal(t, tc.want, got) 1792 }) 1793 } 1794 } 1795 1796 func TestShardMigrateNoAvailableTabletsForReverseReplication(t *testing.T) { 1797 ctx := context.Background() 1798 tme := newTestShardMigrater(ctx, t, []string{"-40", "40-"}, []string{"-80", "80-"}) 1799 defer tme.stopTablets(t) 1800 1801 // Initial check 1802 checkServedTypes(t, tme.ts, "ks:-40", 3) 1803 checkServedTypes(t, tme.ts, "ks:40-", 3) 1804 checkServedTypes(t, tme.ts, "ks:-80", 0) 1805 checkServedTypes(t, tme.ts, "ks:80-", 0) 1806 1807 tme.expectNoPreviousJournals() 1808 //------------------------------------------------------------------------------------------------------------------- 1809 // Single cell RDONLY migration. 1810 _, err := tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, []string{"cell1"}, workflow.DirectionForward, false) 1811 if err != nil { 1812 t.Fatal(err) 1813 } 1814 checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) 1815 checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) 1816 checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) 1817 checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) 1818 checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 3) 1819 checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 3) 1820 checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 0) 1821 checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 0) 1822 verifyQueries(t, tme.allDBClients) 1823 1824 tme.expectNoPreviousJournals() 1825 //------------------------------------------------------------------------------------------------------------------- 1826 // Other cell REPLICA migration. 1827 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionForward, false) 1828 if err != nil { 1829 t.Fatal(err) 1830 } 1831 checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) 1832 checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) 1833 checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) 1834 checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) 1835 checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 1) 1836 checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 1) 1837 checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 2) 1838 checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 2) 1839 verifyQueries(t, tme.allDBClients) 1840 1841 tme.expectNoPreviousJournals() 1842 //------------------------------------------------------------------------------------------------------------------- 1843 // Single cell backward REPLICA migration. 1844 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, []string{"cell2"}, workflow.DirectionBackward, false) 1845 if err != nil { 1846 t.Fatal(err) 1847 } 1848 checkCellServedTypes(t, tme.ts, "ks:-40", "cell1", 2) 1849 checkCellServedTypes(t, tme.ts, "ks:40-", "cell1", 2) 1850 checkCellServedTypes(t, tme.ts, "ks:-80", "cell1", 1) 1851 checkCellServedTypes(t, tme.ts, "ks:80-", "cell1", 1) 1852 checkCellServedTypes(t, tme.ts, "ks:-40", "cell2", 3) 1853 checkCellServedTypes(t, tme.ts, "ks:40-", "cell2", 3) 1854 checkCellServedTypes(t, tme.ts, "ks:-80", "cell2", 0) 1855 checkCellServedTypes(t, tme.ts, "ks:80-", "cell2", 0) 1856 verifyQueries(t, tme.allDBClients) 1857 1858 tme.expectNoPreviousJournals() 1859 //------------------------------------------------------------------------------------------------------------------- 1860 // Switch all RDONLY. 1861 // This is an extra step that does not exist in the tables test. 1862 // The per-cell migration mechanism is different for tables. So, this 1863 // extra step is needed to bring things in sync. 1864 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1865 if err != nil { 1866 t.Fatal(err) 1867 } 1868 checkServedTypes(t, tme.ts, "ks:-40", 2) 1869 checkServedTypes(t, tme.ts, "ks:40-", 2) 1870 checkServedTypes(t, tme.ts, "ks:-80", 1) 1871 checkServedTypes(t, tme.ts, "ks:80-", 1) 1872 verifyQueries(t, tme.allDBClients) 1873 1874 tme.expectNoPreviousJournals() 1875 //------------------------------------------------------------------------------------------------------------------- 1876 // Switch all REPLICA. 1877 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_REPLICA}, nil, workflow.DirectionForward, false) 1878 if err != nil { 1879 t.Fatal(err) 1880 } 1881 checkServedTypes(t, tme.ts, "ks:-40", 1) 1882 checkServedTypes(t, tme.ts, "ks:40-", 1) 1883 checkServedTypes(t, tme.ts, "ks:-80", 2) 1884 checkServedTypes(t, tme.ts, "ks:80-", 2) 1885 verifyQueries(t, tme.allDBClients) 1886 1887 tme.expectNoPreviousJournals() 1888 //------------------------------------------------------------------------------------------------------------------- 1889 // All cells RDONLY backward migration. 1890 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionBackward, false) 1891 if err != nil { 1892 t.Fatal(err) 1893 } 1894 checkServedTypes(t, tme.ts, "ks:-40", 2) 1895 checkServedTypes(t, tme.ts, "ks:40-", 2) 1896 checkServedTypes(t, tme.ts, "ks:-80", 1) 1897 checkServedTypes(t, tme.ts, "ks:80-", 1) 1898 verifyQueries(t, tme.allDBClients) 1899 1900 //------------------------------------------------------------------------------------------------------------------- 1901 // Can't switch primary with SwitchReads. 1902 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_PRIMARY}, nil, workflow.DirectionForward, false) 1903 want := "tablet type must be REPLICA or RDONLY: PRIMARY" 1904 if err == nil || err.Error() != want { 1905 t.Errorf("SwitchReads(primary) err: %v, want %v", err, want) 1906 } 1907 verifyQueries(t, tme.allDBClients) 1908 1909 //------------------------------------------------------------------------------------------------------------------- 1910 // Test SwitchWrites cancelation on failure. 1911 1912 tme.expectNoPreviousJournals() 1913 // Switch all the reads first. 1914 _, err = tme.wr.SwitchReads(ctx, tme.targetKeyspace, "test", []topodatapb.TabletType{topodatapb.TabletType_RDONLY}, nil, workflow.DirectionForward, false) 1915 if err != nil { 1916 t.Fatal(err) 1917 } 1918 checkServedTypes(t, tme.ts, "ks:-40", 1) 1919 checkServedTypes(t, tme.ts, "ks:40-", 1) 1920 checkServedTypes(t, tme.ts, "ks:-80", 2) 1921 checkServedTypes(t, tme.ts, "ks:80-", 2) 1922 checkIfPrimaryServing(t, tme.ts, "ks:-40", true) 1923 checkIfPrimaryServing(t, tme.ts, "ks:40-", true) 1924 checkIfPrimaryServing(t, tme.ts, "ks:-80", false) 1925 checkIfPrimaryServing(t, tme.ts, "ks:80-", false) 1926 1927 checkJournals := func() { 1928 tme.dbSourceClients[0].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) 1929 tme.dbSourceClients[1].addQuery("select val from _vt.resharding_journal where id=6432976123657117097", &sqltypes.Result{}, nil) 1930 } 1931 checkJournals() 1932 1933 stopStreams := func() { 1934 tme.dbSourceClients[0].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse' and state = 'Stopped' and message != 'FROZEN'", &sqltypes.Result{}, nil) 1935 tme.dbSourceClients[1].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse' and state = 'Stopped' and message != 'FROZEN'", &sqltypes.Result{}, nil) 1936 tme.dbSourceClients[0].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 1937 tme.dbSourceClients[1].addQuery("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name='vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 1938 } 1939 stopStreams() 1940 1941 deleteReverseReplicaion := func() { 1942 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test_reverse'", resultid3, nil) 1943 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test_reverse'", resultid34, nil) 1944 tme.dbSourceClients[0].addQuery("delete from _vt.vreplication where id in (3)", &sqltypes.Result{}, nil) 1945 tme.dbSourceClients[1].addQuery("delete from _vt.vreplication where id in (3, 4)", &sqltypes.Result{}, nil) 1946 tme.dbSourceClients[0].addQuery("delete from _vt.copy_state where vrepl_id in (3)", &sqltypes.Result{}, nil) 1947 tme.dbSourceClients[0].addQuery("delete from _vt.post_copy_action where vrepl_id in (3)", &sqltypes.Result{}, nil) 1948 tme.dbSourceClients[1].addQuery("delete from _vt.copy_state where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1949 tme.dbSourceClients[1].addQuery("delete from _vt.post_copy_action where vrepl_id in (3, 4)", &sqltypes.Result{}, nil) 1950 } 1951 cancelMigration := func() { 1952 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 1953 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow != 'test_reverse'", &sqltypes.Result{}, nil) 1954 1955 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) 1956 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) 1957 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (1, 2)", &sqltypes.Result{}, nil) 1958 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (2)", &sqltypes.Result{}, nil) 1959 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", runningResult(1), nil) 1960 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 1961 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", runningResult(2), nil) 1962 1963 deleteReverseReplicaion() 1964 } 1965 cancelMigration() 1966 1967 switchWrites(tme) 1968 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 0*time.Second, false, false, true, false) 1969 want = "DeadlineExceeded" 1970 if err == nil || !strings.Contains(err.Error(), want) { 1971 t.Errorf("SwitchWrites(0 timeout) err: %v, must contain %v", err, want) 1972 } 1973 1974 verifyQueries(t, tme.allDBClients) 1975 checkServedTypes(t, tme.ts, "ks:-40", 1) 1976 checkServedTypes(t, tme.ts, "ks:40-", 1) 1977 checkServedTypes(t, tme.ts, "ks:-80", 2) 1978 checkServedTypes(t, tme.ts, "ks:80-", 2) 1979 checkIfPrimaryServing(t, tme.ts, "ks:-40", true) 1980 checkIfPrimaryServing(t, tme.ts, "ks:40-", true) 1981 checkIfPrimaryServing(t, tme.ts, "ks:-80", false) 1982 checkIfPrimaryServing(t, tme.ts, "ks:80-", false) 1983 1984 //------------------------------------------------------------------------------------------------------------------- 1985 // Test successful SwitchWrites. 1986 1987 checkJournals() 1988 stopStreams() 1989 1990 waitForCatchup := func() { 1991 // mi.waitForCatchup-> mi.wr.tmc.VReplicationWaitForPos 1992 state := sqltypes.MakeTestResult(sqltypes.MakeTestFields( 1993 "pos|state|message", 1994 "varchar|varchar|varchar"), 1995 "MariaDB/5-456-892|Running", 1996 ) 1997 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=1", state, nil) 1998 tme.dbTargetClients[1].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 1999 tme.dbTargetClients[0].addQuery("select pos, state, message from _vt.vreplication where id=2", state, nil) 2000 2001 // mi.waitForCatchup-> mi.wr.tmc.VReplicationExec('stopped for cutover') 2002 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 1", resultid1, nil) 2003 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (1)", &sqltypes.Result{}, nil) 2004 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 2005 tme.dbTargetClients[0].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 2006 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where id = 2", resultid2, nil) 2007 tme.dbTargetClients[1].addQuery("update _vt.vreplication set state = 'Stopped', message = 'stopped for cutover' where id in (2)", &sqltypes.Result{}, nil) 2008 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 2009 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 2010 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 2011 } 2012 waitForCatchup() 2013 2014 createReverseVReplication := func() { 2015 deleteReverseReplicaion() 2016 2017 tme.dbSourceClients[0].addQueryRE("insert into _vt.vreplication.*-80.*-40.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 2018 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*-80.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 1}, nil) 2019 tme.dbSourceClients[1].addQueryRE("insert into _vt.vreplication.*80-.*40-.*MariaDB/5-456-893.*Stopped", &sqltypes.Result{InsertID: 2}, nil) 2020 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 2021 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 2022 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 2023 } 2024 createReverseVReplication() 2025 2026 createJournals := func() { 2027 journal1 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*-80.*MariaDB/5-456-893.*participants.*40.*40" 2028 tme.dbSourceClients[0].addQueryRE(journal1, &sqltypes.Result{}, nil) 2029 journal2 := "insert into _vt.resharding_journal.*6432976123657117097.*migration_type:SHARDS.*local_position.*MariaDB/5-456-892.*shard_gtids.*80.*MariaDB/5-456-893.*shard_gtids.*80.*MariaDB/5-456-893.*participants.*40.*40" 2030 tme.dbSourceClients[1].addQueryRE(journal2, &sqltypes.Result{}, nil) 2031 } 2032 createJournals() 2033 2034 startReverseVReplication := func() { 2035 tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) 2036 tme.dbSourceClients[0].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 2037 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 2038 tme.dbSourceClients[0].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 2039 tme.dbSourceClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks'", resultid34, nil) 2040 tme.dbSourceClients[1].addQuery("update _vt.vreplication set state = 'Running', message = '' where id in (3, 4)", &sqltypes.Result{}, nil) 2041 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 3", runningResult(3), nil) 2042 tme.dbSourceClients[1].addQuery("select * from _vt.vreplication where id = 4", runningResult(4), nil) 2043 } 2044 startReverseVReplication() 2045 2046 freezeTargetVReplication := func() { 2047 tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid12, nil) 2048 tme.dbTargetClients[0].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (1, 2)", &sqltypes.Result{}, nil) 2049 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 1", stoppedResult(1), nil) 2050 tme.dbTargetClients[0].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 2051 tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks' and workflow = 'test'", resultid2, nil) 2052 tme.dbTargetClients[1].addQuery("update _vt.vreplication set message = 'FROZEN' where id in (2)", &sqltypes.Result{}, nil) 2053 tme.dbTargetClients[1].addQuery("select * from _vt.vreplication where id = 2", stoppedResult(2), nil) 2054 } 2055 freezeTargetVReplication() 2056 2057 // Temporarily set tablet types to RDONLY to test that SwitchWrites fails if no tablets of rdonly are available 2058 invariants := make(map[string]*sqltypes.Result) 2059 for i := range tme.targetShards { 2060 invariants[fmt.Sprintf("%s-%d", streamInfoKs, i)] = tme.dbTargetClients[i].getInvariant(streamInfoKs) 2061 tme.dbTargetClients[i].addInvariant(streamInfoKs, tme.dbTargetClients[i].getInvariant(streamInfoKs+"-rdonly")) 2062 } 2063 _, _, err = tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 2064 require.Error(t, err) 2065 require.True(t, strings.Contains(err.Error(), "no tablet found")) 2066 require.True(t, strings.Contains(err.Error(), "-80")) 2067 require.True(t, strings.Contains(err.Error(), "80-")) 2068 require.False(t, strings.Contains(err.Error(), "40")) 2069 for i := range tme.targetShards { 2070 tme.dbTargetClients[i].addInvariant(streamInfoKs, invariants[fmt.Sprintf("%s-%d", streamInfoKs, i)]) 2071 } 2072 2073 journalID, _, err := tme.wr.SwitchWrites(ctx, tme.targetKeyspace, "test", 1*time.Second, false, false, true, false) 2074 if err != nil { 2075 t.Fatal(err) 2076 } 2077 if journalID != 6432976123657117097 { 2078 t.Errorf("journal id: %d, want 6432976123657117097", journalID) 2079 } 2080 2081 verifyQueries(t, tme.allDBClients) 2082 2083 checkServedTypes(t, tme.ts, "ks:-40", 0) 2084 checkServedTypes(t, tme.ts, "ks:40-", 0) 2085 checkServedTypes(t, tme.ts, "ks:-80", 3) 2086 checkServedTypes(t, tme.ts, "ks:80-", 3) 2087 2088 checkIfPrimaryServing(t, tme.ts, "ks:-40", false) 2089 checkIfPrimaryServing(t, tme.ts, "ks:40-", false) 2090 checkIfPrimaryServing(t, tme.ts, "ks:-80", true) 2091 checkIfPrimaryServing(t, tme.ts, "ks:80-", true) 2092 2093 verifyQueries(t, tme.allDBClients) 2094 } 2095 2096 func TestIsPartialMoveTables(t *testing.T) { 2097 ts := &trafficSwitcher{} 2098 type testCase struct { 2099 name string 2100 sourceShards, targetShards []string 2101 want bool 2102 } 2103 testCases := []testCase{ 2104 { 2105 name: "-80", 2106 sourceShards: []string{"-80"}, 2107 targetShards: []string{"-80"}, 2108 want: true, 2109 }, 2110 { 2111 name: "80-", 2112 sourceShards: []string{"80-"}, 2113 targetShards: []string{"80-"}, 2114 want: true, 2115 }, 2116 { 2117 name: "-80,80-", 2118 sourceShards: []string{"-80", "80-"}, 2119 targetShards: []string{"-80", "80-"}, 2120 want: false, 2121 }, 2122 { 2123 name: "mismatch", 2124 sourceShards: []string{"-c0", "c0-"}, 2125 targetShards: []string{"-80", "80-"}, 2126 want: false, 2127 }, 2128 { 2129 name: "different number of shards", 2130 sourceShards: []string{"-a0", "a0-c0", "c0-"}, 2131 targetShards: []string{"-80", "80-"}, 2132 want: false, 2133 }, 2134 } 2135 2136 for _, tc := range testCases { 2137 t.Run(tc.name, func(t *testing.T) { 2138 got, err := ts.isPartialMoveTables(tc.sourceShards, tc.targetShards) 2139 require.NoError(t, err) 2140 require.Equal(t, tc.want, got) 2141 }) 2142 2143 } 2144 } 2145 2146 func checkRouting(t *testing.T, wr *Wrangler, want map[string][]string) { 2147 t.Helper() 2148 ctx := context.Background() 2149 got, err := topotools.GetRoutingRules(ctx, wr.ts) 2150 if err != nil { 2151 t.Fatal(err) 2152 } 2153 if !reflect.DeepEqual(got, want) { 2154 t.Errorf("rules:\n%v, want\n%v", got, want) 2155 } 2156 cells, err := wr.ts.GetCellInfoNames(ctx) 2157 if err != nil { 2158 t.Fatal(err) 2159 } 2160 for _, cell := range cells { 2161 checkCellRouting(t, wr, cell, want) 2162 } 2163 } 2164 2165 func checkCellRouting(t *testing.T, wr *Wrangler, cell string, want map[string][]string) { 2166 t.Helper() 2167 ctx := context.Background() 2168 svs, err := wr.ts.GetSrvVSchema(ctx, cell) 2169 if err != nil { 2170 t.Fatal(err) 2171 } 2172 got := make(map[string][]string) 2173 for _, rr := range svs.RoutingRules.Rules { 2174 got[rr.FromTable] = append(got[rr.FromTable], rr.ToTables...) 2175 } 2176 if !reflect.DeepEqual(got, want) { 2177 t.Fatalf("ERROR: routing rules don't match for cell %s:got\n%v, want\n%v", cell, got, want) 2178 } 2179 } 2180 2181 func checkDenyList(t *testing.T, ts *topo.Server, keyspaceShard string, want []string) { 2182 t.Helper() 2183 ctx := context.Background() 2184 splits := strings.Split(keyspaceShard, ":") 2185 si, err := ts.GetShard(ctx, splits[0], splits[1]) 2186 if err != nil { 2187 t.Fatal(err) 2188 } 2189 tc := si.GetTabletControl(topodatapb.TabletType_PRIMARY) 2190 var got []string 2191 if tc != nil { 2192 got = tc.DeniedTables 2193 } 2194 if !reflect.DeepEqual(got, want) { 2195 t.Errorf("Denied tables for %v: %v, want %v", keyspaceShard, got, want) 2196 } 2197 } 2198 2199 func checkServedTypes(t *testing.T, ts *topo.Server, keyspaceShard string, want int) { 2200 t.Helper() 2201 ctx := context.Background() 2202 splits := strings.Split(keyspaceShard, ":") 2203 si, err := ts.GetShard(ctx, splits[0], splits[1]) 2204 if err != nil { 2205 t.Fatal(err) 2206 } 2207 2208 servedTypes, err := ts.GetShardServingTypes(ctx, si) 2209 if err != nil { 2210 t.Fatal(err) 2211 } 2212 require.Equal(t, want, len(servedTypes), fmt.Sprintf("shard %v has wrong served types: got: %v, want: %v", 2213 keyspaceShard, len(servedTypes), want)) 2214 } 2215 2216 func checkCellServedTypes(t *testing.T, ts *topo.Server, keyspaceShard, cell string, want int) { 2217 t.Helper() 2218 ctx := context.Background() 2219 splits := strings.Split(keyspaceShard, ":") 2220 srvKeyspace, err := ts.GetSrvKeyspace(ctx, cell, splits[0]) 2221 if err != nil { 2222 t.Fatal(err) 2223 } 2224 count := 0 2225 outer: 2226 for _, partition := range srvKeyspace.GetPartitions() { 2227 for _, ref := range partition.ShardReferences { 2228 if ref.Name == splits[1] { 2229 count++ 2230 continue outer 2231 } 2232 } 2233 } 2234 require.Equal(t, want, count, fmt.Sprintf("serving types for keyspaceShard %s, cell %s: %d, want %d", 2235 keyspaceShard, cell, count, want)) 2236 } 2237 2238 func checkIfPrimaryServing(t *testing.T, ts *topo.Server, keyspaceShard string, want bool) { 2239 t.Helper() 2240 ctx := context.Background() 2241 splits := strings.Split(keyspaceShard, ":") 2242 si, err := ts.GetShard(ctx, splits[0], splits[1]) 2243 if err != nil { 2244 t.Fatal(err) 2245 } 2246 if want != si.IsPrimaryServing { 2247 t.Errorf("IsPrimaryServing(%v): %v, want %v", keyspaceShard, si.IsPrimaryServing, want) 2248 } 2249 } 2250 2251 func getResult(id int, state string, keyspace string, shard string) *sqltypes.Result { 2252 return sqltypes.MakeTestResult(sqltypes.MakeTestFields( 2253 "id|state|cell|tablet_types|source", 2254 "int64|varchar|varchar|varchar|varchar"), 2255 fmt.Sprintf("%d|%s|cell1|PRIMARY|keyspace:\"%s\" shard:\"%s\"", id, state, keyspace, shard), 2256 ) 2257 } 2258 2259 func stoppedResult(id int) *sqltypes.Result { 2260 return getResult(id, "Stopped", tpChoice.keyspace, tpChoice.shard) 2261 } 2262 2263 func runningResult(id int) *sqltypes.Result { 2264 return getResult(id, "Running", tpChoice.keyspace, tpChoice.shard) 2265 } 2266 2267 func switchWrites(tmeT any) { 2268 if tme, ok := tmeT.(*testMigraterEnv); ok { 2269 tme.tmeDB.AddQuery("lock tables `t1` read,`t2` read", &sqltypes.Result{}) 2270 } else if tme, ok := tmeT.(*testShardMigraterEnv); ok { 2271 tme.tmeDB.AddQuery("lock tables `t1` read,`t2` read", &sqltypes.Result{}) 2272 } 2273 }