vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/vreplication/controller_test.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vreplication 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "testing" 24 "time" 25 26 querypb "vitess.io/vitess/go/vt/proto/query" 27 28 "vitess.io/vitess/go/sqltypes" 29 "vitess.io/vitess/go/sync2" 30 "vitess.io/vitess/go/vt/binlog/binlogplayer" 31 "vitess.io/vitess/go/vt/mysqlctl/fakemysqldaemon" 32 "vitess.io/vitess/go/vt/mysqlctl/tmutils" 33 34 tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" 35 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 36 ) 37 38 var ( 39 testSettingsResponse = &sqltypes.Result{ 40 Fields: []*querypb.Field{ 41 {Name: "pos", Type: sqltypes.VarBinary}, 42 {Name: "stop_pos", Type: sqltypes.VarBinary}, 43 {Name: "max_tps", Type: sqltypes.Int64}, 44 {Name: "max_replication_lag", Type: sqltypes.Int64}, 45 {Name: "state", Type: sqltypes.VarBinary}, 46 {Name: "workflow_type", Type: sqltypes.Int64}, 47 {Name: "workflow", Type: sqltypes.VarChar}, 48 {Name: "workflow_sub_type", Type: sqltypes.Int64}, 49 {Name: "defer_secondary_keys", Type: sqltypes.Int64}, 50 }, 51 InsertID: 0, 52 Rows: [][]sqltypes.Value{ 53 { 54 sqltypes.NewVarBinary("MariaDB/0-1-1083"), // pos 55 sqltypes.NULL, // stop_pos 56 sqltypes.NewInt64(9223372036854775807), // max_tps 57 sqltypes.NewInt64(9223372036854775807), // max_replication_lag 58 sqltypes.NewVarBinary("Running"), // state 59 sqltypes.NewInt64(1), // workflow_type 60 sqltypes.NewVarChar("wf"), // workflow 61 sqltypes.NewInt64(0), // workflow_sub_type 62 sqltypes.NewInt64(0), // defer_secondary_keys 63 }, 64 }, 65 } 66 testSelectorResponse1 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(1)}}} 67 testSelectorResponse2 = &sqltypes.Result{Rows: [][]sqltypes.Value{{sqltypes.NewInt64(1)}, {sqltypes.NewInt64(2)}}} 68 testDMLResponse = &sqltypes.Result{RowsAffected: 1} 69 testPos = "MariaDB/0-1-1083" 70 ) 71 72 func TestControllerKeyRange(t *testing.T) { 73 resetBinlogClient() 74 wantTablet := addTablet(100) 75 defer deleteTablet(wantTablet) 76 params := map[string]string{ 77 "id": "1", 78 "state": binlogplayer.BlpRunning, 79 "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName), 80 } 81 82 dbClient := binlogplayer.NewMockDBClient(t) 83 dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil) 84 dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) 85 dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil) 86 dbClient.ExpectRequest("begin", nil, nil) 87 dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil) 88 dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil) 89 dbClient.ExpectRequest("commit", nil, nil) 90 91 dbClientFactory := func() binlogplayer.DBClient { return dbClient } 92 mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} 93 94 ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) 95 if err != nil { 96 t.Fatal(err) 97 } 98 defer func() { 99 dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil) 100 ct.Stop() 101 }() 102 103 dbClient.Wait() 104 expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) 105 } 106 107 func TestControllerTables(t *testing.T) { 108 wantTablet := addTablet(100) 109 defer deleteTablet(wantTablet) 110 resetBinlogClient() 111 112 params := map[string]string{ 113 "id": "1", 114 "state": binlogplayer.BlpRunning, 115 "source": fmt.Sprintf(`keyspace:"%s" shard:"0" tables:"table1" tables:"/funtables_/" `, env.KeyspaceName), 116 } 117 118 dbClient := binlogplayer.NewMockDBClient(t) 119 dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil) 120 dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) 121 dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil) 122 dbClient.ExpectRequest("begin", nil, nil) 123 dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil) 124 dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil) 125 dbClient.ExpectRequest("commit", nil, nil) 126 127 dbClientFactory := func() binlogplayer.DBClient { return dbClient } 128 mysqld := &fakemysqldaemon.FakeMysqlDaemon{ 129 MysqlPort: sync2.NewAtomicInt32(3306), 130 Schema: &tabletmanagerdatapb.SchemaDefinition{ 131 DatabaseSchema: "", 132 TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ 133 { 134 Name: "table1", 135 Columns: []string{"id", "msg", "keyspace_id"}, 136 PrimaryKeyColumns: []string{"id"}, 137 Type: tmutils.TableBaseTable, 138 }, 139 { 140 Name: "funtables_one", 141 Columns: []string{"id", "msg", "keyspace_id"}, 142 PrimaryKeyColumns: []string{"id"}, 143 Type: tmutils.TableBaseTable, 144 }, 145 { 146 Name: "excluded_table", 147 Columns: []string{"id", "msg", "keyspace_id"}, 148 PrimaryKeyColumns: []string{"id"}, 149 Type: tmutils.TableBaseTable, 150 }, 151 }, 152 }, 153 } 154 155 ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) 156 if err != nil { 157 t.Fatal(err) 158 } 159 defer func() { 160 dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil) 161 ct.Stop() 162 }() 163 164 dbClient.Wait() 165 expectFBCRequest(t, wantTablet, testPos, []string{"table1", "funtables_one"}, nil) 166 } 167 168 func TestControllerBadID(t *testing.T) { 169 params := map[string]string{ 170 "id": "bad", 171 } 172 _, err := newController(context.Background(), params, nil, nil, nil, "", "", nil, nil) 173 want := `strconv.Atoi: parsing "bad": invalid syntax` 174 if err == nil || err.Error() != want { 175 t.Errorf("newController err: %v, want %v", err, want) 176 } 177 } 178 179 func TestControllerStopped(t *testing.T) { 180 params := map[string]string{ 181 "id": "1", 182 "state": binlogplayer.BlpStopped, 183 } 184 185 ct, err := newController(context.Background(), params, nil, nil, nil, "", "", nil, nil) 186 if err != nil { 187 t.Fatal(err) 188 } 189 defer ct.Stop() 190 191 select { 192 case <-ct.done: 193 default: 194 t.Errorf("context should be closed, but is not: %v", ct) 195 } 196 } 197 198 func TestControllerOverrides(t *testing.T) { 199 resetBinlogClient() 200 wantTablet := addTablet(100) 201 defer deleteTablet(wantTablet) 202 203 params := map[string]string{ 204 "id": "1", 205 "state": binlogplayer.BlpRunning, 206 "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName), 207 "cell": env.Cells[0], 208 "tablet_types": "replica", 209 } 210 211 dbClient := binlogplayer.NewMockDBClient(t) 212 dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil) 213 dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) 214 dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil) 215 dbClient.ExpectRequest("begin", nil, nil) 216 dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil) 217 dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil) 218 dbClient.ExpectRequest("commit", nil, nil) 219 220 dbClientFactory := func() binlogplayer.DBClient { return dbClient } 221 mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} 222 223 ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil) 224 if err != nil { 225 t.Fatal(err) 226 } 227 defer func() { 228 dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil) 229 ct.Stop() 230 }() 231 232 dbClient.Wait() 233 expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) 234 } 235 236 func TestControllerCanceledContext(t *testing.T) { 237 defer deleteTablet(addTablet(100)) 238 239 params := map[string]string{ 240 "id": "1", 241 "state": binlogplayer.BlpRunning, 242 "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName), 243 } 244 245 ctx, cancel := context.WithCancel(context.Background()) 246 cancel() 247 ct, err := newController(ctx, params, nil, nil, env.TopoServ, env.Cells[0], "rdonly", nil, nil) 248 if err != nil { 249 t.Fatal(err) 250 } 251 defer ct.Stop() 252 253 select { 254 case <-ct.done: 255 case <-time.After(1 * time.Second): 256 t.Errorf("context should be closed, but is not: %v", ct) 257 } 258 } 259 260 func TestControllerRetry(t *testing.T) { 261 savedDelay := retryDelay 262 defer func() { retryDelay = savedDelay }() 263 retryDelay = 10 * time.Millisecond 264 265 resetBinlogClient() 266 defer deleteTablet(addTablet(100)) 267 268 params := map[string]string{ 269 "id": "1", 270 "state": binlogplayer.BlpRunning, 271 "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName), 272 "cell": env.Cells[0], 273 "tablet_types": "replica", 274 } 275 276 dbClient := binlogplayer.NewMockDBClient(t) 277 dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil) 278 dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) 279 dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", nil, errors.New("(expected error)")) 280 dbClient.ExpectRequest("update _vt.vreplication set state='Error', message='error (expected error) in selecting vreplication settings select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1' where id=1", testDMLResponse, nil) 281 dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil) 282 dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) 283 dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", testSettingsResponse, nil) 284 dbClient.ExpectRequest("begin", nil, nil) 285 dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil) 286 dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil) 287 dbClient.ExpectRequest("commit", nil, nil) 288 dbClientFactory := func() binlogplayer.DBClient { return dbClient } 289 mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} 290 291 ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "rdonly", nil, nil) 292 if err != nil { 293 t.Fatal(err) 294 } 295 defer ct.Stop() 296 297 dbClient.Wait() 298 } 299 300 func TestControllerStopPosition(t *testing.T) { 301 resetBinlogClient() 302 wantTablet := addTablet(100) 303 defer deleteTablet(wantTablet) 304 305 params := map[string]string{ 306 "id": "1", 307 "state": binlogplayer.BlpRunning, 308 "source": fmt.Sprintf(`keyspace:"%s" shard:"0" key_range:{end:"\x80"}`, env.KeyspaceName), 309 } 310 311 dbClient := binlogplayer.NewMockDBClient(t) 312 dbClient.ExpectRequestRE("update _vt.vreplication set message='Picked source tablet.*", testDMLResponse, nil) 313 dbClient.ExpectRequest("update _vt.vreplication set state='Running', message='' where id=1", testDMLResponse, nil) 314 withStop := &sqltypes.Result{ 315 Fields: []*querypb.Field{ 316 {Name: "pos", Type: sqltypes.VarBinary}, 317 {Name: "stop_pos", Type: sqltypes.VarBinary}, 318 {Name: "max_tps", Type: sqltypes.Int64}, 319 {Name: "max_replication_lag", Type: sqltypes.Int64}, 320 {Name: "state", Type: sqltypes.VarBinary}, 321 {Name: "workflow_type", Type: sqltypes.Int64}, 322 {Name: "workflow", Type: sqltypes.VarChar}, 323 {Name: "workflow_sub_type", Type: sqltypes.Int64}, 324 {Name: "defer_secondary_keys", Type: sqltypes.Int64}, 325 }, 326 InsertID: 0, 327 Rows: [][]sqltypes.Value{ 328 { 329 sqltypes.NewVarBinary("MariaDB/0-1-1083"), // pos 330 sqltypes.NewVarBinary("MariaDB/0-1-1235"), // stop_pos 331 sqltypes.NewInt64(9223372036854775807), // max_tps 332 sqltypes.NewInt64(9223372036854775807), // max_replication_lag 333 sqltypes.NewVarBinary("Running"), // state 334 sqltypes.NewInt64(1), // workflow_type 335 sqltypes.NewVarChar("wf"), // workflow 336 sqltypes.NewInt64(1), // workflow_sub_type 337 sqltypes.NewInt64(1), // defer_secondary_keys 338 }, 339 }, 340 } 341 dbClient.ExpectRequest("select pos, stop_pos, max_tps, max_replication_lag, state, workflow_type, workflow, workflow_sub_type, defer_secondary_keys from _vt.vreplication where id=1", withStop, nil) 342 dbClient.ExpectRequest("begin", nil, nil) 343 dbClient.ExpectRequest("insert into t values(1)", testDMLResponse, nil) 344 dbClient.ExpectRequestRE("update _vt.vreplication set pos='MariaDB/0-1-1235', time_updated=.*", testDMLResponse, nil) 345 dbClient.ExpectRequest("commit", nil, nil) 346 dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='Reached stopping position, done playing logs' where id=1", testDMLResponse, nil) 347 348 dbClientFactory := func() binlogplayer.DBClient { return dbClient } 349 mysqld := &fakemysqldaemon.FakeMysqlDaemon{MysqlPort: sync2.NewAtomicInt32(3306)} 350 351 ct, err := newController(context.Background(), params, dbClientFactory, mysqld, env.TopoServ, env.Cells[0], "replica", nil, nil) 352 if err != nil { 353 t.Fatal(err) 354 } 355 defer func() { 356 dbClient.ExpectRequest("update _vt.vreplication set state='Stopped', message='context canceled' where id=1", testDMLResponse, nil) 357 ct.Stop() 358 }() 359 360 // Also confirm that replication stopped. 361 select { 362 case <-ct.done: 363 case <-time.After(1 * time.Second): 364 t.Errorf("context should be closed, but is not: %v", ct) 365 } 366 367 dbClient.Wait() 368 expectFBCRequest(t, wantTablet, testPos, nil, &topodatapb.KeyRange{End: []byte{0x80}}) 369 }