vitess.io/vitess@v0.16.2/go/vt/vtgate/vstream_manager_test.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package vtgate 18 19 import ( 20 "context" 21 "fmt" 22 "strings" 23 "sync" 24 "testing" 25 "time" 26 27 "vitess.io/vitess/go/sync2" 28 29 "vitess.io/vitess/go/vt/topo" 30 31 vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" 32 vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" 33 "vitess.io/vitess/go/vt/vterrors" 34 35 "vitess.io/vitess/go/stats" 36 "vitess.io/vitess/go/vt/vttablet/sandboxconn" 37 38 "github.com/stretchr/testify/assert" 39 "github.com/stretchr/testify/require" 40 "google.golang.org/protobuf/proto" 41 42 "vitess.io/vitess/go/vt/discovery" 43 "vitess.io/vitess/go/vt/proto/binlogdata" 44 binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" 45 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 46 "vitess.io/vitess/go/vt/srvtopo" 47 ) 48 49 var mu sync.Mutex 50 51 func TestVStreamSkew(t *testing.T) { 52 stream := func(conn *sandboxconn.SandboxConn, keyspace, shard string, count, idx int64) { 53 vevents := getVEvents(keyspace, shard, count, idx) 54 for _, ev := range vevents { 55 conn.VStreamCh <- ev 56 time.Sleep(time.Duration(idx*100) * time.Millisecond) 57 } 58 } 59 type skewTestCase struct { 60 numEventsPerShard int64 61 shard0idx, shard1idx int64 62 expectedDelays int64 63 } 64 tcases := []*skewTestCase{ 65 // shard0 events are all attempted to be sent first along with the first event of shard1 due to the increased sleep 66 // for shard1 in stream(). Third event and fourth events of shard0 need to wait for shard1 to catch up 67 {numEventsPerShard: 4, shard0idx: 1, shard1idx: 2, expectedDelays: 2}, 68 69 // no delays if streams are aligned or if only one stream is present 70 {numEventsPerShard: 4, shard0idx: 1, shard1idx: 1, expectedDelays: 0}, 71 {numEventsPerShard: 4, shard0idx: 0, shard1idx: 1, expectedDelays: 0}, 72 {numEventsPerShard: 4, shard0idx: 1, shard1idx: 0, expectedDelays: 0}, 73 } 74 previousDelays := int64(0) 75 if vstreamSkewDelayCount == nil { 76 // HACK: without a mutex we are not guaranteed that this will avoid the panic caused by a race 77 // between this initialization and the one in vtgate.go 78 vstreamSkewDelayCount = stats.NewCounter("VStreamEventsDelayedBySkewAlignment", 79 "Number of events that had to wait because the skew across shards was too high") 80 } 81 82 cell := "aa" 83 for idx, tcase := range tcases { 84 t.Run("", func(t *testing.T) { 85 ctx, cancel := context.WithCancel(context.Background()) 86 defer cancel() 87 88 ks := fmt.Sprintf("TestVStreamSkew-%d", idx) 89 _ = createSandbox(ks) 90 hc := discovery.NewFakeHealthCheck(nil) 91 st := getSandboxTopo(ctx, cell, ks, []string{"-20", "20-40"}) 92 vsm := newTestVStreamManager(hc, st, cell) 93 vgtid := &binlogdatapb.VGtid{ShardGtids: []*binlogdatapb.ShardGtid{}} 94 want := int64(0) 95 var sbc0, sbc1 *sandboxconn.SandboxConn 96 if tcase.shard0idx != 0 { 97 sbc0 = hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 98 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 99 sbc0.VStreamCh = make(chan *binlogdatapb.VEvent) 100 want += 2 * tcase.numEventsPerShard 101 vgtid.ShardGtids = append(vgtid.ShardGtids, &binlogdatapb.ShardGtid{Keyspace: ks, Gtid: "pos", Shard: "-20"}) 102 go stream(sbc0, ks, "-20", tcase.numEventsPerShard, tcase.shard0idx) 103 } 104 if tcase.shard1idx != 0 { 105 sbc1 = hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_PRIMARY, true, 1, nil) 106 addTabletToSandboxTopo(t, st, ks, "20-40", sbc1.Tablet()) 107 sbc1.VStreamCh = make(chan *binlogdatapb.VEvent) 108 want += 2 * tcase.numEventsPerShard 109 vgtid.ShardGtids = append(vgtid.ShardGtids, &binlogdatapb.ShardGtid{Keyspace: ks, Gtid: "pos", Shard: "20-40"}) 110 go stream(sbc1, ks, "20-40", tcase.numEventsPerShard, tcase.shard1idx) 111 } 112 ch := startVStream(ctx, t, vsm, vgtid, &vtgatepb.VStreamFlags{MinimizeSkew: true}) 113 var receivedEvents []*binlogdatapb.VEvent 114 for len(receivedEvents) < int(want) { 115 select { 116 case <-time.After(1 * time.Minute): 117 require.FailNow(t, "test timed out") 118 case response := <-ch: 119 receivedEvents = append(receivedEvents, response.Events...) 120 } 121 } 122 require.Equal(t, int(want), int(len(receivedEvents))) 123 require.Equal(t, tcase.expectedDelays, vsm.GetTotalStreamDelay()-previousDelays) 124 previousDelays = vsm.GetTotalStreamDelay() 125 }) 126 } 127 } 128 129 func TestVStreamEvents(t *testing.T) { 130 ctx, cancel := context.WithCancel(context.Background()) 131 defer cancel() 132 cell := "aa" 133 ks := "TestVStream" 134 _ = createSandbox(ks) 135 hc := discovery.NewFakeHealthCheck(nil) 136 st := getSandboxTopo(ctx, cell, ks, []string{"-20"}) 137 138 vsm := newTestVStreamManager(hc, st, cell) 139 sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 140 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 141 142 send1 := []*binlogdatapb.VEvent{ 143 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"}, 144 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}}, 145 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}, 146 {Type: binlogdatapb.VEventType_COMMIT}, 147 } 148 want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 149 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 150 ShardGtids: []*binlogdatapb.ShardGtid{{ 151 Keyspace: ks, 152 Shard: "-20", 153 Gtid: "gtid01", 154 }}, 155 }}, 156 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}}, 157 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}}, 158 {Type: binlogdatapb.VEventType_COMMIT}, 159 }} 160 sbc0.AddVStreamEvents(send1, nil) 161 162 send2 := []*binlogdatapb.VEvent{ 163 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"}, 164 {Type: binlogdatapb.VEventType_DDL}, 165 } 166 want2 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 167 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 168 ShardGtids: []*binlogdatapb.ShardGtid{{ 169 Keyspace: ks, 170 Shard: "-20", 171 Gtid: "gtid02", 172 }}, 173 }}, 174 {Type: binlogdatapb.VEventType_DDL}, 175 }} 176 sbc0.AddVStreamEvents(send2, nil) 177 178 vgtid := &binlogdatapb.VGtid{ 179 ShardGtids: []*binlogdatapb.ShardGtid{{ 180 Keyspace: ks, 181 Shard: "-20", 182 Gtid: "pos", 183 }}, 184 } 185 ch := make(chan *binlogdatapb.VStreamResponse) 186 go func() { 187 err := vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error { 188 ch <- &binlogdatapb.VStreamResponse{Events: events} 189 return nil 190 }) 191 wantErr := "context canceled" 192 if err == nil || !strings.Contains(err.Error(), wantErr) { 193 t.Errorf("vstream end: %v, must contain %v", err.Error(), wantErr) 194 } 195 ch <- nil 196 }() 197 verifyEvents(t, ch, want1, want2) 198 199 // Ensure the go func error return was verified. 200 cancel() 201 <-ch 202 } 203 204 // TestVStreamChunks ensures that a transaction that's broken 205 // into chunks is sent together. 206 func TestVStreamChunks(t *testing.T) { 207 ctx, cancel := context.WithCancel(context.Background()) 208 defer cancel() 209 210 ks := "TestVStream" 211 cell := "aa" 212 _ = createSandbox(ks) 213 hc := discovery.NewFakeHealthCheck(nil) 214 st := getSandboxTopo(ctx, cell, ks, []string{"-20", "20-40"}) 215 vsm := newTestVStreamManager(hc, st, cell) 216 sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 217 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 218 sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_PRIMARY, true, 1, nil) 219 addTabletToSandboxTopo(t, st, ks, "20-40", sbc1.Tablet()) 220 221 for i := 0; i < 100; i++ { 222 sbc0.AddVStreamEvents([]*binlogdatapb.VEvent{{Type: binlogdatapb.VEventType_DDL}}, nil) 223 sbc1.AddVStreamEvents([]*binlogdatapb.VEvent{{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}}, nil) 224 } 225 sbc1.AddVStreamEvents([]*binlogdatapb.VEvent{{Type: binlogdatapb.VEventType_COMMIT}}, nil) 226 227 rowEncountered := false 228 doneCounting := false 229 var rowCount, ddlCount sync2.AtomicInt32 230 rowCount.Set(0) 231 ddlCount.Set(0) 232 vgtid := &binlogdatapb.VGtid{ 233 ShardGtids: []*binlogdatapb.ShardGtid{{ 234 Keyspace: ks, 235 Shard: "-20", 236 Gtid: "pos", 237 }, { 238 Keyspace: ks, 239 Shard: "20-40", 240 Gtid: "pos", 241 }}, 242 } 243 _ = vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error { 244 switch events[0].Type { 245 case binlogdatapb.VEventType_ROW: 246 if doneCounting { 247 t.Errorf("Unexpected event, only expecting DDL: %v", events[0]) 248 return fmt.Errorf("unexpected event: %v", events[0]) 249 } 250 rowEncountered = true 251 rowCount.Add(1) 252 case binlogdatapb.VEventType_COMMIT: 253 if !rowEncountered { 254 t.Errorf("Unexpected event, COMMIT after non-rows: %v", events[0]) 255 return fmt.Errorf("unexpected event: %v", events[0]) 256 } 257 doneCounting = true 258 case binlogdatapb.VEventType_DDL: 259 if !doneCounting && rowEncountered { 260 t.Errorf("Unexpected event, DDL during ROW events: %v", events[0]) 261 return fmt.Errorf("unexpected event: %v", events[0]) 262 } 263 ddlCount.Add(1) 264 default: 265 t.Errorf("Unexpected event: %v", events[0]) 266 return fmt.Errorf("unexpected event: %v", events[0]) 267 } 268 if rowCount.Get() == int32(100) && ddlCount.Get() == int32(100) { 269 cancel() 270 } 271 return nil 272 }) 273 assert.Equal(t, int32(100), rowCount.Get()) 274 assert.Equal(t, int32(100), ddlCount.Get()) 275 } 276 277 func TestVStreamMulti(t *testing.T) { 278 ctx, cancel := context.WithCancel(context.Background()) 279 defer cancel() 280 cell := "aa" 281 ks := "TestVStream" 282 _ = createSandbox(ks) 283 hc := discovery.NewFakeHealthCheck(nil) 284 st := getSandboxTopo(ctx, cell, ks, []string{"-20", "20-40"}) 285 vsm := newTestVStreamManager(hc, st, "aa") 286 sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 287 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 288 sbc1 := hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_PRIMARY, true, 1, nil) 289 addTabletToSandboxTopo(t, st, ks, "20-40", sbc1.Tablet()) 290 291 send0 := []*binlogdatapb.VEvent{ 292 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"}, 293 {Type: binlogdatapb.VEventType_COMMIT}, 294 } 295 sbc0.AddVStreamEvents(send0, nil) 296 297 send1 := []*binlogdatapb.VEvent{ 298 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"}, 299 {Type: binlogdatapb.VEventType_COMMIT}, 300 } 301 sbc1.AddVStreamEvents(send1, nil) 302 303 vgtid := &binlogdatapb.VGtid{ 304 ShardGtids: []*binlogdatapb.ShardGtid{{ 305 Keyspace: ks, 306 Shard: "-20", 307 Gtid: "pos", 308 }, { 309 Keyspace: ks, 310 Shard: "20-40", 311 Gtid: "pos", 312 }}, 313 } 314 ch := startVStream(ctx, t, vsm, vgtid, nil) 315 <-ch 316 response := <-ch 317 var got *binlogdatapb.VGtid 318 for _, ev := range response.Events { 319 if ev.Type == binlogdatapb.VEventType_VGTID { 320 got = ev.Vgtid 321 } 322 } 323 want := &binlogdatapb.VGtid{ 324 ShardGtids: []*binlogdatapb.ShardGtid{{ 325 Keyspace: ks, 326 Shard: "-20", 327 Gtid: "gtid01", 328 }, { 329 Keyspace: ks, 330 Shard: "20-40", 331 Gtid: "gtid02", 332 }}, 333 } 334 if !proto.Equal(got, want) { 335 t.Errorf("VGtid:\n%v, want\n%v", got, want) 336 } 337 } 338 339 func TestVStreamRetry(t *testing.T) { 340 ctx, cancel := context.WithCancel(context.Background()) 341 defer cancel() 342 343 cell := "aa" 344 ks := "TestVStream" 345 _ = createSandbox(ks) 346 hc := discovery.NewFakeHealthCheck(nil) 347 348 st := getSandboxTopo(ctx, cell, ks, []string{"-20"}) 349 vsm := newTestVStreamManager(hc, st, "aa") 350 sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 351 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 352 commit := []*binlogdatapb.VEvent{ 353 {Type: binlogdatapb.VEventType_COMMIT}, 354 } 355 sbc0.AddVStreamEvents(commit, nil) 356 sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "aa")) 357 sbc0.AddVStreamEvents(commit, nil) 358 sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "bb")) 359 sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cc")) 360 sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "final error")) 361 var count sync2.AtomicInt32 362 count.Set(0) 363 vgtid := &binlogdatapb.VGtid{ 364 ShardGtids: []*binlogdatapb.ShardGtid{{ 365 Keyspace: ks, 366 Shard: "-20", 367 Gtid: "pos", 368 }}, 369 } 370 err := vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error { 371 count.Add(1) 372 return nil 373 }) 374 wantErr := "final error" 375 if err == nil || !strings.Contains(err.Error(), wantErr) { 376 t.Errorf("vstream end: %v, must contain %v", err.Error(), wantErr) 377 } 378 time.Sleep(100 * time.Millisecond) // wait for goroutine within VStream to finish 379 assert.Equal(t, int32(2), count.Get()) 380 } 381 382 func TestVStreamShouldNotSendSourceHeartbeats(t *testing.T) { 383 ctx, cancel := context.WithCancel(context.Background()) 384 defer cancel() 385 cell := "aa" 386 ks := "TestVStream" 387 _ = createSandbox(ks) 388 hc := discovery.NewFakeHealthCheck(nil) 389 st := getSandboxTopo(ctx, cell, ks, []string{"-20"}) 390 vsm := newTestVStreamManager(hc, st, cell) 391 sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 392 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 393 394 send0 := []*binlogdatapb.VEvent{ 395 {Type: binlogdatapb.VEventType_HEARTBEAT}, 396 } 397 sbc0.AddVStreamEvents(send0, nil) 398 399 send1 := []*binlogdatapb.VEvent{ 400 {Type: binlogdatapb.VEventType_HEARTBEAT}, 401 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"}, 402 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}}, 403 {Type: binlogdatapb.VEventType_HEARTBEAT}, 404 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}, 405 {Type: binlogdatapb.VEventType_COMMIT}, 406 } 407 want := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 408 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 409 ShardGtids: []*binlogdatapb.ShardGtid{{ 410 Keyspace: ks, 411 Shard: "-20", 412 Gtid: "gtid01", 413 }}, 414 }}, 415 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}}, 416 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}}, 417 {Type: binlogdatapb.VEventType_COMMIT}, 418 }} 419 sbc0.AddVStreamEvents(send1, nil) 420 421 vgtid := &binlogdatapb.VGtid{ 422 ShardGtids: []*binlogdatapb.ShardGtid{{ 423 Keyspace: ks, 424 Shard: "-20", 425 Gtid: "pos", 426 }}, 427 } 428 ch := startVStream(ctx, t, vsm, vgtid, nil) 429 verifyEvents(t, ch, want) 430 } 431 432 func TestVStreamJournalOneToMany(t *testing.T) { 433 ctx, cancel := context.WithCancel(context.Background()) 434 defer cancel() 435 cell := "aa" 436 ks := "TestVStream" 437 _ = createSandbox(ks) 438 hc := discovery.NewFakeHealthCheck(nil) 439 st := getSandboxTopo(ctx, cell, ks, []string{"-20", "-10", "10-20"}) 440 vsm := newTestVStreamManager(hc, st, "aa") 441 sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 442 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 443 sbc1 := hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "-10", topodatapb.TabletType_PRIMARY, true, 1, nil) 444 addTabletToSandboxTopo(t, st, ks, "-10", sbc1.Tablet()) 445 sbc2 := hc.AddTestTablet(cell, "1.1.1.1", 1003, ks, "10-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 446 addTabletToSandboxTopo(t, st, ks, "10-20", sbc2.Tablet()) 447 448 send1 := []*binlogdatapb.VEvent{ 449 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"}, 450 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}}, 451 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}, 452 {Type: binlogdatapb.VEventType_COMMIT}, 453 } 454 want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 455 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 456 ShardGtids: []*binlogdatapb.ShardGtid{{ 457 Keyspace: ks, 458 Shard: "-20", 459 Gtid: "gtid01", 460 }}, 461 }}, 462 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}}, 463 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}}, 464 {Type: binlogdatapb.VEventType_COMMIT}, 465 }} 466 sbc0.AddVStreamEvents(send1, nil) 467 468 send2 := []*binlogdatapb.VEvent{ 469 {Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{ 470 Id: 1, 471 MigrationType: binlogdatapb.MigrationType_SHARDS, 472 ShardGtids: []*binlogdatapb.ShardGtid{{ 473 Keyspace: ks, 474 Shard: "-10", 475 Gtid: "pos10", 476 }, { 477 Keyspace: ks, 478 Shard: "10-20", 479 Gtid: "pos1020", 480 }}, 481 Participants: []*binlogdatapb.KeyspaceShard{{ 482 Keyspace: ks, 483 Shard: "-20", 484 }}, 485 }}, 486 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"}, 487 {Type: binlogdatapb.VEventType_COMMIT}, 488 } 489 sbc0.AddVStreamEvents(send2, nil) 490 491 send3 := []*binlogdatapb.VEvent{ 492 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid03"}, 493 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t3"}}, 494 {Type: binlogdatapb.VEventType_COMMIT}, 495 } 496 sbc1.ExpectVStreamStartPos("pos10") 497 sbc1.AddVStreamEvents(send3, nil) 498 499 send4 := []*binlogdatapb.VEvent{ 500 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid04"}, 501 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t4"}}, 502 {Type: binlogdatapb.VEventType_COMMIT}, 503 } 504 sbc2.ExpectVStreamStartPos("pos1020") 505 sbc2.AddVStreamEvents(send4, nil) 506 507 vgtid := &binlogdatapb.VGtid{ 508 ShardGtids: []*binlogdatapb.ShardGtid{{ 509 Keyspace: ks, 510 Shard: "-20", 511 Gtid: "pos", 512 }}, 513 } 514 ch := startVStream(ctx, t, vsm, vgtid, nil) 515 verifyEvents(t, ch, want1) 516 517 // The following two events from the different shards can come in any order. 518 // But the resulting VGTID should be the same after both are received. 519 <-ch 520 got := <-ch 521 wantevent := &binlogdatapb.VEvent{ 522 Type: binlogdatapb.VEventType_VGTID, 523 Vgtid: &binlogdatapb.VGtid{ 524 ShardGtids: []*binlogdatapb.ShardGtid{{ 525 Keyspace: ks, 526 Shard: "-10", 527 Gtid: "gtid03", 528 }, { 529 Keyspace: ks, 530 Shard: "10-20", 531 Gtid: "gtid04", 532 }}, 533 }, 534 } 535 gotEvent := got.Events[0] 536 gotEvent.Keyspace = "" 537 gotEvent.Shard = "" 538 if !proto.Equal(gotEvent, wantevent) { 539 t.Errorf("vgtid: %v, want %v", got.Events[0], wantevent) 540 } 541 } 542 543 func TestVStreamJournalManyToOne(t *testing.T) { 544 ctx, cancel := context.WithCancel(context.Background()) 545 defer cancel() 546 547 // Variable names are maintained like in OneToMany, but order is different. 548 ks := "TestVStream" 549 cell := "aa" 550 _ = createSandbox(ks) 551 hc := discovery.NewFakeHealthCheck(nil) 552 st := getSandboxTopo(ctx, cell, ks, []string{"-20", "-10", "10-20"}) 553 vsm := newTestVStreamManager(hc, st, cell) 554 sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 555 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 556 sbc1 := hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "-10", topodatapb.TabletType_PRIMARY, true, 1, nil) 557 addTabletToSandboxTopo(t, st, ks, "-10", sbc1.Tablet()) 558 sbc2 := hc.AddTestTablet(cell, "1.1.1.1", 1003, ks, "10-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 559 addTabletToSandboxTopo(t, st, ks, "10-20", sbc2.Tablet()) 560 561 send3 := []*binlogdatapb.VEvent{ 562 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid03"}, 563 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t3"}}, 564 {Type: binlogdatapb.VEventType_COMMIT}, 565 } 566 sbc1.ExpectVStreamStartPos("pos10") 567 sbc1.AddVStreamEvents(send3, nil) 568 569 send4 := []*binlogdatapb.VEvent{ 570 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid04"}, 571 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t4"}}, 572 {Type: binlogdatapb.VEventType_COMMIT}, 573 } 574 sbc2.ExpectVStreamStartPos("pos1020") 575 sbc2.AddVStreamEvents(send4, nil) 576 577 send2 := []*binlogdatapb.VEvent{ 578 {Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{ 579 Id: 1, 580 MigrationType: binlogdatapb.MigrationType_SHARDS, 581 ShardGtids: []*binlogdatapb.ShardGtid{{ 582 Keyspace: ks, 583 Shard: "-20", 584 Gtid: "pos20", 585 }}, 586 Participants: []*binlogdatapb.KeyspaceShard{{ 587 Keyspace: ks, 588 Shard: "-10", 589 }, { 590 Keyspace: ks, 591 Shard: "10-20", 592 }}, 593 }}, 594 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"}, 595 {Type: binlogdatapb.VEventType_COMMIT}, 596 } 597 // Journal event has to be sent by both shards. 598 sbc1.AddVStreamEvents(send2, nil) 599 sbc2.AddVStreamEvents(send2, nil) 600 601 send1 := []*binlogdatapb.VEvent{ 602 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"}, 603 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}}, 604 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}, 605 {Type: binlogdatapb.VEventType_COMMIT}, 606 } 607 want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 608 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 609 ShardGtids: []*binlogdatapb.ShardGtid{{ 610 Keyspace: ks, 611 Shard: "-20", 612 Gtid: "gtid01", 613 }}, 614 }}, 615 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}}, 616 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}}, 617 {Type: binlogdatapb.VEventType_COMMIT}, 618 }} 619 sbc0.ExpectVStreamStartPos("pos20") 620 sbc0.AddVStreamEvents(send1, nil) 621 622 vgtid := &binlogdatapb.VGtid{ 623 ShardGtids: []*binlogdatapb.ShardGtid{{ 624 Keyspace: ks, 625 Shard: "-10", 626 Gtid: "pos10", 627 }, { 628 Keyspace: ks, 629 Shard: "10-20", 630 Gtid: "pos1020", 631 }}, 632 } 633 ch := startVStream(ctx, t, vsm, vgtid, nil) 634 // The following two events from the different shards can come in any order. 635 // But the resulting VGTID should be the same after both are received. 636 <-ch 637 got := <-ch 638 wantevent := &binlogdatapb.VEvent{ 639 Type: binlogdatapb.VEventType_VGTID, 640 Vgtid: &binlogdatapb.VGtid{ 641 ShardGtids: []*binlogdatapb.ShardGtid{{ 642 Keyspace: ks, 643 Shard: "-10", 644 Gtid: "gtid03", 645 }, { 646 Keyspace: ks, 647 Shard: "10-20", 648 Gtid: "gtid04", 649 }}, 650 }, 651 } 652 gotEvent := got.Events[0] 653 gotEvent.Keyspace = "" 654 gotEvent.Shard = "" 655 if !proto.Equal(gotEvent, wantevent) { 656 t.Errorf("vgtid: %v, want %v", got.Events[0], wantevent) 657 } 658 verifyEvents(t, ch, want1) 659 } 660 661 func TestVStreamJournalNoMatch(t *testing.T) { 662 ctx, cancel := context.WithCancel(context.Background()) 663 defer cancel() 664 665 ks := "TestVStream" 666 cell := "aa" 667 _ = createSandbox(ks) 668 hc := discovery.NewFakeHealthCheck(nil) 669 st := getSandboxTopo(ctx, cell, ks, []string{"-20"}) 670 vsm := newTestVStreamManager(hc, st, "aa") 671 sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 672 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 673 674 send1 := []*binlogdatapb.VEvent{ 675 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"}, 676 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}}, 677 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}, 678 {Type: binlogdatapb.VEventType_COMMIT}, 679 } 680 want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 681 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 682 ShardGtids: []*binlogdatapb.ShardGtid{{ 683 Keyspace: ks, 684 Shard: "-20", 685 Gtid: "gtid01", 686 }}, 687 }}, 688 {Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}}, 689 {Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}}, 690 {Type: binlogdatapb.VEventType_COMMIT}, 691 }} 692 sbc0.AddVStreamEvents(send1, nil) 693 694 tableJournal := []*binlogdatapb.VEvent{ 695 {Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{ 696 Id: 1, 697 MigrationType: binlogdatapb.MigrationType_TABLES, 698 }}, 699 {Type: binlogdatapb.VEventType_GTID, Gtid: "jn1"}, 700 {Type: binlogdatapb.VEventType_COMMIT}, 701 } 702 wantjn1 := &binlogdata.VStreamResponse{Events: []*binlogdatapb.VEvent{ 703 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 704 ShardGtids: []*binlogdatapb.ShardGtid{{ 705 Keyspace: ks, 706 Shard: "-20", 707 Gtid: "jn1", 708 }}, 709 }}, 710 {Type: binlogdatapb.VEventType_COMMIT}, 711 }} 712 sbc0.AddVStreamEvents(tableJournal, nil) 713 714 send2 := []*binlogdatapb.VEvent{ 715 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"}, 716 {Type: binlogdatapb.VEventType_DDL}, 717 } 718 want2 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 719 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 720 ShardGtids: []*binlogdatapb.ShardGtid{{ 721 Keyspace: ks, 722 Shard: "-20", 723 Gtid: "gtid02", 724 }}, 725 }}, 726 {Type: binlogdatapb.VEventType_DDL}, 727 }} 728 sbc0.AddVStreamEvents(send2, nil) 729 730 shardJournal := []*binlogdatapb.VEvent{ 731 {Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{ 732 Id: 2, 733 MigrationType: binlogdatapb.MigrationType_SHARDS, 734 ShardGtids: []*binlogdatapb.ShardGtid{{ 735 Keyspace: ks, 736 Shard: "c0-", 737 Gtid: "posc0", 738 }}, 739 Participants: []*binlogdatapb.KeyspaceShard{{ 740 Keyspace: ks, 741 Shard: "c0-e0", 742 }, { 743 Keyspace: ks, 744 Shard: "e0-", 745 }}, 746 }}, 747 {Type: binlogdatapb.VEventType_GTID, Gtid: "jn2"}, 748 {Type: binlogdatapb.VEventType_COMMIT}, 749 } 750 wantjn2 := &binlogdata.VStreamResponse{Events: []*binlogdatapb.VEvent{ 751 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 752 ShardGtids: []*binlogdatapb.ShardGtid{{ 753 Keyspace: ks, 754 Shard: "-20", 755 Gtid: "jn2", 756 }}, 757 }}, 758 {Type: binlogdatapb.VEventType_COMMIT}, 759 }} 760 sbc0.AddVStreamEvents(shardJournal, nil) 761 762 send3 := []*binlogdatapb.VEvent{ 763 {Type: binlogdatapb.VEventType_GTID, Gtid: "gtid03"}, 764 {Type: binlogdatapb.VEventType_DDL}, 765 } 766 want3 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{ 767 {Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{ 768 ShardGtids: []*binlogdatapb.ShardGtid{{ 769 Keyspace: ks, 770 Shard: "-20", 771 Gtid: "gtid03", 772 }}, 773 }}, 774 {Type: binlogdatapb.VEventType_DDL}, 775 }} 776 sbc0.AddVStreamEvents(send3, nil) 777 778 vgtid := &binlogdatapb.VGtid{ 779 ShardGtids: []*binlogdatapb.ShardGtid{{ 780 Keyspace: ks, 781 Shard: "-20", 782 Gtid: "pos", 783 }}, 784 } 785 ch := startVStream(ctx, t, vsm, vgtid, nil) 786 verifyEvents(t, ch, want1, wantjn1, want2, wantjn2, want3) 787 } 788 789 func TestVStreamJournalPartialMatch(t *testing.T) { 790 ctx, cancel := context.WithCancel(context.Background()) 791 defer cancel() 792 793 // Variable names are maintained like in OneToMany, but order is different.1 794 ks := "TestVStream" 795 cell := "aa" 796 _ = createSandbox(ks) 797 hc := discovery.NewFakeHealthCheck(nil) 798 st := getSandboxTopo(ctx, cell, ks, []string{"-20", "-10", "10-20"}) 799 vsm := newTestVStreamManager(hc, st, "aa") 800 sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, ks, "-10", topodatapb.TabletType_PRIMARY, true, 1, nil) 801 addTabletToSandboxTopo(t, st, ks, "-10", sbc1.Tablet()) 802 sbc2 := hc.AddTestTablet("aa", "1.1.1.1", 1003, ks, "10-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 803 addTabletToSandboxTopo(t, st, ks, "10-20", sbc2.Tablet()) 804 805 send := []*binlogdatapb.VEvent{ 806 {Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{ 807 Id: 1, 808 MigrationType: binlogdatapb.MigrationType_SHARDS, 809 ShardGtids: []*binlogdatapb.ShardGtid{{ 810 Keyspace: ks, 811 Shard: "10-30", 812 Gtid: "pos1040", 813 }}, 814 Participants: []*binlogdatapb.KeyspaceShard{{ 815 Keyspace: ks, 816 Shard: "10-20", 817 }, { 818 Keyspace: ks, 819 Shard: "20-30", 820 }}, 821 }}, 822 } 823 sbc2.AddVStreamEvents(send, nil) 824 825 vgtid := &binlogdatapb.VGtid{ 826 ShardGtids: []*binlogdatapb.ShardGtid{{ 827 Keyspace: ks, 828 Shard: "-10", 829 Gtid: "pos10", 830 }, { 831 Keyspace: ks, 832 Shard: "10-20", 833 Gtid: "pos1020", 834 }}, 835 } 836 err := vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error { 837 t.Errorf("unexpected events: %v", events) 838 return nil 839 }) 840 wantErr := "not all journaling participants are in the stream" 841 if err == nil || !strings.Contains(err.Error(), wantErr) { 842 t.Errorf("vstream end: %v, must contain %v", err, wantErr) 843 } 844 845 // Try a different order (different code path) 846 send = []*binlogdatapb.VEvent{ 847 {Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{ 848 Id: 1, 849 MigrationType: binlogdatapb.MigrationType_SHARDS, 850 ShardGtids: []*binlogdatapb.ShardGtid{{ 851 Keyspace: ks, 852 Shard: "10-30", 853 Gtid: "pos1040", 854 }}, 855 Participants: []*binlogdatapb.KeyspaceShard{{ 856 Keyspace: ks, 857 Shard: "20-30", 858 }, { 859 Keyspace: ks, 860 Shard: "10-20", 861 }}, 862 }}, 863 } 864 sbc2.AddVStreamEvents(send, nil) 865 err = vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error { 866 t.Errorf("unexpected events: %v", events) 867 return nil 868 }) 869 if err == nil || !strings.Contains(err.Error(), wantErr) { 870 t.Errorf("vstream end: %v, must contain %v", err, wantErr) 871 } 872 cancel() 873 } 874 875 func TestResolveVStreamParams(t *testing.T) { 876 name := "TestVStream" 877 _ = createSandbox(name) 878 hc := discovery.NewFakeHealthCheck(nil) 879 vsm := newTestVStreamManager(hc, newSandboxForCells([]string{"aa"}), "aa") 880 testcases := []struct { 881 input *binlogdatapb.VGtid 882 output *binlogdatapb.VGtid 883 err string 884 }{{ 885 input: nil, 886 err: "vgtid must have at least one value with a starting position", 887 }, { 888 input: &binlogdatapb.VGtid{ 889 ShardGtids: []*binlogdatapb.ShardGtid{{}}, 890 }, 891 err: "for an empty keyspace, the Gtid value must be 'current'", 892 }, { 893 input: &binlogdatapb.VGtid{ 894 ShardGtids: []*binlogdatapb.ShardGtid{{ 895 Keyspace: "TestVStream", 896 }}, 897 }, 898 err: "if shards are unspecified, the Gtid value must be 'current'", 899 }, { 900 input: &binlogdatapb.VGtid{ 901 ShardGtids: []*binlogdatapb.ShardGtid{{ 902 Keyspace: "TestVStream", 903 Gtid: "current", 904 }}, 905 }, 906 output: &binlogdatapb.VGtid{ 907 ShardGtids: []*binlogdatapb.ShardGtid{{ 908 Keyspace: "TestVStream", 909 Shard: "-20", 910 Gtid: "current", 911 }, { 912 Keyspace: "TestVStream", 913 Shard: "20-40", 914 Gtid: "current", 915 }, { 916 Keyspace: "TestVStream", 917 Shard: "40-60", 918 Gtid: "current", 919 }, { 920 Keyspace: "TestVStream", 921 Shard: "60-80", 922 Gtid: "current", 923 }, { 924 Keyspace: "TestVStream", 925 Shard: "80-a0", 926 Gtid: "current", 927 }, { 928 Keyspace: "TestVStream", 929 Shard: "a0-c0", 930 Gtid: "current", 931 }, { 932 Keyspace: "TestVStream", 933 Shard: "c0-e0", 934 Gtid: "current", 935 }, { 936 Keyspace: "TestVStream", 937 Shard: "e0-", 938 Gtid: "current", 939 }}, 940 }, 941 }, { 942 input: &binlogdatapb.VGtid{ 943 ShardGtids: []*binlogdatapb.ShardGtid{{ 944 Keyspace: "TestVStream", 945 Shard: "-20", 946 Gtid: "current", 947 }}, 948 }, 949 output: &binlogdatapb.VGtid{ 950 ShardGtids: []*binlogdatapb.ShardGtid{{ 951 Keyspace: "TestVStream", 952 Shard: "-20", 953 Gtid: "current", 954 }}, 955 }, 956 }, { 957 input: &binlogdatapb.VGtid{ 958 ShardGtids: []*binlogdatapb.ShardGtid{{ 959 Keyspace: "TestVStream", 960 Shard: "-20", 961 Gtid: "other", 962 }}, 963 }, 964 output: &binlogdatapb.VGtid{ 965 ShardGtids: []*binlogdatapb.ShardGtid{{ 966 Keyspace: "TestVStream", 967 Shard: "-20", 968 Gtid: "other", 969 }}, 970 }, 971 }} 972 wantFilter := &binlogdatapb.Filter{ 973 Rules: []*binlogdatapb.Rule{{ 974 Match: "/.*", 975 }}, 976 } 977 for _, tcase := range testcases { 978 vgtid, filter, flags, err := vsm.resolveParams(context.Background(), topodatapb.TabletType_REPLICA, tcase.input, nil, nil) 979 if tcase.err != "" { 980 if err == nil || !strings.Contains(err.Error(), tcase.err) { 981 t.Errorf("resolve(%v) err: %v, must contain %v", tcase.input, err, tcase.err) 982 } 983 continue 984 } 985 require.NoError(t, err, tcase.input) 986 assert.Equal(t, tcase.output, vgtid, tcase.input) 987 assert.Equal(t, wantFilter, filter, tcase.input) 988 require.False(t, flags.MinimizeSkew) 989 } 990 // Special-case: empty keyspace because output is too big. 991 input := &binlogdatapb.VGtid{ 992 ShardGtids: []*binlogdatapb.ShardGtid{{ 993 Gtid: "current", 994 }}, 995 } 996 vgtid, _, _, err := vsm.resolveParams(context.Background(), topodatapb.TabletType_REPLICA, input, nil, nil) 997 require.NoError(t, err, input) 998 if got, want := len(vgtid.ShardGtids), 8; want >= got { 999 t.Errorf("len(vgtid.ShardGtids): %v, must be >%d", got, want) 1000 } 1001 for _, minimizeSkew := range []bool{true, false} { 1002 t.Run(fmt.Sprintf("resolveParams MinimizeSkew %t", minimizeSkew), func(t *testing.T) { 1003 flags := &vtgatepb.VStreamFlags{MinimizeSkew: minimizeSkew} 1004 vgtid := &binlogdatapb.VGtid{ 1005 ShardGtids: []*binlogdatapb.ShardGtid{{ 1006 Keyspace: "TestVStream", 1007 Shard: "-20", 1008 Gtid: "current", 1009 }}, 1010 } 1011 _, _, flags2, err := vsm.resolveParams(context.Background(), topodatapb.TabletType_REPLICA, vgtid, nil, flags) 1012 require.NoError(t, err) 1013 require.Equal(t, minimizeSkew, flags2.MinimizeSkew) 1014 }) 1015 } 1016 1017 } 1018 1019 func TestVStreamIdleHeartbeat(t *testing.T) { 1020 cell := "aa" 1021 ks := "TestVStream" 1022 _ = createSandbox(ks) 1023 hc := discovery.NewFakeHealthCheck(nil) 1024 st := getSandboxTopo(ctx, cell, ks, []string{"-20"}) 1025 vsm := newTestVStreamManager(hc, st, cell) 1026 sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil) 1027 addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet()) 1028 vgtid := &binlogdatapb.VGtid{ 1029 ShardGtids: []*binlogdatapb.ShardGtid{{ 1030 Keyspace: ks, 1031 Shard: "-20", 1032 Gtid: "pos", 1033 }}, 1034 } 1035 1036 type testcase struct { 1037 name string 1038 heartbeatInterval uint32 1039 want int 1040 } 1041 // each test waits for 4.5 seconds, hence expected #heartbeats = floor(4.5/heartbeatInterval) 1042 testcases := []testcase{ 1043 {"off", 0, 0}, 1044 {"on:1s", 1, 4}, 1045 {"on:2s", 2, 2}, 1046 } 1047 for _, tcase := range testcases { 1048 t.Run(tcase.name, func(t *testing.T) { 1049 var mu sync.Mutex 1050 var heartbeatCount int 1051 ctx, cancel := context.WithCancel(context.Background()) 1052 go func() { 1053 vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{HeartbeatInterval: tcase.heartbeatInterval}, 1054 func(events []*binlogdatapb.VEvent) error { 1055 mu.Lock() 1056 defer mu.Unlock() 1057 for _, event := range events { 1058 if event.Type == binlogdatapb.VEventType_HEARTBEAT { 1059 heartbeatCount++ 1060 } 1061 } 1062 return nil 1063 }) 1064 }() 1065 time.Sleep(time.Duration(4500) * time.Millisecond) 1066 mu.Lock() 1067 defer mu.Unlock() 1068 require.Equalf(t, heartbeatCount, tcase.want, "got %d, want %d", heartbeatCount, tcase.want) 1069 cancel() 1070 }) 1071 } 1072 } 1073 1074 func newTestVStreamManager(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vstreamManager { 1075 gw := NewTabletGateway(context.Background(), hc, serv, cell) 1076 srvResolver := srvtopo.NewResolver(serv, gw, cell) 1077 return newVStreamManager(srvResolver, serv, cell) 1078 } 1079 1080 func startVStream(ctx context.Context, t *testing.T, vsm *vstreamManager, vgtid *binlogdatapb.VGtid, flags *vtgatepb.VStreamFlags) <-chan *binlogdatapb.VStreamResponse { 1081 if flags == nil { 1082 flags = &vtgatepb.VStreamFlags{} 1083 } 1084 ch := make(chan *binlogdatapb.VStreamResponse) 1085 go func() { 1086 _ = vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, flags, func(events []*binlogdatapb.VEvent) error { 1087 ch <- &binlogdatapb.VStreamResponse{Events: events} 1088 return nil 1089 }) 1090 }() 1091 return ch 1092 } 1093 1094 func verifyEvents(t *testing.T, ch <-chan *binlogdatapb.VStreamResponse, wants ...*binlogdatapb.VStreamResponse) { 1095 t.Helper() 1096 for i, want := range wants { 1097 got := <-ch 1098 require.NotNil(t, got) 1099 for _, event := range got.Events { 1100 event.Timestamp = 0 1101 } 1102 if !proto.Equal(got, want) { 1103 t.Errorf("vstream(%d):\n%v, want\n%v", i, got, want) 1104 } 1105 } 1106 } 1107 1108 func getVEvents(keyspace, shard string, count, idx int64) []*binlogdatapb.VEvent { 1109 mu.Lock() 1110 defer mu.Unlock() 1111 var vevents []*binlogdatapb.VEvent 1112 var i int64 1113 currentTime := time.Now().Unix() 1114 for i = count; i > 0; i-- { 1115 j := i + idx 1116 vevents = append(vevents, &binlogdatapb.VEvent{ 1117 Type: binlogdatapb.VEventType_GTID, Gtid: fmt.Sprintf("gtid-%s-%d", shard, j), 1118 Timestamp: currentTime - j, 1119 CurrentTime: currentTime * 1e9, 1120 Keyspace: keyspace, 1121 Shard: shard, 1122 }) 1123 1124 vevents = append(vevents, &binlogdatapb.VEvent{ 1125 Type: binlogdatapb.VEventType_COMMIT, 1126 Timestamp: currentTime - j, 1127 CurrentTime: currentTime * 1e9, 1128 Keyspace: keyspace, 1129 Shard: shard, 1130 }) 1131 } 1132 return vevents 1133 } 1134 1135 func getSandboxTopo(ctx context.Context, cell string, keyspace string, shards []string) *sandboxTopo { 1136 st := newSandboxForCells([]string{cell}) 1137 ts := st.topoServer 1138 ts.CreateCellInfo(ctx, cell, &topodatapb.CellInfo{}) 1139 ts.CreateKeyspace(ctx, keyspace, &topodatapb.Keyspace{}) 1140 for _, shard := range shards { 1141 ts.CreateShard(ctx, keyspace, shard) 1142 } 1143 return st 1144 } 1145 1146 func addTabletToSandboxTopo(t *testing.T, st *sandboxTopo, ks, shard string, tablet *topodatapb.Tablet) { 1147 _, err := st.topoServer.UpdateShardFields(ctx, ks, shard, func(si *topo.ShardInfo) error { 1148 si.PrimaryAlias = tablet.Alias 1149 return nil 1150 }) 1151 require.NoError(t, err) 1152 err = st.topoServer.CreateTablet(ctx, tablet) 1153 require.NoError(t, err) 1154 }