vitess.io/vitess@v0.16.2/go/vt/vtgate/vstream_manager_test.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package vtgate
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"sync"
    24  	"testing"
    25  	"time"
    26  
    27  	"vitess.io/vitess/go/sync2"
    28  
    29  	"vitess.io/vitess/go/vt/topo"
    30  
    31  	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
    32  	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
    33  	"vitess.io/vitess/go/vt/vterrors"
    34  
    35  	"vitess.io/vitess/go/stats"
    36  	"vitess.io/vitess/go/vt/vttablet/sandboxconn"
    37  
    38  	"github.com/stretchr/testify/assert"
    39  	"github.com/stretchr/testify/require"
    40  	"google.golang.org/protobuf/proto"
    41  
    42  	"vitess.io/vitess/go/vt/discovery"
    43  	"vitess.io/vitess/go/vt/proto/binlogdata"
    44  	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
    45  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    46  	"vitess.io/vitess/go/vt/srvtopo"
    47  )
    48  
    49  var mu sync.Mutex
    50  
    51  func TestVStreamSkew(t *testing.T) {
    52  	stream := func(conn *sandboxconn.SandboxConn, keyspace, shard string, count, idx int64) {
    53  		vevents := getVEvents(keyspace, shard, count, idx)
    54  		for _, ev := range vevents {
    55  			conn.VStreamCh <- ev
    56  			time.Sleep(time.Duration(idx*100) * time.Millisecond)
    57  		}
    58  	}
    59  	type skewTestCase struct {
    60  		numEventsPerShard    int64
    61  		shard0idx, shard1idx int64
    62  		expectedDelays       int64
    63  	}
    64  	tcases := []*skewTestCase{
    65  		// shard0 events are all attempted to be sent first along with the first event of shard1 due to the increased sleep
    66  		// for shard1 in stream(). Third event and fourth events of shard0 need to wait for shard1 to catch up
    67  		{numEventsPerShard: 4, shard0idx: 1, shard1idx: 2, expectedDelays: 2},
    68  
    69  		// no delays if streams are aligned or if only one stream is present
    70  		{numEventsPerShard: 4, shard0idx: 1, shard1idx: 1, expectedDelays: 0},
    71  		{numEventsPerShard: 4, shard0idx: 0, shard1idx: 1, expectedDelays: 0},
    72  		{numEventsPerShard: 4, shard0idx: 1, shard1idx: 0, expectedDelays: 0},
    73  	}
    74  	previousDelays := int64(0)
    75  	if vstreamSkewDelayCount == nil {
    76  		// HACK: without a mutex we are not guaranteed that this will avoid the panic caused by a race
    77  		// between this initialization and the one in vtgate.go
    78  		vstreamSkewDelayCount = stats.NewCounter("VStreamEventsDelayedBySkewAlignment",
    79  			"Number of events that had to wait because the skew across shards was too high")
    80  	}
    81  
    82  	cell := "aa"
    83  	for idx, tcase := range tcases {
    84  		t.Run("", func(t *testing.T) {
    85  			ctx, cancel := context.WithCancel(context.Background())
    86  			defer cancel()
    87  
    88  			ks := fmt.Sprintf("TestVStreamSkew-%d", idx)
    89  			_ = createSandbox(ks)
    90  			hc := discovery.NewFakeHealthCheck(nil)
    91  			st := getSandboxTopo(ctx, cell, ks, []string{"-20", "20-40"})
    92  			vsm := newTestVStreamManager(hc, st, cell)
    93  			vgtid := &binlogdatapb.VGtid{ShardGtids: []*binlogdatapb.ShardGtid{}}
    94  			want := int64(0)
    95  			var sbc0, sbc1 *sandboxconn.SandboxConn
    96  			if tcase.shard0idx != 0 {
    97  				sbc0 = hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
    98  				addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
    99  				sbc0.VStreamCh = make(chan *binlogdatapb.VEvent)
   100  				want += 2 * tcase.numEventsPerShard
   101  				vgtid.ShardGtids = append(vgtid.ShardGtids, &binlogdatapb.ShardGtid{Keyspace: ks, Gtid: "pos", Shard: "-20"})
   102  				go stream(sbc0, ks, "-20", tcase.numEventsPerShard, tcase.shard0idx)
   103  			}
   104  			if tcase.shard1idx != 0 {
   105  				sbc1 = hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_PRIMARY, true, 1, nil)
   106  				addTabletToSandboxTopo(t, st, ks, "20-40", sbc1.Tablet())
   107  				sbc1.VStreamCh = make(chan *binlogdatapb.VEvent)
   108  				want += 2 * tcase.numEventsPerShard
   109  				vgtid.ShardGtids = append(vgtid.ShardGtids, &binlogdatapb.ShardGtid{Keyspace: ks, Gtid: "pos", Shard: "20-40"})
   110  				go stream(sbc1, ks, "20-40", tcase.numEventsPerShard, tcase.shard1idx)
   111  			}
   112  			ch := startVStream(ctx, t, vsm, vgtid, &vtgatepb.VStreamFlags{MinimizeSkew: true})
   113  			var receivedEvents []*binlogdatapb.VEvent
   114  			for len(receivedEvents) < int(want) {
   115  				select {
   116  				case <-time.After(1 * time.Minute):
   117  					require.FailNow(t, "test timed out")
   118  				case response := <-ch:
   119  					receivedEvents = append(receivedEvents, response.Events...)
   120  				}
   121  			}
   122  			require.Equal(t, int(want), int(len(receivedEvents)))
   123  			require.Equal(t, tcase.expectedDelays, vsm.GetTotalStreamDelay()-previousDelays)
   124  			previousDelays = vsm.GetTotalStreamDelay()
   125  		})
   126  	}
   127  }
   128  
   129  func TestVStreamEvents(t *testing.T) {
   130  	ctx, cancel := context.WithCancel(context.Background())
   131  	defer cancel()
   132  	cell := "aa"
   133  	ks := "TestVStream"
   134  	_ = createSandbox(ks)
   135  	hc := discovery.NewFakeHealthCheck(nil)
   136  	st := getSandboxTopo(ctx, cell, ks, []string{"-20"})
   137  
   138  	vsm := newTestVStreamManager(hc, st, cell)
   139  	sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   140  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   141  
   142  	send1 := []*binlogdatapb.VEvent{
   143  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"},
   144  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}},
   145  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}},
   146  		{Type: binlogdatapb.VEventType_COMMIT},
   147  	}
   148  	want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   149  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   150  			ShardGtids: []*binlogdatapb.ShardGtid{{
   151  				Keyspace: ks,
   152  				Shard:    "-20",
   153  				Gtid:     "gtid01",
   154  			}},
   155  		}},
   156  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}},
   157  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}},
   158  		{Type: binlogdatapb.VEventType_COMMIT},
   159  	}}
   160  	sbc0.AddVStreamEvents(send1, nil)
   161  
   162  	send2 := []*binlogdatapb.VEvent{
   163  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"},
   164  		{Type: binlogdatapb.VEventType_DDL},
   165  	}
   166  	want2 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   167  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   168  			ShardGtids: []*binlogdatapb.ShardGtid{{
   169  				Keyspace: ks,
   170  				Shard:    "-20",
   171  				Gtid:     "gtid02",
   172  			}},
   173  		}},
   174  		{Type: binlogdatapb.VEventType_DDL},
   175  	}}
   176  	sbc0.AddVStreamEvents(send2, nil)
   177  
   178  	vgtid := &binlogdatapb.VGtid{
   179  		ShardGtids: []*binlogdatapb.ShardGtid{{
   180  			Keyspace: ks,
   181  			Shard:    "-20",
   182  			Gtid:     "pos",
   183  		}},
   184  	}
   185  	ch := make(chan *binlogdatapb.VStreamResponse)
   186  	go func() {
   187  		err := vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error {
   188  			ch <- &binlogdatapb.VStreamResponse{Events: events}
   189  			return nil
   190  		})
   191  		wantErr := "context canceled"
   192  		if err == nil || !strings.Contains(err.Error(), wantErr) {
   193  			t.Errorf("vstream end: %v, must contain %v", err.Error(), wantErr)
   194  		}
   195  		ch <- nil
   196  	}()
   197  	verifyEvents(t, ch, want1, want2)
   198  
   199  	// Ensure the go func error return was verified.
   200  	cancel()
   201  	<-ch
   202  }
   203  
   204  // TestVStreamChunks ensures that a transaction that's broken
   205  // into chunks is sent together.
   206  func TestVStreamChunks(t *testing.T) {
   207  	ctx, cancel := context.WithCancel(context.Background())
   208  	defer cancel()
   209  
   210  	ks := "TestVStream"
   211  	cell := "aa"
   212  	_ = createSandbox(ks)
   213  	hc := discovery.NewFakeHealthCheck(nil)
   214  	st := getSandboxTopo(ctx, cell, ks, []string{"-20", "20-40"})
   215  	vsm := newTestVStreamManager(hc, st, cell)
   216  	sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   217  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   218  	sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_PRIMARY, true, 1, nil)
   219  	addTabletToSandboxTopo(t, st, ks, "20-40", sbc1.Tablet())
   220  
   221  	for i := 0; i < 100; i++ {
   222  		sbc0.AddVStreamEvents([]*binlogdatapb.VEvent{{Type: binlogdatapb.VEventType_DDL}}, nil)
   223  		sbc1.AddVStreamEvents([]*binlogdatapb.VEvent{{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}}}, nil)
   224  	}
   225  	sbc1.AddVStreamEvents([]*binlogdatapb.VEvent{{Type: binlogdatapb.VEventType_COMMIT}}, nil)
   226  
   227  	rowEncountered := false
   228  	doneCounting := false
   229  	var rowCount, ddlCount sync2.AtomicInt32
   230  	rowCount.Set(0)
   231  	ddlCount.Set(0)
   232  	vgtid := &binlogdatapb.VGtid{
   233  		ShardGtids: []*binlogdatapb.ShardGtid{{
   234  			Keyspace: ks,
   235  			Shard:    "-20",
   236  			Gtid:     "pos",
   237  		}, {
   238  			Keyspace: ks,
   239  			Shard:    "20-40",
   240  			Gtid:     "pos",
   241  		}},
   242  	}
   243  	_ = vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error {
   244  		switch events[0].Type {
   245  		case binlogdatapb.VEventType_ROW:
   246  			if doneCounting {
   247  				t.Errorf("Unexpected event, only expecting DDL: %v", events[0])
   248  				return fmt.Errorf("unexpected event: %v", events[0])
   249  			}
   250  			rowEncountered = true
   251  			rowCount.Add(1)
   252  		case binlogdatapb.VEventType_COMMIT:
   253  			if !rowEncountered {
   254  				t.Errorf("Unexpected event, COMMIT after non-rows: %v", events[0])
   255  				return fmt.Errorf("unexpected event: %v", events[0])
   256  			}
   257  			doneCounting = true
   258  		case binlogdatapb.VEventType_DDL:
   259  			if !doneCounting && rowEncountered {
   260  				t.Errorf("Unexpected event, DDL during ROW events: %v", events[0])
   261  				return fmt.Errorf("unexpected event: %v", events[0])
   262  			}
   263  			ddlCount.Add(1)
   264  		default:
   265  			t.Errorf("Unexpected event: %v", events[0])
   266  			return fmt.Errorf("unexpected event: %v", events[0])
   267  		}
   268  		if rowCount.Get() == int32(100) && ddlCount.Get() == int32(100) {
   269  			cancel()
   270  		}
   271  		return nil
   272  	})
   273  	assert.Equal(t, int32(100), rowCount.Get())
   274  	assert.Equal(t, int32(100), ddlCount.Get())
   275  }
   276  
   277  func TestVStreamMulti(t *testing.T) {
   278  	ctx, cancel := context.WithCancel(context.Background())
   279  	defer cancel()
   280  	cell := "aa"
   281  	ks := "TestVStream"
   282  	_ = createSandbox(ks)
   283  	hc := discovery.NewFakeHealthCheck(nil)
   284  	st := getSandboxTopo(ctx, cell, ks, []string{"-20", "20-40"})
   285  	vsm := newTestVStreamManager(hc, st, "aa")
   286  	sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   287  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   288  	sbc1 := hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "20-40", topodatapb.TabletType_PRIMARY, true, 1, nil)
   289  	addTabletToSandboxTopo(t, st, ks, "20-40", sbc1.Tablet())
   290  
   291  	send0 := []*binlogdatapb.VEvent{
   292  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"},
   293  		{Type: binlogdatapb.VEventType_COMMIT},
   294  	}
   295  	sbc0.AddVStreamEvents(send0, nil)
   296  
   297  	send1 := []*binlogdatapb.VEvent{
   298  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"},
   299  		{Type: binlogdatapb.VEventType_COMMIT},
   300  	}
   301  	sbc1.AddVStreamEvents(send1, nil)
   302  
   303  	vgtid := &binlogdatapb.VGtid{
   304  		ShardGtids: []*binlogdatapb.ShardGtid{{
   305  			Keyspace: ks,
   306  			Shard:    "-20",
   307  			Gtid:     "pos",
   308  		}, {
   309  			Keyspace: ks,
   310  			Shard:    "20-40",
   311  			Gtid:     "pos",
   312  		}},
   313  	}
   314  	ch := startVStream(ctx, t, vsm, vgtid, nil)
   315  	<-ch
   316  	response := <-ch
   317  	var got *binlogdatapb.VGtid
   318  	for _, ev := range response.Events {
   319  		if ev.Type == binlogdatapb.VEventType_VGTID {
   320  			got = ev.Vgtid
   321  		}
   322  	}
   323  	want := &binlogdatapb.VGtid{
   324  		ShardGtids: []*binlogdatapb.ShardGtid{{
   325  			Keyspace: ks,
   326  			Shard:    "-20",
   327  			Gtid:     "gtid01",
   328  		}, {
   329  			Keyspace: ks,
   330  			Shard:    "20-40",
   331  			Gtid:     "gtid02",
   332  		}},
   333  	}
   334  	if !proto.Equal(got, want) {
   335  		t.Errorf("VGtid:\n%v, want\n%v", got, want)
   336  	}
   337  }
   338  
   339  func TestVStreamRetry(t *testing.T) {
   340  	ctx, cancel := context.WithCancel(context.Background())
   341  	defer cancel()
   342  
   343  	cell := "aa"
   344  	ks := "TestVStream"
   345  	_ = createSandbox(ks)
   346  	hc := discovery.NewFakeHealthCheck(nil)
   347  
   348  	st := getSandboxTopo(ctx, cell, ks, []string{"-20"})
   349  	vsm := newTestVStreamManager(hc, st, "aa")
   350  	sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   351  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   352  	commit := []*binlogdatapb.VEvent{
   353  		{Type: binlogdatapb.VEventType_COMMIT},
   354  	}
   355  	sbc0.AddVStreamEvents(commit, nil)
   356  	sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "aa"))
   357  	sbc0.AddVStreamEvents(commit, nil)
   358  	sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "bb"))
   359  	sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cc"))
   360  	sbc0.AddVStreamEvents(nil, vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "final error"))
   361  	var count sync2.AtomicInt32
   362  	count.Set(0)
   363  	vgtid := &binlogdatapb.VGtid{
   364  		ShardGtids: []*binlogdatapb.ShardGtid{{
   365  			Keyspace: ks,
   366  			Shard:    "-20",
   367  			Gtid:     "pos",
   368  		}},
   369  	}
   370  	err := vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error {
   371  		count.Add(1)
   372  		return nil
   373  	})
   374  	wantErr := "final error"
   375  	if err == nil || !strings.Contains(err.Error(), wantErr) {
   376  		t.Errorf("vstream end: %v, must contain %v", err.Error(), wantErr)
   377  	}
   378  	time.Sleep(100 * time.Millisecond) // wait for goroutine within VStream to finish
   379  	assert.Equal(t, int32(2), count.Get())
   380  }
   381  
   382  func TestVStreamShouldNotSendSourceHeartbeats(t *testing.T) {
   383  	ctx, cancel := context.WithCancel(context.Background())
   384  	defer cancel()
   385  	cell := "aa"
   386  	ks := "TestVStream"
   387  	_ = createSandbox(ks)
   388  	hc := discovery.NewFakeHealthCheck(nil)
   389  	st := getSandboxTopo(ctx, cell, ks, []string{"-20"})
   390  	vsm := newTestVStreamManager(hc, st, cell)
   391  	sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   392  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   393  
   394  	send0 := []*binlogdatapb.VEvent{
   395  		{Type: binlogdatapb.VEventType_HEARTBEAT},
   396  	}
   397  	sbc0.AddVStreamEvents(send0, nil)
   398  
   399  	send1 := []*binlogdatapb.VEvent{
   400  		{Type: binlogdatapb.VEventType_HEARTBEAT},
   401  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"},
   402  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}},
   403  		{Type: binlogdatapb.VEventType_HEARTBEAT},
   404  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}},
   405  		{Type: binlogdatapb.VEventType_COMMIT},
   406  	}
   407  	want := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   408  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   409  			ShardGtids: []*binlogdatapb.ShardGtid{{
   410  				Keyspace: ks,
   411  				Shard:    "-20",
   412  				Gtid:     "gtid01",
   413  			}},
   414  		}},
   415  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}},
   416  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}},
   417  		{Type: binlogdatapb.VEventType_COMMIT},
   418  	}}
   419  	sbc0.AddVStreamEvents(send1, nil)
   420  
   421  	vgtid := &binlogdatapb.VGtid{
   422  		ShardGtids: []*binlogdatapb.ShardGtid{{
   423  			Keyspace: ks,
   424  			Shard:    "-20",
   425  			Gtid:     "pos",
   426  		}},
   427  	}
   428  	ch := startVStream(ctx, t, vsm, vgtid, nil)
   429  	verifyEvents(t, ch, want)
   430  }
   431  
   432  func TestVStreamJournalOneToMany(t *testing.T) {
   433  	ctx, cancel := context.WithCancel(context.Background())
   434  	defer cancel()
   435  	cell := "aa"
   436  	ks := "TestVStream"
   437  	_ = createSandbox(ks)
   438  	hc := discovery.NewFakeHealthCheck(nil)
   439  	st := getSandboxTopo(ctx, cell, ks, []string{"-20", "-10", "10-20"})
   440  	vsm := newTestVStreamManager(hc, st, "aa")
   441  	sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   442  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   443  	sbc1 := hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "-10", topodatapb.TabletType_PRIMARY, true, 1, nil)
   444  	addTabletToSandboxTopo(t, st, ks, "-10", sbc1.Tablet())
   445  	sbc2 := hc.AddTestTablet(cell, "1.1.1.1", 1003, ks, "10-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   446  	addTabletToSandboxTopo(t, st, ks, "10-20", sbc2.Tablet())
   447  
   448  	send1 := []*binlogdatapb.VEvent{
   449  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"},
   450  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}},
   451  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}},
   452  		{Type: binlogdatapb.VEventType_COMMIT},
   453  	}
   454  	want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   455  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   456  			ShardGtids: []*binlogdatapb.ShardGtid{{
   457  				Keyspace: ks,
   458  				Shard:    "-20",
   459  				Gtid:     "gtid01",
   460  			}},
   461  		}},
   462  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}},
   463  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}},
   464  		{Type: binlogdatapb.VEventType_COMMIT},
   465  	}}
   466  	sbc0.AddVStreamEvents(send1, nil)
   467  
   468  	send2 := []*binlogdatapb.VEvent{
   469  		{Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{
   470  			Id:            1,
   471  			MigrationType: binlogdatapb.MigrationType_SHARDS,
   472  			ShardGtids: []*binlogdatapb.ShardGtid{{
   473  				Keyspace: ks,
   474  				Shard:    "-10",
   475  				Gtid:     "pos10",
   476  			}, {
   477  				Keyspace: ks,
   478  				Shard:    "10-20",
   479  				Gtid:     "pos1020",
   480  			}},
   481  			Participants: []*binlogdatapb.KeyspaceShard{{
   482  				Keyspace: ks,
   483  				Shard:    "-20",
   484  			}},
   485  		}},
   486  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"},
   487  		{Type: binlogdatapb.VEventType_COMMIT},
   488  	}
   489  	sbc0.AddVStreamEvents(send2, nil)
   490  
   491  	send3 := []*binlogdatapb.VEvent{
   492  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid03"},
   493  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t3"}},
   494  		{Type: binlogdatapb.VEventType_COMMIT},
   495  	}
   496  	sbc1.ExpectVStreamStartPos("pos10")
   497  	sbc1.AddVStreamEvents(send3, nil)
   498  
   499  	send4 := []*binlogdatapb.VEvent{
   500  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid04"},
   501  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t4"}},
   502  		{Type: binlogdatapb.VEventType_COMMIT},
   503  	}
   504  	sbc2.ExpectVStreamStartPos("pos1020")
   505  	sbc2.AddVStreamEvents(send4, nil)
   506  
   507  	vgtid := &binlogdatapb.VGtid{
   508  		ShardGtids: []*binlogdatapb.ShardGtid{{
   509  			Keyspace: ks,
   510  			Shard:    "-20",
   511  			Gtid:     "pos",
   512  		}},
   513  	}
   514  	ch := startVStream(ctx, t, vsm, vgtid, nil)
   515  	verifyEvents(t, ch, want1)
   516  
   517  	// The following two events from the different shards can come in any order.
   518  	// But the resulting VGTID should be the same after both are received.
   519  	<-ch
   520  	got := <-ch
   521  	wantevent := &binlogdatapb.VEvent{
   522  		Type: binlogdatapb.VEventType_VGTID,
   523  		Vgtid: &binlogdatapb.VGtid{
   524  			ShardGtids: []*binlogdatapb.ShardGtid{{
   525  				Keyspace: ks,
   526  				Shard:    "-10",
   527  				Gtid:     "gtid03",
   528  			}, {
   529  				Keyspace: ks,
   530  				Shard:    "10-20",
   531  				Gtid:     "gtid04",
   532  			}},
   533  		},
   534  	}
   535  	gotEvent := got.Events[0]
   536  	gotEvent.Keyspace = ""
   537  	gotEvent.Shard = ""
   538  	if !proto.Equal(gotEvent, wantevent) {
   539  		t.Errorf("vgtid: %v, want %v", got.Events[0], wantevent)
   540  	}
   541  }
   542  
   543  func TestVStreamJournalManyToOne(t *testing.T) {
   544  	ctx, cancel := context.WithCancel(context.Background())
   545  	defer cancel()
   546  
   547  	// Variable names are maintained like in OneToMany, but order is different.
   548  	ks := "TestVStream"
   549  	cell := "aa"
   550  	_ = createSandbox(ks)
   551  	hc := discovery.NewFakeHealthCheck(nil)
   552  	st := getSandboxTopo(ctx, cell, ks, []string{"-20", "-10", "10-20"})
   553  	vsm := newTestVStreamManager(hc, st, cell)
   554  	sbc0 := hc.AddTestTablet(cell, "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   555  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   556  	sbc1 := hc.AddTestTablet(cell, "1.1.1.1", 1002, ks, "-10", topodatapb.TabletType_PRIMARY, true, 1, nil)
   557  	addTabletToSandboxTopo(t, st, ks, "-10", sbc1.Tablet())
   558  	sbc2 := hc.AddTestTablet(cell, "1.1.1.1", 1003, ks, "10-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   559  	addTabletToSandboxTopo(t, st, ks, "10-20", sbc2.Tablet())
   560  
   561  	send3 := []*binlogdatapb.VEvent{
   562  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid03"},
   563  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t3"}},
   564  		{Type: binlogdatapb.VEventType_COMMIT},
   565  	}
   566  	sbc1.ExpectVStreamStartPos("pos10")
   567  	sbc1.AddVStreamEvents(send3, nil)
   568  
   569  	send4 := []*binlogdatapb.VEvent{
   570  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid04"},
   571  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t4"}},
   572  		{Type: binlogdatapb.VEventType_COMMIT},
   573  	}
   574  	sbc2.ExpectVStreamStartPos("pos1020")
   575  	sbc2.AddVStreamEvents(send4, nil)
   576  
   577  	send2 := []*binlogdatapb.VEvent{
   578  		{Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{
   579  			Id:            1,
   580  			MigrationType: binlogdatapb.MigrationType_SHARDS,
   581  			ShardGtids: []*binlogdatapb.ShardGtid{{
   582  				Keyspace: ks,
   583  				Shard:    "-20",
   584  				Gtid:     "pos20",
   585  			}},
   586  			Participants: []*binlogdatapb.KeyspaceShard{{
   587  				Keyspace: ks,
   588  				Shard:    "-10",
   589  			}, {
   590  				Keyspace: ks,
   591  				Shard:    "10-20",
   592  			}},
   593  		}},
   594  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"},
   595  		{Type: binlogdatapb.VEventType_COMMIT},
   596  	}
   597  	// Journal event has to be sent by both shards.
   598  	sbc1.AddVStreamEvents(send2, nil)
   599  	sbc2.AddVStreamEvents(send2, nil)
   600  
   601  	send1 := []*binlogdatapb.VEvent{
   602  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"},
   603  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}},
   604  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}},
   605  		{Type: binlogdatapb.VEventType_COMMIT},
   606  	}
   607  	want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   608  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   609  			ShardGtids: []*binlogdatapb.ShardGtid{{
   610  				Keyspace: ks,
   611  				Shard:    "-20",
   612  				Gtid:     "gtid01",
   613  			}},
   614  		}},
   615  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}},
   616  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}},
   617  		{Type: binlogdatapb.VEventType_COMMIT},
   618  	}}
   619  	sbc0.ExpectVStreamStartPos("pos20")
   620  	sbc0.AddVStreamEvents(send1, nil)
   621  
   622  	vgtid := &binlogdatapb.VGtid{
   623  		ShardGtids: []*binlogdatapb.ShardGtid{{
   624  			Keyspace: ks,
   625  			Shard:    "-10",
   626  			Gtid:     "pos10",
   627  		}, {
   628  			Keyspace: ks,
   629  			Shard:    "10-20",
   630  			Gtid:     "pos1020",
   631  		}},
   632  	}
   633  	ch := startVStream(ctx, t, vsm, vgtid, nil)
   634  	// The following two events from the different shards can come in any order.
   635  	// But the resulting VGTID should be the same after both are received.
   636  	<-ch
   637  	got := <-ch
   638  	wantevent := &binlogdatapb.VEvent{
   639  		Type: binlogdatapb.VEventType_VGTID,
   640  		Vgtid: &binlogdatapb.VGtid{
   641  			ShardGtids: []*binlogdatapb.ShardGtid{{
   642  				Keyspace: ks,
   643  				Shard:    "-10",
   644  				Gtid:     "gtid03",
   645  			}, {
   646  				Keyspace: ks,
   647  				Shard:    "10-20",
   648  				Gtid:     "gtid04",
   649  			}},
   650  		},
   651  	}
   652  	gotEvent := got.Events[0]
   653  	gotEvent.Keyspace = ""
   654  	gotEvent.Shard = ""
   655  	if !proto.Equal(gotEvent, wantevent) {
   656  		t.Errorf("vgtid: %v, want %v", got.Events[0], wantevent)
   657  	}
   658  	verifyEvents(t, ch, want1)
   659  }
   660  
   661  func TestVStreamJournalNoMatch(t *testing.T) {
   662  	ctx, cancel := context.WithCancel(context.Background())
   663  	defer cancel()
   664  
   665  	ks := "TestVStream"
   666  	cell := "aa"
   667  	_ = createSandbox(ks)
   668  	hc := discovery.NewFakeHealthCheck(nil)
   669  	st := getSandboxTopo(ctx, cell, ks, []string{"-20"})
   670  	vsm := newTestVStreamManager(hc, st, "aa")
   671  	sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   672  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
   673  
   674  	send1 := []*binlogdatapb.VEvent{
   675  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid01"},
   676  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "f0"}},
   677  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "t0"}},
   678  		{Type: binlogdatapb.VEventType_COMMIT},
   679  	}
   680  	want1 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   681  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   682  			ShardGtids: []*binlogdatapb.ShardGtid{{
   683  				Keyspace: ks,
   684  				Shard:    "-20",
   685  				Gtid:     "gtid01",
   686  			}},
   687  		}},
   688  		{Type: binlogdatapb.VEventType_FIELD, FieldEvent: &binlogdatapb.FieldEvent{TableName: "TestVStream.f0"}},
   689  		{Type: binlogdatapb.VEventType_ROW, RowEvent: &binlogdatapb.RowEvent{TableName: "TestVStream.t0"}},
   690  		{Type: binlogdatapb.VEventType_COMMIT},
   691  	}}
   692  	sbc0.AddVStreamEvents(send1, nil)
   693  
   694  	tableJournal := []*binlogdatapb.VEvent{
   695  		{Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{
   696  			Id:            1,
   697  			MigrationType: binlogdatapb.MigrationType_TABLES,
   698  		}},
   699  		{Type: binlogdatapb.VEventType_GTID, Gtid: "jn1"},
   700  		{Type: binlogdatapb.VEventType_COMMIT},
   701  	}
   702  	wantjn1 := &binlogdata.VStreamResponse{Events: []*binlogdatapb.VEvent{
   703  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   704  			ShardGtids: []*binlogdatapb.ShardGtid{{
   705  				Keyspace: ks,
   706  				Shard:    "-20",
   707  				Gtid:     "jn1",
   708  			}},
   709  		}},
   710  		{Type: binlogdatapb.VEventType_COMMIT},
   711  	}}
   712  	sbc0.AddVStreamEvents(tableJournal, nil)
   713  
   714  	send2 := []*binlogdatapb.VEvent{
   715  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid02"},
   716  		{Type: binlogdatapb.VEventType_DDL},
   717  	}
   718  	want2 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   719  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   720  			ShardGtids: []*binlogdatapb.ShardGtid{{
   721  				Keyspace: ks,
   722  				Shard:    "-20",
   723  				Gtid:     "gtid02",
   724  			}},
   725  		}},
   726  		{Type: binlogdatapb.VEventType_DDL},
   727  	}}
   728  	sbc0.AddVStreamEvents(send2, nil)
   729  
   730  	shardJournal := []*binlogdatapb.VEvent{
   731  		{Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{
   732  			Id:            2,
   733  			MigrationType: binlogdatapb.MigrationType_SHARDS,
   734  			ShardGtids: []*binlogdatapb.ShardGtid{{
   735  				Keyspace: ks,
   736  				Shard:    "c0-",
   737  				Gtid:     "posc0",
   738  			}},
   739  			Participants: []*binlogdatapb.KeyspaceShard{{
   740  				Keyspace: ks,
   741  				Shard:    "c0-e0",
   742  			}, {
   743  				Keyspace: ks,
   744  				Shard:    "e0-",
   745  			}},
   746  		}},
   747  		{Type: binlogdatapb.VEventType_GTID, Gtid: "jn2"},
   748  		{Type: binlogdatapb.VEventType_COMMIT},
   749  	}
   750  	wantjn2 := &binlogdata.VStreamResponse{Events: []*binlogdatapb.VEvent{
   751  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   752  			ShardGtids: []*binlogdatapb.ShardGtid{{
   753  				Keyspace: ks,
   754  				Shard:    "-20",
   755  				Gtid:     "jn2",
   756  			}},
   757  		}},
   758  		{Type: binlogdatapb.VEventType_COMMIT},
   759  	}}
   760  	sbc0.AddVStreamEvents(shardJournal, nil)
   761  
   762  	send3 := []*binlogdatapb.VEvent{
   763  		{Type: binlogdatapb.VEventType_GTID, Gtid: "gtid03"},
   764  		{Type: binlogdatapb.VEventType_DDL},
   765  	}
   766  	want3 := &binlogdatapb.VStreamResponse{Events: []*binlogdatapb.VEvent{
   767  		{Type: binlogdatapb.VEventType_VGTID, Vgtid: &binlogdatapb.VGtid{
   768  			ShardGtids: []*binlogdatapb.ShardGtid{{
   769  				Keyspace: ks,
   770  				Shard:    "-20",
   771  				Gtid:     "gtid03",
   772  			}},
   773  		}},
   774  		{Type: binlogdatapb.VEventType_DDL},
   775  	}}
   776  	sbc0.AddVStreamEvents(send3, nil)
   777  
   778  	vgtid := &binlogdatapb.VGtid{
   779  		ShardGtids: []*binlogdatapb.ShardGtid{{
   780  			Keyspace: ks,
   781  			Shard:    "-20",
   782  			Gtid:     "pos",
   783  		}},
   784  	}
   785  	ch := startVStream(ctx, t, vsm, vgtid, nil)
   786  	verifyEvents(t, ch, want1, wantjn1, want2, wantjn2, want3)
   787  }
   788  
   789  func TestVStreamJournalPartialMatch(t *testing.T) {
   790  	ctx, cancel := context.WithCancel(context.Background())
   791  	defer cancel()
   792  
   793  	// Variable names are maintained like in OneToMany, but order is different.1
   794  	ks := "TestVStream"
   795  	cell := "aa"
   796  	_ = createSandbox(ks)
   797  	hc := discovery.NewFakeHealthCheck(nil)
   798  	st := getSandboxTopo(ctx, cell, ks, []string{"-20", "-10", "10-20"})
   799  	vsm := newTestVStreamManager(hc, st, "aa")
   800  	sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, ks, "-10", topodatapb.TabletType_PRIMARY, true, 1, nil)
   801  	addTabletToSandboxTopo(t, st, ks, "-10", sbc1.Tablet())
   802  	sbc2 := hc.AddTestTablet("aa", "1.1.1.1", 1003, ks, "10-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
   803  	addTabletToSandboxTopo(t, st, ks, "10-20", sbc2.Tablet())
   804  
   805  	send := []*binlogdatapb.VEvent{
   806  		{Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{
   807  			Id:            1,
   808  			MigrationType: binlogdatapb.MigrationType_SHARDS,
   809  			ShardGtids: []*binlogdatapb.ShardGtid{{
   810  				Keyspace: ks,
   811  				Shard:    "10-30",
   812  				Gtid:     "pos1040",
   813  			}},
   814  			Participants: []*binlogdatapb.KeyspaceShard{{
   815  				Keyspace: ks,
   816  				Shard:    "10-20",
   817  			}, {
   818  				Keyspace: ks,
   819  				Shard:    "20-30",
   820  			}},
   821  		}},
   822  	}
   823  	sbc2.AddVStreamEvents(send, nil)
   824  
   825  	vgtid := &binlogdatapb.VGtid{
   826  		ShardGtids: []*binlogdatapb.ShardGtid{{
   827  			Keyspace: ks,
   828  			Shard:    "-10",
   829  			Gtid:     "pos10",
   830  		}, {
   831  			Keyspace: ks,
   832  			Shard:    "10-20",
   833  			Gtid:     "pos1020",
   834  		}},
   835  	}
   836  	err := vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error {
   837  		t.Errorf("unexpected events: %v", events)
   838  		return nil
   839  	})
   840  	wantErr := "not all journaling participants are in the stream"
   841  	if err == nil || !strings.Contains(err.Error(), wantErr) {
   842  		t.Errorf("vstream end: %v, must contain %v", err, wantErr)
   843  	}
   844  
   845  	// Try a different order (different code path)
   846  	send = []*binlogdatapb.VEvent{
   847  		{Type: binlogdatapb.VEventType_JOURNAL, Journal: &binlogdatapb.Journal{
   848  			Id:            1,
   849  			MigrationType: binlogdatapb.MigrationType_SHARDS,
   850  			ShardGtids: []*binlogdatapb.ShardGtid{{
   851  				Keyspace: ks,
   852  				Shard:    "10-30",
   853  				Gtid:     "pos1040",
   854  			}},
   855  			Participants: []*binlogdatapb.KeyspaceShard{{
   856  				Keyspace: ks,
   857  				Shard:    "20-30",
   858  			}, {
   859  				Keyspace: ks,
   860  				Shard:    "10-20",
   861  			}},
   862  		}},
   863  	}
   864  	sbc2.AddVStreamEvents(send, nil)
   865  	err = vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{}, func(events []*binlogdatapb.VEvent) error {
   866  		t.Errorf("unexpected events: %v", events)
   867  		return nil
   868  	})
   869  	if err == nil || !strings.Contains(err.Error(), wantErr) {
   870  		t.Errorf("vstream end: %v, must contain %v", err, wantErr)
   871  	}
   872  	cancel()
   873  }
   874  
   875  func TestResolveVStreamParams(t *testing.T) {
   876  	name := "TestVStream"
   877  	_ = createSandbox(name)
   878  	hc := discovery.NewFakeHealthCheck(nil)
   879  	vsm := newTestVStreamManager(hc, newSandboxForCells([]string{"aa"}), "aa")
   880  	testcases := []struct {
   881  		input  *binlogdatapb.VGtid
   882  		output *binlogdatapb.VGtid
   883  		err    string
   884  	}{{
   885  		input: nil,
   886  		err:   "vgtid must have at least one value with a starting position",
   887  	}, {
   888  		input: &binlogdatapb.VGtid{
   889  			ShardGtids: []*binlogdatapb.ShardGtid{{}},
   890  		},
   891  		err: "for an empty keyspace, the Gtid value must be 'current'",
   892  	}, {
   893  		input: &binlogdatapb.VGtid{
   894  			ShardGtids: []*binlogdatapb.ShardGtid{{
   895  				Keyspace: "TestVStream",
   896  			}},
   897  		},
   898  		err: "if shards are unspecified, the Gtid value must be 'current'",
   899  	}, {
   900  		input: &binlogdatapb.VGtid{
   901  			ShardGtids: []*binlogdatapb.ShardGtid{{
   902  				Keyspace: "TestVStream",
   903  				Gtid:     "current",
   904  			}},
   905  		},
   906  		output: &binlogdatapb.VGtid{
   907  			ShardGtids: []*binlogdatapb.ShardGtid{{
   908  				Keyspace: "TestVStream",
   909  				Shard:    "-20",
   910  				Gtid:     "current",
   911  			}, {
   912  				Keyspace: "TestVStream",
   913  				Shard:    "20-40",
   914  				Gtid:     "current",
   915  			}, {
   916  				Keyspace: "TestVStream",
   917  				Shard:    "40-60",
   918  				Gtid:     "current",
   919  			}, {
   920  				Keyspace: "TestVStream",
   921  				Shard:    "60-80",
   922  				Gtid:     "current",
   923  			}, {
   924  				Keyspace: "TestVStream",
   925  				Shard:    "80-a0",
   926  				Gtid:     "current",
   927  			}, {
   928  				Keyspace: "TestVStream",
   929  				Shard:    "a0-c0",
   930  				Gtid:     "current",
   931  			}, {
   932  				Keyspace: "TestVStream",
   933  				Shard:    "c0-e0",
   934  				Gtid:     "current",
   935  			}, {
   936  				Keyspace: "TestVStream",
   937  				Shard:    "e0-",
   938  				Gtid:     "current",
   939  			}},
   940  		},
   941  	}, {
   942  		input: &binlogdatapb.VGtid{
   943  			ShardGtids: []*binlogdatapb.ShardGtid{{
   944  				Keyspace: "TestVStream",
   945  				Shard:    "-20",
   946  				Gtid:     "current",
   947  			}},
   948  		},
   949  		output: &binlogdatapb.VGtid{
   950  			ShardGtids: []*binlogdatapb.ShardGtid{{
   951  				Keyspace: "TestVStream",
   952  				Shard:    "-20",
   953  				Gtid:     "current",
   954  			}},
   955  		},
   956  	}, {
   957  		input: &binlogdatapb.VGtid{
   958  			ShardGtids: []*binlogdatapb.ShardGtid{{
   959  				Keyspace: "TestVStream",
   960  				Shard:    "-20",
   961  				Gtid:     "other",
   962  			}},
   963  		},
   964  		output: &binlogdatapb.VGtid{
   965  			ShardGtids: []*binlogdatapb.ShardGtid{{
   966  				Keyspace: "TestVStream",
   967  				Shard:    "-20",
   968  				Gtid:     "other",
   969  			}},
   970  		},
   971  	}}
   972  	wantFilter := &binlogdatapb.Filter{
   973  		Rules: []*binlogdatapb.Rule{{
   974  			Match: "/.*",
   975  		}},
   976  	}
   977  	for _, tcase := range testcases {
   978  		vgtid, filter, flags, err := vsm.resolveParams(context.Background(), topodatapb.TabletType_REPLICA, tcase.input, nil, nil)
   979  		if tcase.err != "" {
   980  			if err == nil || !strings.Contains(err.Error(), tcase.err) {
   981  				t.Errorf("resolve(%v) err: %v, must contain %v", tcase.input, err, tcase.err)
   982  			}
   983  			continue
   984  		}
   985  		require.NoError(t, err, tcase.input)
   986  		assert.Equal(t, tcase.output, vgtid, tcase.input)
   987  		assert.Equal(t, wantFilter, filter, tcase.input)
   988  		require.False(t, flags.MinimizeSkew)
   989  	}
   990  	// Special-case: empty keyspace because output is too big.
   991  	input := &binlogdatapb.VGtid{
   992  		ShardGtids: []*binlogdatapb.ShardGtid{{
   993  			Gtid: "current",
   994  		}},
   995  	}
   996  	vgtid, _, _, err := vsm.resolveParams(context.Background(), topodatapb.TabletType_REPLICA, input, nil, nil)
   997  	require.NoError(t, err, input)
   998  	if got, want := len(vgtid.ShardGtids), 8; want >= got {
   999  		t.Errorf("len(vgtid.ShardGtids): %v, must be >%d", got, want)
  1000  	}
  1001  	for _, minimizeSkew := range []bool{true, false} {
  1002  		t.Run(fmt.Sprintf("resolveParams MinimizeSkew %t", minimizeSkew), func(t *testing.T) {
  1003  			flags := &vtgatepb.VStreamFlags{MinimizeSkew: minimizeSkew}
  1004  			vgtid := &binlogdatapb.VGtid{
  1005  				ShardGtids: []*binlogdatapb.ShardGtid{{
  1006  					Keyspace: "TestVStream",
  1007  					Shard:    "-20",
  1008  					Gtid:     "current",
  1009  				}},
  1010  			}
  1011  			_, _, flags2, err := vsm.resolveParams(context.Background(), topodatapb.TabletType_REPLICA, vgtid, nil, flags)
  1012  			require.NoError(t, err)
  1013  			require.Equal(t, minimizeSkew, flags2.MinimizeSkew)
  1014  		})
  1015  	}
  1016  
  1017  }
  1018  
  1019  func TestVStreamIdleHeartbeat(t *testing.T) {
  1020  	cell := "aa"
  1021  	ks := "TestVStream"
  1022  	_ = createSandbox(ks)
  1023  	hc := discovery.NewFakeHealthCheck(nil)
  1024  	st := getSandboxTopo(ctx, cell, ks, []string{"-20"})
  1025  	vsm := newTestVStreamManager(hc, st, cell)
  1026  	sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, ks, "-20", topodatapb.TabletType_PRIMARY, true, 1, nil)
  1027  	addTabletToSandboxTopo(t, st, ks, "-20", sbc0.Tablet())
  1028  	vgtid := &binlogdatapb.VGtid{
  1029  		ShardGtids: []*binlogdatapb.ShardGtid{{
  1030  			Keyspace: ks,
  1031  			Shard:    "-20",
  1032  			Gtid:     "pos",
  1033  		}},
  1034  	}
  1035  
  1036  	type testcase struct {
  1037  		name              string
  1038  		heartbeatInterval uint32
  1039  		want              int
  1040  	}
  1041  	// each test waits for 4.5 seconds, hence expected #heartbeats = floor(4.5/heartbeatInterval)
  1042  	testcases := []testcase{
  1043  		{"off", 0, 0},
  1044  		{"on:1s", 1, 4},
  1045  		{"on:2s", 2, 2},
  1046  	}
  1047  	for _, tcase := range testcases {
  1048  		t.Run(tcase.name, func(t *testing.T) {
  1049  			var mu sync.Mutex
  1050  			var heartbeatCount int
  1051  			ctx, cancel := context.WithCancel(context.Background())
  1052  			go func() {
  1053  				vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, &vtgatepb.VStreamFlags{HeartbeatInterval: tcase.heartbeatInterval},
  1054  					func(events []*binlogdatapb.VEvent) error {
  1055  						mu.Lock()
  1056  						defer mu.Unlock()
  1057  						for _, event := range events {
  1058  							if event.Type == binlogdatapb.VEventType_HEARTBEAT {
  1059  								heartbeatCount++
  1060  							}
  1061  						}
  1062  						return nil
  1063  					})
  1064  			}()
  1065  			time.Sleep(time.Duration(4500) * time.Millisecond)
  1066  			mu.Lock()
  1067  			defer mu.Unlock()
  1068  			require.Equalf(t, heartbeatCount, tcase.want, "got %d, want %d", heartbeatCount, tcase.want)
  1069  			cancel()
  1070  		})
  1071  	}
  1072  }
  1073  
  1074  func newTestVStreamManager(hc discovery.HealthCheck, serv srvtopo.Server, cell string) *vstreamManager {
  1075  	gw := NewTabletGateway(context.Background(), hc, serv, cell)
  1076  	srvResolver := srvtopo.NewResolver(serv, gw, cell)
  1077  	return newVStreamManager(srvResolver, serv, cell)
  1078  }
  1079  
  1080  func startVStream(ctx context.Context, t *testing.T, vsm *vstreamManager, vgtid *binlogdatapb.VGtid, flags *vtgatepb.VStreamFlags) <-chan *binlogdatapb.VStreamResponse {
  1081  	if flags == nil {
  1082  		flags = &vtgatepb.VStreamFlags{}
  1083  	}
  1084  	ch := make(chan *binlogdatapb.VStreamResponse)
  1085  	go func() {
  1086  		_ = vsm.VStream(ctx, topodatapb.TabletType_PRIMARY, vgtid, nil, flags, func(events []*binlogdatapb.VEvent) error {
  1087  			ch <- &binlogdatapb.VStreamResponse{Events: events}
  1088  			return nil
  1089  		})
  1090  	}()
  1091  	return ch
  1092  }
  1093  
  1094  func verifyEvents(t *testing.T, ch <-chan *binlogdatapb.VStreamResponse, wants ...*binlogdatapb.VStreamResponse) {
  1095  	t.Helper()
  1096  	for i, want := range wants {
  1097  		got := <-ch
  1098  		require.NotNil(t, got)
  1099  		for _, event := range got.Events {
  1100  			event.Timestamp = 0
  1101  		}
  1102  		if !proto.Equal(got, want) {
  1103  			t.Errorf("vstream(%d):\n%v, want\n%v", i, got, want)
  1104  		}
  1105  	}
  1106  }
  1107  
  1108  func getVEvents(keyspace, shard string, count, idx int64) []*binlogdatapb.VEvent {
  1109  	mu.Lock()
  1110  	defer mu.Unlock()
  1111  	var vevents []*binlogdatapb.VEvent
  1112  	var i int64
  1113  	currentTime := time.Now().Unix()
  1114  	for i = count; i > 0; i-- {
  1115  		j := i + idx
  1116  		vevents = append(vevents, &binlogdatapb.VEvent{
  1117  			Type: binlogdatapb.VEventType_GTID, Gtid: fmt.Sprintf("gtid-%s-%d", shard, j),
  1118  			Timestamp:   currentTime - j,
  1119  			CurrentTime: currentTime * 1e9,
  1120  			Keyspace:    keyspace,
  1121  			Shard:       shard,
  1122  		})
  1123  
  1124  		vevents = append(vevents, &binlogdatapb.VEvent{
  1125  			Type:        binlogdatapb.VEventType_COMMIT,
  1126  			Timestamp:   currentTime - j,
  1127  			CurrentTime: currentTime * 1e9,
  1128  			Keyspace:    keyspace,
  1129  			Shard:       shard,
  1130  		})
  1131  	}
  1132  	return vevents
  1133  }
  1134  
  1135  func getSandboxTopo(ctx context.Context, cell string, keyspace string, shards []string) *sandboxTopo {
  1136  	st := newSandboxForCells([]string{cell})
  1137  	ts := st.topoServer
  1138  	ts.CreateCellInfo(ctx, cell, &topodatapb.CellInfo{})
  1139  	ts.CreateKeyspace(ctx, keyspace, &topodatapb.Keyspace{})
  1140  	for _, shard := range shards {
  1141  		ts.CreateShard(ctx, keyspace, shard)
  1142  	}
  1143  	return st
  1144  }
  1145  
  1146  func addTabletToSandboxTopo(t *testing.T, st *sandboxTopo, ks, shard string, tablet *topodatapb.Tablet) {
  1147  	_, err := st.topoServer.UpdateShardFields(ctx, ks, shard, func(si *topo.ShardInfo) error {
  1148  		si.PrimaryAlias = tablet.Alias
  1149  		return nil
  1150  	})
  1151  	require.NoError(t, err)
  1152  	err = st.topoServer.CreateTablet(ctx, tablet)
  1153  	require.NoError(t, err)
  1154  }