vitess.io/vitess@v0.16.2/go/vt/vtctl/workflow/stream_migrator.go (about)

     1  /*
     2  Copyright 2021 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8  	http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package workflow
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"sync"
    24  	"text/template"
    25  
    26  	"google.golang.org/protobuf/encoding/prototext"
    27  
    28  	"vitess.io/vitess/go/mysql"
    29  	"vitess.io/vitess/go/sqltypes"
    30  	"vitess.io/vitess/go/vt/binlog/binlogplayer"
    31  	"vitess.io/vitess/go/vt/concurrency"
    32  	"vitess.io/vitess/go/vt/key"
    33  	"vitess.io/vitess/go/vt/logutil"
    34  	"vitess.io/vitess/go/vt/schema"
    35  	"vitess.io/vitess/go/vt/sqlparser"
    36  	"vitess.io/vitess/go/vt/topo"
    37  	"vitess.io/vitess/go/vt/vterrors"
    38  	"vitess.io/vitess/go/vt/vtgate/vindexes"
    39  	"vitess.io/vitess/go/vt/vttablet/tabletmanager/vreplication"
    40  
    41  	binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
    42  )
    43  
    44  // StreamType is an enum representing the kind of stream.
    45  //
    46  // (TODO:@ajm188) This should be made package-private once the last references
    47  // in package wrangler are removed.
    48  type StreamType int
    49  
    50  // StreamType values.
    51  const (
    52  	StreamTypeUnknown = StreamType(iota)
    53  	StreamTypeSharded
    54  	StreamTypeReference
    55  )
    56  
    57  // StreamMigrator contains information needed to migrate a stream
    58  type StreamMigrator struct {
    59  	streams   map[string][]*VReplicationStream
    60  	workflows []string
    61  	templates []*VReplicationStream
    62  	ts        ITrafficSwitcher
    63  	logger    logutil.Logger
    64  }
    65  
    66  // BuildStreamMigrator creates a new StreamMigrator based on the given
    67  // TrafficSwitcher.
    68  func BuildStreamMigrator(ctx context.Context, ts ITrafficSwitcher, cancelMigrate bool) (*StreamMigrator, error) {
    69  	sm := &StreamMigrator{
    70  		ts:     ts,
    71  		logger: ts.Logger(),
    72  	}
    73  
    74  	if sm.ts.MigrationType() == binlogdatapb.MigrationType_TABLES {
    75  		// Source streams should be stopped only for shard migrations.
    76  		return sm, nil
    77  	}
    78  
    79  	var err error
    80  
    81  	sm.streams, err = sm.readSourceStreams(ctx, cancelMigrate)
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  
    86  	// Loop executes only once.
    87  	for _, tabletStreams := range sm.streams {
    88  		tmpl, err := sm.templatize(ctx, tabletStreams)
    89  		if err != nil {
    90  			return nil, err
    91  		}
    92  
    93  		sm.workflows = VReplicationStreams(tmpl).Workflows()
    94  		break
    95  	}
    96  
    97  	return sm, nil
    98  }
    99  
   100  // StreamMigratorFinalize finalizes the stream migration.
   101  //
   102  // (TODO:@ajm88) in the original implementation, "it's a standalone function
   103  // because it does not use the streamMigrater state". That's still true, but
   104  // moving this to a method on StreamMigrator would provide a cleaner namespacing
   105  // in package workflow. But, we would have to update more callers in order to
   106  // do that (*wrangler.switcher's streamMigrateFinalize would need to change its
   107  // signature to also take a *workflow.StreamMigrator), so we will do that in a
   108  // second PR.
   109  func StreamMigratorFinalize(ctx context.Context, ts ITrafficSwitcher, workflows []string) error {
   110  	if len(workflows) == 0 {
   111  		return nil
   112  	}
   113  
   114  	workflowList := stringListify(workflows)
   115  	err := ts.ForAllSources(func(source *MigrationSource) error {
   116  		query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow in (%s)", encodeString(source.GetPrimary().DbName()), workflowList)
   117  		_, err := ts.VReplicationExec(ctx, source.GetPrimary().Alias, query)
   118  		return err
   119  	})
   120  
   121  	if err != nil {
   122  		return err
   123  	}
   124  
   125  	err = ts.ForAllTargets(func(target *MigrationTarget) error {
   126  		query := fmt.Sprintf("update _vt.vreplication set state='Running' where db_name=%s and workflow in (%s)", encodeString(target.GetPrimary().DbName()), workflowList)
   127  		_, err := ts.VReplicationExec(ctx, target.GetPrimary().Alias, query)
   128  		return err
   129  	})
   130  
   131  	return err
   132  }
   133  
   134  // Streams returns a deep-copy of the StreamMigrator's streams map.
   135  func (sm *StreamMigrator) Streams() map[string][]*VReplicationStream {
   136  	streams := make(map[string][]*VReplicationStream, len(sm.streams))
   137  
   138  	for k, v := range sm.streams {
   139  		streams[k] = VReplicationStreams(v).Copy().ToSlice()
   140  	}
   141  
   142  	return streams
   143  }
   144  
   145  // Templates returns a copy of the StreamMigrator's template streams.
   146  func (sm *StreamMigrator) Templates() []*VReplicationStream {
   147  	return VReplicationStreams(sm.templates).Copy().ToSlice()
   148  }
   149  
   150  // CancelMigration cancels a migration
   151  func (sm *StreamMigrator) CancelMigration(ctx context.Context) {
   152  	if sm.streams == nil {
   153  		return
   154  	}
   155  
   156  	_ = sm.deleteTargetStreams(ctx)
   157  
   158  	err := sm.ts.ForAllSources(func(source *MigrationSource) error {
   159  		query := fmt.Sprintf("update _vt.vreplication set state='Running', stop_pos=null, message='' where db_name=%s and workflow != %s", encodeString(source.GetPrimary().DbName()), encodeString(sm.ts.ReverseWorkflowName()))
   160  		_, err := sm.ts.VReplicationExec(ctx, source.GetPrimary().Alias, query)
   161  		return err
   162  	})
   163  	if err != nil {
   164  		sm.logger.Errorf("Cancel migration failed: could not restart source streams: %v", err)
   165  	}
   166  }
   167  
   168  // MigrateStreams migrates N streams
   169  func (sm *StreamMigrator) MigrateStreams(ctx context.Context) error {
   170  	if sm.streams == nil {
   171  		return nil
   172  	}
   173  
   174  	if err := sm.deleteTargetStreams(ctx); err != nil {
   175  		return err
   176  	}
   177  
   178  	return sm.createTargetStreams(ctx, sm.templates)
   179  }
   180  
   181  // StopStreams stops streams
   182  func (sm *StreamMigrator) StopStreams(ctx context.Context) ([]string, error) {
   183  	if sm.streams == nil {
   184  		return nil, nil
   185  	}
   186  
   187  	if err := sm.stopSourceStreams(ctx); err != nil {
   188  		return nil, err
   189  	}
   190  
   191  	positions, err := sm.syncSourceStreams(ctx)
   192  	if err != nil {
   193  		return nil, err
   194  	}
   195  
   196  	return sm.verifyStreamPositions(ctx, positions)
   197  }
   198  
   199  /* tablet streams */
   200  
   201  func (sm *StreamMigrator) readTabletStreams(ctx context.Context, ti *topo.TabletInfo, constraint string) ([]*VReplicationStream, error) {
   202  	query := fmt.Sprintf("select id, workflow, source, pos, workflow_type, workflow_sub_type, defer_secondary_keys from _vt.vreplication where db_name=%s and workflow != %s",
   203  		encodeString(ti.DbName()), encodeString(sm.ts.ReverseWorkflowName()))
   204  	if constraint != "" {
   205  		query += fmt.Sprintf(" and %s", constraint)
   206  	}
   207  
   208  	p3qr, err := sm.ts.TabletManagerClient().VReplicationExec(ctx, ti.Tablet, query)
   209  	if err != nil {
   210  		return nil, err
   211  	}
   212  
   213  	qr := sqltypes.Proto3ToResult(p3qr)
   214  	tabletStreams := make([]*VReplicationStream, 0, len(qr.Rows))
   215  
   216  	for _, row := range qr.Named().Rows {
   217  		id, err := row["id"].ToInt64()
   218  		if err != nil {
   219  			return nil, err
   220  		}
   221  
   222  		workflowName := row["workflow"].ToString()
   223  		switch workflowName {
   224  		case "":
   225  			return nil, fmt.Errorf("VReplication streams must have named workflows for migration: shard: %s:%s, stream: %d",
   226  				ti.Keyspace, ti.Shard, id)
   227  		case sm.ts.WorkflowName():
   228  			return nil, fmt.Errorf("VReplication stream has the same workflow name as the resharding workflow: shard: %s:%s, stream: %d",
   229  				ti.Keyspace, ti.Shard, id)
   230  		}
   231  
   232  		workflowType, err := row["workflow_type"].ToInt64()
   233  		if err != nil {
   234  			return nil, err
   235  		}
   236  		workflowSubType, err := row["workflow_sub_type"].ToInt64()
   237  		if err != nil {
   238  			return nil, err
   239  		}
   240  
   241  		deferSecondaryKeys, err := row["defer_secondary_keys"].ToBool()
   242  		if err != nil {
   243  			return nil, err
   244  		}
   245  
   246  		var bls binlogdatapb.BinlogSource
   247  		rowBytes, err := row["source"].ToBytes()
   248  		if err != nil {
   249  			return nil, err
   250  		}
   251  		if err := prototext.Unmarshal(rowBytes, &bls); err != nil {
   252  			return nil, err
   253  		}
   254  
   255  		isReference, err := sm.blsIsReference(&bls)
   256  		if err != nil {
   257  			return nil, vterrors.Wrap(err, "blsIsReference")
   258  		}
   259  
   260  		if isReference {
   261  			sm.ts.Logger().Infof("readTabletStreams: ignoring reference table %+v", &bls)
   262  			continue
   263  		}
   264  
   265  		pos, err := mysql.DecodePosition(row["pos"].ToString())
   266  		if err != nil {
   267  			return nil, err
   268  		}
   269  
   270  		tabletStreams = append(tabletStreams, &VReplicationStream{
   271  			ID:                 uint32(id),
   272  			Workflow:           workflowName,
   273  			BinlogSource:       &bls,
   274  			Position:           pos,
   275  			WorkflowType:       binlogdatapb.VReplicationWorkflowType(workflowType),
   276  			WorkflowSubType:    binlogdatapb.VReplicationWorkflowSubType(workflowSubType),
   277  			DeferSecondaryKeys: deferSecondaryKeys,
   278  		})
   279  	}
   280  	return tabletStreams, nil
   281  }
   282  
   283  /* source streams */
   284  
   285  func (sm *StreamMigrator) readSourceStreams(ctx context.Context, cancelMigrate bool) (map[string][]*VReplicationStream, error) {
   286  	var (
   287  		mu      sync.Mutex
   288  		streams = make(map[string][]*VReplicationStream)
   289  	)
   290  
   291  	err := sm.ts.ForAllSources(func(source *MigrationSource) error {
   292  		if !cancelMigrate {
   293  			// This flow protects us from the following scenario: When we create streams,
   294  			// we always do it in two phases. We start them off as Stopped, and then
   295  			// update them to Running. If such an operation fails, we may be left with
   296  			// lingering Stopped streams. They should actually be cleaned up by the user.
   297  			// In the current workflow, we stop streams and restart them.
   298  			// Once existing streams are stopped, there will be confusion about which of
   299  			// them can be restarted because they will be no different from the lingering streams.
   300  			// To prevent this confusion, we first check if there are any stopped streams.
   301  			// If so, we request the operator to clean them up, or restart them before going ahead.
   302  			// This allows us to assume that all stopped streams can be safely restarted
   303  			// if we cancel the operation.
   304  			stoppedStreams, err := sm.readTabletStreams(ctx, source.GetPrimary(), "state = 'Stopped' and message != 'FROZEN'")
   305  			if err != nil {
   306  				return err
   307  			}
   308  
   309  			if len(stoppedStreams) != 0 {
   310  				return fmt.Errorf("cannot migrate until all streams are running: %s: %d", source.GetShard().ShardName(), source.GetPrimary().Alias.Uid)
   311  			}
   312  		}
   313  
   314  		tabletStreams, err := sm.readTabletStreams(ctx, source.GetPrimary(), "")
   315  		if err != nil {
   316  			return err
   317  		}
   318  
   319  		if len(tabletStreams) == 0 {
   320  			// No VReplication is running. So, we have no work to do.
   321  			return nil
   322  		}
   323  
   324  		query := fmt.Sprintf("select distinct vrepl_id from _vt.copy_state where vrepl_id in %s", VReplicationStreams(tabletStreams).Values())
   325  		p3qr, err := sm.ts.TabletManagerClient().VReplicationExec(ctx, source.GetPrimary().Tablet, query)
   326  		switch {
   327  		case err != nil:
   328  			return err
   329  		case len(p3qr.Rows) != 0:
   330  			return fmt.Errorf("cannot migrate while vreplication streams in source shards are still copying: %s", source.GetShard().ShardName())
   331  		}
   332  
   333  		mu.Lock()
   334  		defer mu.Unlock()
   335  		streams[source.GetShard().ShardName()] = tabletStreams
   336  		return nil
   337  	})
   338  
   339  	if err != nil {
   340  		return nil, err
   341  	}
   342  
   343  	// Validate that streams match across source shards.
   344  	var (
   345  		reference []*VReplicationStream
   346  		refshard  string
   347  		streams2  = make(map[string][]*VReplicationStream)
   348  	)
   349  
   350  	for k, v := range streams {
   351  		if reference == nil {
   352  			refshard = k
   353  			reference = v
   354  			continue
   355  		}
   356  
   357  		streams2[k] = append([]*VReplicationStream(nil), v...)
   358  	}
   359  
   360  	for shard, tabletStreams := range streams2 {
   361  		for _, refStream := range reference {
   362  			err := func() error {
   363  				for i := 0; i < len(tabletStreams); i++ {
   364  					vrs := tabletStreams[i]
   365  
   366  					if refStream.Workflow == vrs.Workflow &&
   367  						refStream.BinlogSource.Keyspace == vrs.BinlogSource.Keyspace &&
   368  						refStream.BinlogSource.Shard == vrs.BinlogSource.Shard {
   369  						// Delete the matched item and scan for the next stream.
   370  						tabletStreams = append(tabletStreams[:i], tabletStreams[i+1:]...)
   371  						return nil
   372  					}
   373  				}
   374  
   375  				return fmt.Errorf("streams are mismatched across source shards: %s vs %s", refshard, shard)
   376  			}()
   377  
   378  			if err != nil {
   379  				return nil, err
   380  			}
   381  		}
   382  
   383  		if len(tabletStreams) != 0 {
   384  			return nil, fmt.Errorf("streams are mismatched across source shards: %s vs %s", refshard, shard)
   385  		}
   386  	}
   387  
   388  	return streams, nil
   389  }
   390  
   391  func (sm *StreamMigrator) stopSourceStreams(ctx context.Context) error {
   392  	var (
   393  		mu             sync.Mutex
   394  		stoppedStreams = make(map[string][]*VReplicationStream)
   395  	)
   396  
   397  	err := sm.ts.ForAllSources(func(source *MigrationSource) error {
   398  		tabletStreams := sm.streams[source.GetShard().ShardName()]
   399  		if len(tabletStreams) == 0 {
   400  			return nil
   401  		}
   402  
   403  		query := fmt.Sprintf("update _vt.vreplication set state='Stopped', message='for cutover' where id in %s", VReplicationStreams(tabletStreams).Values())
   404  		_, err := sm.ts.TabletManagerClient().VReplicationExec(ctx, source.GetPrimary().Tablet, query)
   405  		if err != nil {
   406  			return err
   407  		}
   408  
   409  		tabletStreams, err = sm.readTabletStreams(ctx, source.GetPrimary(), fmt.Sprintf("id in %s", VReplicationStreams(tabletStreams).Values()))
   410  		if err != nil {
   411  			return err
   412  		}
   413  
   414  		mu.Lock()
   415  		defer mu.Unlock()
   416  		stoppedStreams[source.GetShard().ShardName()] = tabletStreams
   417  
   418  		return nil
   419  	})
   420  
   421  	if err != nil {
   422  		return err
   423  	}
   424  
   425  	sm.streams = stoppedStreams
   426  	return nil
   427  }
   428  
   429  func (sm *StreamMigrator) syncSourceStreams(ctx context.Context) (map[string]mysql.Position, error) {
   430  	stopPositions := make(map[string]mysql.Position)
   431  
   432  	for _, tabletStreams := range sm.streams {
   433  		for _, vrs := range tabletStreams {
   434  			key := fmt.Sprintf("%s:%s", vrs.BinlogSource.Keyspace, vrs.BinlogSource.Shard)
   435  			if pos, ok := stopPositions[key]; !ok || vrs.Position.AtLeast(pos) {
   436  				sm.ts.Logger().Infof("syncSourceStreams setting stopPositions +%s %+v %d", key, vrs.Position, vrs.ID)
   437  				stopPositions[key] = vrs.Position
   438  			}
   439  		}
   440  	}
   441  
   442  	var (
   443  		wg        sync.WaitGroup
   444  		allErrors concurrency.AllErrorRecorder
   445  	)
   446  
   447  	for shard, tabletStreams := range sm.streams {
   448  		for _, vrs := range tabletStreams {
   449  			key := fmt.Sprintf("%s:%s", vrs.BinlogSource.Keyspace, vrs.BinlogSource.Shard)
   450  			pos := stopPositions[key]
   451  			sm.ts.Logger().Infof("syncSourceStreams before go func +%s %+v %d", key, pos, vrs.ID)
   452  
   453  			if vrs.Position.Equal(pos) {
   454  				continue
   455  			}
   456  
   457  			wg.Add(1)
   458  			go func(vrs *VReplicationStream, shard string, pos mysql.Position) {
   459  				defer wg.Done()
   460  				sm.ts.Logger().Infof("syncSourceStreams beginning of go func %s %s %+v %d", shard, vrs.BinlogSource.Shard, pos, vrs.ID)
   461  
   462  				si, err := sm.ts.TopoServer().GetShard(ctx, sm.ts.SourceKeyspaceName(), shard)
   463  				if err != nil {
   464  					allErrors.RecordError(err)
   465  					return
   466  				}
   467  
   468  				primary, err := sm.ts.TopoServer().GetTablet(ctx, si.PrimaryAlias)
   469  				if err != nil {
   470  					allErrors.RecordError(err)
   471  					return
   472  				}
   473  
   474  				query := fmt.Sprintf("update _vt.vreplication set state='Running', stop_pos='%s', message='synchronizing for cutover' where id=%d", mysql.EncodePosition(pos), vrs.ID)
   475  				if _, err := sm.ts.TabletManagerClient().VReplicationExec(ctx, primary.Tablet, query); err != nil {
   476  					allErrors.RecordError(err)
   477  					return
   478  				}
   479  
   480  				sm.ts.Logger().Infof("Waiting for keyspace:shard: %v:%v, position %v", sm.ts.SourceKeyspaceName(), shard, pos)
   481  				if err := sm.ts.TabletManagerClient().VReplicationWaitForPos(ctx, primary.Tablet, int(vrs.ID), mysql.EncodePosition(pos)); err != nil {
   482  					allErrors.RecordError(err)
   483  					return
   484  				}
   485  
   486  				sm.ts.Logger().Infof("Position for keyspace:shard: %v:%v reached", sm.ts.SourceKeyspaceName(), shard)
   487  			}(vrs, shard, pos)
   488  		}
   489  	}
   490  
   491  	wg.Wait()
   492  
   493  	return stopPositions, allErrors.AggrError(vterrors.Aggregate)
   494  }
   495  
   496  func (sm *StreamMigrator) verifyStreamPositions(ctx context.Context, stopPositions map[string]mysql.Position) ([]string, error) {
   497  	var (
   498  		mu             sync.Mutex
   499  		stoppedStreams = make(map[string][]*VReplicationStream)
   500  	)
   501  
   502  	err := sm.ts.ForAllSources(func(source *MigrationSource) error {
   503  		tabletStreams := sm.streams[source.GetShard().ShardName()]
   504  		if len(tabletStreams) == 0 {
   505  			return nil
   506  		}
   507  
   508  		tabletStreams, err := sm.readTabletStreams(ctx, source.GetPrimary(), fmt.Sprintf("id in %s", VReplicationStreams(tabletStreams).Values()))
   509  		if err != nil {
   510  			return err
   511  		}
   512  
   513  		mu.Lock()
   514  		defer mu.Unlock()
   515  		stoppedStreams[source.GetShard().ShardName()] = tabletStreams
   516  
   517  		return nil
   518  	})
   519  
   520  	if err != nil {
   521  		return nil, err
   522  	}
   523  
   524  	// This is not really required because it's not used later.
   525  	// But we keep it up-to-date for good measure.
   526  	sm.streams = stoppedStreams
   527  
   528  	var (
   529  		oneSet    []*VReplicationStream
   530  		allErrors concurrency.AllErrorRecorder
   531  	)
   532  
   533  	for _, tabletStreams := range stoppedStreams {
   534  		if oneSet == nil {
   535  			oneSet = tabletStreams
   536  		}
   537  
   538  		for _, vrs := range tabletStreams {
   539  			key := fmt.Sprintf("%s:%s", vrs.BinlogSource.Keyspace, vrs.BinlogSource.Shard)
   540  			if pos := stopPositions[key]; !vrs.Position.Equal(pos) {
   541  				allErrors.RecordError(fmt.Errorf("%s: stream %d position: %s does not match %s", key, vrs.ID, mysql.EncodePosition(vrs.Position), mysql.EncodePosition(pos)))
   542  			}
   543  		}
   544  	}
   545  
   546  	if allErrors.HasErrors() {
   547  		return nil, allErrors.AggrError(vterrors.Aggregate)
   548  	}
   549  
   550  	sm.templates, err = sm.templatize(ctx, oneSet)
   551  	if err != nil {
   552  		// Unreachable: we've already templatized this before.
   553  		return nil, err
   554  	}
   555  
   556  	return VReplicationStreams(sm.templates).Workflows(), allErrors.AggrError(vterrors.Aggregate)
   557  }
   558  
   559  /* target streams */
   560  
   561  func (sm *StreamMigrator) createTargetStreams(ctx context.Context, tmpl []*VReplicationStream) error {
   562  	if len(tmpl) == 0 {
   563  		return nil
   564  	}
   565  
   566  	return sm.ts.ForAllTargets(func(target *MigrationTarget) error {
   567  		ig := vreplication.NewInsertGenerator(binlogplayer.BlpStopped, target.GetPrimary().DbName())
   568  		tabletStreams := VReplicationStreams(tmpl).Copy().ToSlice()
   569  
   570  		for _, vrs := range tabletStreams {
   571  			for _, rule := range vrs.BinlogSource.Filter.Rules {
   572  				buf := &strings.Builder{}
   573  
   574  				t := template.Must(template.New("").Parse(rule.Filter))
   575  				if err := t.Execute(buf, key.KeyRangeString(target.GetShard().KeyRange)); err != nil {
   576  					return err
   577  				}
   578  
   579  				rule.Filter = buf.String()
   580  			}
   581  
   582  			ig.AddRow(vrs.Workflow, vrs.BinlogSource, mysql.EncodePosition(vrs.Position), "", "",
   583  				int64(vrs.WorkflowType), int64(vrs.WorkflowSubType), vrs.DeferSecondaryKeys)
   584  		}
   585  
   586  		_, err := sm.ts.VReplicationExec(ctx, target.GetPrimary().GetAlias(), ig.String())
   587  		return err
   588  	})
   589  }
   590  
   591  func (sm *StreamMigrator) deleteTargetStreams(ctx context.Context) error {
   592  	if len(sm.workflows) == 0 {
   593  		return nil
   594  	}
   595  
   596  	workflows := stringListify(sm.workflows)
   597  	err := sm.ts.ForAllTargets(func(target *MigrationTarget) error {
   598  		query := fmt.Sprintf("delete from _vt.vreplication where db_name=%s and workflow in (%s)", encodeString(target.GetPrimary().DbName()), workflows)
   599  		_, err := sm.ts.VReplicationExec(ctx, target.GetPrimary().Alias, query)
   600  		return err
   601  	})
   602  
   603  	if err != nil {
   604  		sm.logger.Warningf("Could not delete migrated streams: %v", err)
   605  	}
   606  
   607  	return err
   608  }
   609  
   610  /* templatizing */
   611  
   612  func (sm *StreamMigrator) templatize(ctx context.Context, tabletStreams []*VReplicationStream) ([]*VReplicationStream, error) {
   613  	var shardedStreams []*VReplicationStream
   614  
   615  	tabletStreams = VReplicationStreams(tabletStreams).Copy().ToSlice()
   616  	for _, vrs := range tabletStreams {
   617  		streamType := StreamTypeUnknown
   618  
   619  		for _, rule := range vrs.BinlogSource.Filter.Rules {
   620  			typ, err := sm.templatizeRule(ctx, rule)
   621  			if err != nil {
   622  				return nil, err
   623  			}
   624  
   625  			switch typ {
   626  			case StreamTypeSharded:
   627  				if streamType == StreamTypeReference {
   628  					return nil, fmt.Errorf("cannot migrate streams with a mix of reference and sharded tables: %v", vrs.BinlogSource)
   629  				}
   630  				streamType = StreamTypeSharded
   631  			case StreamTypeReference:
   632  				if streamType == StreamTypeSharded {
   633  					return nil, fmt.Errorf("cannot migrate streams with a mix of reference and sharded tables: %v", vrs.BinlogSource)
   634  				}
   635  				streamType = StreamTypeReference
   636  			}
   637  		}
   638  
   639  		if streamType == StreamTypeSharded {
   640  			shardedStreams = append(shardedStreams, vrs)
   641  		}
   642  	}
   643  
   644  	return shardedStreams, nil
   645  }
   646  
   647  // templatizeRule replaces keyrange values with {{.}}.
   648  // This can then be used by go's template package to substitute other keyrange values.
   649  func (sm *StreamMigrator) templatizeRule(ctx context.Context, rule *binlogdatapb.Rule) (StreamType, error) {
   650  	vtable, ok := sm.ts.SourceKeyspaceSchema().Tables[rule.Match]
   651  	if !ok && !schema.IsInternalOperationTableName(rule.Match) {
   652  		return StreamTypeUnknown, fmt.Errorf("table %v not found in vschema", rule.Match)
   653  	}
   654  
   655  	if vtable != nil && vtable.Type == vindexes.TypeReference {
   656  		return StreamTypeReference, nil
   657  	}
   658  
   659  	switch {
   660  	case rule.Filter == "":
   661  		return StreamTypeUnknown, fmt.Errorf("rule %v does not have a select expression in vreplication", rule)
   662  	case key.IsKeyRange(rule.Filter):
   663  		rule.Filter = "{{.}}"
   664  		return StreamTypeSharded, nil
   665  	case rule.Filter == vreplication.ExcludeStr:
   666  		return StreamTypeUnknown, fmt.Errorf("unexpected rule in vreplication: %v", rule)
   667  	default:
   668  		if err := sm.templatizeKeyRange(ctx, rule); err != nil {
   669  			return StreamTypeUnknown, err
   670  		}
   671  
   672  		return StreamTypeSharded, nil
   673  	}
   674  }
   675  
   676  func (sm *StreamMigrator) templatizeKeyRange(ctx context.Context, rule *binlogdatapb.Rule) error {
   677  	statement, err := sqlparser.Parse(rule.Filter)
   678  	if err != nil {
   679  		return err
   680  	}
   681  
   682  	sel, ok := statement.(*sqlparser.Select)
   683  	if !ok {
   684  		return fmt.Errorf("unexpected query: %v", rule.Filter)
   685  	}
   686  
   687  	var expr sqlparser.Expr
   688  	if sel.Where != nil {
   689  		expr = sel.Where.Expr
   690  	}
   691  
   692  	exprs := sqlparser.SplitAndExpression(nil, expr)
   693  	for _, subexpr := range exprs {
   694  		funcExpr, ok := subexpr.(*sqlparser.FuncExpr)
   695  		if !ok || !funcExpr.Name.EqualString("in_keyrange") {
   696  			continue
   697  		}
   698  
   699  		var krExpr sqlparser.SelectExpr
   700  		switch len(funcExpr.Exprs) {
   701  		case 1:
   702  			krExpr = funcExpr.Exprs[0]
   703  		case 3:
   704  			krExpr = funcExpr.Exprs[2]
   705  		default:
   706  			return fmt.Errorf("unexpected in_keyrange parameters: %v", sqlparser.String(funcExpr))
   707  		}
   708  
   709  		aliased, ok := krExpr.(*sqlparser.AliasedExpr)
   710  		if !ok {
   711  			return fmt.Errorf("unexpected in_keyrange parameters: %v", sqlparser.String(funcExpr))
   712  		}
   713  
   714  		val, ok := aliased.Expr.(*sqlparser.Literal)
   715  		if !ok {
   716  			return fmt.Errorf("unexpected in_keyrange parameters: %v", sqlparser.String(funcExpr))
   717  		}
   718  
   719  		if strings.Contains(rule.Filter, "{{") {
   720  			return fmt.Errorf("cannot migrate queries that contain '{{' in their string: %s", rule.Filter)
   721  		}
   722  
   723  		val.Val = "{{.}}"
   724  		rule.Filter = sqlparser.String(statement)
   725  		return nil
   726  	}
   727  
   728  	// There was no in_keyrange expression. Create a new one.
   729  	vtable := sm.ts.SourceKeyspaceSchema().Tables[rule.Match]
   730  	inkr := &sqlparser.FuncExpr{
   731  		Name: sqlparser.NewIdentifierCI("in_keyrange"),
   732  		Exprs: sqlparser.SelectExprs{
   733  			&sqlparser.AliasedExpr{Expr: &sqlparser.ColName{Name: vtable.ColumnVindexes[0].Columns[0]}},
   734  			&sqlparser.AliasedExpr{Expr: sqlparser.NewStrLiteral(vtable.ColumnVindexes[0].Type)},
   735  			&sqlparser.AliasedExpr{Expr: sqlparser.NewStrLiteral("{{.}}")},
   736  		},
   737  	}
   738  	sel.AddWhere(inkr)
   739  	rule.Filter = sqlparser.String(statement)
   740  	return nil
   741  }
   742  
   743  /* misc */
   744  
   745  func (sm *StreamMigrator) blsIsReference(bls *binlogdatapb.BinlogSource) (bool, error) {
   746  	streamType := StreamTypeUnknown
   747  	for _, rule := range bls.Filter.Rules {
   748  		typ, err := sm.identifyRuleType(rule)
   749  		if err != nil {
   750  			return false, err
   751  		}
   752  
   753  		switch typ {
   754  		case StreamTypeSharded:
   755  			if streamType == StreamTypeReference {
   756  				return false, fmt.Errorf("cannot reshard streams with a mix of reference and sharded tables: %v", bls)
   757  			}
   758  
   759  			streamType = StreamTypeSharded
   760  		case StreamTypeReference:
   761  			if streamType == StreamTypeSharded {
   762  				return false, fmt.Errorf("cannot reshard streams with a mix of reference and sharded tables: %v", bls)
   763  			}
   764  
   765  			streamType = StreamTypeReference
   766  		}
   767  	}
   768  
   769  	return streamType == StreamTypeReference, nil
   770  }
   771  
   772  func (sm *StreamMigrator) identifyRuleType(rule *binlogdatapb.Rule) (StreamType, error) {
   773  	vtable, ok := sm.ts.SourceKeyspaceSchema().Tables[rule.Match]
   774  	if !ok && !schema.IsInternalOperationTableName(rule.Match) {
   775  		return 0, fmt.Errorf("table %v not found in vschema", rule.Match)
   776  	}
   777  
   778  	if vtable != nil && vtable.Type == vindexes.TypeReference {
   779  		return StreamTypeReference, nil
   780  	}
   781  
   782  	// In this case, 'sharded' means that it's not a reference
   783  	// table. We don't care about any other subtleties.
   784  	return StreamTypeSharded, nil
   785  }
   786  
   787  func stringListify(ss []string) string {
   788  	var buf strings.Builder
   789  
   790  	prefix := ""
   791  	for _, s := range ss {
   792  		fmt.Fprintf(&buf, "%s%s", prefix, encodeString(s))
   793  		prefix = ", "
   794  	}
   795  
   796  	return buf.String()
   797  }