vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/tm_state.go (about)

     1  /*
     2  Copyright 2020 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package tabletmanager
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"strings"
    23  	"sync"
    24  	"syscall"
    25  	"time"
    26  
    27  	"github.com/spf13/pflag"
    28  	"google.golang.org/protobuf/proto"
    29  
    30  	"vitess.io/vitess/go/trace"
    31  	"vitess.io/vitess/go/vt/key"
    32  	"vitess.io/vitess/go/vt/log"
    33  	"vitess.io/vitess/go/vt/logutil"
    34  	"vitess.io/vitess/go/vt/mysqlctl"
    35  	"vitess.io/vitess/go/vt/servenv"
    36  	"vitess.io/vitess/go/vt/topo"
    37  	"vitess.io/vitess/go/vt/topo/topoproto"
    38  	"vitess.io/vitess/go/vt/topotools"
    39  	"vitess.io/vitess/go/vt/vterrors"
    40  	"vitess.io/vitess/go/vt/vttablet/tabletserver/rules"
    41  
    42  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    43  )
    44  
    45  var publishRetryInterval = 30 * time.Second
    46  
    47  func registerStateFlags(fs *pflag.FlagSet) {
    48  	fs.DurationVar(&publishRetryInterval, "publish_retry_interval", publishRetryInterval, "how long vttablet waits to retry publishing the tablet record")
    49  }
    50  
    51  func init() {
    52  	servenv.OnParseFor("vtcombo", registerStateFlags)
    53  	servenv.OnParseFor("vttablet", registerStateFlags)
    54  }
    55  
    56  // tmState manages the state of the TabletManager.
    57  type tmState struct {
    58  	tm     *TabletManager
    59  	ctx    context.Context
    60  	cancel context.CancelFunc
    61  
    62  	// mu must be held while accessing the following members and
    63  	// while changing the state of the system to match these values.
    64  	// This can be held for many seconds while tmState connects to
    65  	// external components to change their state.
    66  	// Obtaining tm.actionSema before calling a tmState function is
    67  	// not required.
    68  	// Because mu can be held for long, we publish the current state
    69  	// of these variables into displayState, which can be accessed
    70  	// more freely even while tmState is busy transitioning.
    71  	mu              sync.Mutex
    72  	isOpen          bool
    73  	isOpening       bool
    74  	isResharding    bool
    75  	isInSrvKeyspace bool
    76  	isShardServing  map[topodatapb.TabletType]bool
    77  	tabletControls  map[topodatapb.TabletType]bool
    78  	deniedTables    map[topodatapb.TabletType][]string
    79  	tablet          *topodatapb.Tablet
    80  	isPublishing    bool
    81  
    82  	// displayState contains the current snapshot of the internal state
    83  	// and has its own mutex.
    84  	displayState displayState
    85  }
    86  
    87  func newTMState(tm *TabletManager, tablet *topodatapb.Tablet) *tmState {
    88  	ctx, cancel := context.WithCancel(tm.BatchCtx)
    89  	return &tmState{
    90  		tm: tm,
    91  		displayState: displayState{
    92  			tablet: proto.Clone(tablet).(*topodatapb.Tablet),
    93  		},
    94  		tablet: tablet,
    95  		ctx:    ctx,
    96  		cancel: cancel,
    97  	}
    98  }
    99  
   100  func (ts *tmState) Open() {
   101  	log.Infof("In tmState.Open()")
   102  	ts.mu.Lock()
   103  	defer ts.mu.Unlock()
   104  	if ts.isOpen {
   105  		return
   106  	}
   107  
   108  	ts.isOpen = true
   109  	ts.isOpening = true
   110  	_ = ts.updateLocked(ts.ctx)
   111  	ts.isOpening = false
   112  	ts.publishStateLocked(ts.ctx)
   113  }
   114  
   115  func (ts *tmState) Close() {
   116  	log.Infof("In tmState.Close()")
   117  	ts.mu.Lock()
   118  	defer ts.mu.Unlock()
   119  
   120  	ts.isOpen = false
   121  	ts.cancel()
   122  }
   123  
   124  func (ts *tmState) RefreshFromTopo(ctx context.Context) error {
   125  	span, ctx := trace.NewSpan(ctx, "tmState.refreshFromTopo")
   126  	defer span.Finish()
   127  	log.Info("Refreshing from Topo")
   128  
   129  	shardInfo, err := ts.tm.TopoServer.GetShard(ctx, ts.Keyspace(), ts.Shard())
   130  	if err != nil {
   131  		return err
   132  	}
   133  
   134  	srvKeyspace, err := ts.tm.TopoServer.GetSrvKeyspace(ctx, ts.tm.tabletAlias.Cell, ts.Keyspace())
   135  	if err != nil {
   136  		return err
   137  	}
   138  	ts.RefreshFromTopoInfo(ctx, shardInfo, srvKeyspace)
   139  	return nil
   140  }
   141  
   142  func (ts *tmState) RefreshFromTopoInfo(ctx context.Context, shardInfo *topo.ShardInfo, srvKeyspace *topodatapb.SrvKeyspace) {
   143  	ts.mu.Lock()
   144  	defer ts.mu.Unlock()
   145  
   146  	if shardInfo != nil {
   147  		ts.isResharding = len(shardInfo.SourceShards) > 0
   148  
   149  		ts.deniedTables = make(map[topodatapb.TabletType][]string)
   150  		for _, tc := range shardInfo.TabletControls {
   151  			if topo.InCellList(ts.tm.tabletAlias.Cell, tc.Cells) {
   152  				ts.deniedTables[tc.TabletType] = tc.DeniedTables
   153  			}
   154  		}
   155  	}
   156  
   157  	if srvKeyspace != nil {
   158  		ts.isShardServing = make(map[topodatapb.TabletType]bool)
   159  		ts.tabletControls = make(map[topodatapb.TabletType]bool)
   160  
   161  		for _, partition := range srvKeyspace.GetPartitions() {
   162  
   163  			for _, shard := range partition.GetShardReferences() {
   164  				if key.KeyRangeEqual(shard.GetKeyRange(), ts.tablet.KeyRange) {
   165  					ts.isShardServing[partition.GetServedType()] = true
   166  				}
   167  			}
   168  
   169  			for _, tabletControl := range partition.GetShardTabletControls() {
   170  				if key.KeyRangeEqual(tabletControl.GetKeyRange(), ts.KeyRange()) {
   171  					if tabletControl.QueryServiceDisabled {
   172  						ts.tabletControls[partition.GetServedType()] = true
   173  					}
   174  					break
   175  				}
   176  			}
   177  		}
   178  	}
   179  
   180  	_ = ts.updateLocked(ctx)
   181  }
   182  
   183  func (ts *tmState) ChangeTabletType(ctx context.Context, tabletType topodatapb.TabletType, action DBAction) error {
   184  	ts.mu.Lock()
   185  	defer ts.mu.Unlock()
   186  	log.Infof("Changing Tablet Type: %v for %s", tabletType, ts.tablet.Alias.String())
   187  
   188  	if tabletType == topodatapb.TabletType_PRIMARY {
   189  		PrimaryTermStartTime := logutil.TimeToProto(time.Now())
   190  
   191  		// Update the tablet record first.
   192  		_, err := topotools.ChangeType(ctx, ts.tm.TopoServer, ts.tm.tabletAlias, tabletType, PrimaryTermStartTime)
   193  		if err != nil {
   194  			log.Errorf("Error changing type in topo record for tablet %s :- %v\nWill keep trying to read from the toposerver", topoproto.TabletAliasString(ts.tm.tabletAlias), err)
   195  			// In case of a topo error, we aren't sure if the data has been written or not.
   196  			// We must read the data again and verify whether the previous write succeeded or not.
   197  			// The only way to guarantee safety is to keep retrying read until we succeed
   198  			for {
   199  				if ctx.Err() != nil {
   200  					return fmt.Errorf("context canceled updating tablet_type for %s in the topo, please retry", ts.tm.tabletAlias)
   201  				}
   202  				ti, errInReading := ts.tm.TopoServer.GetTablet(ctx, ts.tm.tabletAlias)
   203  				if errInReading != nil {
   204  					<-time.After(100 * time.Millisecond)
   205  					continue
   206  				}
   207  				if ti.Type == tabletType && proto.Equal(ti.PrimaryTermStartTime, PrimaryTermStartTime) {
   208  					log.Infof("Tablet record in toposerver matches, continuing operation")
   209  					break
   210  				}
   211  				log.Errorf("Tablet record read from toposerver does not match what we attempted to write, canceling operation")
   212  				return err
   213  			}
   214  		}
   215  
   216  		if action == DBActionSetReadWrite {
   217  			// We call SetReadOnly only after the topo has been updated to avoid
   218  			// situations where two tablets are primary at the DB level but not at the vitess level
   219  			if err := ts.tm.MysqlDaemon.SetReadOnly(false); err != nil {
   220  				return err
   221  			}
   222  		}
   223  
   224  		ts.tablet.Type = tabletType
   225  		ts.tablet.PrimaryTermStartTime = PrimaryTermStartTime
   226  	} else {
   227  		ts.tablet.Type = tabletType
   228  		ts.tablet.PrimaryTermStartTime = nil
   229  	}
   230  
   231  	s := topoproto.TabletTypeLString(tabletType)
   232  	statsTabletType.Set(s)
   233  	statsTabletTypeCount.Add(s, 1)
   234  
   235  	err := ts.updateLocked(ctx)
   236  	// No need to short circuit. Apply all steps and return error in the end.
   237  	ts.publishStateLocked(ctx)
   238  	ts.tm.notifyShardSync()
   239  	return err
   240  }
   241  
   242  func (ts *tmState) SetMysqlPort(mport int32) {
   243  	ts.mu.Lock()
   244  	defer ts.mu.Unlock()
   245  
   246  	ts.tablet.MysqlPort = mport
   247  	ts.publishStateLocked(ts.ctx)
   248  }
   249  
   250  // UpdateTablet must be called during initialization only.
   251  func (ts *tmState) UpdateTablet(update func(tablet *topodatapb.Tablet)) {
   252  	ts.mu.Lock()
   253  	defer ts.mu.Unlock()
   254  	update(ts.tablet)
   255  	ts.publishForDisplay()
   256  }
   257  
   258  func (ts *tmState) updateLocked(ctx context.Context) error {
   259  	span, ctx := trace.NewSpan(ctx, "tmState.update")
   260  	defer span.Finish()
   261  	ts.publishForDisplay()
   262  	var returnErr error
   263  	if !ts.isOpen {
   264  		return nil
   265  	}
   266  
   267  	terTime := logutil.ProtoToTime(ts.tablet.PrimaryTermStartTime)
   268  
   269  	// Disable TabletServer first so the nonserving state gets advertised
   270  	// before other services are shutdown.
   271  	reason := ts.canServe(ts.tablet.Type)
   272  	if reason != "" {
   273  		log.Infof("Disabling query service: %v", reason)
   274  		// SetServingType can result in error. Although we have forever retries to fix these transient errors
   275  		// but, under certain conditions these errors are non-transient (see https://github.com/vitessio/vitess/issues/10145).
   276  		// There is no way to distinguish between retry (transient) and non-retryable errors, therefore we will
   277  		// always return error from 'SetServingType' and 'applyDenyList' to our client. It is up to them to handle it accordingly.
   278  		// UpdateLock is called from 'ChangeTabletType', 'Open' and 'RefreshFromTopoInfo'. For 'Open' and 'RefreshFromTopoInfo' we don't need
   279  		// to propagate error to client hence no changes there but we will propagate error from 'ChangeTabletType' to client.
   280  		if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, false, reason); err != nil {
   281  			errStr := fmt.Sprintf("SetServingType(serving=false) failed: %v", err)
   282  			log.Errorf(errStr)
   283  			// No need to short circuit. Apply all steps and return error in the end.
   284  			returnErr = vterrors.Wrapf(err, errStr)
   285  		}
   286  	}
   287  
   288  	if err := ts.applyDenyList(ctx); err != nil {
   289  		errStr := fmt.Sprintf("Cannot update denied tables rule: %v", err)
   290  		log.Errorf(errStr)
   291  		// No need to short circuit. Apply all steps and return error in the end.
   292  		returnErr = vterrors.Wrapf(err, errStr)
   293  	}
   294  
   295  	if ts.tm.UpdateStream != nil {
   296  		if topo.IsRunningUpdateStream(ts.tablet.Type) {
   297  			ts.tm.UpdateStream.Enable()
   298  		} else {
   299  			ts.tm.UpdateStream.Disable()
   300  		}
   301  	}
   302  
   303  	if ts.tm.VREngine != nil {
   304  		if ts.tablet.Type == topodatapb.TabletType_PRIMARY {
   305  			ts.tm.VREngine.Open(ts.tm.BatchCtx)
   306  		} else {
   307  			ts.tm.VREngine.Close()
   308  		}
   309  	}
   310  
   311  	if ts.tm.VDiffEngine != nil {
   312  		if ts.tablet.Type == topodatapb.TabletType_PRIMARY {
   313  			ts.tm.VDiffEngine.Open(ts.tm.BatchCtx, ts.tm.VREngine)
   314  		} else {
   315  			ts.tm.VDiffEngine.Close()
   316  		}
   317  	}
   318  
   319  	if ts.isShardServing[ts.tablet.Type] {
   320  		ts.isInSrvKeyspace = true
   321  		statsIsInSrvKeyspace.Set(1)
   322  	} else {
   323  		ts.isInSrvKeyspace = false
   324  		statsIsInSrvKeyspace.Set(0)
   325  	}
   326  
   327  	// Open TabletServer last so that it advertises serving after all other services are up.
   328  	if reason == "" {
   329  		if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, true, ""); err != nil {
   330  			errStr := fmt.Sprintf("Cannot start query service: %v", err)
   331  			log.Errorf(errStr)
   332  			returnErr = vterrors.Wrapf(err, errStr)
   333  		}
   334  	}
   335  
   336  	return returnErr
   337  }
   338  
   339  func (ts *tmState) canServe(tabletType topodatapb.TabletType) string {
   340  	if !topo.IsRunningQueryService(tabletType) {
   341  		return fmt.Sprintf("not a serving tablet type(%v)", tabletType)
   342  	}
   343  	if ts.tabletControls[tabletType] {
   344  		return "TabletControl.DisableQueryService set"
   345  	}
   346  	if tabletType == topodatapb.TabletType_PRIMARY && ts.isResharding {
   347  		return "primary tablet with filtered replication on"
   348  	}
   349  	return ""
   350  }
   351  
   352  func (ts *tmState) applyDenyList(ctx context.Context) (err error) {
   353  	denyListRules := rules.New()
   354  	deniedTables := ts.deniedTables[ts.tablet.Type]
   355  	if len(deniedTables) > 0 {
   356  		tables, err := mysqlctl.ResolveTables(ctx, ts.tm.MysqlDaemon, topoproto.TabletDbName(ts.tablet), deniedTables)
   357  		if err != nil {
   358  			return err
   359  		}
   360  
   361  		// Verify that at least one table matches the wildcards, so
   362  		// that we don't add a rule to deny all tables
   363  		if len(tables) > 0 {
   364  			log.Infof("Denying tables %v", strings.Join(tables, ", "))
   365  			qr := rules.NewQueryRule("enforce denied tables", "denied_table", rules.QRFailRetry)
   366  			for _, t := range tables {
   367  				qr.AddTableCond(t)
   368  			}
   369  			denyListRules.Add(qr)
   370  		}
   371  	}
   372  
   373  	loadRuleErr := ts.tm.QueryServiceControl.SetQueryRules(denyListQueryList, denyListRules)
   374  	if loadRuleErr != nil {
   375  		log.Warningf("Fail to load query rule set %s: %s", denyListQueryList, loadRuleErr)
   376  	}
   377  	return nil
   378  }
   379  
   380  func (ts *tmState) publishStateLocked(ctx context.Context) {
   381  	log.Infof("Publishing state: %v", ts.tablet)
   382  	// If retry is in progress, there's nothing to do.
   383  	if ts.isPublishing {
   384  		return
   385  	}
   386  	// Fast path: publish immediately.
   387  	ctx, cancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout)
   388  	defer cancel()
   389  	_, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tm.tabletAlias, func(tablet *topodatapb.Tablet) error {
   390  		if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil {
   391  			log.Error(err)
   392  			return topo.NewError(topo.NoUpdateNeeded, "")
   393  		}
   394  		proto.Reset(tablet)
   395  		proto.Merge(tablet, ts.tablet)
   396  		return nil
   397  	})
   398  	if err != nil {
   399  		if topo.IsErrType(err, topo.NoNode) { // Someone deleted the tablet record under us. Shut down gracefully.
   400  			log.Error("Tablet record has disappeared, shutting down")
   401  			servenv.ExitChan <- syscall.SIGTERM
   402  			return
   403  		}
   404  		log.Errorf("Unable to publish state to topo, will keep retrying: %v", err)
   405  		ts.isPublishing = true
   406  		// Keep retrying until success.
   407  		go ts.retryPublish()
   408  	}
   409  }
   410  
   411  func (ts *tmState) retryPublish() {
   412  	ts.mu.Lock()
   413  	defer ts.mu.Unlock()
   414  
   415  	defer func() { ts.isPublishing = false }()
   416  
   417  	for {
   418  		// Retry immediately the first time because the previous failure might have been
   419  		// due to an expired context.
   420  		ctx, cancel := context.WithTimeout(ts.ctx, topo.RemoteOperationTimeout)
   421  		_, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tm.tabletAlias, func(tablet *topodatapb.Tablet) error {
   422  			if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil {
   423  				log.Error(err)
   424  				return topo.NewError(topo.NoUpdateNeeded, "")
   425  			}
   426  			proto.Reset(tablet)
   427  			proto.Merge(tablet, ts.tablet)
   428  			return nil
   429  		})
   430  		cancel()
   431  		if err != nil {
   432  			if topo.IsErrType(err, topo.NoNode) { // Someone deleted the tablet record under us. Shut down gracefully.
   433  				log.Error("Tablet record has disappeared, shutting down")
   434  				servenv.ExitChan <- syscall.SIGTERM
   435  				return
   436  			}
   437  			log.Errorf("Unable to publish state to topo, will keep retrying: %v", err)
   438  			ts.mu.Unlock()
   439  			time.Sleep(publishRetryInterval)
   440  			ts.mu.Lock()
   441  			continue
   442  		}
   443  		log.Infof("Published state: %v", ts.tablet)
   444  		return
   445  	}
   446  }
   447  
   448  // displayState is the externalized version of tmState
   449  // that can be used for observability. The internal version
   450  // of tmState may not be accessible due to longer mutex holds.
   451  // tmState uses publishForDisplay to keep these values uptodate.
   452  type displayState struct {
   453  	mu           sync.Mutex
   454  	tablet       *topodatapb.Tablet
   455  	deniedTables []string
   456  }
   457  
   458  // Note that the methods for displayState are all in tmState.
   459  func (ts *tmState) publishForDisplay() {
   460  	ts.displayState.mu.Lock()
   461  	defer ts.displayState.mu.Unlock()
   462  
   463  	ts.displayState.tablet = proto.Clone(ts.tablet).(*topodatapb.Tablet)
   464  	ts.displayState.deniedTables = ts.deniedTables[ts.tablet.Type]
   465  }
   466  
   467  func (ts *tmState) Tablet() *topodatapb.Tablet {
   468  	ts.displayState.mu.Lock()
   469  	defer ts.displayState.mu.Unlock()
   470  	return proto.Clone(ts.displayState.tablet).(*topodatapb.Tablet)
   471  }
   472  
   473  func (ts *tmState) DeniedTables() []string {
   474  	ts.displayState.mu.Lock()
   475  	defer ts.displayState.mu.Unlock()
   476  	return ts.displayState.deniedTables
   477  }
   478  
   479  func (ts *tmState) Keyspace() string {
   480  	ts.displayState.mu.Lock()
   481  	defer ts.displayState.mu.Unlock()
   482  	return ts.displayState.tablet.Keyspace
   483  }
   484  
   485  func (ts *tmState) Shard() string {
   486  	ts.displayState.mu.Lock()
   487  	defer ts.displayState.mu.Unlock()
   488  	return ts.displayState.tablet.Shard
   489  }
   490  
   491  func (ts *tmState) KeyRange() *topodatapb.KeyRange {
   492  	ts.displayState.mu.Lock()
   493  	defer ts.displayState.mu.Unlock()
   494  	return ts.displayState.tablet.KeyRange
   495  }