vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletmanager/tm_state.go (about) 1 /* 2 Copyright 2020 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package tabletmanager 18 19 import ( 20 "context" 21 "fmt" 22 "strings" 23 "sync" 24 "syscall" 25 "time" 26 27 "github.com/spf13/pflag" 28 "google.golang.org/protobuf/proto" 29 30 "vitess.io/vitess/go/trace" 31 "vitess.io/vitess/go/vt/key" 32 "vitess.io/vitess/go/vt/log" 33 "vitess.io/vitess/go/vt/logutil" 34 "vitess.io/vitess/go/vt/mysqlctl" 35 "vitess.io/vitess/go/vt/servenv" 36 "vitess.io/vitess/go/vt/topo" 37 "vitess.io/vitess/go/vt/topo/topoproto" 38 "vitess.io/vitess/go/vt/topotools" 39 "vitess.io/vitess/go/vt/vterrors" 40 "vitess.io/vitess/go/vt/vttablet/tabletserver/rules" 41 42 topodatapb "vitess.io/vitess/go/vt/proto/topodata" 43 ) 44 45 var publishRetryInterval = 30 * time.Second 46 47 func registerStateFlags(fs *pflag.FlagSet) { 48 fs.DurationVar(&publishRetryInterval, "publish_retry_interval", publishRetryInterval, "how long vttablet waits to retry publishing the tablet record") 49 } 50 51 func init() { 52 servenv.OnParseFor("vtcombo", registerStateFlags) 53 servenv.OnParseFor("vttablet", registerStateFlags) 54 } 55 56 // tmState manages the state of the TabletManager. 57 type tmState struct { 58 tm *TabletManager 59 ctx context.Context 60 cancel context.CancelFunc 61 62 // mu must be held while accessing the following members and 63 // while changing the state of the system to match these values. 64 // This can be held for many seconds while tmState connects to 65 // external components to change their state. 66 // Obtaining tm.actionSema before calling a tmState function is 67 // not required. 68 // Because mu can be held for long, we publish the current state 69 // of these variables into displayState, which can be accessed 70 // more freely even while tmState is busy transitioning. 71 mu sync.Mutex 72 isOpen bool 73 isOpening bool 74 isResharding bool 75 isInSrvKeyspace bool 76 isShardServing map[topodatapb.TabletType]bool 77 tabletControls map[topodatapb.TabletType]bool 78 deniedTables map[topodatapb.TabletType][]string 79 tablet *topodatapb.Tablet 80 isPublishing bool 81 82 // displayState contains the current snapshot of the internal state 83 // and has its own mutex. 84 displayState displayState 85 } 86 87 func newTMState(tm *TabletManager, tablet *topodatapb.Tablet) *tmState { 88 ctx, cancel := context.WithCancel(tm.BatchCtx) 89 return &tmState{ 90 tm: tm, 91 displayState: displayState{ 92 tablet: proto.Clone(tablet).(*topodatapb.Tablet), 93 }, 94 tablet: tablet, 95 ctx: ctx, 96 cancel: cancel, 97 } 98 } 99 100 func (ts *tmState) Open() { 101 log.Infof("In tmState.Open()") 102 ts.mu.Lock() 103 defer ts.mu.Unlock() 104 if ts.isOpen { 105 return 106 } 107 108 ts.isOpen = true 109 ts.isOpening = true 110 _ = ts.updateLocked(ts.ctx) 111 ts.isOpening = false 112 ts.publishStateLocked(ts.ctx) 113 } 114 115 func (ts *tmState) Close() { 116 log.Infof("In tmState.Close()") 117 ts.mu.Lock() 118 defer ts.mu.Unlock() 119 120 ts.isOpen = false 121 ts.cancel() 122 } 123 124 func (ts *tmState) RefreshFromTopo(ctx context.Context) error { 125 span, ctx := trace.NewSpan(ctx, "tmState.refreshFromTopo") 126 defer span.Finish() 127 log.Info("Refreshing from Topo") 128 129 shardInfo, err := ts.tm.TopoServer.GetShard(ctx, ts.Keyspace(), ts.Shard()) 130 if err != nil { 131 return err 132 } 133 134 srvKeyspace, err := ts.tm.TopoServer.GetSrvKeyspace(ctx, ts.tm.tabletAlias.Cell, ts.Keyspace()) 135 if err != nil { 136 return err 137 } 138 ts.RefreshFromTopoInfo(ctx, shardInfo, srvKeyspace) 139 return nil 140 } 141 142 func (ts *tmState) RefreshFromTopoInfo(ctx context.Context, shardInfo *topo.ShardInfo, srvKeyspace *topodatapb.SrvKeyspace) { 143 ts.mu.Lock() 144 defer ts.mu.Unlock() 145 146 if shardInfo != nil { 147 ts.isResharding = len(shardInfo.SourceShards) > 0 148 149 ts.deniedTables = make(map[topodatapb.TabletType][]string) 150 for _, tc := range shardInfo.TabletControls { 151 if topo.InCellList(ts.tm.tabletAlias.Cell, tc.Cells) { 152 ts.deniedTables[tc.TabletType] = tc.DeniedTables 153 } 154 } 155 } 156 157 if srvKeyspace != nil { 158 ts.isShardServing = make(map[topodatapb.TabletType]bool) 159 ts.tabletControls = make(map[topodatapb.TabletType]bool) 160 161 for _, partition := range srvKeyspace.GetPartitions() { 162 163 for _, shard := range partition.GetShardReferences() { 164 if key.KeyRangeEqual(shard.GetKeyRange(), ts.tablet.KeyRange) { 165 ts.isShardServing[partition.GetServedType()] = true 166 } 167 } 168 169 for _, tabletControl := range partition.GetShardTabletControls() { 170 if key.KeyRangeEqual(tabletControl.GetKeyRange(), ts.KeyRange()) { 171 if tabletControl.QueryServiceDisabled { 172 ts.tabletControls[partition.GetServedType()] = true 173 } 174 break 175 } 176 } 177 } 178 } 179 180 _ = ts.updateLocked(ctx) 181 } 182 183 func (ts *tmState) ChangeTabletType(ctx context.Context, tabletType topodatapb.TabletType, action DBAction) error { 184 ts.mu.Lock() 185 defer ts.mu.Unlock() 186 log.Infof("Changing Tablet Type: %v for %s", tabletType, ts.tablet.Alias.String()) 187 188 if tabletType == topodatapb.TabletType_PRIMARY { 189 PrimaryTermStartTime := logutil.TimeToProto(time.Now()) 190 191 // Update the tablet record first. 192 _, err := topotools.ChangeType(ctx, ts.tm.TopoServer, ts.tm.tabletAlias, tabletType, PrimaryTermStartTime) 193 if err != nil { 194 log.Errorf("Error changing type in topo record for tablet %s :- %v\nWill keep trying to read from the toposerver", topoproto.TabletAliasString(ts.tm.tabletAlias), err) 195 // In case of a topo error, we aren't sure if the data has been written or not. 196 // We must read the data again and verify whether the previous write succeeded or not. 197 // The only way to guarantee safety is to keep retrying read until we succeed 198 for { 199 if ctx.Err() != nil { 200 return fmt.Errorf("context canceled updating tablet_type for %s in the topo, please retry", ts.tm.tabletAlias) 201 } 202 ti, errInReading := ts.tm.TopoServer.GetTablet(ctx, ts.tm.tabletAlias) 203 if errInReading != nil { 204 <-time.After(100 * time.Millisecond) 205 continue 206 } 207 if ti.Type == tabletType && proto.Equal(ti.PrimaryTermStartTime, PrimaryTermStartTime) { 208 log.Infof("Tablet record in toposerver matches, continuing operation") 209 break 210 } 211 log.Errorf("Tablet record read from toposerver does not match what we attempted to write, canceling operation") 212 return err 213 } 214 } 215 216 if action == DBActionSetReadWrite { 217 // We call SetReadOnly only after the topo has been updated to avoid 218 // situations where two tablets are primary at the DB level but not at the vitess level 219 if err := ts.tm.MysqlDaemon.SetReadOnly(false); err != nil { 220 return err 221 } 222 } 223 224 ts.tablet.Type = tabletType 225 ts.tablet.PrimaryTermStartTime = PrimaryTermStartTime 226 } else { 227 ts.tablet.Type = tabletType 228 ts.tablet.PrimaryTermStartTime = nil 229 } 230 231 s := topoproto.TabletTypeLString(tabletType) 232 statsTabletType.Set(s) 233 statsTabletTypeCount.Add(s, 1) 234 235 err := ts.updateLocked(ctx) 236 // No need to short circuit. Apply all steps and return error in the end. 237 ts.publishStateLocked(ctx) 238 ts.tm.notifyShardSync() 239 return err 240 } 241 242 func (ts *tmState) SetMysqlPort(mport int32) { 243 ts.mu.Lock() 244 defer ts.mu.Unlock() 245 246 ts.tablet.MysqlPort = mport 247 ts.publishStateLocked(ts.ctx) 248 } 249 250 // UpdateTablet must be called during initialization only. 251 func (ts *tmState) UpdateTablet(update func(tablet *topodatapb.Tablet)) { 252 ts.mu.Lock() 253 defer ts.mu.Unlock() 254 update(ts.tablet) 255 ts.publishForDisplay() 256 } 257 258 func (ts *tmState) updateLocked(ctx context.Context) error { 259 span, ctx := trace.NewSpan(ctx, "tmState.update") 260 defer span.Finish() 261 ts.publishForDisplay() 262 var returnErr error 263 if !ts.isOpen { 264 return nil 265 } 266 267 terTime := logutil.ProtoToTime(ts.tablet.PrimaryTermStartTime) 268 269 // Disable TabletServer first so the nonserving state gets advertised 270 // before other services are shutdown. 271 reason := ts.canServe(ts.tablet.Type) 272 if reason != "" { 273 log.Infof("Disabling query service: %v", reason) 274 // SetServingType can result in error. Although we have forever retries to fix these transient errors 275 // but, under certain conditions these errors are non-transient (see https://github.com/vitessio/vitess/issues/10145). 276 // There is no way to distinguish between retry (transient) and non-retryable errors, therefore we will 277 // always return error from 'SetServingType' and 'applyDenyList' to our client. It is up to them to handle it accordingly. 278 // UpdateLock is called from 'ChangeTabletType', 'Open' and 'RefreshFromTopoInfo'. For 'Open' and 'RefreshFromTopoInfo' we don't need 279 // to propagate error to client hence no changes there but we will propagate error from 'ChangeTabletType' to client. 280 if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, false, reason); err != nil { 281 errStr := fmt.Sprintf("SetServingType(serving=false) failed: %v", err) 282 log.Errorf(errStr) 283 // No need to short circuit. Apply all steps and return error in the end. 284 returnErr = vterrors.Wrapf(err, errStr) 285 } 286 } 287 288 if err := ts.applyDenyList(ctx); err != nil { 289 errStr := fmt.Sprintf("Cannot update denied tables rule: %v", err) 290 log.Errorf(errStr) 291 // No need to short circuit. Apply all steps and return error in the end. 292 returnErr = vterrors.Wrapf(err, errStr) 293 } 294 295 if ts.tm.UpdateStream != nil { 296 if topo.IsRunningUpdateStream(ts.tablet.Type) { 297 ts.tm.UpdateStream.Enable() 298 } else { 299 ts.tm.UpdateStream.Disable() 300 } 301 } 302 303 if ts.tm.VREngine != nil { 304 if ts.tablet.Type == topodatapb.TabletType_PRIMARY { 305 ts.tm.VREngine.Open(ts.tm.BatchCtx) 306 } else { 307 ts.tm.VREngine.Close() 308 } 309 } 310 311 if ts.tm.VDiffEngine != nil { 312 if ts.tablet.Type == topodatapb.TabletType_PRIMARY { 313 ts.tm.VDiffEngine.Open(ts.tm.BatchCtx, ts.tm.VREngine) 314 } else { 315 ts.tm.VDiffEngine.Close() 316 } 317 } 318 319 if ts.isShardServing[ts.tablet.Type] { 320 ts.isInSrvKeyspace = true 321 statsIsInSrvKeyspace.Set(1) 322 } else { 323 ts.isInSrvKeyspace = false 324 statsIsInSrvKeyspace.Set(0) 325 } 326 327 // Open TabletServer last so that it advertises serving after all other services are up. 328 if reason == "" { 329 if err := ts.tm.QueryServiceControl.SetServingType(ts.tablet.Type, terTime, true, ""); err != nil { 330 errStr := fmt.Sprintf("Cannot start query service: %v", err) 331 log.Errorf(errStr) 332 returnErr = vterrors.Wrapf(err, errStr) 333 } 334 } 335 336 return returnErr 337 } 338 339 func (ts *tmState) canServe(tabletType topodatapb.TabletType) string { 340 if !topo.IsRunningQueryService(tabletType) { 341 return fmt.Sprintf("not a serving tablet type(%v)", tabletType) 342 } 343 if ts.tabletControls[tabletType] { 344 return "TabletControl.DisableQueryService set" 345 } 346 if tabletType == topodatapb.TabletType_PRIMARY && ts.isResharding { 347 return "primary tablet with filtered replication on" 348 } 349 return "" 350 } 351 352 func (ts *tmState) applyDenyList(ctx context.Context) (err error) { 353 denyListRules := rules.New() 354 deniedTables := ts.deniedTables[ts.tablet.Type] 355 if len(deniedTables) > 0 { 356 tables, err := mysqlctl.ResolveTables(ctx, ts.tm.MysqlDaemon, topoproto.TabletDbName(ts.tablet), deniedTables) 357 if err != nil { 358 return err 359 } 360 361 // Verify that at least one table matches the wildcards, so 362 // that we don't add a rule to deny all tables 363 if len(tables) > 0 { 364 log.Infof("Denying tables %v", strings.Join(tables, ", ")) 365 qr := rules.NewQueryRule("enforce denied tables", "denied_table", rules.QRFailRetry) 366 for _, t := range tables { 367 qr.AddTableCond(t) 368 } 369 denyListRules.Add(qr) 370 } 371 } 372 373 loadRuleErr := ts.tm.QueryServiceControl.SetQueryRules(denyListQueryList, denyListRules) 374 if loadRuleErr != nil { 375 log.Warningf("Fail to load query rule set %s: %s", denyListQueryList, loadRuleErr) 376 } 377 return nil 378 } 379 380 func (ts *tmState) publishStateLocked(ctx context.Context) { 381 log.Infof("Publishing state: %v", ts.tablet) 382 // If retry is in progress, there's nothing to do. 383 if ts.isPublishing { 384 return 385 } 386 // Fast path: publish immediately. 387 ctx, cancel := context.WithTimeout(ctx, topo.RemoteOperationTimeout) 388 defer cancel() 389 _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tm.tabletAlias, func(tablet *topodatapb.Tablet) error { 390 if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil { 391 log.Error(err) 392 return topo.NewError(topo.NoUpdateNeeded, "") 393 } 394 proto.Reset(tablet) 395 proto.Merge(tablet, ts.tablet) 396 return nil 397 }) 398 if err != nil { 399 if topo.IsErrType(err, topo.NoNode) { // Someone deleted the tablet record under us. Shut down gracefully. 400 log.Error("Tablet record has disappeared, shutting down") 401 servenv.ExitChan <- syscall.SIGTERM 402 return 403 } 404 log.Errorf("Unable to publish state to topo, will keep retrying: %v", err) 405 ts.isPublishing = true 406 // Keep retrying until success. 407 go ts.retryPublish() 408 } 409 } 410 411 func (ts *tmState) retryPublish() { 412 ts.mu.Lock() 413 defer ts.mu.Unlock() 414 415 defer func() { ts.isPublishing = false }() 416 417 for { 418 // Retry immediately the first time because the previous failure might have been 419 // due to an expired context. 420 ctx, cancel := context.WithTimeout(ts.ctx, topo.RemoteOperationTimeout) 421 _, err := ts.tm.TopoServer.UpdateTabletFields(ctx, ts.tm.tabletAlias, func(tablet *topodatapb.Tablet) error { 422 if err := topotools.CheckOwnership(tablet, ts.tablet); err != nil { 423 log.Error(err) 424 return topo.NewError(topo.NoUpdateNeeded, "") 425 } 426 proto.Reset(tablet) 427 proto.Merge(tablet, ts.tablet) 428 return nil 429 }) 430 cancel() 431 if err != nil { 432 if topo.IsErrType(err, topo.NoNode) { // Someone deleted the tablet record under us. Shut down gracefully. 433 log.Error("Tablet record has disappeared, shutting down") 434 servenv.ExitChan <- syscall.SIGTERM 435 return 436 } 437 log.Errorf("Unable to publish state to topo, will keep retrying: %v", err) 438 ts.mu.Unlock() 439 time.Sleep(publishRetryInterval) 440 ts.mu.Lock() 441 continue 442 } 443 log.Infof("Published state: %v", ts.tablet) 444 return 445 } 446 } 447 448 // displayState is the externalized version of tmState 449 // that can be used for observability. The internal version 450 // of tmState may not be accessible due to longer mutex holds. 451 // tmState uses publishForDisplay to keep these values uptodate. 452 type displayState struct { 453 mu sync.Mutex 454 tablet *topodatapb.Tablet 455 deniedTables []string 456 } 457 458 // Note that the methods for displayState are all in tmState. 459 func (ts *tmState) publishForDisplay() { 460 ts.displayState.mu.Lock() 461 defer ts.displayState.mu.Unlock() 462 463 ts.displayState.tablet = proto.Clone(ts.tablet).(*topodatapb.Tablet) 464 ts.displayState.deniedTables = ts.deniedTables[ts.tablet.Type] 465 } 466 467 func (ts *tmState) Tablet() *topodatapb.Tablet { 468 ts.displayState.mu.Lock() 469 defer ts.displayState.mu.Unlock() 470 return proto.Clone(ts.displayState.tablet).(*topodatapb.Tablet) 471 } 472 473 func (ts *tmState) DeniedTables() []string { 474 ts.displayState.mu.Lock() 475 defer ts.displayState.mu.Unlock() 476 return ts.displayState.deniedTables 477 } 478 479 func (ts *tmState) Keyspace() string { 480 ts.displayState.mu.Lock() 481 defer ts.displayState.mu.Unlock() 482 return ts.displayState.tablet.Keyspace 483 } 484 485 func (ts *tmState) Shard() string { 486 ts.displayState.mu.Lock() 487 defer ts.displayState.mu.Unlock() 488 return ts.displayState.tablet.Shard 489 } 490 491 func (ts *tmState) KeyRange() *topodatapb.KeyRange { 492 ts.displayState.mu.Lock() 493 defer ts.displayState.mu.Unlock() 494 return ts.displayState.tablet.KeyRange 495 }