vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletserver/tx_engine_test.go (about) 1 /* 2 Copyright 2019 The Vitess Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package tabletserver 18 19 import ( 20 "context" 21 "errors" 22 "fmt" 23 "strings" 24 "sync" 25 "testing" 26 "time" 27 28 "vitess.io/vitess/go/vt/vttablet/tabletserver/tx" 29 30 "github.com/stretchr/testify/assert" 31 32 "github.com/stretchr/testify/require" 33 34 "vitess.io/vitess/go/mysql/fakesqldb" 35 "vitess.io/vitess/go/sqltypes" 36 37 "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" 38 39 querypb "vitess.io/vitess/go/vt/proto/query" 40 ) 41 42 func TestTxEngineClose(t *testing.T) { 43 db := setUpQueryExecutorTest(t) 44 defer db.Close() 45 ctx := context.Background() 46 config := tabletenv.NewDefaultConfig() 47 config.DB = newDBConfigs(db) 48 config.TxPool.Size = 10 49 config.Oltp.TxTimeoutSeconds = 0.1 50 config.GracePeriods.ShutdownSeconds = 0 51 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) 52 53 // Normal close. 54 te.AcceptReadWrite() 55 start := time.Now() 56 te.Close() 57 assert.Greater(t, int64(50*time.Millisecond), int64(time.Since(start))) 58 59 // Normal close with timeout wait. 60 te.AcceptReadWrite() 61 c, beginSQL, _, err := te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) 62 require.NoError(t, err) 63 require.Equal(t, "begin", beginSQL) 64 c.Unlock() 65 c, beginSQL, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) 66 require.NoError(t, err) 67 require.Equal(t, "begin", beginSQL) 68 c.Unlock() 69 start = time.Now() 70 te.Close() 71 assert.Less(t, int64(50*time.Millisecond), int64(time.Since(start))) 72 assert.EqualValues(t, 2, te.txPool.env.Stats().KillCounters.Counts()["Transactions"]) 73 te.txPool.env.Stats().KillCounters.ResetAll() 74 75 // Immediate close. 76 te.AcceptReadOnly() 77 c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) 78 if err != nil { 79 t.Fatal(err) 80 } 81 c.Unlock() 82 start = time.Now() 83 te.Close() 84 assert.Greater(t, int64(50*time.Millisecond), int64(time.Since(start))) 85 86 // Normal close with short grace period. 87 te.shutdownGracePeriod = 25 * time.Millisecond 88 te.AcceptReadWrite() 89 c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) 90 require.NoError(t, err) 91 c.Unlock() 92 start = time.Now() 93 te.Close() 94 assert.Less(t, int64(1*time.Millisecond), int64(time.Since(start))) 95 assert.Greater(t, int64(50*time.Millisecond), int64(time.Since(start))) 96 97 // Normal close with short grace period, but pool gets empty early. 98 te.shutdownGracePeriod = 25 * time.Millisecond 99 te.AcceptReadWrite() 100 c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) 101 require.NoError(t, err) 102 c.Unlock() 103 go func() { 104 time.Sleep(10 * time.Millisecond) 105 _, err := te.txPool.GetAndLock(c.ReservedID(), "return") 106 assert.NoError(t, err) 107 te.txPool.RollbackAndRelease(ctx, c) 108 }() 109 start = time.Now() 110 te.Close() 111 assert.Less(t, int64(10*time.Millisecond), int64(time.Since(start))) 112 assert.Greater(t, int64(25*time.Millisecond), int64(time.Since(start))) 113 114 // Immediate close, but connection is in use. 115 te.AcceptReadOnly() 116 c, _, _, err = te.txPool.Begin(ctx, &querypb.ExecuteOptions{}, false, 0, nil, nil) 117 require.NoError(t, err) 118 go func() { 119 time.Sleep(100 * time.Millisecond) 120 te.txPool.RollbackAndRelease(ctx, c) 121 }() 122 start = time.Now() 123 te.Close() 124 if diff := time.Since(start); diff > 250*time.Millisecond { 125 t.Errorf("Close time: %v, must be under 0.25s", diff) 126 } 127 if diff := time.Since(start); diff < 100*time.Millisecond { 128 t.Errorf("Close time: %v, must be over 0.1", diff) 129 } 130 131 // Normal close with Reserved connection timeout wait. 132 te.shutdownGracePeriod = 0 * time.Millisecond 133 te.AcceptReadWrite() 134 te.AcceptReadWrite() 135 _, err = te.Reserve(ctx, &querypb.ExecuteOptions{}, 0, nil) 136 require.NoError(t, err) 137 _, _, err = te.ReserveBegin(ctx, &querypb.ExecuteOptions{}, nil, nil) 138 require.NoError(t, err) 139 start = time.Now() 140 te.Close() 141 assert.Less(t, int64(50*time.Millisecond), int64(time.Since(start))) 142 assert.EqualValues(t, 1, te.txPool.env.Stats().KillCounters.Counts()["Transactions"]) 143 assert.EqualValues(t, 1, te.txPool.env.Stats().KillCounters.Counts()["ReservedConnection"]) 144 } 145 146 func TestTxEngineBegin(t *testing.T) { 147 db := setUpQueryExecutorTest(t) 148 defer db.Close() 149 db.AddQueryPattern(".*", &sqltypes.Result{}) 150 config := tabletenv.NewDefaultConfig() 151 config.DB = newDBConfigs(db) 152 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) 153 154 for _, exec := range []func() (int64, string, error){ 155 func() (int64, string, error) { 156 tx, _, schemaStateChanges, err := te.Begin(ctx, nil, 0, nil, &querypb.ExecuteOptions{}) 157 return tx, schemaStateChanges, err 158 }, 159 func() (int64, string, error) { 160 return te.ReserveBegin(ctx, &querypb.ExecuteOptions{}, nil, nil) 161 }, 162 } { 163 te.AcceptReadOnly() 164 tx1, _, err := exec() 165 require.NoError(t, err) 166 _, _, err = te.Commit(ctx, tx1) 167 require.NoError(t, err) 168 requireLogs(t, db.QueryLog(), "start transaction read only", "commit") 169 db.ResetQueryLog() 170 171 te.AcceptReadWrite() 172 tx2, _, err := exec() 173 require.NoError(t, err) 174 _, _, err = te.Commit(ctx, tx2) 175 require.NoError(t, err) 176 requireLogs(t, db.QueryLog(), "begin", "commit") 177 db.ResetQueryLog() 178 179 te.transition(Transitioning) 180 _, _, err = exec() 181 assert.EqualError(t, err, "tx engine can't accept new connections in state Transitioning") 182 183 te.transition(NotServing) 184 _, _, err = exec() 185 assert.EqualError(t, err, "tx engine can't accept new connections in state NotServing") 186 } 187 188 } 189 190 func TestTxEngineRenewFails(t *testing.T) { 191 db := setUpQueryExecutorTest(t) 192 defer db.Close() 193 db.AddQueryPattern(".*", &sqltypes.Result{}) 194 config := tabletenv.NewDefaultConfig() 195 config.DB = newDBConfigs(db) 196 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) 197 te.AcceptReadOnly() 198 options := &querypb.ExecuteOptions{} 199 connID, _, err := te.ReserveBegin(ctx, options, nil, nil) 200 require.NoError(t, err) 201 202 conn, err := te.txPool.GetAndLock(connID, "for test") 203 require.NoError(t, err) 204 conn.Unlock() // but we keep holding on to it... sneaky.... 205 206 // this next bit sets up the scp so our renew will fail 207 conn2, err := te.txPool.scp.NewConn(ctx, options, nil) 208 require.NoError(t, err) 209 defer conn2.Release(tx.TxCommit) 210 te.txPool.scp.lastID.Set(conn2.ConnID - 1) 211 212 // commit will do a renew 213 dbConn := conn.dbConn 214 _, _, err = te.Commit(ctx, connID) 215 require.Error(t, err) 216 assert.True(t, conn.IsClosed(), "connection was not closed") 217 assert.True(t, dbConn.IsClosed(), "underlying connection was not closed") 218 } 219 220 type TxType int 221 222 const ( 223 NoTx TxType = iota 224 ReadOnlyAccepted 225 WriteAccepted 226 ReadOnlyRejected 227 WriteRejected 228 ) 229 230 func (t TxType) String() string { 231 names := [...]string{ 232 "no transaction", 233 "read only transaction accepted", 234 "write transaction accepted", 235 "read only transaction rejected", 236 "write transaction rejected", 237 } 238 239 if t < NoTx || t > WriteRejected { 240 return "unknown" 241 } 242 243 return names[t] 244 } 245 246 type TestCase struct { 247 startState txEngineState 248 TxEngineStates []txEngineState 249 tx TxType 250 stateAssertion func(state txEngineState) error 251 } 252 253 func (test TestCase) String() string { 254 var sb strings.Builder 255 sb.WriteString("start from ") 256 sb.WriteString(test.startState.String()) 257 sb.WriteString(" with ") 258 sb.WriteString(test.tx.String()) 259 260 for _, change := range test.TxEngineStates { 261 sb.WriteString(" change state to ") 262 sb.WriteString(change.String()) 263 } 264 265 return sb.String() 266 } 267 268 func changeState(te *TxEngine, state txEngineState) { 269 switch state { 270 case AcceptingReadAndWrite: 271 te.AcceptReadWrite() 272 case AcceptingReadOnly: 273 te.AcceptReadOnly() 274 case NotServing: 275 te.Close() 276 } 277 } 278 279 func TestWithInnerTests(outerT *testing.T) { 280 281 tests := []TestCase{ 282 // Start from RW and test all single hop transitions with and without tx 283 {AcceptingReadAndWrite, []txEngineState{ 284 NotServing}, 285 NoTx, assertEndStateIs(NotServing)}, 286 287 {AcceptingReadAndWrite, []txEngineState{ 288 AcceptingReadAndWrite}, 289 NoTx, assertEndStateIs(AcceptingReadAndWrite)}, 290 291 {AcceptingReadAndWrite, []txEngineState{ 292 AcceptingReadOnly}, 293 NoTx, assertEndStateIs(AcceptingReadOnly)}, 294 295 {AcceptingReadAndWrite, []txEngineState{ 296 NotServing}, 297 WriteAccepted, assertEndStateIs(NotServing)}, 298 299 {AcceptingReadAndWrite, []txEngineState{ 300 AcceptingReadAndWrite}, 301 WriteAccepted, assertEndStateIs(AcceptingReadAndWrite)}, 302 303 {AcceptingReadAndWrite, []txEngineState{ 304 AcceptingReadOnly}, 305 WriteAccepted, assertEndStateIs(AcceptingReadOnly)}, 306 307 {AcceptingReadAndWrite, []txEngineState{ 308 NotServing}, 309 ReadOnlyAccepted, assertEndStateIs(NotServing)}, 310 311 {AcceptingReadAndWrite, []txEngineState{ 312 AcceptingReadAndWrite}, 313 ReadOnlyAccepted, assertEndStateIs(AcceptingReadAndWrite)}, 314 315 {AcceptingReadAndWrite, []txEngineState{ 316 AcceptingReadOnly}, 317 ReadOnlyAccepted, assertEndStateIs(AcceptingReadOnly)}, 318 319 // Start from RW and test all transitions with and without tx, plus a concurrent Stop() 320 {AcceptingReadAndWrite, []txEngineState{ 321 NotServing, 322 NotServing}, 323 NoTx, assertEndStateIs(NotServing)}, 324 325 {AcceptingReadAndWrite, []txEngineState{ 326 AcceptingReadAndWrite, 327 NotServing}, 328 NoTx, assertEndStateIs(NotServing)}, 329 330 {AcceptingReadAndWrite, []txEngineState{ 331 AcceptingReadOnly, 332 NotServing}, 333 NoTx, assertEndStateIs(NotServing)}, 334 335 {AcceptingReadAndWrite, []txEngineState{ 336 NotServing, 337 NotServing}, 338 WriteAccepted, assertEndStateIs(NotServing)}, 339 340 {AcceptingReadAndWrite, []txEngineState{ 341 AcceptingReadAndWrite, 342 NotServing}, 343 WriteAccepted, assertEndStateIs(NotServing)}, 344 345 {AcceptingReadAndWrite, []txEngineState{ 346 AcceptingReadOnly, 347 NotServing}, 348 WriteAccepted, assertEndStateIs(NotServing)}, 349 350 // Start from RW and test all transitions with and without tx, plus a concurrent ReadOnly() 351 {AcceptingReadAndWrite, []txEngineState{ 352 NotServing, 353 AcceptingReadOnly}, 354 NoTx, assertEndStateIs(AcceptingReadOnly)}, 355 356 {AcceptingReadAndWrite, []txEngineState{ 357 AcceptingReadAndWrite, 358 AcceptingReadOnly}, 359 NoTx, assertEndStateIs(AcceptingReadOnly)}, 360 361 {AcceptingReadAndWrite, []txEngineState{ 362 AcceptingReadOnly, 363 AcceptingReadOnly}, 364 NoTx, assertEndStateIs(AcceptingReadOnly)}, 365 366 {AcceptingReadAndWrite, []txEngineState{ 367 NotServing, 368 AcceptingReadOnly}, 369 WriteAccepted, assertEndStateIs(AcceptingReadOnly)}, 370 371 {AcceptingReadAndWrite, []txEngineState{ 372 AcceptingReadAndWrite, 373 AcceptingReadOnly}, 374 WriteAccepted, assertEndStateIs(AcceptingReadOnly)}, 375 376 {AcceptingReadAndWrite, []txEngineState{ 377 AcceptingReadOnly, 378 AcceptingReadOnly}, 379 WriteAccepted, assertEndStateIs(AcceptingReadOnly)}, 380 381 // Start from RO and test all single hop transitions with and without tx 382 {AcceptingReadOnly, []txEngineState{ 383 NotServing}, 384 NoTx, assertEndStateIs(NotServing)}, 385 386 {AcceptingReadOnly, []txEngineState{ 387 AcceptingReadAndWrite}, 388 NoTx, assertEndStateIs(AcceptingReadAndWrite)}, 389 390 {AcceptingReadOnly, []txEngineState{ 391 AcceptingReadOnly}, 392 NoTx, assertEndStateIs(AcceptingReadOnly)}, 393 394 {AcceptingReadOnly, []txEngineState{ 395 NotServing}, 396 WriteRejected, assertEndStateIs(NotServing)}, 397 398 {AcceptingReadOnly, []txEngineState{ 399 AcceptingReadAndWrite}, 400 WriteRejected, assertEndStateIs(AcceptingReadAndWrite)}, 401 402 {AcceptingReadOnly, []txEngineState{ 403 AcceptingReadOnly}, 404 WriteRejected, assertEndStateIs(AcceptingReadOnly)}, 405 406 // Start from RO and test all transitions with and without tx, plus a concurrent Stop() 407 {AcceptingReadOnly, []txEngineState{ 408 NotServing, 409 NotServing}, 410 NoTx, assertEndStateIs(NotServing)}, 411 412 {AcceptingReadOnly, []txEngineState{ 413 AcceptingReadAndWrite, 414 NotServing}, 415 NoTx, assertEndStateIs(NotServing)}, 416 417 {AcceptingReadOnly, []txEngineState{ 418 AcceptingReadOnly, 419 NotServing}, 420 NoTx, assertEndStateIs(NotServing)}, 421 422 {AcceptingReadOnly, []txEngineState{ 423 NotServing, 424 NotServing}, 425 WriteRejected, assertEndStateIs(NotServing)}, 426 427 {AcceptingReadOnly, []txEngineState{ 428 AcceptingReadAndWrite, 429 NotServing}, 430 WriteRejected, assertEndStateIs(NotServing)}, 431 432 {AcceptingReadOnly, []txEngineState{ 433 AcceptingReadOnly, 434 NotServing}, 435 WriteRejected, assertEndStateIs(NotServing)}, 436 437 // Start from RO and test all transitions with and without tx, plus a concurrent ReadWrite() 438 {AcceptingReadOnly, []txEngineState{ 439 NotServing, 440 AcceptingReadAndWrite}, 441 NoTx, assertEndStateIs(AcceptingReadAndWrite)}, 442 443 {AcceptingReadOnly, []txEngineState{ 444 AcceptingReadAndWrite, 445 AcceptingReadAndWrite}, 446 NoTx, assertEndStateIs(AcceptingReadAndWrite)}, 447 448 {AcceptingReadOnly, []txEngineState{ 449 AcceptingReadOnly, 450 AcceptingReadAndWrite}, 451 NoTx, assertEndStateIs(AcceptingReadAndWrite)}, 452 453 {AcceptingReadOnly, []txEngineState{ 454 NotServing, 455 AcceptingReadAndWrite}, 456 WriteRejected, assertEndStateIs(AcceptingReadAndWrite)}, 457 458 {AcceptingReadOnly, []txEngineState{ 459 AcceptingReadAndWrite, 460 AcceptingReadAndWrite}, 461 WriteRejected, assertEndStateIs(AcceptingReadAndWrite)}, 462 463 {AcceptingReadOnly, []txEngineState{ 464 AcceptingReadOnly, 465 AcceptingReadAndWrite}, 466 WriteRejected, assertEndStateIs(AcceptingReadAndWrite)}, 467 468 // Make sure that all transactions are rejected when we are not serving 469 {NotServing, []txEngineState{}, 470 WriteRejected, assertEndStateIs(NotServing)}, 471 472 {NotServing, []txEngineState{}, 473 ReadOnlyRejected, assertEndStateIs(NotServing)}, 474 } 475 476 for _, test := range tests { 477 outerT.Run(test.String(), func(t *testing.T) { 478 479 db := setUpQueryExecutorTest(t) 480 db.AddQuery("set transaction isolation level REPEATABLE READ", &sqltypes.Result{}) 481 db.AddQuery("start transaction with consistent snapshot, read only", &sqltypes.Result{}) 482 defer db.Close() 483 te := setupTxEngine(db) 484 485 changeState(te, test.startState) 486 487 switch test.tx { 488 case NoTx: 489 // nothing to do 490 case WriteAccepted: 491 require.NoError(t, 492 startTx(te, true)) 493 case ReadOnlyAccepted: 494 require.NoError(t, 495 startTx(te, false)) 496 case WriteRejected: 497 err := startTx(te, true) 498 require.Error(t, err) 499 case ReadOnlyRejected: 500 err := startTx(te, false) 501 require.Error(t, err) 502 default: 503 t.Fatalf("don't know how to [%v]", test.tx) 504 } 505 506 wg := sync.WaitGroup{} 507 for _, newState := range test.TxEngineStates { 508 wg.Add(1) 509 go func(s txEngineState) { 510 defer wg.Done() 511 512 changeState(te, s) 513 }(newState) 514 515 // We give the state changes a chance to get started 516 time.Sleep(10 * time.Millisecond) 517 } 518 519 // Let's wait for all transitions to wrap up 520 wg.Wait() 521 522 require.NoError(t, 523 test.stateAssertion(te.state)) 524 }) 525 } 526 } 527 528 func setupTxEngine(db *fakesqldb.DB) *TxEngine { 529 config := tabletenv.NewDefaultConfig() 530 config.DB = newDBConfigs(db) 531 config.TxPool.Size = 10 532 config.Oltp.TxTimeoutSeconds = 0.1 533 config.GracePeriods.ShutdownSeconds = 0 534 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) 535 return te 536 } 537 538 func assertEndStateIs(expected txEngineState) func(actual txEngineState) error { 539 return func(actual txEngineState) error { 540 if actual != expected { 541 return fmt.Errorf("expected the end state to be %v, but it was %v", expected, actual) 542 } 543 return nil 544 } 545 } 546 547 func startTx(te *TxEngine, writeTransaction bool) error { 548 options := &querypb.ExecuteOptions{} 549 if writeTransaction { 550 options.TransactionIsolation = querypb.ExecuteOptions_DEFAULT 551 } else { 552 options.TransactionIsolation = querypb.ExecuteOptions_CONSISTENT_SNAPSHOT_READ_ONLY 553 } 554 _, _, _, err := te.Begin(context.Background(), nil, 0, nil, options) 555 return err 556 } 557 558 func TestTxEngineFailReserve(t *testing.T) { 559 db := setUpQueryExecutorTest(t) 560 defer db.Close() 561 db.AddQueryPattern(".*", &sqltypes.Result{}) 562 config := tabletenv.NewDefaultConfig() 563 config.DB = newDBConfigs(db) 564 te := NewTxEngine(tabletenv.NewEnv(config, "TabletServerTest")) 565 566 options := &querypb.ExecuteOptions{} 567 _, err := te.Reserve(ctx, options, 0, nil) 568 assert.EqualError(t, err, "tx engine can't accept new connections in state NotServing") 569 570 _, _, err = te.ReserveBegin(ctx, options, nil, nil) 571 assert.EqualError(t, err, "tx engine can't accept new connections in state NotServing") 572 573 te.AcceptReadOnly() 574 575 db.AddRejectedQuery("dummy_query", errors.New("failed executing dummy_query")) 576 _, err = te.Reserve(ctx, options, 0, []string{"dummy_query"}) 577 assert.EqualError(t, err, "unknown error: failed executing dummy_query (errno 1105) (sqlstate HY000) during query: dummy_query") 578 579 _, _, err = te.ReserveBegin(ctx, options, []string{"dummy_query"}, nil) 580 assert.EqualError(t, err, "unknown error: failed executing dummy_query (errno 1105) (sqlstate HY000) during query: dummy_query") 581 582 nonExistingID := int64(42) 583 _, err = te.Reserve(ctx, options, nonExistingID, nil) 584 assert.EqualError(t, err, "transaction 42: not found") 585 586 txID, _, _, err := te.Begin(ctx, nil, 0, nil, options) 587 require.NoError(t, err) 588 conn, err := te.txPool.GetAndLock(txID, "for test") 589 require.NoError(t, err) 590 conn.Unlock() // but we keep holding on to it... sneaky.... 591 592 _, err = te.Reserve(ctx, options, txID, []string{"dummy_query"}) 593 assert.EqualError(t, err, "unknown error: failed executing dummy_query (errno 1105) (sqlstate HY000) during query: dummy_query") 594 595 connID, _, err := te.Commit(ctx, txID) 596 require.Error(t, err) 597 assert.Zero(t, connID) 598 }