vitess.io/vitess@v0.16.2/go/vt/vttablet/tabletserver/twopc.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package tabletserver
    18  
    19  import (
    20  	"fmt"
    21  	"time"
    22  
    23  	"vitess.io/vitess/go/vt/vttablet/tabletserver/tx"
    24  
    25  	"vitess.io/vitess/go/vt/vtgate/evalengine"
    26  
    27  	"context"
    28  
    29  	"vitess.io/vitess/go/sqltypes"
    30  	"vitess.io/vitess/go/vt/dbconfigs"
    31  	"vitess.io/vitess/go/vt/dbconnpool"
    32  	"vitess.io/vitess/go/vt/log"
    33  	"vitess.io/vitess/go/vt/sqlparser"
    34  	"vitess.io/vitess/go/vt/vterrors"
    35  	"vitess.io/vitess/go/vt/vttablet/tabletserver/connpool"
    36  
    37  	querypb "vitess.io/vitess/go/vt/proto/query"
    38  	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
    39  	vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc"
    40  )
    41  
    42  const (
    43  	// RedoStateFailed represents the Failed state for redo_state.
    44  	RedoStateFailed = 0
    45  	// RedoStatePrepared represents the Prepared state for redo_state.
    46  	RedoStatePrepared = 1
    47  	// DTStatePrepare represents the PREPARE state for dt_state.
    48  	DTStatePrepare = querypb.TransactionState_PREPARE
    49  	// DTStateCommit represents the COMMIT state for dt_state.
    50  	DTStateCommit = querypb.TransactionState_COMMIT
    51  	// DTStateRollback represents the ROLLBACK state for dt_state.
    52  	DTStateRollback = querypb.TransactionState_ROLLBACK
    53  
    54  	sqlReadAllRedo = `select t.dtid, t.state, t.time_created, s.statement
    55  	from %s.redo_state t
    56    join %s.redo_statement s on t.dtid = s.dtid
    57  	order by t.dtid, s.id`
    58  
    59  	sqlReadAllTransactions = `select t.dtid, t.state, t.time_created, p.keyspace, p.shard
    60  	from %s.dt_state t
    61    join %s.dt_participant p on t.dtid = p.dtid
    62  	order by t.dtid, p.id`
    63  )
    64  
    65  // TwoPC performs 2PC metadata management (MM) functions.
    66  type TwoPC struct {
    67  	readPool *connpool.Pool
    68  
    69  	insertRedoTx        *sqlparser.ParsedQuery
    70  	insertRedoStmt      *sqlparser.ParsedQuery
    71  	updateRedoTx        *sqlparser.ParsedQuery
    72  	deleteRedoTx        *sqlparser.ParsedQuery
    73  	deleteRedoStmt      *sqlparser.ParsedQuery
    74  	readAllRedo         string
    75  	countUnresolvedRedo *sqlparser.ParsedQuery
    76  
    77  	insertTransaction   *sqlparser.ParsedQuery
    78  	insertParticipants  *sqlparser.ParsedQuery
    79  	transition          *sqlparser.ParsedQuery
    80  	deleteTransaction   *sqlparser.ParsedQuery
    81  	deleteParticipants  *sqlparser.ParsedQuery
    82  	readTransaction     *sqlparser.ParsedQuery
    83  	readParticipants    *sqlparser.ParsedQuery
    84  	readAbandoned       *sqlparser.ParsedQuery
    85  	readAllTransactions string
    86  }
    87  
    88  // NewTwoPC creates a TwoPC variable.
    89  func NewTwoPC(readPool *connpool.Pool) *TwoPC {
    90  	tpc := &TwoPC{readPool: readPool}
    91  	dbname := "_vt"
    92  	tpc.insertRedoTx = sqlparser.BuildParsedQuery(
    93  		"insert into %s.redo_state(dtid, state, time_created) values (%a, %a, %a)",
    94  		dbname, ":dtid", ":state", ":time_created")
    95  	tpc.insertRedoStmt = sqlparser.BuildParsedQuery(
    96  		"insert into %s.redo_statement(dtid, id, statement) values %a",
    97  		dbname, ":vals")
    98  	tpc.updateRedoTx = sqlparser.BuildParsedQuery(
    99  		"update %s.redo_state set state = %a where dtid = %a",
   100  		dbname, ":state", ":dtid")
   101  	tpc.deleteRedoTx = sqlparser.BuildParsedQuery(
   102  		"delete from %s.redo_state where dtid = %a",
   103  		dbname, ":dtid")
   104  	tpc.deleteRedoStmt = sqlparser.BuildParsedQuery(
   105  		"delete from %s.redo_statement where dtid = %a",
   106  		dbname, ":dtid")
   107  	tpc.readAllRedo = fmt.Sprintf(sqlReadAllRedo, dbname, dbname)
   108  	tpc.countUnresolvedRedo = sqlparser.BuildParsedQuery(
   109  		"select count(*) from %s.redo_state where time_created < %a",
   110  		dbname, ":time_created")
   111  
   112  	tpc.insertTransaction = sqlparser.BuildParsedQuery(
   113  		"insert into %s.dt_state(dtid, state, time_created) values (%a, %a, %a)",
   114  		dbname, ":dtid", ":state", ":cur_time")
   115  	tpc.insertParticipants = sqlparser.BuildParsedQuery(
   116  		"insert into %s.dt_participant(dtid, id, keyspace, shard) values %a",
   117  		dbname, ":vals")
   118  	tpc.transition = sqlparser.BuildParsedQuery(
   119  		"update %s.dt_state set state = %a where dtid = %a and state = %a",
   120  		dbname, ":state", ":dtid", ":prepare")
   121  	tpc.deleteTransaction = sqlparser.BuildParsedQuery(
   122  		"delete from %s.dt_state where dtid = %a",
   123  		dbname, ":dtid")
   124  	tpc.deleteParticipants = sqlparser.BuildParsedQuery(
   125  		"delete from %s.dt_participant where dtid = %a",
   126  		dbname, ":dtid")
   127  	tpc.readTransaction = sqlparser.BuildParsedQuery(
   128  		"select dtid, state, time_created from %s.dt_state where dtid = %a",
   129  		dbname, ":dtid")
   130  	tpc.readParticipants = sqlparser.BuildParsedQuery(
   131  		"select keyspace, shard from %s.dt_participant where dtid = %a",
   132  		dbname, ":dtid")
   133  	tpc.readAbandoned = sqlparser.BuildParsedQuery(
   134  		"select dtid, time_created from %s.dt_state where time_created < %a",
   135  		dbname, ":time_created")
   136  	tpc.readAllTransactions = fmt.Sprintf(sqlReadAllTransactions, dbname, dbname)
   137  	return tpc
   138  }
   139  
   140  // Open starts the TwoPC service.
   141  func (tpc *TwoPC) Open(dbconfigs *dbconfigs.DBConfigs) error {
   142  	conn, err := dbconnpool.NewDBConnection(context.TODO(), dbconfigs.DbaWithDB())
   143  	if err != nil {
   144  		return err
   145  	}
   146  	defer conn.Close()
   147  	tpc.readPool.Open(dbconfigs.AppWithDB(), dbconfigs.DbaWithDB(), dbconfigs.DbaWithDB())
   148  	log.Infof("TwoPC: Engine open succeeded")
   149  	return nil
   150  }
   151  
   152  // Close closes the TwoPC service.
   153  func (tpc *TwoPC) Close() {
   154  	tpc.readPool.Close()
   155  }
   156  
   157  // SaveRedo saves the statements in the redo log using the supplied connection.
   158  func (tpc *TwoPC) SaveRedo(ctx context.Context, conn *StatefulConnection, dtid string, queries []string) error {
   159  	bindVars := map[string]*querypb.BindVariable{
   160  		"dtid":         sqltypes.StringBindVariable(dtid),
   161  		"state":        sqltypes.Int64BindVariable(RedoStatePrepared),
   162  		"time_created": sqltypes.Int64BindVariable(time.Now().UnixNano()),
   163  	}
   164  	_, err := tpc.exec(ctx, conn, tpc.insertRedoTx, bindVars)
   165  	if err != nil {
   166  		return err
   167  	}
   168  
   169  	rows := make([][]sqltypes.Value, len(queries))
   170  	for i, query := range queries {
   171  		rows[i] = []sqltypes.Value{
   172  			sqltypes.NewVarBinary(dtid),
   173  			sqltypes.NewInt64(int64(i + 1)),
   174  			sqltypes.NewVarBinary(query),
   175  		}
   176  	}
   177  	extras := map[string]sqlparser.Encodable{
   178  		"vals": sqlparser.InsertValues(rows),
   179  	}
   180  	q, err := tpc.insertRedoStmt.GenerateQuery(nil, extras)
   181  	if err != nil {
   182  		return err
   183  	}
   184  	_, err = conn.Exec(ctx, q, 1, false)
   185  	return err
   186  }
   187  
   188  // UpdateRedo changes the state of the redo log for the dtid.
   189  func (tpc *TwoPC) UpdateRedo(ctx context.Context, conn *StatefulConnection, dtid string, state int) error {
   190  	bindVars := map[string]*querypb.BindVariable{
   191  		"dtid":  sqltypes.StringBindVariable(dtid),
   192  		"state": sqltypes.Int64BindVariable(int64(state)),
   193  	}
   194  	_, err := tpc.exec(ctx, conn, tpc.updateRedoTx, bindVars)
   195  	return err
   196  }
   197  
   198  // DeleteRedo deletes the redo log for the dtid.
   199  func (tpc *TwoPC) DeleteRedo(ctx context.Context, conn *StatefulConnection, dtid string) error {
   200  	bindVars := map[string]*querypb.BindVariable{
   201  		"dtid": sqltypes.StringBindVariable(dtid),
   202  	}
   203  	_, err := tpc.exec(ctx, conn, tpc.deleteRedoTx, bindVars)
   204  	if err != nil {
   205  		return err
   206  	}
   207  	_, err = tpc.exec(ctx, conn, tpc.deleteRedoStmt, bindVars)
   208  	return err
   209  }
   210  
   211  // ReadAllRedo returns all the prepared transactions from the redo logs.
   212  func (tpc *TwoPC) ReadAllRedo(ctx context.Context) (prepared, failed []*tx.PreparedTx, err error) {
   213  	conn, err := tpc.readPool.Get(ctx, nil)
   214  	if err != nil {
   215  		return nil, nil, err
   216  	}
   217  	defer conn.Recycle()
   218  
   219  	qr, err := conn.Exec(ctx, tpc.readAllRedo, 10000, false)
   220  	if err != nil {
   221  		return nil, nil, err
   222  	}
   223  
   224  	var curTx *tx.PreparedTx
   225  	for _, row := range qr.Rows {
   226  		dtid := row[0].ToString()
   227  		if curTx == nil || dtid != curTx.Dtid {
   228  			// Initialize the new element.
   229  			// A failure in time parsing will show up as a very old time,
   230  			// which is harmless.
   231  			tm, _ := evalengine.ToInt64(row[2])
   232  			curTx = &tx.PreparedTx{
   233  				Dtid: dtid,
   234  				Time: time.Unix(0, tm),
   235  			}
   236  			st, err := evalengine.ToInt64(row[1])
   237  			if err != nil {
   238  				log.Errorf("Error parsing state for dtid %s: %v.", dtid, err)
   239  			}
   240  			switch st {
   241  			case RedoStatePrepared:
   242  				prepared = append(prepared, curTx)
   243  			default:
   244  				if st != RedoStateFailed {
   245  					log.Errorf("Unexpected state for dtid %s: %d. Treating it as a failure.", dtid, st)
   246  				}
   247  				failed = append(failed, curTx)
   248  			}
   249  		}
   250  		curTx.Queries = append(curTx.Queries, row[3].ToString())
   251  	}
   252  	return prepared, failed, nil
   253  }
   254  
   255  // CountUnresolvedRedo returns the number of prepared transactions that are still unresolved.
   256  func (tpc *TwoPC) CountUnresolvedRedo(ctx context.Context, unresolvedTime time.Time) (int64, error) {
   257  	conn, err := tpc.readPool.Get(ctx, nil)
   258  	if err != nil {
   259  		return 0, err
   260  	}
   261  	defer conn.Recycle()
   262  
   263  	bindVars := map[string]*querypb.BindVariable{
   264  		"time_created": sqltypes.Int64BindVariable(unresolvedTime.UnixNano()),
   265  	}
   266  	qr, err := tpc.read(ctx, conn, tpc.countUnresolvedRedo, bindVars)
   267  	if err != nil {
   268  		return 0, err
   269  	}
   270  	if len(qr.Rows) < 1 {
   271  		return 0, nil
   272  	}
   273  	v, _ := evalengine.ToInt64(qr.Rows[0][0])
   274  	return v, nil
   275  }
   276  
   277  // CreateTransaction saves the metadata of a 2pc transaction as Prepared.
   278  func (tpc *TwoPC) CreateTransaction(ctx context.Context, conn *StatefulConnection, dtid string, participants []*querypb.Target) error {
   279  	bindVars := map[string]*querypb.BindVariable{
   280  		"dtid":     sqltypes.StringBindVariable(dtid),
   281  		"state":    sqltypes.Int64BindVariable(int64(DTStatePrepare)),
   282  		"cur_time": sqltypes.Int64BindVariable(time.Now().UnixNano()),
   283  	}
   284  	_, err := tpc.exec(ctx, conn, tpc.insertTransaction, bindVars)
   285  	if err != nil {
   286  		return err
   287  	}
   288  
   289  	rows := make([][]sqltypes.Value, len(participants))
   290  	for i, participant := range participants {
   291  		rows[i] = []sqltypes.Value{
   292  			sqltypes.NewVarBinary(dtid),
   293  			sqltypes.NewInt64(int64(i + 1)),
   294  			sqltypes.NewVarBinary(participant.Keyspace),
   295  			sqltypes.NewVarBinary(participant.Shard),
   296  		}
   297  	}
   298  	extras := map[string]sqlparser.Encodable{
   299  		"vals": sqlparser.InsertValues(rows),
   300  	}
   301  	q, err := tpc.insertParticipants.GenerateQuery(nil, extras)
   302  	if err != nil {
   303  		return err
   304  	}
   305  	_, err = conn.Exec(ctx, q, 1, false)
   306  	return err
   307  }
   308  
   309  // Transition performs a transition from Prepare to the specified state.
   310  // If the transaction is not a in the Prepare state, an error is returned.
   311  func (tpc *TwoPC) Transition(ctx context.Context, conn *StatefulConnection, dtid string, state querypb.TransactionState) error {
   312  	bindVars := map[string]*querypb.BindVariable{
   313  		"dtid":    sqltypes.StringBindVariable(dtid),
   314  		"state":   sqltypes.Int64BindVariable(int64(state)),
   315  		"prepare": sqltypes.Int64BindVariable(int64(querypb.TransactionState_PREPARE)),
   316  	}
   317  	qr, err := tpc.exec(ctx, conn, tpc.transition, bindVars)
   318  	if err != nil {
   319  		return err
   320  	}
   321  	if qr.RowsAffected != 1 {
   322  		return vterrors.Errorf(vtrpcpb.Code_NOT_FOUND, "could not transition to %v: %s", state, dtid)
   323  	}
   324  	return nil
   325  }
   326  
   327  // DeleteTransaction deletes the metadata for the specified transaction.
   328  func (tpc *TwoPC) DeleteTransaction(ctx context.Context, conn *StatefulConnection, dtid string) error {
   329  	bindVars := map[string]*querypb.BindVariable{
   330  		"dtid": sqltypes.StringBindVariable(dtid),
   331  	}
   332  	_, err := tpc.exec(ctx, conn, tpc.deleteTransaction, bindVars)
   333  	if err != nil {
   334  		return err
   335  	}
   336  	_, err = tpc.exec(ctx, conn, tpc.deleteParticipants, bindVars)
   337  	return err
   338  }
   339  
   340  // ReadTransaction returns the metadata for the transaction.
   341  func (tpc *TwoPC) ReadTransaction(ctx context.Context, dtid string) (*querypb.TransactionMetadata, error) {
   342  	conn, err := tpc.readPool.Get(ctx, nil)
   343  	if err != nil {
   344  		return nil, err
   345  	}
   346  	defer conn.Recycle()
   347  
   348  	result := &querypb.TransactionMetadata{}
   349  	bindVars := map[string]*querypb.BindVariable{
   350  		"dtid": sqltypes.StringBindVariable(dtid),
   351  	}
   352  	qr, err := tpc.read(ctx, conn, tpc.readTransaction, bindVars)
   353  	if err != nil {
   354  		return nil, err
   355  	}
   356  	if len(qr.Rows) == 0 {
   357  		return result, nil
   358  	}
   359  	result.Dtid = qr.Rows[0][0].ToString()
   360  	st, err := evalengine.ToInt64(qr.Rows[0][1])
   361  	if err != nil {
   362  		return nil, vterrors.Wrapf(err, "error parsing state for dtid %s", dtid)
   363  	}
   364  	result.State = querypb.TransactionState(st)
   365  	if result.State < querypb.TransactionState_PREPARE || result.State > querypb.TransactionState_ROLLBACK {
   366  		return nil, fmt.Errorf("unexpected state for dtid %s: %v", dtid, result.State)
   367  	}
   368  	// A failure in time parsing will show up as a very old time,
   369  	// which is harmless.
   370  	tm, _ := evalengine.ToInt64(qr.Rows[0][2])
   371  	result.TimeCreated = tm
   372  
   373  	qr, err = tpc.read(ctx, conn, tpc.readParticipants, bindVars)
   374  	if err != nil {
   375  		return nil, err
   376  	}
   377  	participants := make([]*querypb.Target, 0, len(qr.Rows))
   378  	for _, row := range qr.Rows {
   379  		participants = append(participants, &querypb.Target{
   380  			Keyspace:   row[0].ToString(),
   381  			Shard:      row[1].ToString(),
   382  			TabletType: topodatapb.TabletType_PRIMARY,
   383  		})
   384  	}
   385  	result.Participants = participants
   386  	return result, nil
   387  }
   388  
   389  // ReadAbandoned returns the list of abandoned transactions
   390  // and their associated start time.
   391  func (tpc *TwoPC) ReadAbandoned(ctx context.Context, abandonTime time.Time) (map[string]time.Time, error) {
   392  	conn, err := tpc.readPool.Get(ctx, nil)
   393  	if err != nil {
   394  		return nil, err
   395  	}
   396  	defer conn.Recycle()
   397  
   398  	bindVars := map[string]*querypb.BindVariable{
   399  		"time_created": sqltypes.Int64BindVariable(abandonTime.UnixNano()),
   400  	}
   401  	qr, err := tpc.read(ctx, conn, tpc.readAbandoned, bindVars)
   402  	if err != nil {
   403  		return nil, err
   404  	}
   405  	txs := make(map[string]time.Time, len(qr.Rows))
   406  	for _, row := range qr.Rows {
   407  		t, err := evalengine.ToInt64(row[1])
   408  		if err != nil {
   409  			return nil, err
   410  		}
   411  		txs[row[0].ToString()] = time.Unix(0, t)
   412  	}
   413  	return txs, nil
   414  }
   415  
   416  // ReadAllTransactions returns info about all distributed transactions.
   417  func (tpc *TwoPC) ReadAllTransactions(ctx context.Context) ([]*tx.DistributedTx, error) {
   418  	conn, err := tpc.readPool.Get(ctx, nil)
   419  	if err != nil {
   420  		return nil, err
   421  	}
   422  	defer conn.Recycle()
   423  
   424  	qr, err := conn.Exec(ctx, tpc.readAllTransactions, 10000, false)
   425  	if err != nil {
   426  		return nil, err
   427  	}
   428  
   429  	var curTx *tx.DistributedTx
   430  	var distributed []*tx.DistributedTx
   431  	for _, row := range qr.Rows {
   432  		dtid := row[0].ToString()
   433  		if curTx == nil || dtid != curTx.Dtid {
   434  			// Initialize the new element.
   435  			// A failure in time parsing will show up as a very old time,
   436  			// which is harmless.
   437  			tm, _ := evalengine.ToInt64(row[2])
   438  			st, err := evalengine.ToInt64(row[1])
   439  			// Just log on error and continue. The state will show up as UNKNOWN
   440  			// on the display.
   441  			if err != nil {
   442  				log.Errorf("Error parsing state for dtid %s: %v.", dtid, err)
   443  			}
   444  			protostate := querypb.TransactionState(st)
   445  			if protostate < querypb.TransactionState_UNKNOWN || protostate > querypb.TransactionState_ROLLBACK {
   446  				log.Errorf("Unexpected state for dtid %s: %v.", dtid, protostate)
   447  			}
   448  			curTx = &tx.DistributedTx{
   449  				Dtid:    dtid,
   450  				State:   querypb.TransactionState(st).String(),
   451  				Created: time.Unix(0, tm),
   452  			}
   453  			distributed = append(distributed, curTx)
   454  		}
   455  		curTx.Participants = append(curTx.Participants, querypb.Target{
   456  			Keyspace: row[3].ToString(),
   457  			Shard:    row[4].ToString(),
   458  		})
   459  	}
   460  	return distributed, nil
   461  }
   462  
   463  func (tpc *TwoPC) exec(ctx context.Context, conn *StatefulConnection, pq *sqlparser.ParsedQuery, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) {
   464  	q, err := pq.GenerateQuery(bindVars, nil)
   465  	if err != nil {
   466  		return nil, err
   467  	}
   468  	return conn.Exec(ctx, q, 1, false)
   469  }
   470  
   471  func (tpc *TwoPC) read(ctx context.Context, conn *connpool.DBConn, pq *sqlparser.ParsedQuery, bindVars map[string]*querypb.BindVariable) (*sqltypes.Result, error) {
   472  	q, err := pq.GenerateQuery(bindVars, nil)
   473  	if err != nil {
   474  		return nil, err
   475  	}
   476  	return conn.Exec(ctx, q, 10000, false)
   477  }