vitess.io/vitess@v0.16.2/go/test/endtoend/cluster/vttablet_process.go (about)

     1  /*
     2  Copyright 2019 The Vitess Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  
    16  */
    17  
    18  package cluster
    19  
    20  import (
    21  	"bufio"
    22  	"context"
    23  	"encoding/json"
    24  	"errors"
    25  	"fmt"
    26  	"io"
    27  	"net/http"
    28  	"os"
    29  	"os/exec"
    30  	"path"
    31  	"reflect"
    32  	"strconv"
    33  	"strings"
    34  	"syscall"
    35  	"testing"
    36  	"time"
    37  
    38  	"vitess.io/vitess/go/mysql"
    39  	"vitess.io/vitess/go/sqltypes"
    40  	"vitess.io/vitess/go/vt/log"
    41  )
    42  
    43  // VttabletProcess is a generic handle for a running vttablet .
    44  // It can be spawned manually
    45  type VttabletProcess struct {
    46  	Name                        string
    47  	Binary                      string
    48  	FileToLogQueries            string
    49  	TabletUID                   int
    50  	TabletPath                  string
    51  	Cell                        string
    52  	Port                        int
    53  	GrpcPort                    int
    54  	Shard                       string
    55  	CommonArg                   VtctlProcess
    56  	LogDir                      string
    57  	TabletHostname              string
    58  	Keyspace                    string
    59  	TabletType                  string
    60  	HealthCheckInterval         int
    61  	BackupStorageImplementation string
    62  	FileBackupStorageRoot       string
    63  	ServiceMap                  string
    64  	VtctldAddress               string
    65  	Directory                   string
    66  	VerifyURL                   string
    67  	QueryzURL                   string
    68  	StatusDetailsURL            string
    69  	SupportsBackup              bool
    70  	ServingStatus               string
    71  	DbPassword                  string
    72  	DbPort                      int
    73  	VreplicationTabletType      string
    74  	DbFlavor                    string
    75  	Charset                     string
    76  	ConsolidationsURL           string
    77  
    78  	//Extra Args to be set before starting the vttablet process
    79  	ExtraArgs []string
    80  
    81  	proc *exec.Cmd
    82  	exit chan error
    83  }
    84  
    85  // Setup starts vttablet process with required arguements
    86  func (vttablet *VttabletProcess) Setup() (err error) {
    87  
    88  	vttablet.proc = exec.Command(
    89  		vttablet.Binary,
    90  		"--topo_implementation", vttablet.CommonArg.TopoImplementation,
    91  		"--topo_global_server_address", vttablet.CommonArg.TopoGlobalAddress,
    92  		"--topo_global_root", vttablet.CommonArg.TopoGlobalRoot,
    93  		"--log_queries_to_file", vttablet.FileToLogQueries,
    94  		"--tablet-path", vttablet.TabletPath,
    95  		"--port", fmt.Sprintf("%d", vttablet.Port),
    96  		"--grpc_port", fmt.Sprintf("%d", vttablet.GrpcPort),
    97  		"--init_shard", vttablet.Shard,
    98  		"--log_dir", vttablet.LogDir,
    99  		"--tablet_hostname", vttablet.TabletHostname,
   100  		"--init_keyspace", vttablet.Keyspace,
   101  		"--init_tablet_type", vttablet.TabletType,
   102  		"--health_check_interval", fmt.Sprintf("%ds", vttablet.HealthCheckInterval),
   103  		"--enable_replication_reporter",
   104  		"--backup_storage_implementation", vttablet.BackupStorageImplementation,
   105  		"--file_backup_storage_root", vttablet.FileBackupStorageRoot,
   106  		"--service_map", vttablet.ServiceMap,
   107  		"--vtctld_addr", vttablet.VtctldAddress,
   108  		"--vtctld_addr", vttablet.VtctldAddress,
   109  		"--vreplication_tablet_type", vttablet.VreplicationTabletType,
   110  		"--db_charset", vttablet.Charset,
   111  	)
   112  	if *isCoverage {
   113  		vttablet.proc.Args = append(vttablet.proc.Args, "--test.coverprofile="+getCoveragePath("vttablet.out"))
   114  	}
   115  	if *PerfTest {
   116  		vttablet.proc.Args = append(vttablet.proc.Args, "--pprof", fmt.Sprintf("cpu,waitSig,path=vttablet_pprof_%s", vttablet.Name))
   117  	}
   118  
   119  	if vttablet.SupportsBackup {
   120  		vttablet.proc.Args = append(vttablet.proc.Args, "--restore_from_backup")
   121  	}
   122  	if vttablet.DbFlavor != "" {
   123  		vttablet.proc.Args = append(vttablet.proc.Args, fmt.Sprintf("--db_flavor=%s", vttablet.DbFlavor))
   124  	}
   125  
   126  	vttablet.proc.Args = append(vttablet.proc.Args, vttablet.ExtraArgs...)
   127  	fname := path.Join(vttablet.LogDir, vttablet.TabletPath+"-vttablet-stderr.txt")
   128  	errFile, _ := os.Create(fname)
   129  	vttablet.proc.Stderr = errFile
   130  
   131  	vttablet.proc.Env = append(vttablet.proc.Env, os.Environ()...)
   132  
   133  	log.Infof("Running vttablet with command: %v", strings.Join(vttablet.proc.Args, " "))
   134  
   135  	err = vttablet.proc.Start()
   136  	if err != nil {
   137  		return
   138  	}
   139  
   140  	vttablet.exit = make(chan error)
   141  	go func() {
   142  		if vttablet.proc != nil {
   143  			vttablet.exit <- vttablet.proc.Wait()
   144  			close(vttablet.exit)
   145  		}
   146  	}()
   147  
   148  	if vttablet.ServingStatus != "" {
   149  		if err = vttablet.WaitForTabletStatus(vttablet.ServingStatus); err != nil {
   150  			errFileContent, _ := os.ReadFile(fname)
   151  			if errFileContent != nil {
   152  				log.Infof("vttablet error:\n%s\n", string(errFileContent))
   153  			}
   154  			return fmt.Errorf("process '%s' timed out after 10s (err: %s)", vttablet.Name, err)
   155  		}
   156  	}
   157  	return nil
   158  }
   159  
   160  // GetStatus returns /debug/status endpoint result
   161  func (vttablet *VttabletProcess) GetStatus() string {
   162  	URL := fmt.Sprintf("http://%s:%d/debug/status", vttablet.TabletHostname, vttablet.Port)
   163  	resp, err := http.Get(URL)
   164  	if err != nil {
   165  		return ""
   166  	}
   167  	defer resp.Body.Close()
   168  	if resp.StatusCode == 200 {
   169  		respByte, _ := io.ReadAll(resp.Body)
   170  		return string(respByte)
   171  	}
   172  	return ""
   173  }
   174  
   175  // GetVars gets the debug vars as map
   176  func (vttablet *VttabletProcess) GetVars() map[string]any {
   177  	resp, err := http.Get(vttablet.VerifyURL)
   178  	if err != nil {
   179  		return nil
   180  	}
   181  	defer resp.Body.Close()
   182  
   183  	if resp.StatusCode == 200 {
   184  		resultMap := make(map[string]any)
   185  		respByte, _ := io.ReadAll(resp.Body)
   186  		err := json.Unmarshal(respByte, &resultMap)
   187  		if err != nil {
   188  			return nil
   189  		}
   190  		return resultMap
   191  	}
   192  	return nil
   193  }
   194  
   195  // GetStatusDetails gets the status details
   196  func (vttablet *VttabletProcess) GetStatusDetails() string {
   197  	resp, err := http.Get(vttablet.StatusDetailsURL)
   198  	if err != nil {
   199  		return fmt.Sprintf("Status details failed: %v", err.Error())
   200  	}
   201  	defer resp.Body.Close()
   202  
   203  	respByte, _ := io.ReadAll(resp.Body)
   204  	return string(respByte)
   205  }
   206  
   207  // GetConsolidations gets consolidations
   208  func (vttablet *VttabletProcess) GetConsolidations() (map[string]int, error) {
   209  	resp, err := http.Get(vttablet.ConsolidationsURL)
   210  	if err != nil {
   211  		return nil, fmt.Errorf("failed to get consolidations: %v", err)
   212  	}
   213  	defer resp.Body.Close()
   214  
   215  	result := make(map[string]int)
   216  
   217  	scanner := bufio.NewScanner(resp.Body)
   218  	for scanner.Scan() {
   219  		line := scanner.Text()
   220  		splits := strings.SplitN(line, ":", 2)
   221  		if len(splits) != 2 {
   222  			return nil, fmt.Errorf("failed to split consolidations line: %v", err)
   223  		}
   224  		// Discard "Length: [N]" lines.
   225  		if splits[0] == "Length" {
   226  			continue
   227  		}
   228  		countS := splits[0]
   229  		countI64, err := strconv.ParseInt(countS, 10, 32)
   230  		if err != nil {
   231  			return nil, fmt.Errorf("failed to parse consolidations count: %v", err)
   232  		}
   233  		result[strings.TrimSpace(splits[1])] = int(countI64)
   234  	}
   235  	if err := scanner.Err(); err != nil && !errors.Is(err, io.EOF) {
   236  		return nil, fmt.Errorf("failed to read consolidations: %v", err)
   237  	}
   238  
   239  	return result, nil
   240  }
   241  
   242  // WaitForStatus waits till desired status of tablet is reached
   243  func (vttablet *VttabletProcess) WaitForStatus(status string, howLong time.Duration) bool {
   244  	ticker := time.NewTicker(howLong)
   245  	for range ticker.C {
   246  		if vttablet.GetTabletStatus() == status {
   247  			return true
   248  		}
   249  	}
   250  	return false
   251  }
   252  
   253  // GetTabletStatus returns the tablet state as seen in /debug/vars TabletStateName
   254  func (vttablet *VttabletProcess) GetTabletStatus() string {
   255  	resultMap := vttablet.GetVars()
   256  	if resultMap != nil {
   257  		return reflect.ValueOf(resultMap["TabletStateName"]).String()
   258  	}
   259  	return ""
   260  }
   261  
   262  // GetTabletType returns the tablet type as seen in /debug/vars TabletType
   263  func (vttablet *VttabletProcess) GetTabletType() string {
   264  	resultMap := vttablet.GetVars()
   265  	if resultMap != nil {
   266  		return reflect.ValueOf(resultMap["TabletType"]).String()
   267  	}
   268  	return ""
   269  }
   270  
   271  // WaitForTabletStatus waits for 10 second till expected status is reached
   272  func (vttablet *VttabletProcess) WaitForTabletStatus(expectedStatus string) error {
   273  	return vttablet.WaitForTabletStatusesForTimeout([]string{expectedStatus}, 10*time.Second)
   274  }
   275  
   276  // WaitForTabletStatuses waits for 10 second till one of expected statuses is reached
   277  func (vttablet *VttabletProcess) WaitForTabletStatuses(expectedStatuses []string) error {
   278  	return vttablet.WaitForTabletStatusesForTimeout(expectedStatuses, 10*time.Second)
   279  }
   280  
   281  // WaitForTabletTypes waits for 10 second till one of expected statuses is reached
   282  func (vttablet *VttabletProcess) WaitForTabletTypes(expectedTypes []string) error {
   283  	return vttablet.WaitForTabletTypesForTimeout(expectedTypes, 10*time.Second)
   284  }
   285  
   286  // WaitForTabletStatusesForTimeout waits till the tablet reaches to any of the provided statuses
   287  func (vttablet *VttabletProcess) WaitForTabletStatusesForTimeout(expectedStatuses []string, timeout time.Duration) error {
   288  	waitUntil := time.Now().Add(timeout)
   289  	var status string
   290  	for time.Now().Before(waitUntil) {
   291  		status = vttablet.GetTabletStatus()
   292  		if contains(expectedStatuses, status) {
   293  			return nil
   294  		}
   295  		select {
   296  		case err := <-vttablet.exit:
   297  			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
   298  		default:
   299  			time.Sleep(300 * time.Millisecond)
   300  		}
   301  	}
   302  	return fmt.Errorf("Vttablet %s, current status = %s, expected status [%s] not reached, details: %v",
   303  		vttablet.TabletPath, status, strings.Join(expectedStatuses, ","), vttablet.GetStatusDetails())
   304  }
   305  
   306  // WaitForTabletTypesForTimeout waits till the tablet reaches to any of the provided types
   307  func (vttablet *VttabletProcess) WaitForTabletTypesForTimeout(expectedTypes []string, timeout time.Duration) error {
   308  	waitUntil := time.Now().Add(timeout)
   309  	var tabletType string
   310  	for time.Now().Before(waitUntil) {
   311  		tabletType = vttablet.GetTabletType()
   312  		if contains(expectedTypes, tabletType) {
   313  			return nil
   314  		}
   315  		select {
   316  		case err := <-vttablet.exit:
   317  			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
   318  		default:
   319  			time.Sleep(300 * time.Millisecond)
   320  		}
   321  	}
   322  	return fmt.Errorf("Vttablet %s, current type = %s, expected type [%s] not reached, status details: %v",
   323  		vttablet.TabletPath, tabletType, strings.Join(expectedTypes, ","), vttablet.GetStatusDetails())
   324  }
   325  
   326  func contains(arr []string, str string) bool {
   327  	for _, a := range arr {
   328  		if a == str {
   329  			return true
   330  		}
   331  	}
   332  	return false
   333  }
   334  
   335  // WaitForBinLogPlayerCount waits till binlog player count var matches
   336  func (vttablet *VttabletProcess) WaitForBinLogPlayerCount(expectedCount int) error {
   337  	timeout := time.Now().Add(10 * time.Second)
   338  	for time.Now().Before(timeout) {
   339  		if vttablet.getVReplStreamCount() == fmt.Sprintf("%d", expectedCount) {
   340  			return nil
   341  		}
   342  		select {
   343  		case err := <-vttablet.exit:
   344  			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
   345  		default:
   346  			time.Sleep(300 * time.Millisecond)
   347  		}
   348  	}
   349  	return fmt.Errorf("vttablet %s, expected status not reached", vttablet.TabletPath)
   350  }
   351  
   352  // WaitForBinlogServerState wait for the tablet's binlog server to be in the provided state.
   353  func (vttablet *VttabletProcess) WaitForBinlogServerState(expectedStatus string) error {
   354  	timeout := time.Now().Add(10 * time.Second)
   355  	for time.Now().Before(timeout) {
   356  		if vttablet.getVarValue("UpdateStreamState") == expectedStatus {
   357  			return nil
   358  		}
   359  		select {
   360  		case err := <-vttablet.exit:
   361  			return fmt.Errorf("process '%s' exited prematurely (err: %s)", vttablet.Name, err)
   362  		default:
   363  			time.Sleep(300 * time.Millisecond)
   364  		}
   365  	}
   366  	return fmt.Errorf("vttablet %s, expected status not reached", vttablet.TabletPath)
   367  }
   368  
   369  func (vttablet *VttabletProcess) getVReplStreamCount() string {
   370  	return vttablet.getVarValue("VReplicationStreamCount")
   371  }
   372  
   373  func (vttablet *VttabletProcess) getVarValue(keyname string) string {
   374  	resultMap := vttablet.GetVars()
   375  	object := reflect.ValueOf(resultMap[keyname])
   376  	return fmt.Sprintf("%v", object)
   377  }
   378  
   379  // TearDown shuts down the running vttablet service and fails after 10 seconds
   380  func (vttablet *VttabletProcess) TearDown() error {
   381  	return vttablet.TearDownWithTimeout(10 * time.Second)
   382  }
   383  
   384  // TearDownWithTimeout shuts down the running vttablet service and fails once the given
   385  // duration has elapsed.
   386  func (vttablet *VttabletProcess) TearDownWithTimeout(timeout time.Duration) error {
   387  	if vttablet.proc == nil || vttablet.exit == nil {
   388  		return nil
   389  	}
   390  	// Attempt graceful shutdown with SIGTERM first
   391  	vttablet.proc.Process.Signal(syscall.SIGTERM)
   392  
   393  	select {
   394  	case <-vttablet.exit:
   395  		vttablet.proc = nil
   396  		return nil
   397  
   398  	case <-time.After(timeout):
   399  		vttablet.proc.Process.Kill()
   400  		err := <-vttablet.exit
   401  		vttablet.proc = nil
   402  		return err
   403  	}
   404  }
   405  
   406  // CreateDB creates the database for keyspace
   407  func (vttablet *VttabletProcess) CreateDB(keyspace string) error {
   408  	_, _ = vttablet.QueryTablet(fmt.Sprintf("drop database IF EXISTS vt_%s", keyspace), keyspace, false)
   409  	_, err := vttablet.QueryTablet(fmt.Sprintf("create database IF NOT EXISTS vt_%s", keyspace), keyspace, false)
   410  	return err
   411  }
   412  
   413  // QueryTablet lets you execute a query in this tablet and get the result
   414  func (vttablet *VttabletProcess) QueryTablet(query string, keyspace string, useDb bool) (*sqltypes.Result, error) {
   415  	if !useDb {
   416  		keyspace = ""
   417  	}
   418  	dbParams := NewConnParams(vttablet.DbPort, vttablet.DbPassword, path.Join(vttablet.Directory, "mysql.sock"), keyspace)
   419  	conn, err := vttablet.conn(&dbParams)
   420  	if err != nil {
   421  		return nil, err
   422  	}
   423  	defer conn.Close()
   424  	return executeQuery(conn, query)
   425  }
   426  
   427  func (vttablet *VttabletProcess) defaultConn(dbname string) (*mysql.Conn, error) {
   428  	dbParams := mysql.ConnParams{
   429  		Uname:      "vt_dba",
   430  		UnixSocket: path.Join(vttablet.Directory, "mysql.sock"),
   431  		DbName:     dbname,
   432  	}
   433  	if vttablet.DbPassword != "" {
   434  		dbParams.Pass = vttablet.DbPassword
   435  	}
   436  	return vttablet.conn(&dbParams)
   437  }
   438  
   439  func (vttablet *VttabletProcess) conn(dbParams *mysql.ConnParams) (*mysql.Conn, error) {
   440  	ctx := context.Background()
   441  	return mysql.Connect(ctx, dbParams)
   442  }
   443  
   444  // QueryTabletWithDB lets you execute query on a specific DB in this tablet and get the result
   445  func (vttablet *VttabletProcess) QueryTabletWithDB(query string, dbname string) (*sqltypes.Result, error) {
   446  	conn, err := vttablet.defaultConn(dbname)
   447  	if err != nil {
   448  		return nil, err
   449  	}
   450  	defer conn.Close()
   451  	return executeQuery(conn, query)
   452  }
   453  
   454  // executeQuery will retry the query up to 10 times with a small sleep in between each try.
   455  // This allows the tests to be more robust in the face of transient failures.
   456  func executeQuery(dbConn *mysql.Conn, query string) (*sqltypes.Result, error) {
   457  	var (
   458  		err    error
   459  		result *sqltypes.Result
   460  	)
   461  	retries := 10
   462  	retryDelay := 1 * time.Second
   463  	for i := 0; i < retries; i++ {
   464  		if i > 0 {
   465  			// We only audit from 2nd attempt and onwards, otherwise this is just too verbose.
   466  			log.Infof("Executing query %s (attempt %d of %d)", query, (i + 1), retries)
   467  		}
   468  		result, err = dbConn.ExecuteFetch(query, 10000, true)
   469  		if err == nil {
   470  			break
   471  		}
   472  		time.Sleep(retryDelay)
   473  	}
   474  
   475  	return result, err
   476  }
   477  
   478  // GetDBVar returns first matching database variable's value
   479  func (vttablet *VttabletProcess) GetDBVar(varName string, ksName string) (string, error) {
   480  	return vttablet.getDBSystemValues("variables", varName, ksName)
   481  }
   482  
   483  // GetDBStatus returns first matching database variable's value
   484  func (vttablet *VttabletProcess) GetDBStatus(status string, ksName string) (string, error) {
   485  	return vttablet.getDBSystemValues("status", status, ksName)
   486  }
   487  
   488  func (vttablet *VttabletProcess) getDBSystemValues(placeholder string, value string, ksName string) (string, error) {
   489  	output, err := vttablet.QueryTablet(fmt.Sprintf("show %s like '%s'", placeholder, value), ksName, true)
   490  	if err != nil || output.Rows == nil {
   491  		return "", err
   492  	}
   493  	if len(output.Rows) > 0 {
   494  		return output.Rows[0][1].ToString(), nil
   495  	}
   496  	return "", nil
   497  }
   498  
   499  // ToggleProfiling enables or disables the configured CPU profiler on this vttablet
   500  func (vttablet *VttabletProcess) ToggleProfiling() error {
   501  	return vttablet.proc.Process.Signal(syscall.SIGUSR1)
   502  }
   503  
   504  // WaitForVReplicationToCatchup waits for "workflow" to finish copying
   505  func (vttablet *VttabletProcess) WaitForVReplicationToCatchup(t testing.TB, workflow, database string, duration time.Duration) {
   506  	queries := [3]string{
   507  		fmt.Sprintf(`select count(*) from _vt.vreplication where workflow = "%s" and db_name = "%s" and pos = ''`, workflow, database),
   508  		"select count(*) from information_schema.tables where table_schema='_vt' and table_name='copy_state' limit 1;",
   509  		fmt.Sprintf(`select count(*) from _vt.copy_state where vrepl_id in (select id from _vt.vreplication where workflow = "%s" and db_name = "%s" )`, workflow, database),
   510  	}
   511  	results := [3]string{"[INT64(0)]", "[INT64(1)]", "[INT64(0)]"}
   512  
   513  	conn, err := vttablet.defaultConn("")
   514  	if err != nil {
   515  		t.Fatal(err)
   516  	}
   517  	defer conn.Close()
   518  
   519  	var lastChecked time.Time
   520  	for ind, query := range queries {
   521  		waitDuration := 500 * time.Millisecond
   522  		for duration > 0 {
   523  			log.Infof("Executing query %s on %s", query, vttablet.Name)
   524  			lastChecked = time.Now()
   525  			qr, err := executeQuery(conn, query)
   526  			if err != nil {
   527  				t.Fatal(err)
   528  			}
   529  			if qr != nil && qr.Rows != nil && len(qr.Rows) > 0 && fmt.Sprintf("%v", qr.Rows[0]) == string(results[ind]) {
   530  				break
   531  			} else {
   532  				log.Infof("In WaitForVReplicationToCatchup: %s %+v", query, qr.Rows)
   533  			}
   534  			time.Sleep(waitDuration)
   535  			duration -= waitDuration
   536  		}
   537  		if duration <= 0 {
   538  			t.Fatalf("WaitForVReplicationToCatchup timed out for workflow %s, keyspace %s", workflow, database)
   539  		}
   540  	}
   541  	log.Infof("WaitForVReplicationToCatchup succeeded at %v", lastChecked)
   542  }
   543  
   544  // BulkLoad performs a bulk load of rows into a given vttablet.
   545  func (vttablet *VttabletProcess) BulkLoad(t testing.TB, db, table string, bulkInsert func(io.Writer)) {
   546  	tmpbulk, err := os.CreateTemp(path.Join(vttablet.Directory, "tmp"), "bulk_load")
   547  	if err != nil {
   548  		t.Fatalf("failed to create tmp file for loading: %v", err)
   549  	}
   550  	defer os.Remove(tmpbulk.Name())
   551  
   552  	log.Infof("create temporary file for bulk loading %q", tmpbulk.Name())
   553  	bufStart := time.Now()
   554  
   555  	bulkBuffer := bufio.NewWriter(tmpbulk)
   556  	bulkInsert(bulkBuffer)
   557  	bulkBuffer.Flush()
   558  
   559  	pos, _ := tmpbulk.Seek(0, 1)
   560  	bufFinish := time.Now()
   561  	log.Infof("bulk loading %d bytes from %q...", pos, tmpbulk.Name())
   562  
   563  	if err := tmpbulk.Close(); err != nil {
   564  		t.Fatal(err)
   565  	}
   566  
   567  	conn, err := vttablet.defaultConn("vt_" + db)
   568  	if err != nil {
   569  		t.Fatal(err)
   570  	}
   571  	defer conn.Close()
   572  
   573  	query := fmt.Sprintf("LOAD DATA INFILE '%s' INTO TABLE `%s` FIELDS TERMINATED BY ',' ENCLOSED BY '\"'", tmpbulk.Name(), table)
   574  	_, err = executeQuery(conn, query)
   575  	if err != nil {
   576  		t.Fatal(err)
   577  	}
   578  
   579  	end := time.Now()
   580  	log.Infof("bulk insert successful (write tmp file = %v, mysql bulk load = %v, total = %v",
   581  		bufFinish.Sub(bufStart), end.Sub(bufFinish), end.Sub(bufStart))
   582  }
   583  
   584  // IsShutdown returns whether a vttablet is shutdown or not
   585  func (vttablet *VttabletProcess) IsShutdown() bool {
   586  	return vttablet.proc == nil
   587  }
   588  
   589  // VttabletProcessInstance returns a VttabletProcess handle for vttablet process
   590  // configured with the given Config.
   591  // The process must be manually started by calling setup()
   592  func VttabletProcessInstance(port, grpcPort, tabletUID int, cell, shard, keyspace string, vtctldPort int, tabletType string, topoPort int, hostname, tmpDirectory string, extraArgs []string, charset string) *VttabletProcess {
   593  	vtctl := VtctlProcessInstance(topoPort, hostname)
   594  	vttablet := &VttabletProcess{
   595  		Name:                        "vttablet",
   596  		Binary:                      "vttablet",
   597  		FileToLogQueries:            path.Join(tmpDirectory, fmt.Sprintf("/vt_%010d_querylog.txt", tabletUID)),
   598  		Directory:                   path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tabletUID)),
   599  		TabletPath:                  fmt.Sprintf("%s-%010d", cell, tabletUID),
   600  		ServiceMap:                  "grpc-queryservice,grpc-tabletmanager,grpc-updatestream,grpc-throttler",
   601  		LogDir:                      tmpDirectory,
   602  		Shard:                       shard,
   603  		TabletHostname:              hostname,
   604  		Keyspace:                    keyspace,
   605  		TabletType:                  "replica",
   606  		CommonArg:                   *vtctl,
   607  		HealthCheckInterval:         5,
   608  		Port:                        port,
   609  		GrpcPort:                    grpcPort,
   610  		VtctldAddress:               fmt.Sprintf("http://%s:%d", hostname, vtctldPort),
   611  		ExtraArgs:                   extraArgs,
   612  		SupportsBackup:              true,
   613  		ServingStatus:               "NOT_SERVING",
   614  		BackupStorageImplementation: "file",
   615  		FileBackupStorageRoot:       path.Join(os.Getenv("VTDATAROOT"), "/backups"),
   616  		VreplicationTabletType:      "replica",
   617  		TabletUID:                   tabletUID,
   618  		Charset:                     charset,
   619  	}
   620  
   621  	if tabletType == "rdonly" {
   622  		vttablet.TabletType = tabletType
   623  	}
   624  	vttablet.VerifyURL = fmt.Sprintf("http://%s:%d/debug/vars", hostname, port)
   625  	vttablet.QueryzURL = fmt.Sprintf("http://%s:%d/queryz", hostname, port)
   626  	vttablet.StatusDetailsURL = fmt.Sprintf("http://%s:%d/debug/status_details", hostname, port)
   627  	vttablet.ConsolidationsURL = fmt.Sprintf("http://%s:%d/debug/consolidations", hostname, port)
   628  
   629  	return vttablet
   630  }