github.com/KinWaiYuen/client-go/v2@v2.5.4/tikv/kv.go (about)

     1  // Copyright 2021 TiKV Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // NOTE: The code in this file is based on code from the
    16  // TiDB project, licensed under the Apache License v 2.0
    17  //
    18  // https://github.com/pingcap/tidb/tree/cc5e161ac06827589c4966674597c137cc9e809c/store/tikv/kv.go
    19  //
    20  
    21  // Copyright 2016 PingCAP, Inc.
    22  //
    23  // Licensed under the Apache License, Version 2.0 (the "License");
    24  // you may not use this file except in compliance with the License.
    25  // You may obtain a copy of the License at
    26  //
    27  //     http://www.apache.org/licenses/LICENSE-2.0
    28  //
    29  // Unless required by applicable law or agreed to in writing, software
    30  // distributed under the License is distributed on an "AS IS" BASIS,
    31  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    32  // See the License for the specific language governing permissions and
    33  // limitations under the License.
    34  
    35  package tikv
    36  
    37  import (
    38  	"context"
    39  	"crypto/tls"
    40  	"fmt"
    41  	"math"
    42  	"math/rand"
    43  	"strconv"
    44  	"sync"
    45  	"sync/atomic"
    46  	"time"
    47  
    48  	"github.com/KinWaiYuen/client-go/v2/config"
    49  	tikverr "github.com/KinWaiYuen/client-go/v2/error"
    50  	"github.com/KinWaiYuen/client-go/v2/internal/client"
    51  	"github.com/KinWaiYuen/client-go/v2/internal/latch"
    52  	"github.com/KinWaiYuen/client-go/v2/internal/locate"
    53  	"github.com/KinWaiYuen/client-go/v2/internal/logutil"
    54  	"github.com/KinWaiYuen/client-go/v2/internal/retry"
    55  	"github.com/KinWaiYuen/client-go/v2/kv"
    56  	"github.com/KinWaiYuen/client-go/v2/metrics"
    57  	"github.com/KinWaiYuen/client-go/v2/oracle"
    58  	"github.com/KinWaiYuen/client-go/v2/oracle/oracles"
    59  	"github.com/KinWaiYuen/client-go/v2/tikvrpc"
    60  	"github.com/KinWaiYuen/client-go/v2/txnkv/rangetask"
    61  	"github.com/KinWaiYuen/client-go/v2/txnkv/transaction"
    62  	"github.com/KinWaiYuen/client-go/v2/txnkv/txnlock"
    63  	"github.com/KinWaiYuen/client-go/v2/txnkv/txnsnapshot"
    64  	"github.com/KinWaiYuen/client-go/v2/util"
    65  	"github.com/opentracing/opentracing-go"
    66  	"github.com/pingcap/errors"
    67  	"github.com/pingcap/kvproto/pkg/kvrpcpb"
    68  	"github.com/pingcap/kvproto/pkg/metapb"
    69  	pd "github.com/tikv/pd/client"
    70  	"go.etcd.io/etcd/clientv3"
    71  	"go.uber.org/zap"
    72  	"google.golang.org/grpc"
    73  	"google.golang.org/grpc/keepalive"
    74  )
    75  
    76  // DCLabelKey indicates the key of label which represents the dc for Store.
    77  const DCLabelKey = "zone"
    78  
    79  func createEtcdKV(addrs []string, tlsConfig *tls.Config) (*clientv3.Client, error) {
    80  	cfg := config.GetGlobalConfig()
    81  	cli, err := clientv3.New(clientv3.Config{
    82  		Endpoints:            addrs,
    83  		AutoSyncInterval:     30 * time.Second,
    84  		DialTimeout:          5 * time.Second,
    85  		TLS:                  tlsConfig,
    86  		DialKeepAliveTime:    time.Second * time.Duration(cfg.TiKVClient.GrpcKeepAliveTime),
    87  		DialKeepAliveTimeout: time.Second * time.Duration(cfg.TiKVClient.GrpcKeepAliveTimeout),
    88  	})
    89  	if err != nil {
    90  		return nil, errors.Trace(err)
    91  	}
    92  	return cli, nil
    93  }
    94  
    95  // update oracle's lastTS every 2000ms.
    96  var oracleUpdateInterval = 2000
    97  
    98  // KVStore contains methods to interact with a TiKV cluster.
    99  type KVStore struct {
   100  	clusterID uint64
   101  	uuid      string
   102  	oracle    oracle.Oracle
   103  	clientMu  struct {
   104  		sync.RWMutex
   105  		client Client
   106  	}
   107  	pdClient     pd.Client
   108  	regionCache  *locate.RegionCache
   109  	lockResolver *txnlock.LockResolver
   110  	txnLatches   *latch.LatchesScheduler
   111  
   112  	mock bool
   113  
   114  	kv        SafePointKV
   115  	safePoint uint64
   116  	spTime    time.Time
   117  	spMutex   sync.RWMutex // this is used to update safePoint and spTime
   118  
   119  	// storeID -> safeTS, stored as map[uint64]uint64
   120  	// safeTS here will be used during the Stale Read process,
   121  	// it indicates the safe timestamp point that can be used to read consistent but may not the latest data.
   122  	safeTSMap sync.Map
   123  
   124  	replicaReadSeed uint32 // this is used to load balance followers / learners when replica read is enabled
   125  
   126  	ctx    context.Context
   127  	cancel context.CancelFunc
   128  	wg     sync.WaitGroup
   129  }
   130  
   131  // UpdateSPCache updates cached safepoint.
   132  func (s *KVStore) UpdateSPCache(cachedSP uint64, cachedTime time.Time) {
   133  	s.spMutex.Lock()
   134  	s.safePoint = cachedSP
   135  	s.spTime = cachedTime
   136  	s.spMutex.Unlock()
   137  }
   138  
   139  // CheckVisibility checks if it is safe to read using given ts.
   140  func (s *KVStore) CheckVisibility(startTime uint64) error {
   141  	s.spMutex.RLock()
   142  	cachedSafePoint := s.safePoint
   143  	cachedTime := s.spTime
   144  	s.spMutex.RUnlock()
   145  	diff := time.Since(cachedTime)
   146  
   147  	if diff > (GcSafePointCacheInterval - gcCPUTimeInaccuracyBound) {
   148  		return tikverr.NewErrPDServerTimeout("start timestamp may fall behind safe point")
   149  	}
   150  
   151  	if startTime < cachedSafePoint {
   152  		t1 := oracle.GetTimeFromTS(startTime)
   153  		t2 := oracle.GetTimeFromTS(cachedSafePoint)
   154  		return &tikverr.ErrGCTooEarly{
   155  			TxnStartTS:  t1,
   156  			GCSafePoint: t2,
   157  		}
   158  	}
   159  
   160  	return nil
   161  }
   162  
   163  // NewKVStore creates a new TiKV store instance.
   164  func NewKVStore(uuid string, pdClient pd.Client, spkv SafePointKV, tikvclient Client) (*KVStore, error) {
   165  	o, err := oracles.NewPdOracle(pdClient, time.Duration(oracleUpdateInterval)*time.Millisecond)
   166  	if err != nil {
   167  		return nil, errors.Trace(err)
   168  	}
   169  	ctx, cancel := context.WithCancel(context.Background())
   170  	store := &KVStore{
   171  		clusterID:       pdClient.GetClusterID(context.TODO()),
   172  		uuid:            uuid,
   173  		oracle:          o,
   174  		pdClient:        pdClient,
   175  		regionCache:     locate.NewRegionCache(pdClient),
   176  		kv:              spkv,
   177  		safePoint:       0,
   178  		spTime:          time.Now(),
   179  		replicaReadSeed: rand.Uint32(),
   180  		ctx:             ctx,
   181  		cancel:          cancel,
   182  	}
   183  	store.clientMu.client = client.NewReqCollapse(tikvclient)
   184  	store.lockResolver = txnlock.NewLockResolver(store)
   185  
   186  	store.wg.Add(2)
   187  	go store.runSafePointChecker()
   188  	go store.safeTSUpdater()
   189  
   190  	return store, nil
   191  }
   192  
   193  // NewTxnClient creates a txn client with pdAddrs.
   194  func NewTxnClient(pdAddrs []string) (*KVStore, error) {
   195  
   196  	cfg := config.GetGlobalConfig()
   197  	pdClient, err := NewPDClient(pdAddrs)
   198  	if err != nil {
   199  		return nil, errors.Trace(err)
   200  	}
   201  	// init uuid
   202  	// FIXME: uuid will be a very long and ugly string, simplify it.
   203  	uuid := fmt.Sprintf("tikv-%v", pdClient.GetClusterID(context.TODO()))
   204  	tlsConfig, err := cfg.Security.ToTLSConfig()
   205  	if err != nil {
   206  		return nil, errors.Trace(err)
   207  	}
   208  
   209  	spkv, err := NewEtcdSafePointKV(pdAddrs, tlsConfig)
   210  	if err != nil {
   211  		return nil, errors.Trace(err)
   212  	}
   213  
   214  	s, err := NewKVStore(uuid, pdClient, spkv, NewRPCClient(WithSecurity(cfg.Security)))
   215  	if err != nil {
   216  		return nil, errors.Trace(err)
   217  	}
   218  	if cfg.TxnLocalLatches.Enabled {
   219  		s.EnableTxnLocalLatches(cfg.TxnLocalLatches.Capacity)
   220  	}
   221  	return s, nil
   222  }
   223  
   224  // NewPDClient creates pd.Client with pdAddrs.
   225  func NewPDClient(pdAddrs []string) (pd.Client, error) {
   226  	cfg := config.GetGlobalConfig()
   227  	// init pd-client
   228  	pdCli, err := pd.NewClient(pdAddrs, pd.SecurityOption{
   229  		CAPath:   cfg.Security.ClusterSSLCA,
   230  		CertPath: cfg.Security.ClusterSSLCert,
   231  		KeyPath:  cfg.Security.ClusterSSLKey,
   232  	},
   233  		pd.WithGRPCDialOptions(
   234  			grpc.WithKeepaliveParams(keepalive.ClientParameters{
   235  				Time:    time.Duration(cfg.TiKVClient.GrpcKeepAliveTime) * time.Second,
   236  				Timeout: time.Duration(cfg.TiKVClient.GrpcKeepAliveTimeout) * time.Second,
   237  			}),
   238  		),
   239  		pd.WithCustomTimeoutOption(time.Duration(cfg.PDClient.PDServerTimeout)*time.Second),
   240  		pd.WithForwardingOption(config.GetGlobalConfig().EnableForwarding))
   241  
   242  	if err != nil {
   243  		return nil, errors.Trace(err)
   244  	}
   245  	pdClient := &CodecPDClient{Client: util.InterceptedPDClient{Client: pdCli}}
   246  	return pdClient, nil
   247  }
   248  
   249  // EnableTxnLocalLatches enables txn latch. It should be called before using
   250  // the store to serve any requests.
   251  func (s *KVStore) EnableTxnLocalLatches(size uint) {
   252  	s.txnLatches = latch.NewScheduler(size)
   253  }
   254  
   255  // IsLatchEnabled is used by mockstore.TestConfig.
   256  func (s *KVStore) IsLatchEnabled() bool {
   257  	return s.txnLatches != nil
   258  }
   259  
   260  func (s *KVStore) runSafePointChecker() {
   261  	defer s.wg.Done()
   262  	d := gcSafePointUpdateInterval
   263  	for {
   264  		select {
   265  		case spCachedTime := <-time.After(d):
   266  			cachedSafePoint, err := loadSafePoint(s.GetSafePointKV())
   267  			if err == nil {
   268  				metrics.TiKVLoadSafepointCounter.WithLabelValues("ok").Inc()
   269  				s.UpdateSPCache(cachedSafePoint, spCachedTime)
   270  				d = gcSafePointUpdateInterval
   271  			} else {
   272  				metrics.TiKVLoadSafepointCounter.WithLabelValues("fail").Inc()
   273  				logutil.BgLogger().Error("fail to load safepoint from pd", zap.Error(err))
   274  				d = gcSafePointQuickRepeatInterval
   275  			}
   276  		case <-s.ctx.Done():
   277  			return
   278  		}
   279  	}
   280  }
   281  
   282  // Begin a global transaction.
   283  func (s *KVStore) Begin() (*transaction.KVTxn, error) {
   284  	return s.BeginWithOption(DefaultStartTSOption())
   285  }
   286  
   287  // BeginWithOption begins a transaction with the given StartTSOption
   288  func (s *KVStore) BeginWithOption(options StartTSOption) (*transaction.KVTxn, error) {
   289  	if options.TxnScope == "" {
   290  		options.TxnScope = oracle.GlobalTxnScope
   291  	}
   292  
   293  	if options.StartTS != nil {
   294  		snapshot := txnsnapshot.NewTiKVSnapshot(s, *options.StartTS, s.nextReplicaReadSeed())
   295  		return transaction.NewTiKVTxn(s, snapshot, *options.StartTS, options.TxnScope)
   296  	}
   297  
   298  	bo := retry.NewBackofferWithVars(context.Background(), transaction.TsoMaxBackoff, nil)
   299  	startTS, err := s.getTimestampWithRetry(bo, options.TxnScope)
   300  	if err != nil {
   301  		return nil, errors.Trace(err)
   302  	}
   303  	snapshot := txnsnapshot.NewTiKVSnapshot(s, startTS, s.nextReplicaReadSeed())
   304  	return transaction.NewTiKVTxn(s, snapshot, startTS, options.TxnScope)
   305  }
   306  
   307  // DeleteRange delete all versions of all keys in the range[startKey,endKey) immediately.
   308  // Be careful while using this API. This API doesn't keep recent MVCC versions, but will delete all versions of all keys
   309  // in the range immediately. Also notice that frequent invocation to this API may cause performance problems to TiKV.
   310  func (s *KVStore) DeleteRange(ctx context.Context, startKey []byte, endKey []byte, concurrency int) (completedRegions int, err error) {
   311  	task := rangetask.NewDeleteRangeTask(s, startKey, endKey, concurrency)
   312  	err = task.Execute(ctx)
   313  	if err == nil {
   314  		completedRegions = task.CompletedRegions()
   315  	}
   316  	return completedRegions, err
   317  }
   318  
   319  // GetSnapshot gets a snapshot that is able to read any data which data is <= the given ts.
   320  // If the given ts is greater than the current TSO timestamp, the snapshot is not guaranteed
   321  // to be consistent.
   322  // Specially, it is useful to set ts to math.MaxUint64 to point get the latest committed data.
   323  func (s *KVStore) GetSnapshot(ts uint64) *txnsnapshot.KVSnapshot {
   324  	snapshot := txnsnapshot.NewTiKVSnapshot(s, ts, s.nextReplicaReadSeed())
   325  	return snapshot
   326  }
   327  
   328  // Close store
   329  func (s *KVStore) Close() error {
   330  	s.cancel()
   331  	s.wg.Wait()
   332  
   333  	s.oracle.Close()
   334  	s.pdClient.Close()
   335  
   336  	if err := s.GetTiKVClient().Close(); err != nil {
   337  		return errors.Trace(err)
   338  	}
   339  
   340  	if s.txnLatches != nil {
   341  		s.txnLatches.Close()
   342  	}
   343  	s.regionCache.Close()
   344  
   345  	if err := s.kv.Close(); err != nil {
   346  		return errors.Trace(err)
   347  	}
   348  	return nil
   349  }
   350  
   351  // UUID return a unique ID which represents a Storage.
   352  func (s *KVStore) UUID() string {
   353  	return s.uuid
   354  }
   355  
   356  // CurrentTimestamp returns current timestamp with the given txnScope (local or global).
   357  func (s *KVStore) CurrentTimestamp(txnScope string) (uint64, error) {
   358  	bo := retry.NewBackofferWithVars(context.Background(), transaction.TsoMaxBackoff, nil)
   359  	startTS, err := s.getTimestampWithRetry(bo, txnScope)
   360  	if err != nil {
   361  		return 0, errors.Trace(err)
   362  	}
   363  	return startTS, nil
   364  }
   365  
   366  // GetTimestampWithRetry returns latest timestamp.
   367  func (s *KVStore) GetTimestampWithRetry(bo *Backoffer, scope string) (uint64, error) {
   368  	return s.getTimestampWithRetry(bo, scope)
   369  }
   370  
   371  func (s *KVStore) getTimestampWithRetry(bo *Backoffer, txnScope string) (uint64, error) {
   372  	if span := opentracing.SpanFromContext(bo.GetCtx()); span != nil && span.Tracer() != nil {
   373  		span1 := span.Tracer().StartSpan("TiKVStore.getTimestampWithRetry", opentracing.ChildOf(span.Context()))
   374  		defer span1.Finish()
   375  		bo.SetCtx(opentracing.ContextWithSpan(bo.GetCtx(), span1))
   376  	}
   377  
   378  	for {
   379  		startTS, err := s.oracle.GetTimestamp(bo.GetCtx(), &oracle.Option{TxnScope: txnScope})
   380  		// mockGetTSErrorInRetry should wait MockCommitErrorOnce first, then will run into retry() logic.
   381  		// Then mockGetTSErrorInRetry will return retryable error when first retry.
   382  		// Before PR #8743, we don't cleanup txn after meet error such as error like: PD server timeout
   383  		// This may cause duplicate data to be written.
   384  		if val, e := util.EvalFailpoint("mockGetTSErrorInRetry"); e == nil && val.(bool) {
   385  			if _, e := util.EvalFailpoint("mockCommitErrorOpt"); e != nil {
   386  				err = tikverr.NewErrPDServerTimeout("mock PD timeout")
   387  			}
   388  		}
   389  
   390  		if err == nil {
   391  			return startTS, nil
   392  		}
   393  		err = bo.Backoff(retry.BoPDRPC, errors.Errorf("get timestamp failed: %v", err))
   394  		if err != nil {
   395  			return 0, errors.Trace(err)
   396  		}
   397  	}
   398  }
   399  
   400  func (s *KVStore) nextReplicaReadSeed() uint32 {
   401  	return atomic.AddUint32(&s.replicaReadSeed, 1)
   402  }
   403  
   404  // GetOracle gets a timestamp oracle client.
   405  func (s *KVStore) GetOracle() oracle.Oracle {
   406  	return s.oracle
   407  }
   408  
   409  // GetPDClient returns the PD client.
   410  func (s *KVStore) GetPDClient() pd.Client {
   411  	return s.pdClient
   412  }
   413  
   414  // SupportDeleteRange gets the storage support delete range or not.
   415  func (s *KVStore) SupportDeleteRange() (supported bool) {
   416  	return !s.mock
   417  }
   418  
   419  // SendReq sends a request to locate.
   420  func (s *KVStore) SendReq(bo *Backoffer, req *tikvrpc.Request, regionID locate.RegionVerID, timeout time.Duration) (*tikvrpc.Response, error) {
   421  	sender := locate.NewRegionRequestSender(s.regionCache, s.GetTiKVClient())
   422  	return sender.SendReq(bo, req, regionID, timeout)
   423  }
   424  
   425  // GetRegionCache returns the region cache instance.
   426  func (s *KVStore) GetRegionCache() *locate.RegionCache {
   427  	return s.regionCache
   428  }
   429  
   430  // GetLockResolver returns the lock resolver instance.
   431  func (s *KVStore) GetLockResolver() *txnlock.LockResolver {
   432  	return s.lockResolver
   433  }
   434  
   435  // Closed returns a channel that indicates if the store is closed.
   436  func (s *KVStore) Closed() <-chan struct{} {
   437  	return s.ctx.Done()
   438  }
   439  
   440  // GetSafePointKV returns the kv store that used for safepoint.
   441  func (s *KVStore) GetSafePointKV() SafePointKV {
   442  	return s.kv
   443  }
   444  
   445  // SetOracle resets the oracle instance.
   446  func (s *KVStore) SetOracle(oracle oracle.Oracle) {
   447  	s.oracle = oracle
   448  }
   449  
   450  // SetTiKVClient resets the client instance.
   451  func (s *KVStore) SetTiKVClient(client Client) {
   452  	s.clientMu.Lock()
   453  	defer s.clientMu.Unlock()
   454  	s.clientMu.client = client
   455  }
   456  
   457  // GetTiKVClient gets the client instance.
   458  func (s *KVStore) GetTiKVClient() (client Client) {
   459  	s.clientMu.RLock()
   460  	defer s.clientMu.RUnlock()
   461  	return s.clientMu.client
   462  }
   463  
   464  // GetMinSafeTS return the minimal safeTS of the storage with given txnScope.
   465  func (s *KVStore) GetMinSafeTS(txnScope string) uint64 {
   466  	stores := make([]*locate.Store, 0)
   467  	allStores := s.regionCache.GetStoresByType(tikvrpc.TiKV)
   468  	if txnScope != oracle.GlobalTxnScope {
   469  		for _, store := range allStores {
   470  			if store.IsLabelsMatch([]*metapb.StoreLabel{
   471  				{
   472  					Key:   DCLabelKey,
   473  					Value: txnScope,
   474  				},
   475  			}) {
   476  				stores = append(stores, store)
   477  			}
   478  		}
   479  	} else {
   480  		stores = allStores
   481  	}
   482  	return s.getMinSafeTSByStores(stores)
   483  }
   484  
   485  // Ctx returns ctx.
   486  func (s *KVStore) Ctx() context.Context {
   487  	return s.ctx
   488  }
   489  
   490  // WaitGroup returns wg
   491  func (s *KVStore) WaitGroup() *sync.WaitGroup {
   492  	return &s.wg
   493  }
   494  
   495  // TxnLatches returns txnLatches.
   496  func (s *KVStore) TxnLatches() *latch.LatchesScheduler {
   497  	return s.txnLatches
   498  }
   499  
   500  // GetClusterID returns store's cluster id.
   501  func (s *KVStore) GetClusterID() uint64 {
   502  	return s.clusterID
   503  }
   504  
   505  func (s *KVStore) getSafeTS(storeID uint64) uint64 {
   506  	safeTS, ok := s.safeTSMap.Load(storeID)
   507  	if !ok {
   508  		return 0
   509  	}
   510  	return safeTS.(uint64)
   511  }
   512  
   513  // setSafeTS sets safeTs for store storeID, export for testing
   514  func (s *KVStore) setSafeTS(storeID, safeTS uint64) {
   515  	s.safeTSMap.Store(storeID, safeTS)
   516  }
   517  
   518  func (s *KVStore) getMinSafeTSByStores(stores []*locate.Store) uint64 {
   519  	if val, err := util.EvalFailpoint("injectSafeTS"); err == nil {
   520  		injectTS := val.(int)
   521  		return uint64(injectTS)
   522  	}
   523  	minSafeTS := uint64(math.MaxUint64)
   524  	// when there is no store, return 0 in order to let minStartTS become startTS directly
   525  	if len(stores) < 1 {
   526  		return 0
   527  	}
   528  	for _, store := range stores {
   529  		safeTS := s.getSafeTS(store.StoreID())
   530  		if safeTS < minSafeTS {
   531  			minSafeTS = safeTS
   532  		}
   533  	}
   534  	return minSafeTS
   535  }
   536  
   537  func (s *KVStore) safeTSUpdater() {
   538  	defer s.wg.Done()
   539  	t := time.NewTicker(time.Second * 2)
   540  	defer t.Stop()
   541  	ctx, cancel := context.WithCancel(s.ctx)
   542  	defer cancel()
   543  	for {
   544  		select {
   545  		case <-s.ctx.Done():
   546  			return
   547  		case <-t.C:
   548  			s.updateSafeTS(ctx)
   549  		}
   550  	}
   551  }
   552  
   553  func (s *KVStore) updateSafeTS(ctx context.Context) {
   554  	stores := s.regionCache.GetStoresByType(tikvrpc.TiKV)
   555  	tikvClient := s.GetTiKVClient()
   556  	wg := &sync.WaitGroup{}
   557  	wg.Add(len(stores))
   558  	for _, store := range stores {
   559  		storeID := store.StoreID()
   560  		storeAddr := store.GetAddr()
   561  		go func(ctx context.Context, wg *sync.WaitGroup, storeID uint64, storeAddr string) {
   562  			defer wg.Done()
   563  			resp, err := tikvClient.SendRequest(ctx, storeAddr, tikvrpc.NewRequest(tikvrpc.CmdStoreSafeTS, &kvrpcpb.StoreSafeTSRequest{KeyRange: &kvrpcpb.KeyRange{
   564  				StartKey: []byte(""),
   565  				EndKey:   []byte(""),
   566  			}}), client.ReadTimeoutShort)
   567  			storeIDStr := strconv.Itoa(int(storeID))
   568  			if err != nil {
   569  				metrics.TiKVSafeTSUpdateCounter.WithLabelValues("fail", storeIDStr).Inc()
   570  				logutil.BgLogger().Debug("update safeTS failed", zap.Error(err), zap.Uint64("store-id", storeID))
   571  				return
   572  			}
   573  			safeTS := resp.Resp.(*kvrpcpb.StoreSafeTSResponse).GetSafeTs()
   574  			s.setSafeTS(storeID, safeTS)
   575  			metrics.TiKVSafeTSUpdateCounter.WithLabelValues("success", storeIDStr).Inc()
   576  			safeTSTime := oracle.GetTimeFromTS(safeTS)
   577  			metrics.TiKVMinSafeTSGapSeconds.WithLabelValues(storeIDStr).Set(time.Since(safeTSTime).Seconds())
   578  		}(ctx, wg, storeID, storeAddr)
   579  	}
   580  	wg.Wait()
   581  }
   582  
   583  // Variables defines the variables used by TiKV storage.
   584  type Variables = kv.Variables
   585  
   586  // NewLockResolver is exported for other pkg to use, suppress unused warning.
   587  var _ = NewLockResolver
   588  
   589  // NewLockResolver creates a LockResolver.
   590  // It is exported for other pkg to use. For instance, binlog service needs
   591  // to determine a transaction's commit state.
   592  func NewLockResolver(etcdAddrs []string, security config.Security, opts ...pd.ClientOption) (*txnlock.LockResolver, error) {
   593  	pdCli, err := pd.NewClient(etcdAddrs, pd.SecurityOption{
   594  		CAPath:   security.ClusterSSLCA,
   595  		CertPath: security.ClusterSSLCert,
   596  		KeyPath:  security.ClusterSSLKey,
   597  	}, opts...)
   598  	if err != nil {
   599  		return nil, errors.Trace(err)
   600  	}
   601  	pdCli = util.InterceptedPDClient{Client: pdCli}
   602  	uuid := fmt.Sprintf("tikv-%v", pdCli.GetClusterID(context.TODO()))
   603  
   604  	tlsConfig, err := security.ToTLSConfig()
   605  	if err != nil {
   606  		return nil, errors.Trace(err)
   607  	}
   608  
   609  	spkv, err := NewEtcdSafePointKV(etcdAddrs, tlsConfig)
   610  	if err != nil {
   611  		return nil, errors.Trace(err)
   612  	}
   613  
   614  	s, err := NewKVStore(uuid, locate.NewCodeCPDClient(pdCli), spkv, client.NewRPCClient(WithSecurity(security)))
   615  	if err != nil {
   616  		return nil, errors.Trace(err)
   617  	}
   618  	return s.lockResolver, nil
   619  }
   620  
   621  // StartTSOption indicates the option when beginning a transaction
   622  // `TxnScope` must be set for each object
   623  // Every other fields are optional, but currently at most one of them can be set
   624  type StartTSOption struct {
   625  	TxnScope string
   626  	StartTS  *uint64
   627  }
   628  
   629  // DefaultStartTSOption creates a default StartTSOption, ie. Work in GlobalTxnScope and get start ts when got used
   630  func DefaultStartTSOption() StartTSOption {
   631  	return StartTSOption{TxnScope: oracle.GlobalTxnScope}
   632  }
   633  
   634  // SetStartTS returns a new StartTSOption with StartTS set to the given startTS
   635  func (to StartTSOption) SetStartTS(startTS uint64) StartTSOption {
   636  	to.StartTS = &startTS
   637  	return to
   638  }
   639  
   640  // SetTxnScope returns a new StartTSOption with TxnScope set to txnScope
   641  func (to StartTSOption) SetTxnScope(txnScope string) StartTSOption {
   642  	to.TxnScope = txnScope
   643  	return to
   644  }
   645  
   646  // TODO: remove once tidb and br are ready
   647  
   648  // KVTxn contains methods to interact with a TiKV transaction.
   649  type KVTxn = transaction.KVTxn
   650  
   651  // BinlogWriteResult defines the result of prewrite binlog.
   652  type BinlogWriteResult = transaction.BinlogWriteResult
   653  
   654  // KVFilter is a filter that filters out unnecessary KV pairs.
   655  type KVFilter = transaction.KVFilter
   656  
   657  // SchemaLeaseChecker is used to validate schema version is not changed during transaction execution.
   658  type SchemaLeaseChecker = transaction.SchemaLeaseChecker
   659  
   660  // SchemaVer is the infoSchema which will return the schema version.
   661  type SchemaVer = transaction.SchemaVer
   662  
   663  // SchemaAmender is used by pessimistic transactions to amend commit mutations for schema change during 2pc.
   664  type SchemaAmender = transaction.SchemaAmender
   665  
   666  // MaxTxnTimeUse is the max time a Txn may use (in ms) from its begin to commit.
   667  // We use it to abort the transaction to guarantee GC worker will not influence it.
   668  const MaxTxnTimeUse = transaction.MaxTxnTimeUse