github.com/whtcorpsinc/MilevaDB-Prod@v0.0.0-20211104133533-f57f4be3b597/causetstore/store_test.go (about)

     1  // Copyright 2020 WHTCORPS INC, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package causetstore
    15  
    16  import (
    17  	"context"
    18  	"fmt"
    19  	"os"
    20  	"strconv"
    21  	"sync"
    22  	"sync/atomic"
    23  	"testing"
    24  	"time"
    25  
    26  	. "github.com/whtcorpsinc/check"
    27  	"github.com/whtcorpsinc/milevadb/causetstore/mockstore"
    28  	"github.com/whtcorpsinc/milevadb/ekv"
    29  	"github.com/whtcorpsinc/milevadb/soliton/logutil"
    30  	"github.com/whtcorpsinc/milevadb/soliton/testleak"
    31  )
    32  
    33  const (
    34  	startIndex = 0
    35  	testCount  = 2
    36  	indexStep  = 2
    37  )
    38  
    39  type brokenStore struct{}
    40  
    41  func (s *brokenStore) Open(schemaReplicant string) (ekv.CausetStorage, error) {
    42  	return nil, ekv.ErrTxnRetryable
    43  }
    44  
    45  func TestT(t *testing.T) {
    46  	CustomVerboseFlag = true
    47  	logLevel := os.Getenv("log_level")
    48  	logutil.InitLogger(logutil.NewLogConfig(logLevel, logutil.DefaultLogFormat, "", logutil.EmptyFileLogConfig, false))
    49  	TestingT(t)
    50  }
    51  
    52  var _ = Suite(&testKVSuite{})
    53  
    54  type testKVSuite struct {
    55  	s ekv.CausetStorage
    56  }
    57  
    58  func (s *testKVSuite) SetUpSuite(c *C) {
    59  	testleak.BeforeTest()
    60  	causetstore, err := mockstore.NewMockStore()
    61  	c.Assert(err, IsNil)
    62  	s.s = causetstore
    63  }
    64  
    65  func (s *testKVSuite) TearDownSuite(c *C) {
    66  	err := s.s.Close()
    67  	c.Assert(err, IsNil)
    68  	testleak.AfterTest(c)()
    69  }
    70  
    71  func insertData(c *C, txn ekv.Transaction) {
    72  	for i := startIndex; i < testCount; i++ {
    73  		val := encodeInt(i * indexStep)
    74  		err := txn.Set(val, val)
    75  		c.Assert(err, IsNil)
    76  	}
    77  }
    78  
    79  func mustDel(c *C, txn ekv.Transaction) {
    80  	for i := startIndex; i < testCount; i++ {
    81  		val := encodeInt(i * indexStep)
    82  		err := txn.Delete(val)
    83  		c.Assert(err, IsNil)
    84  	}
    85  }
    86  
    87  func encodeInt(n int) []byte {
    88  	return []byte(fmt.Sprintf("%010d", n))
    89  }
    90  
    91  func decodeInt(s []byte) int {
    92  	var n int
    93  	fmt.Sscanf(string(s), "%010d", &n)
    94  	return n
    95  }
    96  
    97  func valToStr(c *C, iter ekv.Iterator) string {
    98  	val := iter.Value()
    99  	return string(val)
   100  }
   101  
   102  func checkSeek(c *C, txn ekv.Transaction) {
   103  	for i := startIndex; i < testCount; i++ {
   104  		val := encodeInt(i * indexStep)
   105  		iter, err := txn.Iter(val, nil)
   106  		c.Assert(err, IsNil)
   107  		c.Assert([]byte(iter.Key()), BytesEquals, val)
   108  		c.Assert(decodeInt([]byte(valToStr(c, iter))), Equals, i*indexStep)
   109  		iter.Close()
   110  	}
   111  
   112  	// Test iterator Next()
   113  	for i := startIndex; i < testCount-1; i++ {
   114  		val := encodeInt(i * indexStep)
   115  		iter, err := txn.Iter(val, nil)
   116  		c.Assert(err, IsNil)
   117  		c.Assert([]byte(iter.Key()), BytesEquals, val)
   118  		c.Assert(valToStr(c, iter), Equals, string(val))
   119  
   120  		err = iter.Next()
   121  		c.Assert(err, IsNil)
   122  		c.Assert(iter.Valid(), IsTrue)
   123  
   124  		val = encodeInt((i + 1) * indexStep)
   125  		c.Assert([]byte(iter.Key()), BytesEquals, val)
   126  		c.Assert(valToStr(c, iter), Equals, string(val))
   127  		iter.Close()
   128  	}
   129  
   130  	// Non exist and beyond maximum seek test
   131  	iter, err := txn.Iter(encodeInt(testCount*indexStep), nil)
   132  	c.Assert(err, IsNil)
   133  	c.Assert(iter.Valid(), IsFalse)
   134  
   135  	// Non exist but between existing keys seek test,
   136  	// it returns the smallest key that larger than the one we are seeking
   137  	inBetween := encodeInt((testCount-1)*indexStep - 1)
   138  	last := encodeInt((testCount - 1) * indexStep)
   139  	iter, err = txn.Iter(inBetween, nil)
   140  	c.Assert(err, IsNil)
   141  	c.Assert(iter.Valid(), IsTrue)
   142  	c.Assert([]byte(iter.Key()), Not(BytesEquals), inBetween)
   143  	c.Assert([]byte(iter.Key()), BytesEquals, last)
   144  	iter.Close()
   145  }
   146  
   147  func mustNotGet(c *C, txn ekv.Transaction) {
   148  	for i := startIndex; i < testCount; i++ {
   149  		s := encodeInt(i * indexStep)
   150  		_, err := txn.Get(context.TODO(), s)
   151  		c.Assert(err, NotNil)
   152  	}
   153  }
   154  
   155  func mustGet(c *C, txn ekv.Transaction) {
   156  	for i := startIndex; i < testCount; i++ {
   157  		s := encodeInt(i * indexStep)
   158  		val, err := txn.Get(context.TODO(), s)
   159  		c.Assert(err, IsNil)
   160  		c.Assert(string(val), Equals, string(s))
   161  	}
   162  }
   163  
   164  func (s *testKVSuite) TestNew(c *C) {
   165  	causetstore, err := New("goleveldb://relative/path")
   166  	c.Assert(err, NotNil)
   167  	c.Assert(causetstore, IsNil)
   168  }
   169  
   170  func (s *testKVSuite) TestGetSet(c *C) {
   171  	txn, err := s.s.Begin()
   172  	c.Assert(err, IsNil)
   173  
   174  	insertData(c, txn)
   175  
   176  	mustGet(c, txn)
   177  
   178  	// Check transaction results
   179  	err = txn.Commit(context.Background())
   180  	c.Assert(err, IsNil)
   181  
   182  	txn, err = s.s.Begin()
   183  	c.Assert(err, IsNil)
   184  	defer txn.Commit(context.Background())
   185  
   186  	mustGet(c, txn)
   187  	mustDel(c, txn)
   188  }
   189  
   190  func (s *testKVSuite) TestSeek(c *C) {
   191  	txn, err := s.s.Begin()
   192  	c.Assert(err, IsNil)
   193  
   194  	insertData(c, txn)
   195  	checkSeek(c, txn)
   196  
   197  	// Check transaction results
   198  	err = txn.Commit(context.Background())
   199  	c.Assert(err, IsNil)
   200  
   201  	txn, err = s.s.Begin()
   202  	c.Assert(err, IsNil)
   203  	defer txn.Commit(context.Background())
   204  
   205  	checkSeek(c, txn)
   206  	mustDel(c, txn)
   207  }
   208  
   209  func (s *testKVSuite) TestInc(c *C) {
   210  	txn, err := s.s.Begin()
   211  	c.Assert(err, IsNil)
   212  
   213  	key := []byte("incKey")
   214  	n, err := ekv.IncInt64(txn, key, 100)
   215  	c.Assert(err, IsNil)
   216  	c.Assert(n, Equals, int64(100))
   217  
   218  	// Check transaction results
   219  	err = txn.Commit(context.Background())
   220  	c.Assert(err, IsNil)
   221  
   222  	txn, err = s.s.Begin()
   223  	c.Assert(err, IsNil)
   224  
   225  	n, err = ekv.IncInt64(txn, key, -200)
   226  	c.Assert(err, IsNil)
   227  	c.Assert(n, Equals, int64(-100))
   228  
   229  	err = txn.Delete(key)
   230  	c.Assert(err, IsNil)
   231  
   232  	n, err = ekv.IncInt64(txn, key, 100)
   233  	c.Assert(err, IsNil)
   234  	c.Assert(n, Equals, int64(100))
   235  
   236  	err = txn.Delete(key)
   237  	c.Assert(err, IsNil)
   238  
   239  	err = txn.Commit(context.Background())
   240  	c.Assert(err, IsNil)
   241  }
   242  
   243  func (s *testKVSuite) TestDelete(c *C) {
   244  	txn, err := s.s.Begin()
   245  	c.Assert(err, IsNil)
   246  
   247  	insertData(c, txn)
   248  
   249  	mustDel(c, txn)
   250  
   251  	mustNotGet(c, txn)
   252  	err = txn.Commit(context.Background())
   253  	c.Assert(err, IsNil)
   254  
   255  	// Try get
   256  	txn, err = s.s.Begin()
   257  	c.Assert(err, IsNil)
   258  
   259  	mustNotGet(c, txn)
   260  
   261  	// Insert again
   262  	insertData(c, txn)
   263  	err = txn.Commit(context.Background())
   264  	c.Assert(err, IsNil)
   265  
   266  	// Delete all
   267  	txn, err = s.s.Begin()
   268  	c.Assert(err, IsNil)
   269  
   270  	mustDel(c, txn)
   271  	err = txn.Commit(context.Background())
   272  	c.Assert(err, IsNil)
   273  
   274  	txn, err = s.s.Begin()
   275  	c.Assert(err, IsNil)
   276  
   277  	mustNotGet(c, txn)
   278  	err = txn.Commit(context.Background())
   279  	c.Assert(err, IsNil)
   280  }
   281  
   282  func (s *testKVSuite) TestDelete2(c *C) {
   283  	txn, err := s.s.Begin()
   284  	c.Assert(err, IsNil)
   285  	val := []byte("test")
   286  	txn.Set([]byte("DATA_test_tbl_department_record__0000000001_0003"), val)
   287  	txn.Set([]byte("DATA_test_tbl_department_record__0000000001_0004"), val)
   288  	txn.Set([]byte("DATA_test_tbl_department_record__0000000002_0003"), val)
   289  	txn.Set([]byte("DATA_test_tbl_department_record__0000000002_0004"), val)
   290  	err = txn.Commit(context.Background())
   291  	c.Assert(err, IsNil)
   292  
   293  	// Delete all
   294  	txn, err = s.s.Begin()
   295  	c.Assert(err, IsNil)
   296  
   297  	it, err := txn.Iter([]byte("DATA_test_tbl_department_record__0000000001_0003"), nil)
   298  	c.Assert(err, IsNil)
   299  	for it.Valid() {
   300  		err = txn.Delete(it.Key())
   301  		c.Assert(err, IsNil)
   302  		err = it.Next()
   303  		c.Assert(err, IsNil)
   304  	}
   305  	err = txn.Commit(context.Background())
   306  	c.Assert(err, IsNil)
   307  
   308  	txn, err = s.s.Begin()
   309  	c.Assert(err, IsNil)
   310  	it, _ = txn.Iter([]byte("DATA_test_tbl_department_record__000000000"), nil)
   311  	c.Assert(it.Valid(), IsFalse)
   312  	err = txn.Commit(context.Background())
   313  	c.Assert(err, IsNil)
   314  }
   315  
   316  func (s *testKVSuite) TestSetNil(c *C) {
   317  	txn, err := s.s.Begin()
   318  	defer txn.Commit(context.Background())
   319  	c.Assert(err, IsNil)
   320  	err = txn.Set([]byte("1"), nil)
   321  	c.Assert(err, NotNil)
   322  }
   323  
   324  func (s *testKVSuite) TestBasicSeek(c *C) {
   325  	txn, err := s.s.Begin()
   326  	c.Assert(err, IsNil)
   327  	txn.Set([]byte("1"), []byte("1"))
   328  	err = txn.Commit(context.Background())
   329  	c.Assert(err, IsNil)
   330  	txn, err = s.s.Begin()
   331  	c.Assert(err, IsNil)
   332  	defer txn.Commit(context.Background())
   333  
   334  	it, err := txn.Iter([]byte("2"), nil)
   335  	c.Assert(err, IsNil)
   336  	c.Assert(it.Valid(), Equals, false)
   337  	txn.Delete([]byte("1"))
   338  }
   339  
   340  func (s *testKVSuite) TestBasicTable(c *C) {
   341  	txn, err := s.s.Begin()
   342  	c.Assert(err, IsNil)
   343  	for i := 1; i < 5; i++ {
   344  		b := []byte(strconv.Itoa(i))
   345  		txn.Set(b, b)
   346  	}
   347  	err = txn.Commit(context.Background())
   348  	c.Assert(err, IsNil)
   349  	txn, err = s.s.Begin()
   350  	c.Assert(err, IsNil)
   351  	defer txn.Commit(context.Background())
   352  
   353  	err = txn.Set([]byte("1"), []byte("1"))
   354  	c.Assert(err, IsNil)
   355  
   356  	it, err := txn.Iter([]byte("0"), nil)
   357  	c.Assert(err, IsNil)
   358  	c.Assert(string(it.Key()), Equals, "1")
   359  
   360  	err = txn.Set([]byte("0"), []byte("0"))
   361  	c.Assert(err, IsNil)
   362  	it, err = txn.Iter([]byte("0"), nil)
   363  	c.Assert(err, IsNil)
   364  	c.Assert(string(it.Key()), Equals, "0")
   365  	err = txn.Delete([]byte("0"))
   366  	c.Assert(err, IsNil)
   367  
   368  	txn.Delete([]byte("1"))
   369  	it, err = txn.Iter([]byte("0"), nil)
   370  	c.Assert(err, IsNil)
   371  	c.Assert(string(it.Key()), Equals, "2")
   372  
   373  	err = txn.Delete([]byte("3"))
   374  	c.Assert(err, IsNil)
   375  	it, err = txn.Iter([]byte("2"), nil)
   376  	c.Assert(err, IsNil)
   377  	c.Assert(string(it.Key()), Equals, "2")
   378  
   379  	it, err = txn.Iter([]byte("3"), nil)
   380  	c.Assert(err, IsNil)
   381  	c.Assert(string(it.Key()), Equals, "4")
   382  	err = txn.Delete([]byte("2"))
   383  	c.Assert(err, IsNil)
   384  	err = txn.Delete([]byte("4"))
   385  	c.Assert(err, IsNil)
   386  }
   387  
   388  func (s *testKVSuite) TestRollback(c *C) {
   389  	txn, err := s.s.Begin()
   390  	c.Assert(err, IsNil)
   391  
   392  	err = txn.Rollback()
   393  	c.Assert(err, IsNil)
   394  
   395  	txn, err = s.s.Begin()
   396  	c.Assert(err, IsNil)
   397  
   398  	insertData(c, txn)
   399  
   400  	mustGet(c, txn)
   401  
   402  	err = txn.Rollback()
   403  	c.Assert(err, IsNil)
   404  
   405  	txn, err = s.s.Begin()
   406  	c.Assert(err, IsNil)
   407  	defer txn.Commit(context.Background())
   408  
   409  	for i := startIndex; i < testCount; i++ {
   410  		_, err := txn.Get(context.TODO(), []byte(strconv.Itoa(i)))
   411  		c.Assert(err, NotNil)
   412  	}
   413  }
   414  
   415  func (s *testKVSuite) TestSeekMin(c *C) {
   416  	rows := []struct {
   417  		key   string
   418  		value string
   419  	}{
   420  		{"DATA_test_main_db_tbl_tbl_test_record__00000000000000000001", "dagger-version"},
   421  		{"DATA_test_main_db_tbl_tbl_test_record__00000000000000000001_0002", "1"},
   422  		{"DATA_test_main_db_tbl_tbl_test_record__00000000000000000001_0003", "hello"},
   423  		{"DATA_test_main_db_tbl_tbl_test_record__00000000000000000002", "dagger-version"},
   424  		{"DATA_test_main_db_tbl_tbl_test_record__00000000000000000002_0002", "2"},
   425  		{"DATA_test_main_db_tbl_tbl_test_record__00000000000000000002_0003", "hello"},
   426  	}
   427  
   428  	txn, err := s.s.Begin()
   429  	c.Assert(err, IsNil)
   430  	for _, event := range rows {
   431  		txn.Set([]byte(event.key), []byte(event.value))
   432  	}
   433  
   434  	it, err := txn.Iter(nil, nil)
   435  	c.Assert(err, IsNil)
   436  	for it.Valid() {
   437  		it.Next()
   438  	}
   439  
   440  	it, err = txn.Iter([]byte("DATA_test_main_db_tbl_tbl_test_record__00000000000000000000"), nil)
   441  	c.Assert(err, IsNil)
   442  	c.Assert(string(it.Key()), Equals, "DATA_test_main_db_tbl_tbl_test_record__00000000000000000001")
   443  
   444  	for _, event := range rows {
   445  		txn.Delete([]byte(event.key))
   446  	}
   447  }
   448  
   449  func (s *testKVSuite) TestConditionIfNotExist(c *C) {
   450  	var success int64
   451  	cnt := 100
   452  	b := []byte("1")
   453  	var wg sync.WaitGroup
   454  	wg.Add(cnt)
   455  	for i := 0; i < cnt; i++ {
   456  		go func() {
   457  			defer wg.Done()
   458  			txn, err := s.s.Begin()
   459  			c.Assert(err, IsNil)
   460  			err = txn.Set(b, b)
   461  			if err != nil {
   462  				return
   463  			}
   464  			err = txn.Commit(context.Background())
   465  			if err == nil {
   466  				atomic.AddInt64(&success, 1)
   467  			}
   468  		}()
   469  	}
   470  	wg.Wait()
   471  	// At least one txn can success.
   472  	c.Assert(success, Greater, int64(0))
   473  
   474  	// Clean up
   475  	txn, err := s.s.Begin()
   476  	c.Assert(err, IsNil)
   477  	err = txn.Delete(b)
   478  	c.Assert(err, IsNil)
   479  	err = txn.Commit(context.Background())
   480  	c.Assert(err, IsNil)
   481  }
   482  
   483  func (s *testKVSuite) TestConditionIfEqual(c *C) {
   484  	var success int64
   485  	cnt := 100
   486  	b := []byte("1")
   487  	var wg sync.WaitGroup
   488  	wg.Add(cnt)
   489  
   490  	txn, err := s.s.Begin()
   491  	c.Assert(err, IsNil)
   492  	txn.Set(b, b)
   493  	err = txn.Commit(context.Background())
   494  	c.Assert(err, IsNil)
   495  
   496  	for i := 0; i < cnt; i++ {
   497  		go func() {
   498  			defer wg.Done()
   499  			// Use txn1/err1 instead of txn/err is
   500  			// to pass `go tool vet -shadow` check.
   501  			txn1, err1 := s.s.Begin()
   502  			c.Assert(err1, IsNil)
   503  			txn1.Set(b, []byte("newValue"))
   504  			err1 = txn1.Commit(context.Background())
   505  			if err1 == nil {
   506  				atomic.AddInt64(&success, 1)
   507  			}
   508  		}()
   509  	}
   510  	wg.Wait()
   511  	c.Assert(success, Greater, int64(0))
   512  
   513  	// Clean up
   514  	txn, err = s.s.Begin()
   515  	c.Assert(err, IsNil)
   516  	err = txn.Delete(b)
   517  	c.Assert(err, IsNil)
   518  	err = txn.Commit(context.Background())
   519  	c.Assert(err, IsNil)
   520  }
   521  
   522  func (s *testKVSuite) TestConditionUFIDelate(c *C) {
   523  	txn, err := s.s.Begin()
   524  	c.Assert(err, IsNil)
   525  	txn.Delete([]byte("b"))
   526  	ekv.IncInt64(txn, []byte("a"), 1)
   527  	err = txn.Commit(context.Background())
   528  	c.Assert(err, IsNil)
   529  }
   530  
   531  func (s *testKVSuite) TestDBClose(c *C) {
   532  	c.Skip("don't know why it fails.")
   533  	causetstore, err := mockstore.NewMockStore()
   534  	c.Assert(err, IsNil)
   535  
   536  	txn, err := causetstore.Begin()
   537  	c.Assert(err, IsNil)
   538  
   539  	err = txn.Set([]byte("a"), []byte("b"))
   540  	c.Assert(err, IsNil)
   541  
   542  	err = txn.Commit(context.Background())
   543  	c.Assert(err, IsNil)
   544  
   545  	ver, err := causetstore.CurrentVersion()
   546  	c.Assert(err, IsNil)
   547  	c.Assert(ekv.MaxVersion.Cmp(ver), Equals, 1)
   548  
   549  	snap, err := causetstore.GetSnapshot(ekv.MaxVersion)
   550  	c.Assert(err, IsNil)
   551  
   552  	_, err = snap.Get(context.TODO(), []byte("a"))
   553  	c.Assert(err, IsNil)
   554  
   555  	txn, err = causetstore.Begin()
   556  	c.Assert(err, IsNil)
   557  
   558  	err = causetstore.Close()
   559  	c.Assert(err, IsNil)
   560  
   561  	_, err = causetstore.Begin()
   562  	c.Assert(err, NotNil)
   563  
   564  	_, err = causetstore.GetSnapshot(ekv.MaxVersion)
   565  	c.Assert(err, NotNil)
   566  
   567  	err = txn.Set([]byte("a"), []byte("b"))
   568  	c.Assert(err, IsNil)
   569  
   570  	err = txn.Commit(context.Background())
   571  	c.Assert(err, NotNil)
   572  }
   573  
   574  func (s *testKVSuite) TestIsolationInc(c *C) {
   575  	threadCnt := 4
   576  
   577  	ids := make(map[int64]struct{}, threadCnt*100)
   578  	var m sync.Mutex
   579  	var wg sync.WaitGroup
   580  
   581  	wg.Add(threadCnt)
   582  	for i := 0; i < threadCnt; i++ {
   583  		go func() {
   584  			defer wg.Done()
   585  			for j := 0; j < 100; j++ {
   586  				var id int64
   587  				err := ekv.RunInNewTxn(s.s, true, func(txn ekv.Transaction) error {
   588  					var err1 error
   589  					id, err1 = ekv.IncInt64(txn, []byte("key"), 1)
   590  					return err1
   591  				})
   592  				c.Assert(err, IsNil)
   593  
   594  				m.Lock()
   595  				_, ok := ids[id]
   596  				ids[id] = struct{}{}
   597  				m.Unlock()
   598  				c.Assert(ok, IsFalse)
   599  			}
   600  		}()
   601  	}
   602  
   603  	wg.Wait()
   604  
   605  	// delete
   606  	txn, err := s.s.Begin()
   607  	c.Assert(err, IsNil)
   608  	defer txn.Commit(context.Background())
   609  	txn.Delete([]byte("key"))
   610  }
   611  
   612  func (s *testKVSuite) TestIsolationMultiInc(c *C) {
   613  	threadCnt := 4
   614  	incCnt := 100
   615  	keyCnt := 4
   616  
   617  	keys := make([][]byte, 0, keyCnt)
   618  	for i := 0; i < keyCnt; i++ {
   619  		keys = append(keys, []byte(fmt.Sprintf("test_key_%d", i)))
   620  	}
   621  
   622  	var wg sync.WaitGroup
   623  
   624  	wg.Add(threadCnt)
   625  	for i := 0; i < threadCnt; i++ {
   626  		go func() {
   627  			defer wg.Done()
   628  			for j := 0; j < incCnt; j++ {
   629  				err := ekv.RunInNewTxn(s.s, true, func(txn ekv.Transaction) error {
   630  					for _, key := range keys {
   631  						_, err1 := ekv.IncInt64(txn, key, 1)
   632  						if err1 != nil {
   633  							return err1
   634  						}
   635  					}
   636  
   637  					return nil
   638  				})
   639  				c.Assert(err, IsNil)
   640  			}
   641  		}()
   642  	}
   643  
   644  	wg.Wait()
   645  
   646  	err := ekv.RunInNewTxn(s.s, false, func(txn ekv.Transaction) error {
   647  		for _, key := range keys {
   648  			id, err1 := ekv.GetInt64(context.TODO(), txn, key)
   649  			if err1 != nil {
   650  				return err1
   651  			}
   652  			c.Assert(id, Equals, int64(threadCnt*incCnt))
   653  			txn.Delete(key)
   654  		}
   655  		return nil
   656  	})
   657  	c.Assert(err, IsNil)
   658  }
   659  
   660  func (s *testKVSuite) TestRetryOpenStore(c *C) {
   661  	begin := time.Now()
   662  	Register("dummy", &brokenStore{})
   663  	causetstore, err := newStoreWithRetry("dummy://dummy-causetstore", 3)
   664  	if causetstore != nil {
   665  		defer causetstore.Close()
   666  	}
   667  	c.Assert(err, NotNil)
   668  	elapse := time.Since(begin)
   669  	c.Assert(uint64(elapse), GreaterEqual, uint64(3*time.Second), Commentf("elapse: %s", elapse))
   670  }
   671  
   672  func (s *testKVSuite) TestOpenStore(c *C) {
   673  	Register("open", &brokenStore{})
   674  	causetstore, err := newStoreWithRetry(":", 3)
   675  	if causetstore != nil {
   676  		defer causetstore.Close()
   677  	}
   678  	c.Assert(err, NotNil)
   679  }
   680  
   681  func (s *testKVSuite) TestRegister(c *C) {
   682  	err := Register("retry", &brokenStore{})
   683  	c.Assert(err, IsNil)
   684  	err = Register("retry", &brokenStore{})
   685  	c.Assert(err, NotNil)
   686  }