github.com/nutsdb/nutsdb@v1.0.4/tx.go (about)

     1  // Copyright 2019 The nutsdb Author. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package nutsdb
    16  
    17  import (
    18  	"bytes"
    19  	"errors"
    20  	"fmt"
    21  	"strings"
    22  	"sync/atomic"
    23  
    24  	"github.com/bwmarrin/snowflake"
    25  	"github.com/xujiajun/utils/strconv2"
    26  )
    27  
    28  const (
    29  	// txStatusRunning means the tx is running
    30  	txStatusRunning = 1
    31  	// txStatusCommitting means the tx is committing
    32  	txStatusCommitting = 2
    33  	// txStatusClosed means the tx is closed, ether committed or rollback
    34  	txStatusClosed = 3
    35  )
    36  
    37  // Tx represents a transaction.
    38  type Tx struct {
    39  	id                uint64
    40  	db                *DB
    41  	writable          bool
    42  	status            atomic.Value
    43  	pendingWrites     *pendingEntryList
    44  	size              int64
    45  	pendingBucketList pendingBucketList
    46  }
    47  
    48  type txnCb struct {
    49  	commit func() error
    50  	user   func(error)
    51  	err    error
    52  }
    53  
    54  func (tx *Tx) submitEntry(ds uint16, bucket string, e *Entry) {
    55  	tx.pendingWrites.submitEntry(ds, bucket, e)
    56  }
    57  
    58  func runTxnCallback(cb *txnCb) {
    59  	switch {
    60  	case cb == nil:
    61  		panic("tx callback is nil")
    62  	case cb.user == nil:
    63  		panic("Must have caught a nil callback for tx.CommitWith")
    64  	case cb.err != nil:
    65  		cb.user(cb.err)
    66  	case cb.commit != nil:
    67  		err := cb.commit()
    68  		cb.user(err)
    69  	default:
    70  		cb.user(nil)
    71  	}
    72  }
    73  
    74  // Begin opens a new transaction.
    75  // Multiple read-only transactions can be opened at the same time but there can
    76  // only be one read/write transaction at a time. Attempting to open a read/write
    77  // transactions while another one is in progress will result in blocking until
    78  // the current read/write transaction is completed.
    79  // All transactions must be closed by calling Commit() or Rollback() when done.
    80  func (db *DB) Begin(writable bool) (tx *Tx, err error) {
    81  	tx, err = newTx(db, writable)
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  
    86  	tx.lock()
    87  	tx.setStatusRunning()
    88  	if db.closed {
    89  		tx.unlock()
    90  		tx.setStatusClosed()
    91  		return nil, ErrDBClosed
    92  	}
    93  
    94  	return
    95  }
    96  
    97  // newTx returns a newly initialized Tx object at given writable.
    98  func newTx(db *DB, writable bool) (tx *Tx, err error) {
    99  	var txID uint64
   100  
   101  	tx = &Tx{
   102  		db:                db,
   103  		writable:          writable,
   104  		pendingWrites:     newPendingEntriesList(),
   105  		pendingBucketList: make(map[Ds]map[BucketName]*Bucket),
   106  	}
   107  
   108  	txID, err = tx.getTxID()
   109  	if err != nil {
   110  		return nil, err
   111  	}
   112  
   113  	tx.id = txID
   114  
   115  	return
   116  }
   117  
   118  func (tx *Tx) CommitWith(cb func(error)) {
   119  	if cb == nil {
   120  		panic("Nil callback provided to CommitWith")
   121  	}
   122  
   123  	if tx.pendingWrites.size == 0 {
   124  		// Do not run these callbacks from here, because the CommitWith and the
   125  		// callback might be acquiring the same locks. Instead run the callback
   126  		// from another goroutine.
   127  		go runTxnCallback(&txnCb{user: cb, err: nil})
   128  		return
   129  	}
   130  	// defer tx.setStatusClosed()  //must not add this code because another process is also accessing tx
   131  	commitCb, err := tx.commitAndSend()
   132  	if err != nil {
   133  		go runTxnCallback(&txnCb{user: cb, err: err})
   134  		return
   135  	}
   136  
   137  	go runTxnCallback(&txnCb{user: cb, commit: commitCb})
   138  }
   139  
   140  func (tx *Tx) commitAndSend() (func() error, error) {
   141  	req, err := tx.db.sendToWriteCh(tx)
   142  	if err != nil {
   143  		return nil, err
   144  	}
   145  	ret := func() error {
   146  		err := req.Wait()
   147  		return err
   148  	}
   149  
   150  	return ret, nil
   151  }
   152  
   153  func (tx *Tx) checkSize() error {
   154  	count := tx.pendingWrites.size
   155  	if int64(count) >= tx.db.getMaxBatchCount() || tx.size >= tx.db.getMaxBatchSize() {
   156  		return ErrTxnTooBig
   157  	}
   158  
   159  	return nil
   160  }
   161  
   162  // getTxID returns the tx id.
   163  func (tx *Tx) getTxID() (id uint64, err error) {
   164  	node, err := snowflake.NewNode(tx.db.opt.NodeNum)
   165  	if err != nil {
   166  		return 0, err
   167  	}
   168  
   169  	id = uint64(node.Generate().Int64())
   170  
   171  	return
   172  }
   173  
   174  // Commit commits the transaction, following these steps:
   175  //
   176  // 1. check the length of pendingWrites.If there are no writes, return immediately.
   177  //
   178  // 2. check if the ActiveFile has not enough space to store entry. if not, call rotateActiveFile function.
   179  //
   180  // 3. write pendingWrites to disk, if a non-nil error,return the error.
   181  //
   182  // 4. build Hint index.
   183  //
   184  // 5. Unlock the database and clear the db field.
   185  func (tx *Tx) Commit() (err error) {
   186  	defer func() {
   187  		if err != nil {
   188  			tx.handleErr(err)
   189  		}
   190  		tx.unlock()
   191  		tx.db = nil
   192  
   193  		tx.pendingWrites = nil
   194  	}()
   195  	if tx.isClosed() {
   196  		return ErrCannotCommitAClosedTx
   197  	}
   198  
   199  	if tx.db == nil {
   200  		tx.setStatusClosed()
   201  		return ErrDBClosed
   202  	}
   203  
   204  	var curWriteCount int64
   205  	if tx.db.opt.MaxWriteRecordCount > 0 {
   206  		curWriteCount, err = tx.getNewAddRecordCount()
   207  		if err != nil {
   208  			return err
   209  		}
   210  
   211  		// judge all write records is whether more than the MaxWriteRecordCount
   212  		if tx.db.RecordCount+curWriteCount > tx.db.opt.MaxWriteRecordCount {
   213  			return ErrTxnExceedWriteLimit
   214  		}
   215  	}
   216  
   217  	tx.setStatusCommitting()
   218  	defer tx.setStatusClosed()
   219  
   220  	writesBucketLen := len(tx.pendingBucketList)
   221  	if tx.pendingWrites.size == 0 && writesBucketLen == 0 {
   222  		return nil
   223  	}
   224  
   225  	buff := tx.allocCommitBuffer()
   226  	defer tx.db.commitBuffer.Reset()
   227  
   228  	var records []*Record
   229  
   230  	pendingWriteList := tx.pendingWrites.toList()
   231  	lastIndex := len(pendingWriteList) - 1
   232  	for i := 0; i < len(pendingWriteList); i++ {
   233  		entry := pendingWriteList[i]
   234  		entrySize := entry.Size()
   235  		if entrySize > tx.db.opt.SegmentSize {
   236  			return ErrDataSizeExceed
   237  		}
   238  
   239  		if tx.db.ActiveFile.ActualSize+int64(buff.Len())+entrySize > tx.db.opt.SegmentSize {
   240  			if _, err := tx.writeData(buff.Bytes()); err != nil {
   241  				return err
   242  			}
   243  			buff.Reset()
   244  
   245  			if err := tx.rotateActiveFile(); err != nil {
   246  				return err
   247  			}
   248  		}
   249  
   250  		offset := tx.db.ActiveFile.writeOff + int64(buff.Len())
   251  
   252  		if i == lastIndex {
   253  			entry.Meta.Status = Committed
   254  		}
   255  
   256  		if _, err := buff.Write(entry.Encode()); err != nil {
   257  			return err
   258  		}
   259  
   260  		if i == lastIndex {
   261  			if _, err := tx.writeData(buff.Bytes()); err != nil {
   262  				return err
   263  			}
   264  		}
   265  
   266  		record := tx.db.createRecordByModeWithFidAndOff(tx.db.ActiveFile.fileID, uint64(offset), entry)
   267  
   268  		// add to cache
   269  		if tx.db.getHintKeyAndRAMIdxCacheSize() > 0 && tx.db.opt.EntryIdxMode == HintKeyAndRAMIdxMode {
   270  			tx.db.hintKeyAndRAMIdxModeLru.Add(record, entry)
   271  		}
   272  
   273  		records = append(records, record)
   274  	}
   275  
   276  	if err := tx.SubmitBucket(); err != nil {
   277  		return err
   278  	}
   279  
   280  	if err := tx.buildIdxes(records, pendingWriteList); err != nil {
   281  		return err
   282  	}
   283  	tx.db.RecordCount += curWriteCount
   284  
   285  	if err := tx.buildBucketInIndex(); err != nil {
   286  		return err
   287  	}
   288  
   289  	return nil
   290  }
   291  
   292  func (tx *Tx) getNewAddRecordCount() (int64, error) {
   293  	var res int64
   294  	changeCountInEntries := tx.getChangeCountInEntriesChanges()
   295  	changeCountInBucket := tx.getChangeCountInBucketChanges()
   296  	res += changeCountInEntries
   297  	res += changeCountInBucket
   298  	return res, nil
   299  }
   300  
   301  func (tx *Tx) getListHeadTailSeq(bucketId BucketId, key string) *HeadTailSeq {
   302  	res := HeadTailSeq{Head: initialListSeq, Tail: initialListSeq + 1}
   303  	if _, ok := tx.db.Index.list.idx[bucketId]; ok {
   304  		if _, ok := tx.db.Index.list.idx[bucketId].Seq[key]; ok {
   305  			res = *tx.db.Index.list.idx[bucketId].Seq[key]
   306  		}
   307  	}
   308  
   309  	return &res
   310  }
   311  
   312  func (tx *Tx) getListEntryNewAddRecordCount(bucketId BucketId, entry *Entry) (int64, error) {
   313  	if entry.Meta.Flag == DataExpireListFlag {
   314  		return 0, nil
   315  	}
   316  
   317  	var res int64
   318  	key := string(entry.Key)
   319  	value := string(entry.Value)
   320  	l := tx.db.Index.list.getWithDefault(bucketId)
   321  
   322  	switch entry.Meta.Flag {
   323  	case DataLPushFlag, DataRPushFlag:
   324  		res++
   325  	case DataLPopFlag, DataRPopFlag:
   326  		res--
   327  	case DataLRemByIndex:
   328  		indexes, _ := UnmarshalInts([]byte(value))
   329  		res -= int64(len(l.getValidIndexes(key, indexes)))
   330  	case DataLRemFlag:
   331  		count, newValue := splitIntStringStr(value, SeparatorForListKey)
   332  		removeIndices, err := l.getRemoveIndexes(key, count, func(r *Record) (bool, error) {
   333  			v, err := tx.db.getValueByRecord(r)
   334  			if err != nil {
   335  				return false, err
   336  			}
   337  			return bytes.Equal([]byte(newValue), v), nil
   338  		})
   339  		if err != nil {
   340  			return 0, err
   341  		}
   342  		res -= int64(len(removeIndices))
   343  	case DataLTrimFlag:
   344  		newKey, start := splitStringIntStr(key, SeparatorForListKey)
   345  		end, _ := strconv2.StrToInt(value)
   346  
   347  		if l.IsExpire(newKey) {
   348  			return 0, nil
   349  		}
   350  
   351  		if _, ok := l.Items[newKey]; !ok {
   352  			return 0, nil
   353  		}
   354  
   355  		items, err := l.LRange(newKey, start, end)
   356  		if err != nil {
   357  			return res, err
   358  		}
   359  
   360  		list := l.Items[newKey]
   361  		res -= int64(list.Count() - len(items))
   362  	}
   363  
   364  	return res, nil
   365  }
   366  
   367  func (tx *Tx) getKvEntryNewAddRecordCount(bucketId BucketId, entry *Entry) (int64, error) {
   368  	var res int64
   369  
   370  	switch entry.Meta.Flag {
   371  	case DataDeleteFlag:
   372  		res--
   373  	case DataSetFlag:
   374  		if idx, ok := tx.db.Index.bTree.exist(bucketId); ok {
   375  			_, found := idx.Find(entry.Key)
   376  			if !found {
   377  				res++
   378  			}
   379  		} else {
   380  			res++
   381  		}
   382  	}
   383  
   384  	return res, nil
   385  }
   386  
   387  func (tx *Tx) getSetEntryNewAddRecordCount(bucketId BucketId, entry *Entry) (int64, error) {
   388  	var res int64
   389  
   390  	if entry.Meta.Flag == DataDeleteFlag {
   391  		res--
   392  	}
   393  
   394  	if entry.Meta.Flag == DataSetFlag {
   395  		res++
   396  	}
   397  
   398  	return res, nil
   399  }
   400  
   401  func (tx *Tx) getSortedSetEntryNewAddRecordCount(bucketId BucketId, entry *Entry) (int64, error) {
   402  	var res int64
   403  	key := string(entry.Key)
   404  	value := string(entry.Value)
   405  
   406  	switch entry.Meta.Flag {
   407  	case DataZAddFlag:
   408  		if !tx.keyExistsInSortedSet(bucketId, key, value) {
   409  			res++
   410  		}
   411  	case DataZRemFlag:
   412  		res--
   413  	case DataZRemRangeByRankFlag:
   414  		start, end := splitIntIntStr(value, SeparatorForZSetKey)
   415  		delNodes, err := tx.db.Index.sortedSet.getWithDefault(bucketId, tx.db).getZRemRangeByRankNodes(key, start, end)
   416  		if err != nil {
   417  			return res, err
   418  		}
   419  		res -= int64(len(delNodes))
   420  	case DataZPopMaxFlag, DataZPopMinFlag:
   421  		res--
   422  	}
   423  
   424  	return res, nil
   425  }
   426  
   427  func (tx *Tx) keyExistsInSortedSet(bucketId BucketId, key, value string) bool {
   428  	if _, exist := tx.db.Index.sortedSet.exist(bucketId); !exist {
   429  		return false
   430  	}
   431  	newKey := key
   432  	if strings.Contains(key, SeparatorForZSetKey) {
   433  		newKey, _ = splitStringFloat64Str(key, SeparatorForZSetKey)
   434  	}
   435  	exists, _ := tx.db.Index.sortedSet.idx[bucketId].ZExist(newKey, []byte(value))
   436  	return exists
   437  }
   438  
   439  func (tx *Tx) getEntryNewAddRecordCount(entry *Entry) (int64, error) {
   440  	var res int64
   441  	var err error
   442  
   443  	bucket, err := tx.db.bm.GetBucketById(entry.Meta.BucketId)
   444  	if err != nil {
   445  		return 0, err
   446  	}
   447  	bucketId := bucket.Id
   448  
   449  	if entry.Meta.Ds == DataStructureBTree {
   450  		res, err = tx.getKvEntryNewAddRecordCount(bucketId, entry)
   451  	}
   452  
   453  	if entry.Meta.Ds == DataStructureList {
   454  		res, err = tx.getListEntryNewAddRecordCount(bucketId, entry)
   455  	}
   456  
   457  	if entry.Meta.Ds == DataStructureSet {
   458  		res, err = tx.getSetEntryNewAddRecordCount(bucketId, entry)
   459  	}
   460  
   461  	if entry.Meta.Ds == DataStructureSortedSet {
   462  		res, err = tx.getSortedSetEntryNewAddRecordCount(bucketId, entry)
   463  	}
   464  
   465  	return res, err
   466  }
   467  
   468  func (tx *Tx) allocCommitBuffer() *bytes.Buffer {
   469  	var buff *bytes.Buffer
   470  
   471  	if tx.size < tx.db.opt.CommitBufferSize {
   472  		buff = tx.db.commitBuffer
   473  	} else {
   474  		buff = new(bytes.Buffer)
   475  		// avoid grow
   476  		buff.Grow(int(tx.size))
   477  	}
   478  
   479  	return buff
   480  }
   481  
   482  // rotateActiveFile rotates log file when active file is not enough space to store the entry.
   483  func (tx *Tx) rotateActiveFile() error {
   484  	var err error
   485  	tx.db.MaxFileID++
   486  
   487  	if !tx.db.opt.SyncEnable && tx.db.opt.RWMode == MMap {
   488  		if err := tx.db.ActiveFile.rwManager.Sync(); err != nil {
   489  			return err
   490  		}
   491  	}
   492  
   493  	if err := tx.db.ActiveFile.rwManager.Release(); err != nil {
   494  		return err
   495  	}
   496  
   497  	// reset ActiveFile
   498  	path := getDataPath(tx.db.MaxFileID, tx.db.opt.Dir)
   499  	tx.db.ActiveFile, err = tx.db.fm.getDataFile(path, tx.db.opt.SegmentSize)
   500  	if err != nil {
   501  		return err
   502  	}
   503  
   504  	tx.db.ActiveFile.fileID = tx.db.MaxFileID
   505  	return nil
   506  }
   507  
   508  func (tx *Tx) writeData(data []byte) (n int, err error) {
   509  	if len(data) == 0 {
   510  		return
   511  	}
   512  
   513  	writeOffset := tx.db.ActiveFile.ActualSize
   514  
   515  	l := len(data)
   516  	if writeOffset+int64(l) > tx.db.opt.SegmentSize {
   517  		return 0, errors.New("not enough file space")
   518  	}
   519  
   520  	if n, err = tx.db.ActiveFile.WriteAt(data, writeOffset); err != nil {
   521  		return
   522  	}
   523  
   524  	tx.db.ActiveFile.writeOff += int64(l)
   525  	tx.db.ActiveFile.ActualSize += int64(l)
   526  
   527  	if tx.db.opt.SyncEnable {
   528  		if err := tx.db.ActiveFile.rwManager.Sync(); err != nil {
   529  			return 0, err
   530  		}
   531  	}
   532  
   533  	return
   534  }
   535  
   536  // Rollback closes the transaction.
   537  func (tx *Tx) Rollback() error {
   538  	if tx.db == nil {
   539  		tx.setStatusClosed()
   540  		return ErrDBClosed
   541  	}
   542  	if tx.isCommitting() {
   543  		return ErrCannotRollbackACommittingTx
   544  	}
   545  
   546  	if tx.isClosed() {
   547  		return ErrCannotRollbackAClosedTx
   548  	}
   549  
   550  	tx.setStatusClosed()
   551  	tx.unlock()
   552  
   553  	tx.db = nil
   554  	tx.pendingWrites = nil
   555  
   556  	return nil
   557  }
   558  
   559  // lock locks the database based on the transaction type.
   560  func (tx *Tx) lock() {
   561  	if tx.writable {
   562  		tx.db.mu.Lock()
   563  	} else {
   564  		tx.db.mu.RLock()
   565  	}
   566  }
   567  
   568  // unlock unlocks the database based on the transaction type.
   569  func (tx *Tx) unlock() {
   570  	if tx.writable {
   571  		tx.db.mu.Unlock()
   572  	} else {
   573  		tx.db.mu.RUnlock()
   574  	}
   575  }
   576  
   577  func (tx *Tx) handleErr(err error) {
   578  	if tx.db.opt.ErrorHandler != nil {
   579  		tx.db.opt.ErrorHandler.HandleError(err)
   580  	}
   581  }
   582  
   583  func (tx *Tx) checkTxIsClosed() error {
   584  	if tx.db == nil {
   585  		return ErrTxClosed
   586  	}
   587  	return nil
   588  }
   589  
   590  // put sets the value for a key in the bucket.
   591  // Returns an error if tx is closed, if performing a write operation on a read-only transaction, if the key is empty.
   592  func (tx *Tx) put(bucket string, key, value []byte, ttl uint32, flag uint16, timestamp uint64, ds uint16) error {
   593  	if err := tx.checkTxIsClosed(); err != nil {
   594  		return err
   595  	}
   596  
   597  	bucketStatus := tx.getBucketStatus(DataStructureBTree, bucket)
   598  	if bucketStatus == BucketStatusDeleted {
   599  		return ErrBucketNotFound
   600  	}
   601  
   602  	if !tx.db.bm.ExistBucket(ds, bucket) {
   603  		return ErrorBucketNotExist
   604  	}
   605  
   606  	if !tx.writable {
   607  		return ErrTxNotWritable
   608  	}
   609  
   610  	bucketId, err := tx.db.bm.GetBucketID(ds, bucket)
   611  	if err != nil {
   612  		return err
   613  	}
   614  
   615  	meta := NewMetaData().WithTimeStamp(timestamp).WithKeySize(uint32(len(key))).WithValueSize(uint32(len(value))).WithFlag(flag).
   616  		WithTTL(ttl).WithStatus(UnCommitted).WithDs(ds).WithTxID(tx.id).WithBucketId(bucketId)
   617  
   618  	e := NewEntry().WithKey(key).WithMeta(meta).WithValue(value)
   619  
   620  	err = e.valid()
   621  	if err != nil {
   622  		return err
   623  	}
   624  	tx.submitEntry(ds, bucket, e)
   625  	if err != nil {
   626  		return err
   627  	}
   628  	tx.size += e.Size()
   629  
   630  	return nil
   631  }
   632  
   633  func (tx *Tx) putDeleteLog(bucketId BucketId, key, value []byte, ttl uint32, flag uint16, timestamp uint64, ds uint16) {
   634  	bucket, err := tx.db.bm.GetBucketById(bucketId)
   635  	if err != nil {
   636  		return
   637  	}
   638  	meta := NewMetaData().WithTimeStamp(timestamp).WithKeySize(uint32(len(key))).WithValueSize(uint32(len(value))).WithFlag(flag).
   639  		WithTTL(ttl).WithStatus(UnCommitted).WithDs(ds).WithTxID(tx.id).WithBucketId(bucket.Id)
   640  
   641  	e := NewEntry().WithKey(key).WithMeta(meta).WithValue(value)
   642  	tx.submitEntry(ds, bucket.Name, e)
   643  	tx.size += e.Size()
   644  }
   645  
   646  // setStatusCommitting will change the tx status to txStatusCommitting
   647  func (tx *Tx) setStatusCommitting() {
   648  	status := txStatusCommitting
   649  	tx.status.Store(status)
   650  }
   651  
   652  // setStatusClosed will change the tx status to txStatusClosed
   653  func (tx *Tx) setStatusClosed() {
   654  	status := txStatusClosed
   655  	tx.status.Store(status)
   656  }
   657  
   658  // setStatusRunning will change the tx status to txStatusRunning
   659  func (tx *Tx) setStatusRunning() {
   660  	status := txStatusRunning
   661  	tx.status.Store(status)
   662  }
   663  
   664  // isRunning will check if the tx status is txStatusRunning
   665  func (tx *Tx) isRunning() bool {
   666  	status := tx.status.Load().(int)
   667  	return status == txStatusRunning
   668  }
   669  
   670  // isCommitting will check if the tx status is txStatusCommitting
   671  func (tx *Tx) isCommitting() bool {
   672  	status := tx.status.Load().(int)
   673  	return status == txStatusCommitting
   674  }
   675  
   676  // isClosed will check if the tx status is txStatusClosed
   677  func (tx *Tx) isClosed() bool {
   678  	status := tx.status.Load().(int)
   679  	return status == txStatusClosed
   680  }
   681  
   682  func (tx *Tx) buildIdxes(records []*Record, entries []*Entry) error {
   683  	for i, entry := range entries {
   684  		meta := entry.Meta
   685  		var err error
   686  		switch meta.Ds {
   687  		case DataStructureBTree:
   688  			err = tx.db.buildBTreeIdx(records[i], entry)
   689  		case DataStructureList:
   690  			err = tx.db.buildListIdx(records[i], entry)
   691  		case DataStructureSet:
   692  			err = tx.db.buildSetIdx(records[i], entry)
   693  		case DataStructureSortedSet:
   694  			err = tx.db.buildSortedSetIdx(records[i], entry)
   695  		}
   696  
   697  		if err != nil {
   698  			return err
   699  		}
   700  
   701  		tx.db.KeyCount++
   702  	}
   703  	return nil
   704  }
   705  
   706  func (tx *Tx) putBucket(b *Bucket) error {
   707  	if _, exist := tx.pendingBucketList[b.Ds]; !exist {
   708  		tx.pendingBucketList[b.Ds] = map[BucketName]*Bucket{}
   709  	}
   710  	bucketInDs := tx.pendingBucketList[b.Ds]
   711  	bucketInDs[b.Name] = b
   712  	return nil
   713  }
   714  
   715  func (tx *Tx) SubmitBucket() error {
   716  	bucketReqs := make([]*bucketSubmitRequest, 0)
   717  	for ds, mapper := range tx.pendingBucketList {
   718  		for name, bucket := range mapper {
   719  			req := &bucketSubmitRequest{
   720  				ds:     ds,
   721  				name:   name,
   722  				bucket: bucket,
   723  			}
   724  			bucketReqs = append(bucketReqs, req)
   725  		}
   726  	}
   727  	return tx.db.bm.SubmitPendingBucketChange(bucketReqs)
   728  }
   729  
   730  // buildBucketInIndex build indexes on creation and deletion of buckets
   731  func (tx *Tx) buildBucketInIndex() error {
   732  	for _, mapper := range tx.pendingBucketList {
   733  		for _, bucket := range mapper {
   734  			if bucket.Meta.Op == BucketInsertOperation {
   735  				switch bucket.Ds {
   736  				case DataStructureBTree:
   737  					tx.db.Index.bTree.getWithDefault(bucket.Id)
   738  				case DataStructureList:
   739  					tx.db.Index.list.getWithDefault(bucket.Id)
   740  				case DataStructureSet:
   741  					tx.db.Index.set.getWithDefault(bucket.Id)
   742  				case DataStructureSortedSet:
   743  					tx.db.Index.sortedSet.getWithDefault(bucket.Id, tx.db)
   744  				default:
   745  					return ErrDataStructureNotSupported
   746  				}
   747  			} else if bucket.Meta.Op == BucketDeleteOperation {
   748  				switch bucket.Ds {
   749  				case DataStructureBTree:
   750  					tx.db.Index.bTree.delete(bucket.Id)
   751  				case DataStructureList:
   752  					tx.db.Index.list.delete(bucket.Id)
   753  				case DataStructureSet:
   754  					tx.db.Index.set.delete(bucket.Id)
   755  				case DataStructureSortedSet:
   756  					tx.db.Index.sortedSet.delete(bucket.Id)
   757  				default:
   758  					return ErrDataStructureNotSupported
   759  				}
   760  			}
   761  		}
   762  	}
   763  	return nil
   764  }
   765  
   766  func (tx *Tx) getChangeCountInEntriesChanges() int64 {
   767  	var res int64
   768  	var err error
   769  	for _, entriesInDS := range tx.pendingWrites.entriesInBTree {
   770  		for _, entry := range entriesInDS {
   771  			curRecordCnt, _ := tx.getEntryNewAddRecordCount(entry)
   772  			if err != nil {
   773  				return res
   774  			}
   775  			res += curRecordCnt
   776  		}
   777  	}
   778  	for _, entriesInDS := range tx.pendingWrites.entries {
   779  		for _, entries := range entriesInDS {
   780  			for _, entry := range entries {
   781  				curRecordCnt, _ := tx.getEntryNewAddRecordCount(entry)
   782  				if err != nil {
   783  					return res
   784  				}
   785  				res += curRecordCnt
   786  			}
   787  		}
   788  	}
   789  	return res
   790  }
   791  
   792  func (tx *Tx) getChangeCountInBucketChanges() int64 {
   793  	var res int64
   794  	var f = func(bucket *Bucket) error {
   795  		bucketId := bucket.Id
   796  		if bucket.Meta.Op == BucketDeleteOperation {
   797  			switch bucket.Ds {
   798  			case DataStructureBTree:
   799  				if bTree, ok := tx.db.Index.bTree.idx[bucketId]; ok {
   800  					res -= int64(bTree.Count())
   801  				}
   802  			case DataStructureSet:
   803  				if set, ok := tx.db.Index.set.idx[bucketId]; ok {
   804  					for key := range set.M {
   805  						res -= int64(set.SCard(key))
   806  					}
   807  				}
   808  			case DataStructureSortedSet:
   809  				if sortedSet, ok := tx.db.Index.sortedSet.idx[bucketId]; ok {
   810  					for key := range sortedSet.M {
   811  						curLen, _ := sortedSet.ZCard(key)
   812  						res -= int64(curLen)
   813  					}
   814  				}
   815  			case DataStructureList:
   816  				if list, ok := tx.db.Index.list.idx[bucketId]; ok {
   817  					for key := range list.Items {
   818  						curLen, _ := list.Size(key)
   819  						res -= int64(curLen)
   820  					}
   821  				}
   822  			default:
   823  				panic(fmt.Sprintf("there is an unexpected data structure that is unimplemented in our database.:%d", bucket.Ds))
   824  			}
   825  		}
   826  		return nil
   827  	}
   828  	_ = tx.pendingBucketList.rangeBucket(f)
   829  	return res
   830  }
   831  
   832  func (tx *Tx) getBucketStatus(ds Ds, name BucketName) BucketStatus {
   833  	if len(tx.pendingBucketList) > 0 {
   834  		if bucketInDs, exist := tx.pendingBucketList[ds]; exist {
   835  			if bucket, exist := bucketInDs[name]; exist {
   836  				switch bucket.Meta.Op {
   837  				case BucketInsertOperation:
   838  					return BucketStatusNew
   839  				case BucketDeleteOperation:
   840  					return BucketStatusDeleted
   841  				case BucketUpdateOperation:
   842  					return BucketStatusUpdated
   843  				}
   844  			}
   845  		}
   846  	}
   847  	if tx.db.bm.ExistBucket(ds, name) {
   848  		return BucketStatusExistAlready
   849  	}
   850  	return BucketStatusUnknown
   851  }
   852  
   853  // findEntryStatus finds the latest status for the certain Entry in Tx
   854  func (tx *Tx) findEntryAndItsStatus(ds Ds, bucket BucketName, key string) (EntryStatus, *Entry) {
   855  	if tx.pendingWrites.size == 0 {
   856  		return NotFoundEntry, nil
   857  	}
   858  	pendingWriteEntries := tx.pendingWrites.entriesInBTree
   859  	if pendingWriteEntries == nil {
   860  		return NotFoundEntry, nil
   861  	}
   862  	if pendingWriteEntries[bucket] == nil {
   863  		return NotFoundEntry, nil
   864  	}
   865  	entries := pendingWriteEntries[bucket]
   866  	if entry, exist := entries[key]; exist {
   867  		switch entry.Meta.Flag {
   868  		case DataDeleteFlag:
   869  			return EntryDeleted, nil
   870  		default:
   871  			return EntryUpdated, entry
   872  		}
   873  	}
   874  	return NotFoundEntry, nil
   875  }