github.com/turingchain2020/turingchain@v1.1.21/system/store/mavl/db/tree.go (about)

     1  // Copyright Turing Corp. 2018 All Rights Reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package mavl 默克尔平衡树算法实现以及裁剪
     6  package mavl
     7  
     8  import (
     9  	"bytes"
    10  	"errors"
    11  	"fmt"
    12  	"sync"
    13  
    14  	"github.com/turingchain2020/turingchain/common"
    15  	dbm "github.com/turingchain2020/turingchain/common/db"
    16  	log "github.com/turingchain2020/turingchain/common/log/log15"
    17  	"github.com/turingchain2020/turingchain/system/store/mavl/db/ticket"
    18  	"github.com/turingchain2020/turingchain/types"
    19  	farm "github.com/dgryski/go-farm"
    20  	"github.com/golang/protobuf/proto"
    21  	lru "github.com/hashicorp/golang-lru"
    22  )
    23  
    24  const (
    25  	hashNodePrefix             = "_mh_"
    26  	leafNodePrefix             = "_mb_"
    27  	curMaxBlockHeight          = "_..mcmbh.._"
    28  	rootHashHeightPrefix       = "_mrhp_"
    29  	tkCloseCacheLen      int32 = 10 * 10000
    30  )
    31  
    32  var (
    33  	// ErrNodeNotExist node is not exist
    34  	ErrNodeNotExist = errors.New("ErrNodeNotExist")
    35  	treelog         = log.New("module", "mavl")
    36  	emptyRoot       [32]byte
    37  	// 当前树的最大高度
    38  	maxBlockHeight int64
    39  	heightMtx      sync.Mutex
    40  	memTree        MemTreeOpera
    41  	tkCloseCache   MemTreeOpera
    42  )
    43  
    44  // InitGlobalMem 初始化全局变量
    45  func InitGlobalMem(treeCfg *TreeConfig) {
    46  	// 使能情况下非空创建当前整tree的缓存
    47  	if treeCfg.EnableMemTree && memTree == nil {
    48  		memTree = NewTreeMap(50 * 10000)
    49  		if treeCfg.TkCloseCacheLen == 0 {
    50  			treeCfg.TkCloseCacheLen = tkCloseCacheLen
    51  		}
    52  		tkCloseCache = NewTreeARC(int(treeCfg.TkCloseCacheLen))
    53  	}
    54  }
    55  
    56  // ReleaseGlobalMem 释放全局缓存
    57  func ReleaseGlobalMem() {
    58  	if memTree != nil {
    59  		memTree = nil
    60  	}
    61  	if tkCloseCache != nil {
    62  		tkCloseCache = nil
    63  	}
    64  }
    65  
    66  // TreeConfig ...
    67  type TreeConfig struct {
    68  	EnableMavlPrefix bool
    69  	EnableMVCC       bool
    70  	EnableMavlPrune  bool
    71  	PruneHeight      int32
    72  	EnableMemTree    bool
    73  	EnableMemVal     bool
    74  	TkCloseCacheLen  int32
    75  }
    76  
    77  type memNode struct {
    78  	data   [][]byte //顺序为lefthash, righthash, key, value
    79  	Height int32
    80  	Size   int32
    81  }
    82  
    83  type uintkey uint64
    84  
    85  // Tree merkle avl tree
    86  type Tree struct {
    87  	root        *Node
    88  	ndb         *nodeDB
    89  	blockHeight int64
    90  	// 树更新之后,废弃的节点(更新缓存中节点先对旧节点进行删除然后再更新)
    91  	obsoleteNode map[uintkey]struct{}
    92  	updateNode   map[uintkey]*memNode
    93  	tkCloseNode  map[uintkey]*memNode
    94  	config       *TreeConfig
    95  }
    96  
    97  // NewTree 新建一个merkle avl 树
    98  func NewTree(db dbm.DB, sync bool, treeCfg *TreeConfig) *Tree {
    99  	if db == nil {
   100  		// In-memory IAVLTree
   101  		return &Tree{
   102  			obsoleteNode: make(map[uintkey]struct{}),
   103  			updateNode:   make(map[uintkey]*memNode),
   104  			tkCloseNode:  make(map[uintkey]*memNode),
   105  			config:       treeCfg,
   106  		}
   107  	}
   108  	// Persistent IAVLTree
   109  	ndb := newNodeDB(db, sync)
   110  	return &Tree{
   111  		ndb:          ndb,
   112  		obsoleteNode: make(map[uintkey]struct{}),
   113  		updateNode:   make(map[uintkey]*memNode),
   114  		tkCloseNode:  make(map[uintkey]*memNode),
   115  		config:       treeCfg,
   116  	}
   117  }
   118  
   119  // Copy copy tree
   120  func (t *Tree) Copy() *Tree {
   121  	if t.root == nil {
   122  		return &Tree{
   123  			root: nil,
   124  			ndb:  t.ndb,
   125  		}
   126  	}
   127  	if t.ndb != nil && !t.root.persisted {
   128  		panic("It is unsafe to Copy() an unpersisted tree.")
   129  	} else if t.ndb == nil && t.root.hash == nil {
   130  		t.root.Hash(t)
   131  	}
   132  	return &Tree{
   133  		root: t.root,
   134  		ndb:  t.ndb,
   135  	}
   136  }
   137  
   138  // Size 获取tree的叶子节点数
   139  func (t *Tree) Size() int32 {
   140  	if t.root == nil {
   141  		return 0
   142  	}
   143  	return t.root.size
   144  }
   145  
   146  // Height 获取tree的高度
   147  func (t *Tree) Height() int32 {
   148  	if t.root == nil {
   149  		return 0
   150  	}
   151  	return t.root.height
   152  }
   153  
   154  // Has 判断key是否存在tree中
   155  func (t *Tree) Has(key []byte) bool {
   156  	if t.root == nil {
   157  		return false
   158  	}
   159  	return t.root.has(t, key)
   160  }
   161  
   162  // Set 设置k:v pair到tree中
   163  func (t *Tree) Set(key []byte, value []byte) (updated bool) {
   164  	//treelog.Info("IAVLTree.Set", "key", key, "value",value)
   165  	if t.root == nil {
   166  		t.root = NewNode(copyBytes(key), copyBytes(value))
   167  		return false
   168  	}
   169  	t.root, updated = t.root.set(t, copyBytes(key), copyBytes(value))
   170  	return updated
   171  }
   172  
   173  func (t *Tree) getObsoleteNode() map[uintkey]struct{} {
   174  	return t.obsoleteNode
   175  }
   176  
   177  // Hash 计算tree 的roothash
   178  func (t *Tree) Hash() []byte {
   179  	if t.root == nil {
   180  		return nil
   181  	}
   182  	hash := t.root.Hash(t)
   183  	// 更新memTree
   184  	if t.config != nil && t.config.EnableMemTree && memTree != nil && tkCloseCache != nil {
   185  		for k := range t.obsoleteNode {
   186  			memTree.Delete(k)
   187  		}
   188  		for k, v := range t.updateNode {
   189  			memTree.Add(k, v)
   190  		}
   191  		for k, v := range t.tkCloseNode {
   192  			tkCloseCache.Add(k, v)
   193  		}
   194  		treelog.Debug("Tree.Hash", "memTree len", memTree.Len(), "tkCloseCache len", tkCloseCache.Len(), "tree height", t.blockHeight)
   195  	}
   196  	return hash
   197  }
   198  
   199  // Save 保存整个tree的节点信息到db中
   200  func (t *Tree) Save() []byte {
   201  	if t.root == nil {
   202  		return nil
   203  	}
   204  	if t.ndb != nil {
   205  		if t.config != nil && t.config.EnableMavlPrune && t.isRemoveLeafCountKey() {
   206  			//DelLeafCountKV 需要先提前将leafcoutkey删除,这里需先于t.ndb.Commit()
   207  			err := DelLeafCountKV(t.ndb.db, t.blockHeight, t.config)
   208  			if err != nil {
   209  				treelog.Error("Tree.Save", "DelLeafCountKV err", err)
   210  			}
   211  		}
   212  		saveNodeNo := t.root.save(t)
   213  		treelog.Debug("Tree.Save", "saveNodeNo", saveNodeNo, "tree height", t.blockHeight)
   214  		// 保存每个高度的roothash
   215  		if t.config != nil && t.config.EnableMavlPrune {
   216  			err := t.root.saveRootHash(t)
   217  			if err != nil {
   218  				treelog.Error("Tree.Save", "saveRootHash err", err)
   219  			}
   220  		}
   221  
   222  		beg := types.Now()
   223  		err := t.ndb.Commit()
   224  		treelog.Debug("tree.commit", "cost", types.Since(beg))
   225  		if err != nil {
   226  			return nil
   227  		}
   228  		// 该线程应只允许一个
   229  		if t.config != nil && t.config.EnableMavlPrune && !isPruning() &&
   230  			t.config.PruneHeight != 0 &&
   231  			t.blockHeight%int64(t.config.PruneHeight) == 0 &&
   232  			t.blockHeight/int64(t.config.PruneHeight) > 1 {
   233  			wg.Add(1)
   234  			go pruning(t.ndb.db, t.blockHeight, t.config)
   235  		}
   236  	}
   237  	return t.root.hash
   238  }
   239  
   240  // Load 从db中加载rootnode
   241  func (t *Tree) Load(hash []byte) (err error) {
   242  	if hash == nil {
   243  		return
   244  	}
   245  	if !bytes.Equal(hash, emptyRoot[:]) {
   246  		t.root, err = t.ndb.GetNode(t, hash)
   247  		return err
   248  	}
   249  	return nil
   250  }
   251  
   252  // SetBlockHeight 设置block高度到tree
   253  func (t *Tree) SetBlockHeight(height int64) {
   254  	t.blockHeight = height
   255  }
   256  
   257  // Get 通过key获取leaf节点信息
   258  func (t *Tree) Get(key []byte) (index int32, value []byte, exists bool) {
   259  	if t.root == nil {
   260  		return 0, nil, false
   261  	}
   262  	return t.root.get(t, key)
   263  }
   264  
   265  // GetHash 通过key获取leaf节点hash信息
   266  func (t *Tree) GetHash(key []byte) (index int32, hash []byte, exists bool) {
   267  	if t.root == nil {
   268  		return 0, nil, false
   269  	}
   270  	return t.root.getHash(t, key)
   271  }
   272  
   273  // GetByIndex 通过index获取leaf节点信息
   274  func (t *Tree) GetByIndex(index int32) (key []byte, value []byte) {
   275  	if t.root == nil {
   276  		return nil, nil
   277  	}
   278  	return t.root.getByIndex(t, index)
   279  }
   280  
   281  // Proof 获取指定k:v pair的proof证明
   282  func (t *Tree) Proof(key []byte) (value []byte, proofBytes []byte, exists bool) {
   283  	value, proof := t.ConstructProof(key)
   284  	if proof == nil {
   285  		return nil, nil, false
   286  	}
   287  	var mavlproof types.MAVLProof
   288  	mavlproof.InnerNodes = proof.InnerNodes
   289  	proofBytes, err := proto.Marshal(&mavlproof)
   290  	if err != nil {
   291  		treelog.Error("Proof proto.Marshal err!", "err", err)
   292  		return nil, nil, false
   293  	}
   294  	return value, proofBytes, true
   295  }
   296  
   297  // Remove 删除key对应的节点
   298  func (t *Tree) Remove(key []byte) (value []byte, removed bool) {
   299  	if t.root == nil {
   300  		return nil, false
   301  	}
   302  	newRootHash, newRoot, _, value, removed := t.root.remove(t, key)
   303  	if !removed {
   304  		return nil, false
   305  	}
   306  	if newRoot == nil && newRootHash != nil {
   307  		root, err := t.ndb.GetNode(t, newRootHash)
   308  		if err != nil {
   309  			panic(err) //数据库已经损坏
   310  		}
   311  		t.root = root
   312  	} else {
   313  		t.root = newRoot
   314  	}
   315  	return value, true
   316  }
   317  
   318  func (t *Tree) getMaxBlockHeight() int64 {
   319  	if t.ndb == nil || t.ndb.db == nil {
   320  		return 0
   321  	}
   322  	value, err := t.ndb.db.Get([]byte(curMaxBlockHeight))
   323  	if len(value) == 0 || err != nil {
   324  		return 0
   325  	}
   326  	h := &types.Int64{}
   327  	err = proto.Unmarshal(value, h)
   328  	if err != nil {
   329  		return 0
   330  	}
   331  	return h.Data
   332  }
   333  
   334  func (t *Tree) setMaxBlockHeight(height int64) error {
   335  	if t.ndb == nil || t.ndb.batch == nil {
   336  		return fmt.Errorf("ndb is nil")
   337  	}
   338  	h := &types.Int64{}
   339  	h.Data = height
   340  	value, err := proto.Marshal(h)
   341  	if err != nil {
   342  		return err
   343  	}
   344  	t.ndb.batch.Set([]byte(curMaxBlockHeight), value)
   345  	return nil
   346  }
   347  
   348  func (t *Tree) isRemoveLeafCountKey() bool {
   349  	if t.ndb == nil || t.ndb.db == nil {
   350  		return false
   351  	}
   352  	heightMtx.Lock()
   353  	defer heightMtx.Unlock()
   354  
   355  	if maxBlockHeight == 0 {
   356  		maxBlockHeight = t.getMaxBlockHeight()
   357  	}
   358  	if t.blockHeight > maxBlockHeight {
   359  		err := t.setMaxBlockHeight(t.blockHeight)
   360  		if err != nil {
   361  			panic(err)
   362  		}
   363  		maxBlockHeight = t.blockHeight
   364  		return false
   365  	}
   366  	return true
   367  }
   368  
   369  // RemoveLeafCountKey 删除叶子节点的索引节点(防止裁剪时候回退产生的误删除)
   370  func (t *Tree) RemoveLeafCountKey(height int64) {
   371  	if t.root == nil || t.ndb == nil {
   372  		return
   373  	}
   374  	prefix := genPrefixHashKey(&Node{}, height)
   375  	it := t.ndb.db.Iterator(prefix, nil, true)
   376  	defer it.Close()
   377  
   378  	var keys [][]byte
   379  	for it.Rewind(); it.Valid(); it.Next() {
   380  		value := make([]byte, len(it.Value()))
   381  		copy(value, it.Value())
   382  		pData := &types.StoreNode{}
   383  		err := proto.Unmarshal(value, pData)
   384  		if err == nil {
   385  			keys = append(keys, pData.Key)
   386  		}
   387  	}
   388  
   389  	batch := t.ndb.db.NewBatch(true)
   390  	for _, k := range keys {
   391  		_, hash, exits := t.GetHash(k)
   392  		if exits {
   393  			batch.Delete(genLeafCountKey(k, hash, height, len(hash)))
   394  			treelog.Debug("RemoveLeafCountKey:", "height", height, "key:", string(k), "hash:", common.ToHex(hash))
   395  		}
   396  	}
   397  	dbm.MustWrite(batch)
   398  }
   399  
   400  // Iterate 依次迭代遍历树的所有键
   401  func (t *Tree) Iterate(fn func(key []byte, value []byte) bool) (stopped bool) {
   402  	if t.root == nil {
   403  		return false
   404  	}
   405  	return t.root.traverse(t, true, func(node *Node) bool {
   406  		if node.height == 0 {
   407  			return fn(node.key, node.value)
   408  		}
   409  		return false
   410  	})
   411  }
   412  
   413  // IterateRange 在start和end之间的键进行迭代回调[start, end)
   414  func (t *Tree) IterateRange(start, end []byte, ascending bool, fn func(key []byte, value []byte) bool) (stopped bool) {
   415  	if t.root == nil {
   416  		return false
   417  	}
   418  	return t.root.traverseInRange(t, start, end, ascending, false, 0, func(node *Node, _ uint8) bool {
   419  		if node.height == 0 {
   420  			return fn(node.key, node.value)
   421  		}
   422  		return false
   423  	})
   424  }
   425  
   426  // IterateRangeInclusive 在start和end之间的键进行迭代回调[start, end]
   427  func (t *Tree) IterateRangeInclusive(start, end []byte, ascending bool, fn func(key, value []byte) bool) (stopped bool) {
   428  	if t.root == nil {
   429  		return false
   430  	}
   431  	return t.root.traverseInRange(t, start, end, ascending, true, 0, func(node *Node, _ uint8) bool {
   432  		if node.height == 0 {
   433  			return fn(node.key, node.value)
   434  		}
   435  		return false
   436  	})
   437  }
   438  
   439  //-----------------------------------------------------------------------------
   440  
   441  type nodeDB struct {
   442  	mtx     sync.Mutex
   443  	cache   *lru.ARCCache
   444  	db      dbm.DB
   445  	batch   dbm.Batch
   446  	orphans map[string]struct{}
   447  }
   448  
   449  func newNodeDB(db dbm.DB, sync bool) *nodeDB {
   450  	ndb := &nodeDB{
   451  		cache:   db.GetCache(),
   452  		db:      db,
   453  		batch:   db.NewBatch(sync),
   454  		orphans: make(map[string]struct{}),
   455  	}
   456  	return ndb
   457  }
   458  
   459  // GetNode 从数据库中查询Node
   460  func (ndb *nodeDB) GetNode(t *Tree, hash []byte) (*Node, error) {
   461  	ndb.mtx.Lock()
   462  	defer ndb.mtx.Unlock()
   463  
   464  	// Check the cache.
   465  
   466  	if ndb.cache != nil {
   467  		elem, ok := ndb.cache.Get(string(hash))
   468  		if ok {
   469  			return elem.(*Node), nil
   470  		}
   471  	}
   472  	//从memtree中获取
   473  	if t != nil && t.config != nil && t.config.EnableMemTree {
   474  		node, err := getNodeMemTree(hash)
   475  		if err == nil {
   476  			return node, nil
   477  		}
   478  	}
   479  	// Doesn't exist, load from db.
   480  	var buf []byte
   481  	buf, err := ndb.db.Get(hash)
   482  
   483  	if len(buf) == 0 || err != nil {
   484  		return nil, ErrNodeNotExist
   485  	}
   486  	node, err := MakeNode(buf, t)
   487  	if err != nil {
   488  		panic(fmt.Sprintf("Error reading IAVLNode. bytes: %X  error: %v", buf, err))
   489  	}
   490  	node.hash = hash
   491  	node.persisted = true
   492  	ndb.cacheNode(node)
   493  	// Save node hashInt64 to memTree
   494  	if t != nil && t.config != nil && t.config.EnableMemTree {
   495  		updateGlobalMemTree(node, t.config)
   496  	}
   497  	return node, nil
   498  }
   499  
   500  // 获取叶子节点的所有父节点
   501  func getHashNode(node *Node) (hashs [][]byte) {
   502  	for {
   503  		if node == nil {
   504  			return
   505  		}
   506  		parN := node.parentNode
   507  		if parN != nil {
   508  			hashs = append(hashs, parN.hash)
   509  			node = parN
   510  		} else {
   511  			break
   512  		}
   513  	}
   514  	return hashs
   515  }
   516  
   517  // SaveNode 保存节点
   518  func (ndb *nodeDB) SaveNode(t *Tree, node *Node) {
   519  	ndb.mtx.Lock()
   520  	defer ndb.mtx.Unlock()
   521  
   522  	if node.hash == nil {
   523  		panic("Expected to find node.hash, but none found.")
   524  	}
   525  	if node.persisted {
   526  		panic("Shouldn't be calling save on an already persisted node.")
   527  	}
   528  	// Save node bytes to db
   529  	storenode := node.storeNode(t)
   530  	ndb.batch.Set(node.hash, storenode)
   531  	if t.config != nil && t.config.EnableMavlPrune && node.height == 0 {
   532  		//save leafnode key&hash
   533  		k := genLeafCountKey(node.key, node.hash, t.blockHeight, len(node.hash))
   534  		data := &types.PruneData{
   535  			Hashs: getHashNode(node),
   536  		}
   537  		v, err := proto.Marshal(data)
   538  		if err != nil {
   539  			panic(err)
   540  		}
   541  		ndb.batch.Set(k, v)
   542  	}
   543  	node.persisted = true
   544  	ndb.cacheNode(node)
   545  	delete(ndb.orphans, string(node.hash))
   546  }
   547  
   548  func getNodeMemTree(hash []byte) (*Node, error) {
   549  	if memTree != nil {
   550  		elem, ok := memTree.Get(uintkey(farm.Hash64(hash)))
   551  		if ok {
   552  			sn := elem.(*memNode)
   553  			node := &Node{
   554  				height:    sn.Height,
   555  				size:      sn.Size,
   556  				hash:      hash,
   557  				persisted: true,
   558  			}
   559  			node.leftHash = sn.data[0]
   560  			node.rightHash = sn.data[1]
   561  			node.key = sn.data[2]
   562  			if len(sn.data) == 4 {
   563  				node.value = sn.data[3]
   564  			}
   565  			return node, nil
   566  		}
   567  	}
   568  
   569  	if tkCloseCache != nil {
   570  		// 从tkCloseCache缓存中获取
   571  		elem, ok := tkCloseCache.Get(uintkey(farm.Hash64(hash)))
   572  		if ok {
   573  			sn := elem.(*memNode)
   574  			node := &Node{
   575  				height:    sn.Height,
   576  				size:      sn.Size,
   577  				hash:      hash,
   578  				persisted: true,
   579  			}
   580  			node.leftHash = sn.data[0]
   581  			node.rightHash = sn.data[1]
   582  			node.key = sn.data[2]
   583  			if len(sn.data) == 4 {
   584  				node.value = sn.data[3]
   585  			}
   586  			return node, nil
   587  		}
   588  	}
   589  	return nil, ErrNodeNotExist
   590  }
   591  
   592  // Save node hashInt64 to memTree
   593  func updateGlobalMemTree(node *Node, treeCfg *TreeConfig) {
   594  	if node == nil || memTree == nil || tkCloseCache == nil || treeCfg == nil {
   595  		return
   596  	}
   597  	if !treeCfg.EnableMemVal && node.height == 0 {
   598  		return
   599  	}
   600  	memN := &memNode{
   601  		Height: node.height,
   602  		Size:   node.size,
   603  	}
   604  	var isTkCloseNode bool
   605  	if node.height == 0 {
   606  		if bytes.HasPrefix(node.key, ticket.TicketPrefix) {
   607  			tk := &ticket.Ticket{}
   608  			err := proto.Unmarshal(node.value, tk)
   609  			if err == nil && tk.Status == ticket.StatusCloseTicket { //ticket为close状态下不做存储
   610  				isTkCloseNode = true
   611  			}
   612  		}
   613  		memN.data = make([][]byte, 4)
   614  		memN.data[3] = node.value
   615  	} else {
   616  		memN.data = make([][]byte, 3)
   617  	}
   618  	memN.data[0] = node.leftHash
   619  	memN.data[1] = node.rightHash
   620  	memN.data[2] = node.key
   621  	if isTkCloseNode {
   622  		tkCloseCache.Add(uintkey(farm.Hash64(node.hash)), memN)
   623  	} else {
   624  		memTree.Add(uintkey(farm.Hash64(node.hash)), memN)
   625  	}
   626  }
   627  
   628  // Save node hashInt64 to localmem
   629  func updateLocalMemTree(t *Tree, node *Node) {
   630  	if t == nil || node == nil {
   631  		return
   632  	}
   633  	if t.config != nil && !t.config.EnableMemVal && node.height == 0 {
   634  		return
   635  	}
   636  	if t.updateNode != nil && t.tkCloseNode != nil {
   637  		memN := &memNode{
   638  			Height: node.height,
   639  			Size:   node.size,
   640  		}
   641  		var isTkCloseNode bool
   642  		if node.height == 0 {
   643  			if bytes.HasPrefix(node.key, ticket.TicketPrefix) {
   644  				tk := &ticket.Ticket{}
   645  				err := proto.Unmarshal(node.value, tk)
   646  				if err == nil && tk.Status == ticket.StatusCloseTicket { //ticket为close状态下不做存储
   647  					isTkCloseNode = true
   648  				}
   649  			}
   650  			memN.data = make([][]byte, 4)
   651  			memN.data[3] = node.value
   652  		} else {
   653  			memN.data = make([][]byte, 3)
   654  		}
   655  		memN.data[0] = node.leftHash
   656  		memN.data[1] = node.rightHash
   657  		memN.data[2] = node.key
   658  		if isTkCloseNode {
   659  			t.tkCloseNode[uintkey(farm.Hash64(node.hash))] = memN
   660  		} else {
   661  			t.updateNode[uintkey(farm.Hash64(node.hash))] = memN
   662  		}
   663  		//treelog.Debug("Tree.SaveNode", "store struct size", unsafe.Sizeof(store), "byte size", len(storenode), "height", node.height)
   664  	}
   665  }
   666  
   667  //cache缓存节点
   668  func (ndb *nodeDB) cacheNode(node *Node) {
   669  	//接进叶子节点,不容易命中cache,就不做cache
   670  	if ndb.cache != nil && node.height > 2 {
   671  		ndb.cache.Add(string(node.hash), node)
   672  		if ndb.cache.Len()%10000 == 0 {
   673  			log.Info("store db cache ", "len", ndb.cache.Len())
   674  		}
   675  	}
   676  }
   677  
   678  // RemoveNode 删除节点
   679  func (ndb *nodeDB) RemoveNode(t *Tree, node *Node) {
   680  	ndb.mtx.Lock()
   681  	defer ndb.mtx.Unlock()
   682  
   683  	if node.hash == nil {
   684  		panic("Expected to find node.hash, but none found.")
   685  	}
   686  	if !node.persisted {
   687  		panic("Shouldn't be calling remove on a non-persisted node.")
   688  	}
   689  	if ndb.cache != nil {
   690  		ndb.cache.Remove(string(node.hash))
   691  	}
   692  	ndb.orphans[string(node.hash)] = struct{}{}
   693  }
   694  
   695  // Commit 提交状态tree,批量写入db中
   696  func (ndb *nodeDB) Commit() error {
   697  
   698  	ndb.mtx.Lock()
   699  	defer ndb.mtx.Unlock()
   700  
   701  	// Write saves
   702  	dbm.MustWrite(ndb.batch)
   703  
   704  	ndb.batch = nil
   705  	ndb.orphans = make(map[string]struct{})
   706  	return nil
   707  }
   708  
   709  // SetKVPair 设置kv对外接口
   710  func SetKVPair(db dbm.DB, storeSet *types.StoreSet, sync bool, treeCfg *TreeConfig) ([]byte, error) {
   711  	tree := NewTree(db, sync, treeCfg)
   712  	tree.SetBlockHeight(storeSet.Height)
   713  	err := tree.Load(storeSet.StateHash)
   714  	if err != nil {
   715  		return nil, err
   716  	}
   717  	for i := 0; i < len(storeSet.KV); i++ {
   718  		tree.Set(storeSet.KV[i].Key, storeSet.KV[i].Value)
   719  	}
   720  	return tree.Save(), nil
   721  }
   722  
   723  // GetKVPair 获取kv对外接口
   724  func GetKVPair(db dbm.DB, storeGet *types.StoreGet, treeCfg *TreeConfig) ([][]byte, error) {
   725  	tree := NewTree(db, true, treeCfg)
   726  	err := tree.Load(storeGet.StateHash)
   727  	values := make([][]byte, len(storeGet.Keys))
   728  	if err != nil {
   729  		return values, err
   730  	}
   731  	for i := 0; i < len(storeGet.Keys); i++ {
   732  		_, value, exit := tree.Get(storeGet.Keys[i])
   733  		if exit {
   734  			values[i] = value
   735  		}
   736  	}
   737  	return values, nil
   738  }
   739  
   740  // GetKVPairProof 获取指定k:v pair的proof证明
   741  func GetKVPairProof(db dbm.DB, roothash []byte, key []byte, treeCfg *TreeConfig) ([]byte, error) {
   742  	tree := NewTree(db, true, treeCfg)
   743  	err := tree.Load(roothash)
   744  	if err != nil {
   745  		return nil, err
   746  	}
   747  	_, proof, exit := tree.Proof(key)
   748  	if exit {
   749  		return proof, nil
   750  	}
   751  	return nil, nil
   752  }
   753  
   754  // DelKVPair 剔除key对应的节点在本次tree中,返回新的roothash和key对应的value
   755  func DelKVPair(db dbm.DB, storeDel *types.StoreGet, treeCfg *TreeConfig) ([]byte, [][]byte, error) {
   756  	tree := NewTree(db, true, treeCfg)
   757  	err := tree.Load(storeDel.StateHash)
   758  	if err != nil {
   759  		return nil, nil, err
   760  	}
   761  	values := make([][]byte, len(storeDel.Keys))
   762  	for i := 0; i < len(storeDel.Keys); i++ {
   763  		value, removed := tree.Remove(storeDel.Keys[i])
   764  		if removed {
   765  			values[i] = value
   766  		}
   767  	}
   768  	return tree.Save(), values, nil
   769  }
   770  
   771  // DelLeafCountKV 回退时候用于删除叶子节点的索引节点
   772  func DelLeafCountKV(db dbm.DB, blockHeight int64, treeCfg *TreeConfig) error {
   773  	treelog.Debug("RemoveLeafCountKey:", "height", blockHeight)
   774  	prefix := genRootHashPrefix(blockHeight)
   775  	it := db.Iterator(prefix, nil, true)
   776  	defer it.Close()
   777  	for it.Rewind(); it.Valid(); it.Next() {
   778  		hash, err := getRootHash(it.Key())
   779  		if err == nil {
   780  			tree := NewTree(db, true, treeCfg)
   781  			err := tree.Load(hash)
   782  			if err == nil {
   783  				treelog.Debug("RemoveLeafCountKey:", "height", blockHeight, "root hash:", common.ToHex(hash))
   784  				tree.RemoveLeafCountKey(blockHeight)
   785  			}
   786  		}
   787  	}
   788  	return nil
   789  }
   790  
   791  // VerifyKVPairProof 验证KVPair 的证明
   792  func VerifyKVPairProof(db dbm.DB, roothash []byte, keyvalue types.KeyValue, proof []byte) bool {
   793  
   794  	//通过传入的keyvalue构造leafnode
   795  	leafNode := types.LeafNode{Key: keyvalue.GetKey(), Value: keyvalue.GetValue(), Height: 0, Size: 1}
   796  	leafHash := leafNode.Hash()
   797  
   798  	proofnode, err := ReadProof(roothash, leafHash, proof)
   799  	if err != nil {
   800  		treelog.Info("VerifyKVPairProof ReadProof err!", "err", err)
   801  	}
   802  	istrue := proofnode.Verify(keyvalue.GetKey(), keyvalue.GetValue(), roothash)
   803  	if !istrue {
   804  		treelog.Info("VerifyKVPairProof fail!", "keyBytes", keyvalue.GetKey(), "valueBytes", keyvalue.GetValue(), "roothash", roothash)
   805  	}
   806  	return istrue
   807  }
   808  
   809  // PrintTreeLeaf 通过roothash打印所有叶子节点
   810  func PrintTreeLeaf(db dbm.DB, roothash []byte, treeCfg *TreeConfig) {
   811  	tree := NewTree(db, true, treeCfg)
   812  	err := tree.Load(roothash)
   813  	if err != nil {
   814  		return
   815  	}
   816  	var i int32
   817  	if tree.root != nil {
   818  		leafs := tree.root.size
   819  		treelog.Info("PrintTreeLeaf info")
   820  		for i = 0; i < leafs; i++ {
   821  			key, value := tree.GetByIndex(i)
   822  			treelog.Info("leaf:", "index:", i, "key", string(key), "value", string(value))
   823  		}
   824  	}
   825  }
   826  
   827  // IterateRangeByStateHash 在start和end之间的键进行迭代回调[start, end)
   828  func IterateRangeByStateHash(db dbm.DB, statehash, start, end []byte, ascending bool, treeCfg *TreeConfig, fn func([]byte, []byte) bool) {
   829  	tree := NewTree(db, true, treeCfg)
   830  	err := tree.Load(statehash)
   831  	if err != nil {
   832  		return
   833  	}
   834  	//treelog.Debug("IterateRangeByStateHash", "statehash", hex.EncodeToString(statehash), "start", string(start), "end", string(end))
   835  
   836  	tree.IterateRange(start, end, ascending, fn)
   837  }
   838  
   839  func genPrefixHashKey(node *Node, blockHeight int64) (key []byte) {
   840  	//leafnode
   841  	if node.height == 0 {
   842  		key = []byte(fmt.Sprintf("%s-%010d-", leafNodePrefix, blockHeight))
   843  	} else {
   844  		key = []byte(fmt.Sprintf("%s-%010d-", hashNodePrefix, blockHeight))
   845  	}
   846  	return key
   847  }
   848  
   849  func genRootHashPrefix(blockHeight int64) (key []byte) {
   850  	key = []byte(fmt.Sprintf("%s%010d", rootHashHeightPrefix, blockHeight))
   851  	return key
   852  }
   853  
   854  func genRootHashHeight(blockHeight int64, hash []byte) (key []byte) {
   855  	key = []byte(fmt.Sprintf("%s%010d%s", rootHashHeightPrefix, blockHeight, string(hash)))
   856  	return key
   857  }
   858  
   859  func getRootHash(hashKey []byte) (hash []byte, err error) {
   860  	if len(hashKey) < len(rootHashHeightPrefix)+blockHeightStrLen+sha256Len {
   861  		return nil, types.ErrSize
   862  	}
   863  	if !bytes.Contains(hashKey, []byte(rootHashHeightPrefix)) {
   864  		return nil, types.ErrSize
   865  	}
   866  	hash = hashKey[len(hashKey)-sha256Len:]
   867  	return hash, nil
   868  }