github.com/okex/exchain@v1.8.0/libs/tendermint/state/txindex/kv/kv.go (about)

     1  package kv
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/hex"
     7  	"fmt"
     8  	"strconv"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/pkg/errors"
    13  
    14  	dbm "github.com/okex/exchain/libs/tm-db"
    15  
    16  	"github.com/okex/exchain/libs/tendermint/libs/pubsub/query"
    17  	tmstring "github.com/okex/exchain/libs/tendermint/libs/strings"
    18  	"github.com/okex/exchain/libs/tendermint/state/txindex"
    19  	"github.com/okex/exchain/libs/tendermint/types"
    20  )
    21  
    22  const (
    23  	tagKeySeparator = "/"
    24  	defaultTimeOut = 5 * time.Second
    25  	maxQueryRange = 256
    26  )
    27  
    28  var _ txindex.TxIndexer = (*TxIndex)(nil)
    29  
    30  // TxIndex is the simplest possible indexer, backed by key-value storage (levelDB).
    31  type TxIndex struct {
    32  	store                dbm.DB
    33  	compositeKeysToIndex []string
    34  	indexAllEvents       bool
    35  }
    36  
    37  // NewTxIndex creates new KV indexer.
    38  func NewTxIndex(store dbm.DB, options ...func(*TxIndex)) *TxIndex {
    39  	txi := &TxIndex{store: store, compositeKeysToIndex: make([]string, 0), indexAllEvents: false}
    40  	for _, o := range options {
    41  		o(txi)
    42  	}
    43  	return txi
    44  }
    45  
    46  // IndexEvents is an option for setting which composite keys to index.
    47  func IndexEvents(compositeKeys []string) func(*TxIndex) {
    48  	return func(txi *TxIndex) {
    49  		txi.compositeKeysToIndex = compositeKeys
    50  	}
    51  }
    52  
    53  // IndexAllEvents is an option for indexing all events.
    54  func IndexAllEvents() func(*TxIndex) {
    55  	return func(txi *TxIndex) {
    56  		txi.indexAllEvents = true
    57  	}
    58  }
    59  
    60  func getTxResultFromBytes(rawBytes []byte) (*types.TxResult, error) {
    61  	if rawBytes == nil {
    62  		return nil, nil
    63  	}
    64  	txResult := new(types.TxResult)
    65  	err := txResult.UnmarshalFromAmino(cdc, rawBytes)
    66  	if err != nil {
    67  		txResult = new(types.TxResult)
    68  		err = cdc.UnmarshalBinaryBare(rawBytes, &txResult)
    69  		if err != nil {
    70  			return nil, fmt.Errorf("error reading TxResult: %v", err)
    71  		}
    72  	}
    73  	return txResult, nil
    74  }
    75  
    76  // Get gets transaction from the TxIndex storage and returns it or nil if the
    77  // transaction is not found.
    78  func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) {
    79  	if len(hash) == 0 {
    80  		return nil, txindex.ErrorEmptyHash
    81  	}
    82  
    83  	v, err := txi.store.GetUnsafeValue(hash, func(rawBytes []byte) (interface{}, error) {
    84  		return getTxResultFromBytes(rawBytes)
    85  	})
    86  	if err != nil {
    87  		txResult, ok := v.(*types.TxResult)
    88  		if !ok {
    89  			panic(err)
    90  		} else {
    91  			return txResult, err
    92  		}
    93  	}
    94  
    95  	txResult := v.(*types.TxResult)
    96  	return txResult, err
    97  }
    98  
    99  // AddBatch indexes a batch of transactions using the given list of events. Each
   100  // key that indexed from the tx's events is a composite of the event type and
   101  // the respective attribute's key delimited by a "." (eg. "account.number").
   102  // Any event with an empty type is not indexed.
   103  func (txi *TxIndex) AddBatch(b *txindex.Batch) error {
   104  	storeBatch := txi.store.NewBatch()
   105  	defer storeBatch.Close()
   106  
   107  	for _, result := range b.Ops {
   108  		hash := result.Tx.Hash(result.Height)
   109  
   110  		// index tx by events
   111  		txi.indexEvents(result, hash, storeBatch)
   112  
   113  		// index tx by height
   114  		if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) {
   115  			storeBatch.Set(keyForHeight(result), hash)
   116  		}
   117  
   118  		// index tx by hash
   119  		rawBytes, err := cdc.MarshalBinaryBare(result)
   120  		if err != nil {
   121  			return err
   122  		}
   123  		storeBatch.Set(hash, rawBytes)
   124  	}
   125  
   126  	storeBatch.WriteSync()
   127  	return nil
   128  }
   129  
   130  // Index indexes a single transaction using the given list of events. Each key
   131  // that indexed from the tx's events is a composite of the event type and the
   132  // respective attribute's key delimited by a "." (eg. "account.number").
   133  // Any event with an empty type is not indexed.
   134  func (txi *TxIndex) Index(result *types.TxResult) error {
   135  	b := txi.store.NewBatch()
   136  	defer b.Close()
   137  
   138  	hash := result.Tx.Hash(result.Height)
   139  
   140  	// index tx by events
   141  	txi.indexEvents(result, hash, b)
   142  
   143  	// index tx by height
   144  	if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) {
   145  		b.Set(keyForHeight(result), hash)
   146  	}
   147  
   148  	// index tx by hash
   149  	rawBytes, err := cdc.MarshalBinaryBare(result)
   150  	if err != nil {
   151  		return err
   152  	}
   153  
   154  	b.Set(hash, rawBytes)
   155  	b.WriteSync()
   156  
   157  	return nil
   158  }
   159  
   160  func (txi *TxIndex) indexEvents(result *types.TxResult, hash []byte, store dbm.SetDeleter) {
   161  	for _, event := range result.Result.Events {
   162  		// only index events with a non-empty type
   163  		if len(event.Type) == 0 {
   164  			continue
   165  		}
   166  
   167  		for _, attr := range event.Attributes {
   168  			if len(attr.Key) == 0 {
   169  				continue
   170  			}
   171  
   172  			compositeTag := fmt.Sprintf("%s.%s", event.Type, string(attr.Key))
   173  			if txi.indexAllEvents || tmstring.StringInSlice(compositeTag, txi.compositeKeysToIndex) {
   174  				store.Set(keyForEvent(compositeTag, attr.Value, result), hash)
   175  			}
   176  		}
   177  	}
   178  }
   179  
   180  // Search performs a search using the given query.
   181  //
   182  // It breaks the query into conditions (like "tx.height > 5"). For each
   183  // condition, it queries the DB index. One special use cases here: (1) if
   184  // "tx.hash" is found, it returns tx result for it (2) for range queries it is
   185  // better for the client to provide both lower and upper bounds, so we are not
   186  // performing a full scan. Results from querying indexes are then intersected
   187  // and returned to the caller, in no particular order.
   188  //
   189  // Search will exit early and return any result fetched so far,
   190  // when a message is received on the context chan.
   191  func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*types.TxResult, error) {
   192  	// Potentially exit early.
   193  	select {
   194  	case <-ctx.Done():
   195  		results := make([]*types.TxResult, 0)
   196  		return results, nil
   197  	default:
   198  	}
   199  
   200  	var hashesInitialized bool
   201  	filteredHashes := make(map[string][]byte)
   202  
   203  	// get a list of conditions (like "tx.height > 5")
   204  	conditions, err := q.Conditions()
   205  	if err != nil {
   206  		return nil, errors.Wrap(err, "error during parsing conditions from query")
   207  	}
   208  
   209  	// if there is a hash condition, return the result immediately
   210  	hash, ok, err := lookForHash(conditions)
   211  	if err != nil {
   212  		return nil, errors.Wrap(err, "error during searching for a hash in the query")
   213  	} else if ok {
   214  		res, err := txi.Get(hash)
   215  		switch {
   216  		case err != nil:
   217  			return []*types.TxResult{}, errors.Wrap(err, "error while retrieving the result")
   218  		case res == nil:
   219  			return []*types.TxResult{}, nil
   220  		default:
   221  			return []*types.TxResult{res}, nil
   222  		}
   223  	}
   224  
   225  	// conditions to skip because they're handled before "everything else"
   226  	skipIndexes := make([]int, 0)
   227  	ctx, cancel := context.WithTimeout(ctx, defaultTimeOut)
   228  	defer cancel()
   229  	// extract ranges
   230  	// if both upper and lower bounds exist, it's better to get them in order not
   231  	// no iterate over kvs that are not within range.
   232  	ranges, rangeIndexes := lookForRanges(conditions)
   233  	if len(ranges) > 0 {
   234  		skipIndexes = append(skipIndexes, rangeIndexes...)
   235  
   236  		for _, r := range ranges {
   237  			if !hashesInitialized {
   238  				filteredHashes, err = txi.matchRange(ctx, r, startKey(r.key), filteredHashes, true)
   239  				if err != nil {
   240  					return []*types.TxResult{}, err
   241  				}
   242  				hashesInitialized = true
   243  
   244  				// Ignore any remaining conditions if the first condition resulted
   245  				// in no matches (assuming implicit AND operand).
   246  				if len(filteredHashes) == 0 {
   247  					break
   248  				}
   249  			} else {
   250  				filteredHashes, err = txi.matchRange(ctx, r, startKey(r.key), filteredHashes, false)
   251  				if err != nil {
   252  					return []*types.TxResult{}, err
   253  				}
   254  			}
   255  			// Potentially exit early.
   256  			select {
   257  			case <-ctx.Done():
   258  				return []*types.TxResult{}, errors.New("request processing timeout, optimize request filter conditions parameter")
   259  			default:
   260  			}
   261  		}
   262  	}
   263  
   264  	// if there is a height condition ("tx.height=3"), extract it
   265  	height := lookForHeight(conditions)
   266  
   267  	// for all other conditions
   268  	for i, c := range conditions {
   269  		if intInSlice(i, skipIndexes) {
   270  			continue
   271  		}
   272  
   273  		if !hashesInitialized {
   274  			filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, true)
   275  			hashesInitialized = true
   276  
   277  			// Ignore any remaining conditions if the first condition resulted
   278  			// in no matches (assuming implicit AND operand).
   279  			if len(filteredHashes) == 0 {
   280  				break
   281  			}
   282  		} else {
   283  			filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, false)
   284  		}
   285  	}
   286  
   287  	results := make([]*types.TxResult, 0, len(filteredHashes))
   288  	for _, h := range filteredHashes {
   289  		res, err := txi.Get(h)
   290  		if err != nil {
   291  			return nil, errors.Wrapf(err, "failed to get Tx{%X}", h)
   292  		}
   293  		results = append(results, res)
   294  
   295  		// Potentially exit early.
   296  		select {
   297  		case <-ctx.Done():
   298  			break
   299  		default:
   300  		}
   301  	}
   302  
   303  	return results, nil
   304  }
   305  
   306  func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) {
   307  	for _, c := range conditions {
   308  		if c.CompositeKey == types.TxHashKey {
   309  			decoded, err := hex.DecodeString(c.Operand.(string))
   310  			return decoded, true, err
   311  		}
   312  	}
   313  	return
   314  }
   315  
   316  // lookForHeight returns a height if there is an "height=X" condition.
   317  func lookForHeight(conditions []query.Condition) (height int64) {
   318  	for _, c := range conditions {
   319  		if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual {
   320  			return c.Operand.(int64)
   321  		}
   322  	}
   323  	return 0
   324  }
   325  
   326  // special map to hold range conditions
   327  // Example: account.number => queryRange{lowerBound: 1, upperBound: 5}
   328  type queryRanges map[string]queryRange
   329  
   330  type queryRange struct {
   331  	lowerBound        interface{} // int || time.Time
   332  	upperBound        interface{} // int || time.Time
   333  	key               string
   334  	includeLowerBound bool
   335  	includeUpperBound bool
   336  }
   337  
   338  func (r queryRange) lowerBoundValue() interface{} {
   339  	if r.lowerBound == nil {
   340  		return nil
   341  	}
   342  
   343  	if r.includeLowerBound {
   344  		return r.lowerBound
   345  	}
   346  
   347  	switch t := r.lowerBound.(type) {
   348  	case int64:
   349  		return t + 1
   350  	case time.Time:
   351  		return t.Unix() + 1
   352  	default:
   353  		panic("not implemented")
   354  	}
   355  }
   356  
   357  func (r queryRange) AnyBound() interface{} {
   358  	if r.lowerBound != nil {
   359  		return r.lowerBound
   360  	}
   361  
   362  	return r.upperBound
   363  }
   364  
   365  func (r queryRange) upperBoundValue() interface{} {
   366  	if r.upperBound == nil {
   367  		return nil
   368  	}
   369  
   370  	if r.includeUpperBound {
   371  		return r.upperBound
   372  	}
   373  
   374  	switch t := r.upperBound.(type) {
   375  	case int64:
   376  		return t - 1
   377  	case time.Time:
   378  		return t.Unix() - 1
   379  	default:
   380  		panic("not implemented")
   381  	}
   382  }
   383  
   384  func lookForRanges(conditions []query.Condition) (ranges queryRanges, indexes []int) {
   385  	ranges = make(queryRanges)
   386  	for i, c := range conditions {
   387  		if isRangeOperation(c.Op) {
   388  			r, ok := ranges[c.CompositeKey]
   389  			if !ok {
   390  				r = queryRange{key: c.CompositeKey}
   391  			}
   392  			switch c.Op {
   393  			case query.OpGreater:
   394  				r.lowerBound = c.Operand
   395  			case query.OpGreaterEqual:
   396  				r.includeLowerBound = true
   397  				r.lowerBound = c.Operand
   398  			case query.OpLess:
   399  				r.upperBound = c.Operand
   400  			case query.OpLessEqual:
   401  				r.includeUpperBound = true
   402  				r.upperBound = c.Operand
   403  			}
   404  			ranges[c.CompositeKey] = r
   405  			indexes = append(indexes, i)
   406  		}
   407  	}
   408  	return ranges, indexes
   409  }
   410  
   411  func isRangeOperation(op query.Operator) bool {
   412  	switch op {
   413  	case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual:
   414  		return true
   415  	default:
   416  		return false
   417  	}
   418  }
   419  
   420  // match returns all matching txs by hash that meet a given condition and start
   421  // key. An already filtered result (filteredHashes) is provided such that any
   422  // non-intersecting matches are removed.
   423  //
   424  // NOTE: filteredHashes may be empty if no previous condition has matched.
   425  func (txi *TxIndex) match(
   426  	ctx context.Context,
   427  	c query.Condition,
   428  	startKeyBz []byte,
   429  	filteredHashes map[string][]byte,
   430  	firstRun bool,
   431  ) map[string][]byte {
   432  	// A previous match was attempted but resulted in no matches, so we return
   433  	// no matches (assuming AND operand).
   434  	if !firstRun && len(filteredHashes) == 0 {
   435  		return filteredHashes
   436  	}
   437  
   438  	tmpHashes := make(map[string][]byte)
   439  
   440  	switch {
   441  	case c.Op == query.OpEqual:
   442  		it, err := dbm.IteratePrefix(txi.store, startKeyBz)
   443  		if err != nil {
   444  			panic(err)
   445  		}
   446  		defer it.Close()
   447  
   448  		for ; it.Valid(); it.Next() {
   449  			tmpHashes[string(it.Value())] = it.Value()
   450  
   451  			// Potentially exit early.
   452  			select {
   453  			case <-ctx.Done():
   454  				break
   455  			default:
   456  			}
   457  		}
   458  
   459  	case c.Op == query.OpContains:
   460  		// XXX: startKey does not apply here.
   461  		// For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an"
   462  		// we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/"
   463  		it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey))
   464  		if err != nil {
   465  			panic(err)
   466  		}
   467  		defer it.Close()
   468  
   469  		for ; it.Valid(); it.Next() {
   470  			if !isTagKey(it.Key()) {
   471  				continue
   472  			}
   473  
   474  			if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) {
   475  				tmpHashes[string(it.Value())] = it.Value()
   476  			}
   477  
   478  			// Potentially exit early.
   479  			select {
   480  			case <-ctx.Done():
   481  				break
   482  			default:
   483  			}
   484  		}
   485  	default:
   486  		panic("other operators should be handled already")
   487  	}
   488  
   489  	if len(tmpHashes) == 0 || firstRun {
   490  		// Either:
   491  		//
   492  		// 1. Regardless if a previous match was attempted, which may have had
   493  		// results, but no match was found for the current condition, then we
   494  		// return no matches (assuming AND operand).
   495  		//
   496  		// 2. A previous match was not attempted, so we return all results.
   497  		return tmpHashes
   498  	}
   499  
   500  	// Remove/reduce matches in filteredHashes that were not found in this
   501  	// match (tmpHashes).
   502  	for k := range filteredHashes {
   503  		if tmpHashes[k] == nil {
   504  			delete(filteredHashes, k)
   505  
   506  			// Potentially exit early.
   507  			select {
   508  			case <-ctx.Done():
   509  				break
   510  			default:
   511  			}
   512  		}
   513  	}
   514  
   515  	return filteredHashes
   516  }
   517  
   518  // matchRange returns all matching txs by hash that meet a given queryRange and
   519  // start key. An already filtered result (filteredHashes) is provided such that
   520  // any non-intersecting matches are removed.
   521  //
   522  // NOTE: filteredHashes may be empty if no previous condition has matched.
   523  func (txi *TxIndex) matchRange(
   524  	ctx context.Context,
   525  	r queryRange,
   526  	startKey []byte,
   527  	filteredHashes map[string][]byte,
   528  	firstRun bool,
   529  ) (map[string][]byte, error) {
   530  	// A previous match was attempted but resulted in no matches, so we return
   531  	// no matches (assuming AND operand).
   532  	if !firstRun && len(filteredHashes) == 0 {
   533  		return filteredHashes, nil
   534  	}
   535  
   536  	tmpHashes := make(map[string][]byte)
   537  	lowerBound := r.lowerBoundValue()
   538  	upperBound := r.upperBoundValue()
   539  
   540  	it, err := dbm.IteratePrefix(txi.store, startKey)
   541  	if err != nil {
   542  		panic(err)
   543  	}
   544  	defer it.Close()
   545  	count := 0
   546  
   547  LOOP:
   548  	for ; it.Valid(); it.Next() {
   549  		if count > maxQueryRange {
   550  			return nil, errors.New("request processing more than max query range, optimize request filter conditions parameter")
   551  		}
   552  		count++
   553  		// Potentially exit early.
   554  		select {
   555  		case <-ctx.Done():
   556  			return nil, errors.New("request processing timeout, optimize request filter conditions parameter")
   557  		default:
   558  		}
   559  		if !isTagKey(it.Key()) {
   560  			continue
   561  		}
   562  
   563  		if _, ok := r.AnyBound().(int64); ok {
   564  			v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64)
   565  			if err != nil {
   566  				continue LOOP
   567  			}
   568  
   569  			include := true
   570  			if lowerBound != nil && v < lowerBound.(int64) {
   571  				include = false
   572  			}
   573  
   574  			if upperBound != nil && v > upperBound.(int64) {
   575  				include = false
   576  			}
   577  
   578  			if include {
   579  				tmpHashes[string(it.Value())] = it.Value()
   580  			}
   581  
   582  			// XXX: passing time in a ABCI Events is not yet implemented
   583  			// case time.Time:
   584  			// 	v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64)
   585  			// 	if v == r.upperBound {
   586  			// 		break
   587  			// 	}
   588  		}
   589  	}
   590  
   591  	if len(tmpHashes) == 0 || firstRun {
   592  		// Either:
   593  		//
   594  		// 1. Regardless if a previous match was attempted, which may have had
   595  		// results, but no match was found for the current condition, then we
   596  		// return no matches (assuming AND operand).
   597  		//
   598  		// 2. A previous match was not attempted, so we return all results.
   599  		return tmpHashes, nil
   600  	}
   601  
   602  	// Remove/reduce matches in filteredHashes that were not found in this
   603  	// match (tmpHashes).
   604  	for k := range filteredHashes {
   605  		if tmpHashes[k] == nil {
   606  			delete(filteredHashes, k)
   607  
   608  			// Potentially exit early.
   609  			select {
   610  			case <-ctx.Done():
   611  				break
   612  			default:
   613  			}
   614  		}
   615  	}
   616  
   617  	return filteredHashes, nil
   618  }
   619  
   620  ///////////////////////////////////////////////////////////////////////////////
   621  // Keys
   622  
   623  func isTagKey(key []byte) bool {
   624  	return strings.Count(string(key), tagKeySeparator) == 3
   625  }
   626  
   627  func extractValueFromKey(key []byte) string {
   628  	parts := strings.SplitN(string(key), tagKeySeparator, 3)
   629  	return parts[1]
   630  }
   631  
   632  func keyForEvent(key string, value []byte, result *types.TxResult) []byte {
   633  	return []byte(fmt.Sprintf("%s/%s/%d/%d",
   634  		key,
   635  		value,
   636  		result.Height,
   637  		result.Index,
   638  	))
   639  }
   640  
   641  func keyForHeight(result *types.TxResult) []byte {
   642  	return []byte(fmt.Sprintf("%s/%d/%d/%d",
   643  		types.TxHeightKey,
   644  		result.Height,
   645  		result.Height,
   646  		result.Index,
   647  	))
   648  }
   649  
   650  func startKeyForCondition(c query.Condition, height int64) []byte {
   651  	if height > 0 {
   652  		return startKey(c.CompositeKey, c.Operand, height)
   653  	}
   654  	return startKey(c.CompositeKey, c.Operand)
   655  }
   656  
   657  func startKey(fields ...interface{}) []byte {
   658  	var b bytes.Buffer
   659  	for _, f := range fields {
   660  		b.Write([]byte(fmt.Sprintf("%v", f) + tagKeySeparator))
   661  	}
   662  	return b.Bytes()
   663  }