github.com/okex/exchain@v1.8.0/libs/tendermint/state/txindex/kv/kv.go (about) 1 package kv 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/hex" 7 "fmt" 8 "strconv" 9 "strings" 10 "time" 11 12 "github.com/pkg/errors" 13 14 dbm "github.com/okex/exchain/libs/tm-db" 15 16 "github.com/okex/exchain/libs/tendermint/libs/pubsub/query" 17 tmstring "github.com/okex/exchain/libs/tendermint/libs/strings" 18 "github.com/okex/exchain/libs/tendermint/state/txindex" 19 "github.com/okex/exchain/libs/tendermint/types" 20 ) 21 22 const ( 23 tagKeySeparator = "/" 24 defaultTimeOut = 5 * time.Second 25 maxQueryRange = 256 26 ) 27 28 var _ txindex.TxIndexer = (*TxIndex)(nil) 29 30 // TxIndex is the simplest possible indexer, backed by key-value storage (levelDB). 31 type TxIndex struct { 32 store dbm.DB 33 compositeKeysToIndex []string 34 indexAllEvents bool 35 } 36 37 // NewTxIndex creates new KV indexer. 38 func NewTxIndex(store dbm.DB, options ...func(*TxIndex)) *TxIndex { 39 txi := &TxIndex{store: store, compositeKeysToIndex: make([]string, 0), indexAllEvents: false} 40 for _, o := range options { 41 o(txi) 42 } 43 return txi 44 } 45 46 // IndexEvents is an option for setting which composite keys to index. 47 func IndexEvents(compositeKeys []string) func(*TxIndex) { 48 return func(txi *TxIndex) { 49 txi.compositeKeysToIndex = compositeKeys 50 } 51 } 52 53 // IndexAllEvents is an option for indexing all events. 54 func IndexAllEvents() func(*TxIndex) { 55 return func(txi *TxIndex) { 56 txi.indexAllEvents = true 57 } 58 } 59 60 func getTxResultFromBytes(rawBytes []byte) (*types.TxResult, error) { 61 if rawBytes == nil { 62 return nil, nil 63 } 64 txResult := new(types.TxResult) 65 err := txResult.UnmarshalFromAmino(cdc, rawBytes) 66 if err != nil { 67 txResult = new(types.TxResult) 68 err = cdc.UnmarshalBinaryBare(rawBytes, &txResult) 69 if err != nil { 70 return nil, fmt.Errorf("error reading TxResult: %v", err) 71 } 72 } 73 return txResult, nil 74 } 75 76 // Get gets transaction from the TxIndex storage and returns it or nil if the 77 // transaction is not found. 78 func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { 79 if len(hash) == 0 { 80 return nil, txindex.ErrorEmptyHash 81 } 82 83 v, err := txi.store.GetUnsafeValue(hash, func(rawBytes []byte) (interface{}, error) { 84 return getTxResultFromBytes(rawBytes) 85 }) 86 if err != nil { 87 txResult, ok := v.(*types.TxResult) 88 if !ok { 89 panic(err) 90 } else { 91 return txResult, err 92 } 93 } 94 95 txResult := v.(*types.TxResult) 96 return txResult, err 97 } 98 99 // AddBatch indexes a batch of transactions using the given list of events. Each 100 // key that indexed from the tx's events is a composite of the event type and 101 // the respective attribute's key delimited by a "." (eg. "account.number"). 102 // Any event with an empty type is not indexed. 103 func (txi *TxIndex) AddBatch(b *txindex.Batch) error { 104 storeBatch := txi.store.NewBatch() 105 defer storeBatch.Close() 106 107 for _, result := range b.Ops { 108 hash := result.Tx.Hash(result.Height) 109 110 // index tx by events 111 txi.indexEvents(result, hash, storeBatch) 112 113 // index tx by height 114 if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) { 115 storeBatch.Set(keyForHeight(result), hash) 116 } 117 118 // index tx by hash 119 rawBytes, err := cdc.MarshalBinaryBare(result) 120 if err != nil { 121 return err 122 } 123 storeBatch.Set(hash, rawBytes) 124 } 125 126 storeBatch.WriteSync() 127 return nil 128 } 129 130 // Index indexes a single transaction using the given list of events. Each key 131 // that indexed from the tx's events is a composite of the event type and the 132 // respective attribute's key delimited by a "." (eg. "account.number"). 133 // Any event with an empty type is not indexed. 134 func (txi *TxIndex) Index(result *types.TxResult) error { 135 b := txi.store.NewBatch() 136 defer b.Close() 137 138 hash := result.Tx.Hash(result.Height) 139 140 // index tx by events 141 txi.indexEvents(result, hash, b) 142 143 // index tx by height 144 if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) { 145 b.Set(keyForHeight(result), hash) 146 } 147 148 // index tx by hash 149 rawBytes, err := cdc.MarshalBinaryBare(result) 150 if err != nil { 151 return err 152 } 153 154 b.Set(hash, rawBytes) 155 b.WriteSync() 156 157 return nil 158 } 159 160 func (txi *TxIndex) indexEvents(result *types.TxResult, hash []byte, store dbm.SetDeleter) { 161 for _, event := range result.Result.Events { 162 // only index events with a non-empty type 163 if len(event.Type) == 0 { 164 continue 165 } 166 167 for _, attr := range event.Attributes { 168 if len(attr.Key) == 0 { 169 continue 170 } 171 172 compositeTag := fmt.Sprintf("%s.%s", event.Type, string(attr.Key)) 173 if txi.indexAllEvents || tmstring.StringInSlice(compositeTag, txi.compositeKeysToIndex) { 174 store.Set(keyForEvent(compositeTag, attr.Value, result), hash) 175 } 176 } 177 } 178 } 179 180 // Search performs a search using the given query. 181 // 182 // It breaks the query into conditions (like "tx.height > 5"). For each 183 // condition, it queries the DB index. One special use cases here: (1) if 184 // "tx.hash" is found, it returns tx result for it (2) for range queries it is 185 // better for the client to provide both lower and upper bounds, so we are not 186 // performing a full scan. Results from querying indexes are then intersected 187 // and returned to the caller, in no particular order. 188 // 189 // Search will exit early and return any result fetched so far, 190 // when a message is received on the context chan. 191 func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*types.TxResult, error) { 192 // Potentially exit early. 193 select { 194 case <-ctx.Done(): 195 results := make([]*types.TxResult, 0) 196 return results, nil 197 default: 198 } 199 200 var hashesInitialized bool 201 filteredHashes := make(map[string][]byte) 202 203 // get a list of conditions (like "tx.height > 5") 204 conditions, err := q.Conditions() 205 if err != nil { 206 return nil, errors.Wrap(err, "error during parsing conditions from query") 207 } 208 209 // if there is a hash condition, return the result immediately 210 hash, ok, err := lookForHash(conditions) 211 if err != nil { 212 return nil, errors.Wrap(err, "error during searching for a hash in the query") 213 } else if ok { 214 res, err := txi.Get(hash) 215 switch { 216 case err != nil: 217 return []*types.TxResult{}, errors.Wrap(err, "error while retrieving the result") 218 case res == nil: 219 return []*types.TxResult{}, nil 220 default: 221 return []*types.TxResult{res}, nil 222 } 223 } 224 225 // conditions to skip because they're handled before "everything else" 226 skipIndexes := make([]int, 0) 227 ctx, cancel := context.WithTimeout(ctx, defaultTimeOut) 228 defer cancel() 229 // extract ranges 230 // if both upper and lower bounds exist, it's better to get them in order not 231 // no iterate over kvs that are not within range. 232 ranges, rangeIndexes := lookForRanges(conditions) 233 if len(ranges) > 0 { 234 skipIndexes = append(skipIndexes, rangeIndexes...) 235 236 for _, r := range ranges { 237 if !hashesInitialized { 238 filteredHashes, err = txi.matchRange(ctx, r, startKey(r.key), filteredHashes, true) 239 if err != nil { 240 return []*types.TxResult{}, err 241 } 242 hashesInitialized = true 243 244 // Ignore any remaining conditions if the first condition resulted 245 // in no matches (assuming implicit AND operand). 246 if len(filteredHashes) == 0 { 247 break 248 } 249 } else { 250 filteredHashes, err = txi.matchRange(ctx, r, startKey(r.key), filteredHashes, false) 251 if err != nil { 252 return []*types.TxResult{}, err 253 } 254 } 255 // Potentially exit early. 256 select { 257 case <-ctx.Done(): 258 return []*types.TxResult{}, errors.New("request processing timeout, optimize request filter conditions parameter") 259 default: 260 } 261 } 262 } 263 264 // if there is a height condition ("tx.height=3"), extract it 265 height := lookForHeight(conditions) 266 267 // for all other conditions 268 for i, c := range conditions { 269 if intInSlice(i, skipIndexes) { 270 continue 271 } 272 273 if !hashesInitialized { 274 filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, true) 275 hashesInitialized = true 276 277 // Ignore any remaining conditions if the first condition resulted 278 // in no matches (assuming implicit AND operand). 279 if len(filteredHashes) == 0 { 280 break 281 } 282 } else { 283 filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, false) 284 } 285 } 286 287 results := make([]*types.TxResult, 0, len(filteredHashes)) 288 for _, h := range filteredHashes { 289 res, err := txi.Get(h) 290 if err != nil { 291 return nil, errors.Wrapf(err, "failed to get Tx{%X}", h) 292 } 293 results = append(results, res) 294 295 // Potentially exit early. 296 select { 297 case <-ctx.Done(): 298 break 299 default: 300 } 301 } 302 303 return results, nil 304 } 305 306 func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) { 307 for _, c := range conditions { 308 if c.CompositeKey == types.TxHashKey { 309 decoded, err := hex.DecodeString(c.Operand.(string)) 310 return decoded, true, err 311 } 312 } 313 return 314 } 315 316 // lookForHeight returns a height if there is an "height=X" condition. 317 func lookForHeight(conditions []query.Condition) (height int64) { 318 for _, c := range conditions { 319 if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { 320 return c.Operand.(int64) 321 } 322 } 323 return 0 324 } 325 326 // special map to hold range conditions 327 // Example: account.number => queryRange{lowerBound: 1, upperBound: 5} 328 type queryRanges map[string]queryRange 329 330 type queryRange struct { 331 lowerBound interface{} // int || time.Time 332 upperBound interface{} // int || time.Time 333 key string 334 includeLowerBound bool 335 includeUpperBound bool 336 } 337 338 func (r queryRange) lowerBoundValue() interface{} { 339 if r.lowerBound == nil { 340 return nil 341 } 342 343 if r.includeLowerBound { 344 return r.lowerBound 345 } 346 347 switch t := r.lowerBound.(type) { 348 case int64: 349 return t + 1 350 case time.Time: 351 return t.Unix() + 1 352 default: 353 panic("not implemented") 354 } 355 } 356 357 func (r queryRange) AnyBound() interface{} { 358 if r.lowerBound != nil { 359 return r.lowerBound 360 } 361 362 return r.upperBound 363 } 364 365 func (r queryRange) upperBoundValue() interface{} { 366 if r.upperBound == nil { 367 return nil 368 } 369 370 if r.includeUpperBound { 371 return r.upperBound 372 } 373 374 switch t := r.upperBound.(type) { 375 case int64: 376 return t - 1 377 case time.Time: 378 return t.Unix() - 1 379 default: 380 panic("not implemented") 381 } 382 } 383 384 func lookForRanges(conditions []query.Condition) (ranges queryRanges, indexes []int) { 385 ranges = make(queryRanges) 386 for i, c := range conditions { 387 if isRangeOperation(c.Op) { 388 r, ok := ranges[c.CompositeKey] 389 if !ok { 390 r = queryRange{key: c.CompositeKey} 391 } 392 switch c.Op { 393 case query.OpGreater: 394 r.lowerBound = c.Operand 395 case query.OpGreaterEqual: 396 r.includeLowerBound = true 397 r.lowerBound = c.Operand 398 case query.OpLess: 399 r.upperBound = c.Operand 400 case query.OpLessEqual: 401 r.includeUpperBound = true 402 r.upperBound = c.Operand 403 } 404 ranges[c.CompositeKey] = r 405 indexes = append(indexes, i) 406 } 407 } 408 return ranges, indexes 409 } 410 411 func isRangeOperation(op query.Operator) bool { 412 switch op { 413 case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: 414 return true 415 default: 416 return false 417 } 418 } 419 420 // match returns all matching txs by hash that meet a given condition and start 421 // key. An already filtered result (filteredHashes) is provided such that any 422 // non-intersecting matches are removed. 423 // 424 // NOTE: filteredHashes may be empty if no previous condition has matched. 425 func (txi *TxIndex) match( 426 ctx context.Context, 427 c query.Condition, 428 startKeyBz []byte, 429 filteredHashes map[string][]byte, 430 firstRun bool, 431 ) map[string][]byte { 432 // A previous match was attempted but resulted in no matches, so we return 433 // no matches (assuming AND operand). 434 if !firstRun && len(filteredHashes) == 0 { 435 return filteredHashes 436 } 437 438 tmpHashes := make(map[string][]byte) 439 440 switch { 441 case c.Op == query.OpEqual: 442 it, err := dbm.IteratePrefix(txi.store, startKeyBz) 443 if err != nil { 444 panic(err) 445 } 446 defer it.Close() 447 448 for ; it.Valid(); it.Next() { 449 tmpHashes[string(it.Value())] = it.Value() 450 451 // Potentially exit early. 452 select { 453 case <-ctx.Done(): 454 break 455 default: 456 } 457 } 458 459 case c.Op == query.OpContains: 460 // XXX: startKey does not apply here. 461 // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" 462 // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" 463 it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) 464 if err != nil { 465 panic(err) 466 } 467 defer it.Close() 468 469 for ; it.Valid(); it.Next() { 470 if !isTagKey(it.Key()) { 471 continue 472 } 473 474 if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { 475 tmpHashes[string(it.Value())] = it.Value() 476 } 477 478 // Potentially exit early. 479 select { 480 case <-ctx.Done(): 481 break 482 default: 483 } 484 } 485 default: 486 panic("other operators should be handled already") 487 } 488 489 if len(tmpHashes) == 0 || firstRun { 490 // Either: 491 // 492 // 1. Regardless if a previous match was attempted, which may have had 493 // results, but no match was found for the current condition, then we 494 // return no matches (assuming AND operand). 495 // 496 // 2. A previous match was not attempted, so we return all results. 497 return tmpHashes 498 } 499 500 // Remove/reduce matches in filteredHashes that were not found in this 501 // match (tmpHashes). 502 for k := range filteredHashes { 503 if tmpHashes[k] == nil { 504 delete(filteredHashes, k) 505 506 // Potentially exit early. 507 select { 508 case <-ctx.Done(): 509 break 510 default: 511 } 512 } 513 } 514 515 return filteredHashes 516 } 517 518 // matchRange returns all matching txs by hash that meet a given queryRange and 519 // start key. An already filtered result (filteredHashes) is provided such that 520 // any non-intersecting matches are removed. 521 // 522 // NOTE: filteredHashes may be empty if no previous condition has matched. 523 func (txi *TxIndex) matchRange( 524 ctx context.Context, 525 r queryRange, 526 startKey []byte, 527 filteredHashes map[string][]byte, 528 firstRun bool, 529 ) (map[string][]byte, error) { 530 // A previous match was attempted but resulted in no matches, so we return 531 // no matches (assuming AND operand). 532 if !firstRun && len(filteredHashes) == 0 { 533 return filteredHashes, nil 534 } 535 536 tmpHashes := make(map[string][]byte) 537 lowerBound := r.lowerBoundValue() 538 upperBound := r.upperBoundValue() 539 540 it, err := dbm.IteratePrefix(txi.store, startKey) 541 if err != nil { 542 panic(err) 543 } 544 defer it.Close() 545 count := 0 546 547 LOOP: 548 for ; it.Valid(); it.Next() { 549 if count > maxQueryRange { 550 return nil, errors.New("request processing more than max query range, optimize request filter conditions parameter") 551 } 552 count++ 553 // Potentially exit early. 554 select { 555 case <-ctx.Done(): 556 return nil, errors.New("request processing timeout, optimize request filter conditions parameter") 557 default: 558 } 559 if !isTagKey(it.Key()) { 560 continue 561 } 562 563 if _, ok := r.AnyBound().(int64); ok { 564 v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) 565 if err != nil { 566 continue LOOP 567 } 568 569 include := true 570 if lowerBound != nil && v < lowerBound.(int64) { 571 include = false 572 } 573 574 if upperBound != nil && v > upperBound.(int64) { 575 include = false 576 } 577 578 if include { 579 tmpHashes[string(it.Value())] = it.Value() 580 } 581 582 // XXX: passing time in a ABCI Events is not yet implemented 583 // case time.Time: 584 // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) 585 // if v == r.upperBound { 586 // break 587 // } 588 } 589 } 590 591 if len(tmpHashes) == 0 || firstRun { 592 // Either: 593 // 594 // 1. Regardless if a previous match was attempted, which may have had 595 // results, but no match was found for the current condition, then we 596 // return no matches (assuming AND operand). 597 // 598 // 2. A previous match was not attempted, so we return all results. 599 return tmpHashes, nil 600 } 601 602 // Remove/reduce matches in filteredHashes that were not found in this 603 // match (tmpHashes). 604 for k := range filteredHashes { 605 if tmpHashes[k] == nil { 606 delete(filteredHashes, k) 607 608 // Potentially exit early. 609 select { 610 case <-ctx.Done(): 611 break 612 default: 613 } 614 } 615 } 616 617 return filteredHashes, nil 618 } 619 620 /////////////////////////////////////////////////////////////////////////////// 621 // Keys 622 623 func isTagKey(key []byte) bool { 624 return strings.Count(string(key), tagKeySeparator) == 3 625 } 626 627 func extractValueFromKey(key []byte) string { 628 parts := strings.SplitN(string(key), tagKeySeparator, 3) 629 return parts[1] 630 } 631 632 func keyForEvent(key string, value []byte, result *types.TxResult) []byte { 633 return []byte(fmt.Sprintf("%s/%s/%d/%d", 634 key, 635 value, 636 result.Height, 637 result.Index, 638 )) 639 } 640 641 func keyForHeight(result *types.TxResult) []byte { 642 return []byte(fmt.Sprintf("%s/%d/%d/%d", 643 types.TxHeightKey, 644 result.Height, 645 result.Height, 646 result.Index, 647 )) 648 } 649 650 func startKeyForCondition(c query.Condition, height int64) []byte { 651 if height > 0 { 652 return startKey(c.CompositeKey, c.Operand, height) 653 } 654 return startKey(c.CompositeKey, c.Operand) 655 } 656 657 func startKey(fields ...interface{}) []byte { 658 var b bytes.Buffer 659 for _, f := range fields { 660 b.Write([]byte(fmt.Sprintf("%v", f) + tagKeySeparator)) 661 } 662 return b.Bytes() 663 }