github.com/fiagdao/tendermint@v0.32.11-0.20220824195748-2087fcc480c1/state/txindex/kv/kv.go (about) 1 package kv 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/hex" 7 "fmt" 8 "sort" 9 "strconv" 10 "strings" 11 "time" 12 13 "github.com/pkg/errors" 14 15 dbm "github.com/tendermint/tm-db" 16 17 "github.com/tendermint/tendermint/libs/pubsub/query" 18 tmstring "github.com/tendermint/tendermint/libs/strings" 19 "github.com/tendermint/tendermint/state/txindex" 20 "github.com/tendermint/tendermint/types" 21 ) 22 23 const ( 24 tagKeySeparator = "/" 25 ) 26 27 var _ txindex.TxIndexer = (*TxIndex)(nil) 28 29 // TxIndex is the simplest possible indexer, backed by key-value storage (levelDB). 30 type TxIndex struct { 31 store dbm.DB 32 compositeKeysToIndex []string 33 indexAllEvents bool 34 } 35 36 // NewTxIndex creates new KV indexer. 37 func NewTxIndex(store dbm.DB, options ...func(*TxIndex)) *TxIndex { 38 txi := &TxIndex{store: store, compositeKeysToIndex: make([]string, 0), indexAllEvents: false} 39 for _, o := range options { 40 o(txi) 41 } 42 return txi 43 } 44 45 // IndexEvents is an option for setting which composite keys to index. 46 func IndexEvents(compositeKeys []string) func(*TxIndex) { 47 return func(txi *TxIndex) { 48 txi.compositeKeysToIndex = compositeKeys 49 } 50 } 51 52 // IndexAllEvents is an option for indexing all events. 53 func IndexAllEvents() func(*TxIndex) { 54 return func(txi *TxIndex) { 55 txi.indexAllEvents = true 56 } 57 } 58 59 // Get gets transaction from the TxIndex storage and returns it or nil if the 60 // transaction is not found. 61 func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { 62 if len(hash) == 0 { 63 return nil, txindex.ErrorEmptyHash 64 } 65 66 rawBytes, _ := txi.store.Get(hash) 67 if rawBytes == nil { 68 return nil, nil 69 } 70 71 txResult := new(types.TxResult) 72 err := cdc.UnmarshalBinaryBare(rawBytes, &txResult) 73 if err != nil { 74 return nil, fmt.Errorf("error reading TxResult: %v", err) 75 } 76 77 return txResult, nil 78 } 79 80 // AddBatch indexes a batch of transactions using the given list of events. Each 81 // key that indexed from the tx's events is a composite of the event type and 82 // the respective attribute's key delimited by a "." (eg. "account.number"). 83 // Any event with an empty type is not indexed. 84 func (txi *TxIndex) AddBatch(b *txindex.Batch) error { 85 storeBatch := txi.store.NewBatch() 86 defer storeBatch.Close() 87 88 for _, result := range b.Ops { 89 hash := result.Tx.Hash() 90 91 // index tx by events 92 txi.indexEvents(result, hash, storeBatch) 93 94 // index tx by height 95 if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) { 96 storeBatch.Set(keyForHeight(result), hash) 97 } 98 99 // index tx by hash 100 rawBytes, err := cdc.MarshalBinaryBare(result) 101 if err != nil { 102 return err 103 } 104 storeBatch.Set(hash, rawBytes) 105 } 106 107 storeBatch.WriteSync() 108 return nil 109 } 110 111 // Index indexes a single transaction using the given list of events. Each key 112 // that indexed from the tx's events is a composite of the event type and the 113 // respective attribute's key delimited by a "." (eg. "account.number"). 114 // Any event with an empty type is not indexed. 115 func (txi *TxIndex) Index(result *types.TxResult) error { 116 b := txi.store.NewBatch() 117 defer b.Close() 118 119 hash := result.Tx.Hash() 120 121 // index tx by events 122 txi.indexEvents(result, hash, b) 123 124 // index tx by height 125 if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) { 126 b.Set(keyForHeight(result), hash) 127 } 128 129 // index tx by hash 130 rawBytes, err := cdc.MarshalBinaryBare(result) 131 if err != nil { 132 return err 133 } 134 135 b.Set(hash, rawBytes) 136 b.WriteSync() 137 138 return nil 139 } 140 141 func (txi *TxIndex) indexEvents(result *types.TxResult, hash []byte, store dbm.SetDeleter) { 142 for _, event := range result.Result.Events { 143 // only index events with a non-empty type 144 if len(event.Type) == 0 { 145 continue 146 } 147 148 for _, attr := range event.Attributes { 149 if len(attr.Key) == 0 { 150 continue 151 } 152 153 compositeTag := fmt.Sprintf("%s.%s", event.Type, string(attr.Key)) 154 if txi.indexAllEvents || tmstring.StringInSlice(compositeTag, txi.compositeKeysToIndex) { 155 store.Set(keyForEvent(compositeTag, attr.Value, result), hash) 156 } 157 } 158 } 159 } 160 161 func (txi *TxIndex) deleteEvents(result *types.TxResult, hash []byte, store dbm.SetDeleter) { 162 for _, event := range result.Result.Events { 163 // only index events with a non-empty type 164 if len(event.Type) == 0 { 165 continue 166 } 167 168 for _, attr := range event.Attributes { 169 if len(attr.Key) == 0 { 170 continue 171 } 172 173 compositeTag := fmt.Sprintf("%s.%s", event.Type, string(attr.Key)) 174 //if txi.indexAllTags || cmn.StringInSlice(compositeTag, txi.tagsToIndex) { // defensive 175 store.Delete(keyForEvent(compositeTag, attr.Value, result)) 176 //} 177 } 178 } 179 } 180 181 func (txi *TxIndex) DeleteFromHeight(ctx context.Context, height int64) error { 182 q, err := query.New("tx.height > " + strconv.Itoa(int(height))) 183 if err != nil { 184 return err 185 } 186 res, _, err := txi.Search(ctx, q) 187 if err != nil { 188 return err 189 } 190 b := txi.store.NewBatch() 191 defer b.Close() 192 for _, txRes := range res { 193 hash := txRes.Tx.Hash() 194 // index tx by events 195 txi.deleteEvents(txRes, hash, b) 196 // index tx by height 197 if txi.indexAllEvents || tmstring.StringInSlice(types.TxHeightKey, txi.compositeKeysToIndex) { 198 b.Delete(keyForHeight(txRes)) 199 } 200 b.Delete(hash) 201 } 202 b.WriteSync() 203 return nil 204 } 205 206 // Search performs a search using the given query. It breaks the query into 207 // conditions (like "tx.height > 5"). For each condition, it queries the DB 208 // index. One special use cases here: (1) if "tx.hash" is found, it returns tx 209 // result for it (2) for range queries it is better for the client to provide 210 // both lower and upper bounds, so we are not performing a full scan. Results 211 // from querying indexes are then intersected and returned to the caller. 212 func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*types.TxResult, int, error) { 213 var hashesInitialized bool 214 filteredHashes := make(map[string]*keyAndHash) 215 216 // get a list of conditions (like "tx.height > 5") 217 conditions, err := q.Conditions() 218 if err != nil { 219 return nil, 0, errors.Wrap(err, "error during parsing conditions from query") 220 } 221 222 // if there is a hash condition, return the result immediately 223 hash, ok, err := lookForHash(conditions...) 224 if err != nil { 225 return nil, 0, errors.Wrap(err, "error during searching for a hash in the query") 226 } else if ok { 227 res, err := txi.Get(hash) 228 switch { 229 case err != nil: 230 return []*types.TxResult{}, 0, errors.Wrap(err, "error while retrieving the result") 231 case res == nil: 232 return []*types.TxResult{}, 0, nil 233 default: 234 return []*types.TxResult{res}, 0, nil 235 } 236 } 237 238 // conditions to skip because they're handled before "everything else" 239 skipIndexes := make([]int, 0) 240 241 // extract ranges 242 // if both upper and lower bounds exist, it's better to get them in order not 243 // no iterate over kvs that are not within range. 244 ranges, rangeIndexes := lookForRanges(conditions...) 245 if len(ranges) > 0 { 246 skipIndexes = append(skipIndexes, rangeIndexes...) 247 248 for _, r := range ranges { 249 if !hashesInitialized { 250 filteredHashes = txi.matchRange(ctx, r, startKey(r.key), filteredHashes, true) 251 hashesInitialized = true 252 253 // Ignore any remaining conditions if the first condition resulted 254 // in no matches (assuming implicit AND operand). 255 if len(filteredHashes) == 0 { 256 break 257 } 258 } else { 259 filteredHashes = txi.matchRange(ctx, r, startKey(r.key), filteredHashes, false) 260 } 261 } 262 } 263 264 // if there is a height condition ("tx.height=3"), extract it 265 height := lookForHeight(conditions...) 266 267 // for all other conditions 268 for i, c := range conditions { 269 if intInSlice(i, skipIndexes) { 270 continue 271 } 272 if !hashesInitialized { 273 filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, true) 274 hashesInitialized = true 275 276 // Ignore any remaining conditions if the first condition resulted 277 // in no matches (assuming implicit AND operand). 278 if len(filteredHashes) == 0 { 279 break 280 } 281 } else { 282 filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, false) 283 } 284 } 285 286 results := make([]*types.TxResult, 0, len(filteredHashes)) 287 if q.Pagination != nil { 288 var sortHashes []*keyAndHash 289 for _, hashKeyPair := range filteredHashes { 290 sortHashes = append(sortHashes, hashKeyPair) 291 } 292 filteredHashes = nil // IGNORE MAP PLEASE!! XD 293 294 switch q.Pagination.Sort { 295 case "desc": 296 sort.Slice(sortHashes, func(i, j int) bool { 297 a := strings.Split(sortHashes[i].key, "/") 298 b := strings.Split(sortHashes[j].key, "/") 299 aHeight, _ := strconv.Atoi(a[2]) 300 bHeight, _ := strconv.Atoi(b[2]) 301 if aHeight == bHeight { 302 aIndex, _ := strconv.Atoi(a[3]) 303 bIndex, _ := strconv.Atoi(b[3]) 304 return aIndex < bIndex 305 } 306 return aHeight > bHeight 307 }) 308 case "asc", "": 309 sort.Slice(sortHashes, func(i, j int) bool { 310 a := strings.Split(sortHashes[i].key, "/") 311 b := strings.Split(sortHashes[j].key, "/") 312 aHeight, _ := strconv.Atoi(a[2]) 313 bHeight, _ := strconv.Atoi(b[2]) 314 if aHeight == bHeight { 315 aIndex, _ := strconv.Atoi(a[3]) 316 bIndex, _ := strconv.Atoi(b[3]) 317 return aIndex < bIndex 318 } 319 return aHeight < bHeight 320 }) 321 } 322 skipCount := 0 323 results = make([]*types.TxResult, 0, q.Pagination.Size) 324 for _, hat := range sortHashes { 325 select { 326 case <-ctx.Done(): 327 break 328 default: 329 // skip keys 330 if skipCount > q.Pagination.Skip { 331 skipCount++ 332 continue 333 } 334 res, err := txi.Get(hat.hash) 335 if err != nil { 336 return nil, 0, errors.Wrapf(err, "failed to get Tx{%X}", hat.hash) 337 } 338 results = append(results, res) 339 // Potentially exit early. 340 if len(results) == cap(results) { 341 return results, 0, nil 342 } 343 } 344 } 345 } 346 for _, hashAndKeyPair := range filteredHashes { 347 // Potentially exit early. 348 select { 349 case <-ctx.Done(): 350 break 351 default: 352 res, err := txi.Get(hashAndKeyPair.hash) 353 if err != nil { 354 return nil, 0, errors.Wrapf(err, "failed to get Tx{%X}", hashAndKeyPair.hash) 355 } 356 results = append(results, res) 357 } 358 } 359 return results, 0, nil 360 } 361 362 func lookForHash(conditions ...query.Condition) (hash []byte, ok bool, err error) { 363 for _, c := range conditions { 364 if c.CompositeKey == types.TxHashKey { 365 decoded, err := hex.DecodeString(c.Operand.(string)) 366 return decoded, true, err 367 } 368 } 369 return 370 } 371 372 // lookForHeight returns a height if there is an "height=X" condition. 373 func lookForHeight(conditions ...query.Condition) (height int64) { 374 for _, c := range conditions { 375 if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { 376 return c.Operand.(int64) 377 } 378 } 379 return 0 380 } 381 382 // special map to hold range conditions 383 // Example: account.number => queryRange{lowerBound: 1, upperBound: 5} 384 type queryRanges map[string]queryRange 385 386 type queryRange struct { 387 lowerBound interface{} // int || time.Time 388 upperBound interface{} // int || time.Time 389 key string 390 includeLowerBound bool 391 includeUpperBound bool 392 } 393 394 func (r queryRange) lowerBoundValue() interface{} { 395 if r.lowerBound == nil { 396 return nil 397 } 398 399 if r.includeLowerBound { 400 return r.lowerBound 401 } 402 403 switch t := r.lowerBound.(type) { 404 case int64: 405 return t + 1 406 case time.Time: 407 return t.Unix() + 1 408 default: 409 panic("not implemented") 410 } 411 } 412 413 func (r queryRange) AnyBound() interface{} { 414 if r.lowerBound != nil { 415 return r.lowerBound 416 } 417 418 return r.upperBound 419 } 420 421 func (r queryRange) upperBoundValue() interface{} { 422 if r.upperBound == nil { 423 return nil 424 } 425 426 if r.includeUpperBound { 427 return r.upperBound 428 } 429 430 switch t := r.upperBound.(type) { 431 case int64: 432 return t - 1 433 case time.Time: 434 return t.Unix() - 1 435 default: 436 panic("not implemented") 437 } 438 } 439 440 func lookForRanges(conditions ...query.Condition) (ranges queryRanges, indexes []int) { 441 ranges = make(queryRanges) 442 for i, c := range conditions { 443 if isRangeOperation(c.Op) { 444 r, ok := ranges[c.CompositeKey] 445 if !ok { 446 r = queryRange{key: c.CompositeKey} 447 } 448 switch c.Op { 449 case query.OpGreater: 450 r.lowerBound = c.Operand 451 case query.OpGreaterEqual: 452 r.includeLowerBound = true 453 r.lowerBound = c.Operand 454 case query.OpLess: 455 r.upperBound = c.Operand 456 case query.OpLessEqual: 457 r.includeUpperBound = true 458 r.upperBound = c.Operand 459 } 460 ranges[c.CompositeKey] = r 461 indexes = append(indexes, i) 462 } 463 } 464 return ranges, indexes 465 } 466 467 func isRangeOperation(op query.Operator) bool { 468 switch op { 469 case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: 470 return true 471 default: 472 return false 473 } 474 } 475 476 type hashAndIndexer struct { 477 hash []byte 478 txi *TxIndex 479 } 480 481 type sortByHeight []hashAndIndexer 482 type sortByKey []hashAndIndexer 483 484 type keyAndHash struct { 485 key string 486 hash []byte 487 } 488 489 // Retrieves the keys from the iterator based on condition 490 // NOTE: filteredHashes may be empty if no previous condition has matched. 491 func (txi *TxIndex) keys( 492 ctx context.Context, 493 c query.Condition, 494 startKeyBz []byte, 495 ) map[string][]byte { 496 hashes := make(map[string][]byte) 497 switch { 498 case c.Op == query.OpEqual: 499 it, _ := dbm.IteratePrefix(txi.store, startKeyBz) 500 defer it.Close() 501 502 for ; it.Valid(); it.Next() { 503 // Potentially exit early. 504 select { 505 case <-ctx.Done(): 506 break 507 default: 508 hashes[string(it.Key())] = it.Value() 509 } 510 511 } 512 case c.Op == query.OpContains: 513 // XXX: startKey does not apply here. 514 // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" 515 // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" 516 it, _ := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) 517 defer it.Close() 518 519 for ; it.Valid(); it.Next() { 520 // Potentially exit early. 521 select { 522 case <-ctx.Done(): 523 break 524 default: 525 if !isTagKey(it.Key()) { 526 continue 527 } 528 529 if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { 530 hashes[string(it.Key())] = it.Value() 531 } 532 } 533 } 534 default: 535 panic("other operators should be handled already") 536 } 537 return hashes 538 } 539 540 // match returns all matching txs by hash that meet a given condition and start 541 // key. An already filtered result (filteredHashes) is provided such that any 542 // non-intersecting matches are removed. 543 // 544 // NOTE: filteredHashes may be empty if no previous condition has matched. 545 func (txi *TxIndex) match( 546 ctx context.Context, 547 c query.Condition, 548 startKeyBz []byte, 549 filteredHashes map[string]*keyAndHash, 550 firstRun bool, 551 ) map[string]*keyAndHash { 552 // A previous match was attempted but resulted in no matches, so we return 553 // no matches (assuming AND operand). 554 if !firstRun && len(filteredHashes) == 0 { 555 return filteredHashes 556 } 557 tmpHashes := make(map[string]*keyAndHash) 558 switch { 559 case c.Op == query.OpEqual: 560 it, _ := dbm.IteratePrefix(txi.store, startKeyBz) 561 defer it.Close() 562 563 for ; it.Valid(); it.Next() { 564 // Potentially exit early. 565 select { 566 case <-ctx.Done(): 567 break 568 default: 569 key := string(it.Value()) 570 tmpHashes[key] = &keyAndHash{string(it.Key()), it.Value()} 571 } 572 573 } 574 case c.Op == query.OpContains: 575 // XXX: startKey does not apply here. 576 // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" 577 // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" 578 it, _ := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) 579 defer it.Close() 580 581 for ; it.Valid(); it.Next() { 582 // Potentially exit early. 583 select { 584 case <-ctx.Done(): 585 break 586 default: 587 if !isTagKey(it.Key()) { 588 continue 589 } 590 591 if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { 592 tmpHashes[string(it.Value())] = &keyAndHash{string(it.Key()), it.Value()} 593 } 594 } 595 } 596 default: 597 panic("other operators should be handled already") 598 } 599 600 if len(tmpHashes) == 0 || firstRun { 601 // Either: 602 // 603 // 1. Regardless if a previous match was attempted, which may have had 604 // results, but no match was found for the current condition, then we 605 // return no matches (assuming AND operand). 606 // 607 // 2. A previous match was not attempted, so we return all results. 608 return tmpHashes 609 } 610 611 // Remove/reduce matches in filteredHashes that were not found in this 612 // match (tmpHashes). 613 for k := range filteredHashes { 614 if tmpHashes[k] == nil { 615 // Potentially exit early. 616 select { 617 case <-ctx.Done(): 618 break 619 default: 620 delete(filteredHashes, k) 621 } 622 } 623 } 624 625 return filteredHashes 626 } 627 628 // matchRange returns all matching txs by hash that meet a given queryRange and 629 // start key. An already filtered result (filteredHashes) is provided such that 630 // any non-intersecting matches are removed. 631 // 632 // NOTE: filteredHashes may be empty if no previous condition has matched. 633 func (txi *TxIndex) matchRange( 634 ctx context.Context, 635 r queryRange, 636 startKey []byte, 637 filteredHashes map[string]*keyAndHash, 638 firstRun bool, 639 ) map[string]*keyAndHash { 640 // A previous match was attempted but resulted in no matches, so we return 641 // no matches (assuming AND operand). 642 if !firstRun && len(filteredHashes) == 0 { 643 return filteredHashes 644 } 645 646 tmpHashes := make(map[string]*keyAndHash) 647 lowerBound := r.lowerBoundValue() 648 upperBound := r.upperBoundValue() 649 650 it, _ := dbm.IteratePrefix(txi.store, startKey) 651 defer it.Close() 652 653 LOOP: 654 for ; it.Valid(); it.Next() { 655 if !isTagKey(it.Key()) { 656 continue 657 } 658 659 if _, ok := r.AnyBound().(int64); ok { 660 v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) 661 if err != nil { 662 continue LOOP 663 } 664 665 include := true 666 if lowerBound != nil && v < lowerBound.(int64) { 667 include = false 668 } 669 670 if upperBound != nil && v > upperBound.(int64) { 671 include = false 672 } 673 674 if include { 675 tmpHashes[string(it.Value())] = &keyAndHash{string(it.Key()), it.Value()} 676 } 677 678 // XXX: passing time in a ABCI Events is not yet implemented 679 // case time.Time: 680 // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) 681 // if v == r.upperBound { 682 // break 683 // } 684 } 685 686 // Potentially exit early. 687 select { 688 case <-ctx.Done(): 689 break 690 default: 691 } 692 } 693 694 if len(tmpHashes) == 0 || firstRun { 695 // Either: 696 // 697 // 1. Regardless if a previous match was attempted, which may have had 698 // results, but no match was found for the current condition, then we 699 // return no matches (assuming AND operand). 700 // 701 // 2. A previous match was not attempted, so we return all results. 702 return tmpHashes 703 } 704 705 // Remove/reduce matches in filteredHashes that were not found in this 706 // match (tmpHashes). 707 for k := range filteredHashes { 708 if tmpHashes[k] == nil { 709 delete(filteredHashes, k) 710 711 // Potentially exit early. 712 select { 713 case <-ctx.Done(): 714 break 715 default: 716 } 717 } 718 } 719 720 return filteredHashes 721 } 722 723 /////////////////////////////////////////////////////////////////////////////// 724 // Keys 725 726 func isTagKey(key []byte) bool { 727 return strings.Count(string(key), tagKeySeparator) == 3 728 } 729 730 func extractValueFromKey(key []byte) string { 731 parts := strings.SplitN(string(key), tagKeySeparator, 3) 732 return parts[1] 733 } 734 735 func keyForEvent(key string, value []byte, result *types.TxResult) []byte { 736 return []byte(fmt.Sprintf("%s/%s/%d/%d", 737 key, 738 value, 739 result.Height, 740 result.Index, 741 )) 742 } 743 744 func keyForHeight(result *types.TxResult) []byte { 745 return []byte(fmt.Sprintf("%s/%d/%d/%d", 746 types.TxHeightKey, 747 result.Height, 748 result.Height, 749 result.Index, 750 )) 751 } 752 753 func startKeyForCondition(c query.Condition, height int64) []byte { 754 if height > 0 { 755 return startKey(c.CompositeKey, c.Operand, height) 756 } 757 return startKey(c.CompositeKey, c.Operand) 758 } 759 760 func startKey(fields ...interface{}) []byte { 761 var b bytes.Buffer 762 for _, f := range fields { 763 b.Write([]byte(fmt.Sprintf("%v", f) + tagKeySeparator)) 764 } 765 return b.Bytes() 766 }