github.com/Oyster-zx/tendermint@v0.34.24-fork/state/txindex/kv/kv.go (about) 1 package kv 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/hex" 7 "fmt" 8 "strconv" 9 "strings" 10 11 "github.com/gogo/protobuf/proto" 12 dbm "github.com/tendermint/tm-db" 13 14 abci "github.com/tendermint/tendermint/abci/types" 15 "github.com/tendermint/tendermint/libs/pubsub/query" 16 "github.com/tendermint/tendermint/state/indexer" 17 "github.com/tendermint/tendermint/state/txindex" 18 "github.com/tendermint/tendermint/types" 19 ) 20 21 const ( 22 tagKeySeparator = "/" 23 ) 24 25 var _ txindex.TxIndexer = (*TxIndex)(nil) 26 27 // TxIndex is the simplest possible indexer, backed by key-value storage (levelDB). 28 type TxIndex struct { 29 store dbm.DB 30 } 31 32 // NewTxIndex creates new KV indexer. 33 func NewTxIndex(store dbm.DB) *TxIndex { 34 return &TxIndex{ 35 store: store, 36 } 37 } 38 39 // Get gets transaction from the TxIndex storage and returns it or nil if the 40 // transaction is not found. 41 func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) { 42 if len(hash) == 0 { 43 return nil, txindex.ErrorEmptyHash 44 } 45 46 rawBytes, err := txi.store.Get(hash) 47 if err != nil { 48 panic(err) 49 } 50 if rawBytes == nil { 51 return nil, nil 52 } 53 54 txResult := new(abci.TxResult) 55 err = proto.Unmarshal(rawBytes, txResult) 56 if err != nil { 57 return nil, fmt.Errorf("error reading TxResult: %v", err) 58 } 59 60 return txResult, nil 61 } 62 63 // AddBatch indexes a batch of transactions using the given list of events. Each 64 // key that indexed from the tx's events is a composite of the event type and 65 // the respective attribute's key delimited by a "." (eg. "account.number"). 66 // Any event with an empty type is not indexed. 67 func (txi *TxIndex) AddBatch(b *txindex.Batch) error { 68 storeBatch := txi.store.NewBatch() 69 defer storeBatch.Close() 70 71 for _, result := range b.Ops { 72 hash := types.Tx(result.Tx).Hash() 73 74 // index tx by events 75 err := txi.indexEvents(result, hash, storeBatch) 76 if err != nil { 77 return err 78 } 79 80 // index by height (always) 81 err = storeBatch.Set(keyForHeight(result), hash) 82 if err != nil { 83 return err 84 } 85 86 rawBytes, err := proto.Marshal(result) 87 if err != nil { 88 return err 89 } 90 // index by hash (always) 91 err = storeBatch.Set(hash, rawBytes) 92 if err != nil { 93 return err 94 } 95 } 96 97 return storeBatch.WriteSync() 98 } 99 100 // Index indexes a single transaction using the given list of events. Each key 101 // that indexed from the tx's events is a composite of the event type and the 102 // respective attribute's key delimited by a "." (eg. "account.number"). 103 // Any event with an empty type is not indexed. 104 // 105 // If a transaction is indexed with the same hash as a previous transaction, it will 106 // be overwritten unless the tx result was NOT OK and the prior result was OK i.e. 107 // more transactions that successfully executed overwrite transactions that failed 108 // or successful yet older transactions. 109 func (txi *TxIndex) Index(result *abci.TxResult) error { 110 b := txi.store.NewBatch() 111 defer b.Close() 112 113 hash := types.Tx(result.Tx).Hash() 114 115 if !result.Result.IsOK() { 116 oldResult, err := txi.Get(hash) 117 if err != nil { 118 return err 119 } 120 121 // if the new transaction failed and it's already indexed in an older block and was successful 122 // we skip it as we want users to get the older successful transaction when they query. 123 if oldResult != nil && oldResult.Result.Code == abci.CodeTypeOK { 124 return nil 125 } 126 } 127 128 // index tx by events 129 err := txi.indexEvents(result, hash, b) 130 if err != nil { 131 return err 132 } 133 134 // index by height (always) 135 err = b.Set(keyForHeight(result), hash) 136 if err != nil { 137 return err 138 } 139 140 rawBytes, err := proto.Marshal(result) 141 if err != nil { 142 return err 143 } 144 // index by hash (always) 145 err = b.Set(hash, rawBytes) 146 if err != nil { 147 return err 148 } 149 150 return b.WriteSync() 151 } 152 153 func (txi *TxIndex) indexEvents(result *abci.TxResult, hash []byte, store dbm.Batch) error { 154 for _, event := range result.Result.Events { 155 // only index events with a non-empty type 156 if len(event.Type) == 0 { 157 continue 158 } 159 160 for _, attr := range event.Attributes { 161 if len(attr.Key) == 0 { 162 continue 163 } 164 165 // index if `index: true` is set 166 compositeTag := fmt.Sprintf("%s.%s", event.Type, string(attr.Key)) 167 if attr.GetIndex() { 168 err := store.Set(keyForEvent(compositeTag, attr.Value, result), hash) 169 if err != nil { 170 return err 171 } 172 } 173 } 174 } 175 176 return nil 177 } 178 179 // Search performs a search using the given query. 180 // 181 // It breaks the query into conditions (like "tx.height > 5"). For each 182 // condition, it queries the DB index. One special use cases here: (1) if 183 // "tx.hash" is found, it returns tx result for it (2) for range queries it is 184 // better for the client to provide both lower and upper bounds, so we are not 185 // performing a full scan. Results from querying indexes are then intersected 186 // and returned to the caller, in no particular order. 187 // 188 // Search will exit early and return any result fetched so far, 189 // when a message is received on the context chan. 190 func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResult, error) { 191 select { 192 case <-ctx.Done(): 193 return make([]*abci.TxResult, 0), nil 194 195 default: 196 } 197 198 var hashesInitialized bool 199 filteredHashes := make(map[string][]byte) 200 201 // get a list of conditions (like "tx.height > 5") 202 conditions, err := q.Conditions() 203 if err != nil { 204 return nil, fmt.Errorf("error during parsing conditions from query: %w", err) 205 } 206 207 // if there is a hash condition, return the result immediately 208 hash, ok, err := lookForHash(conditions) 209 if err != nil { 210 return nil, fmt.Errorf("error during searching for a hash in the query: %w", err) 211 } else if ok { 212 res, err := txi.Get(hash) 213 switch { 214 case err != nil: 215 return []*abci.TxResult{}, fmt.Errorf("error while retrieving the result: %w", err) 216 case res == nil: 217 return []*abci.TxResult{}, nil 218 default: 219 return []*abci.TxResult{res}, nil 220 } 221 } 222 223 // conditions to skip because they're handled before "everything else" 224 skipIndexes := make([]int, 0) 225 226 // extract ranges 227 // if both upper and lower bounds exist, it's better to get them in order not 228 // no iterate over kvs that are not within range. 229 ranges, rangeIndexes := indexer.LookForRanges(conditions) 230 if len(ranges) > 0 { 231 skipIndexes = append(skipIndexes, rangeIndexes...) 232 233 for _, qr := range ranges { 234 if !hashesInitialized { 235 filteredHashes = txi.matchRange(ctx, qr, startKey(qr.Key), filteredHashes, true) 236 hashesInitialized = true 237 238 // Ignore any remaining conditions if the first condition resulted 239 // in no matches (assuming implicit AND operand). 240 if len(filteredHashes) == 0 { 241 break 242 } 243 } else { 244 filteredHashes = txi.matchRange(ctx, qr, startKey(qr.Key), filteredHashes, false) 245 } 246 } 247 } 248 249 // if there is a height condition ("tx.height=3"), extract it 250 height := lookForHeight(conditions) 251 252 // for all other conditions 253 for i, c := range conditions { 254 if intInSlice(i, skipIndexes) { 255 continue 256 } 257 258 if !hashesInitialized { 259 filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, true) 260 hashesInitialized = true 261 262 // Ignore any remaining conditions if the first condition resulted 263 // in no matches (assuming implicit AND operand). 264 if len(filteredHashes) == 0 { 265 break 266 } 267 } else { 268 filteredHashes = txi.match(ctx, c, startKeyForCondition(c, height), filteredHashes, false) 269 } 270 } 271 272 results := make([]*abci.TxResult, 0, len(filteredHashes)) 273 for _, h := range filteredHashes { 274 res, err := txi.Get(h) 275 if err != nil { 276 return nil, fmt.Errorf("failed to get Tx{%X}: %w", h, err) 277 } 278 results = append(results, res) 279 280 // Potentially exit early. 281 select { 282 case <-ctx.Done(): 283 break 284 default: 285 } 286 } 287 288 return results, nil 289 } 290 291 func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) { 292 for _, c := range conditions { 293 if c.CompositeKey == types.TxHashKey { 294 decoded, err := hex.DecodeString(c.Operand.(string)) 295 return decoded, true, err 296 } 297 } 298 return 299 } 300 301 // lookForHeight returns a height if there is an "height=X" condition. 302 func lookForHeight(conditions []query.Condition) (height int64) { 303 for _, c := range conditions { 304 if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { 305 return c.Operand.(int64) 306 } 307 } 308 return 0 309 } 310 311 // match returns all matching txs by hash that meet a given condition and start 312 // key. An already filtered result (filteredHashes) is provided such that any 313 // non-intersecting matches are removed. 314 // 315 // NOTE: filteredHashes may be empty if no previous condition has matched. 316 func (txi *TxIndex) match( 317 ctx context.Context, 318 c query.Condition, 319 startKeyBz []byte, 320 filteredHashes map[string][]byte, 321 firstRun bool, 322 ) map[string][]byte { 323 // A previous match was attempted but resulted in no matches, so we return 324 // no matches (assuming AND operand). 325 if !firstRun && len(filteredHashes) == 0 { 326 return filteredHashes 327 } 328 329 tmpHashes := make(map[string][]byte) 330 331 switch { 332 case c.Op == query.OpEqual: 333 it, err := dbm.IteratePrefix(txi.store, startKeyBz) 334 if err != nil { 335 panic(err) 336 } 337 defer it.Close() 338 339 for ; it.Valid(); it.Next() { 340 tmpHashes[string(it.Value())] = it.Value() 341 342 // Potentially exit early. 343 select { 344 case <-ctx.Done(): 345 break 346 default: 347 } 348 } 349 if err := it.Error(); err != nil { 350 panic(err) 351 } 352 353 case c.Op == query.OpExists: 354 // XXX: can't use startKeyBz here because c.Operand is nil 355 // (e.g. "account.owner/<nil>/" won't match w/ a single row) 356 it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) 357 if err != nil { 358 panic(err) 359 } 360 defer it.Close() 361 362 for ; it.Valid(); it.Next() { 363 tmpHashes[string(it.Value())] = it.Value() 364 365 // Potentially exit early. 366 select { 367 case <-ctx.Done(): 368 break 369 default: 370 } 371 } 372 if err := it.Error(); err != nil { 373 panic(err) 374 } 375 376 case c.Op == query.OpContains: 377 // XXX: startKey does not apply here. 378 // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" 379 // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" 380 it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) 381 if err != nil { 382 panic(err) 383 } 384 defer it.Close() 385 386 for ; it.Valid(); it.Next() { 387 if !isTagKey(it.Key()) { 388 continue 389 } 390 391 if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { 392 tmpHashes[string(it.Value())] = it.Value() 393 } 394 395 // Potentially exit early. 396 select { 397 case <-ctx.Done(): 398 break 399 default: 400 } 401 } 402 if err := it.Error(); err != nil { 403 panic(err) 404 } 405 default: 406 panic("other operators should be handled already") 407 } 408 409 if len(tmpHashes) == 0 || firstRun { 410 // Either: 411 // 412 // 1. Regardless if a previous match was attempted, which may have had 413 // results, but no match was found for the current condition, then we 414 // return no matches (assuming AND operand). 415 // 416 // 2. A previous match was not attempted, so we return all results. 417 return tmpHashes 418 } 419 420 // Remove/reduce matches in filteredHashes that were not found in this 421 // match (tmpHashes). 422 for k := range filteredHashes { 423 if tmpHashes[k] == nil { 424 delete(filteredHashes, k) 425 426 // Potentially exit early. 427 select { 428 case <-ctx.Done(): 429 break 430 default: 431 } 432 } 433 } 434 435 return filteredHashes 436 } 437 438 // matchRange returns all matching txs by hash that meet a given queryRange and 439 // start key. An already filtered result (filteredHashes) is provided such that 440 // any non-intersecting matches are removed. 441 // 442 // NOTE: filteredHashes may be empty if no previous condition has matched. 443 func (txi *TxIndex) matchRange( 444 ctx context.Context, 445 qr indexer.QueryRange, 446 startKey []byte, 447 filteredHashes map[string][]byte, 448 firstRun bool, 449 ) map[string][]byte { 450 // A previous match was attempted but resulted in no matches, so we return 451 // no matches (assuming AND operand). 452 if !firstRun && len(filteredHashes) == 0 { 453 return filteredHashes 454 } 455 456 tmpHashes := make(map[string][]byte) 457 lowerBound := qr.LowerBoundValue() 458 upperBound := qr.UpperBoundValue() 459 460 it, err := dbm.IteratePrefix(txi.store, startKey) 461 if err != nil { 462 panic(err) 463 } 464 defer it.Close() 465 466 LOOP: 467 for ; it.Valid(); it.Next() { 468 if !isTagKey(it.Key()) { 469 continue 470 } 471 472 if _, ok := qr.AnyBound().(int64); ok { 473 v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) 474 if err != nil { 475 continue LOOP 476 } 477 478 include := true 479 if lowerBound != nil && v < lowerBound.(int64) { 480 include = false 481 } 482 483 if upperBound != nil && v > upperBound.(int64) { 484 include = false 485 } 486 487 if include { 488 tmpHashes[string(it.Value())] = it.Value() 489 } 490 491 // XXX: passing time in a ABCI Events is not yet implemented 492 // case time.Time: 493 // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) 494 // if v == r.upperBound { 495 // break 496 // } 497 } 498 499 // Potentially exit early. 500 select { 501 case <-ctx.Done(): 502 break 503 default: 504 } 505 } 506 if err := it.Error(); err != nil { 507 panic(err) 508 } 509 510 if len(tmpHashes) == 0 || firstRun { 511 // Either: 512 // 513 // 1. Regardless if a previous match was attempted, which may have had 514 // results, but no match was found for the current condition, then we 515 // return no matches (assuming AND operand). 516 // 517 // 2. A previous match was not attempted, so we return all results. 518 return tmpHashes 519 } 520 521 // Remove/reduce matches in filteredHashes that were not found in this 522 // match (tmpHashes). 523 for k := range filteredHashes { 524 if tmpHashes[k] == nil { 525 delete(filteredHashes, k) 526 527 // Potentially exit early. 528 select { 529 case <-ctx.Done(): 530 break 531 default: 532 } 533 } 534 } 535 536 return filteredHashes 537 } 538 539 // Keys 540 541 func isTagKey(key []byte) bool { 542 return strings.Count(string(key), tagKeySeparator) == 3 543 } 544 545 func extractValueFromKey(key []byte) string { 546 parts := strings.SplitN(string(key), tagKeySeparator, 3) 547 return parts[1] 548 } 549 550 func keyForEvent(key string, value []byte, result *abci.TxResult) []byte { 551 return []byte(fmt.Sprintf("%s/%s/%d/%d", 552 key, 553 value, 554 result.Height, 555 result.Index, 556 )) 557 } 558 559 func keyForHeight(result *abci.TxResult) []byte { 560 return []byte(fmt.Sprintf("%s/%d/%d/%d", 561 types.TxHeightKey, 562 result.Height, 563 result.Height, 564 result.Index, 565 )) 566 } 567 568 func startKeyForCondition(c query.Condition, height int64) []byte { 569 if height > 0 { 570 return startKey(c.CompositeKey, c.Operand, height) 571 } 572 return startKey(c.CompositeKey, c.Operand) 573 } 574 575 func startKey(fields ...interface{}) []byte { 576 var b bytes.Buffer 577 for _, f := range fields { 578 b.Write([]byte(fmt.Sprintf("%v", f) + tagKeySeparator)) 579 } 580 return b.Bytes() 581 }