github.com/aakash4dev/cometbft@v0.38.2/state/indexer/block/kv/kv.go (about) 1 package kv 2 3 import ( 4 "bytes" 5 "context" 6 "errors" 7 "fmt" 8 "math/big" 9 "sort" 10 "strconv" 11 "strings" 12 13 "github.com/google/orderedcode" 14 15 dbm "github.com/aakash4dev/cometbft-db" 16 17 abci "github.com/aakash4dev/cometbft/abci/types" 18 idxutil "github.com/aakash4dev/cometbft/internal/indexer" 19 "github.com/aakash4dev/cometbft/libs/log" 20 "github.com/aakash4dev/cometbft/libs/pubsub/query" 21 "github.com/aakash4dev/cometbft/libs/pubsub/query/syntax" 22 "github.com/aakash4dev/cometbft/state/indexer" 23 "github.com/aakash4dev/cometbft/types" 24 ) 25 26 var _ indexer.BlockIndexer = (*BlockerIndexer)(nil) 27 28 // BlockerIndexer implements a block indexer, indexing FinalizeBlock 29 // events with an underlying KV store. Block events are indexed by their height, 30 // such that matching search criteria returns the respective block height(s). 31 type BlockerIndexer struct { 32 store dbm.DB 33 34 // Add unique event identifier to use when querying 35 // Matching will be done both on height AND eventSeq 36 eventSeq int64 37 log log.Logger 38 } 39 40 func New(store dbm.DB) *BlockerIndexer { 41 return &BlockerIndexer{ 42 store: store, 43 } 44 } 45 46 func (idx *BlockerIndexer) SetLogger(l log.Logger) { 47 idx.log = l 48 } 49 50 // Has returns true if the given height has been indexed. An error is returned 51 // upon database query failure. 52 func (idx *BlockerIndexer) Has(height int64) (bool, error) { 53 key, err := heightKey(height) 54 if err != nil { 55 return false, fmt.Errorf("failed to create block height index key: %w", err) 56 } 57 58 return idx.store.Has(key) 59 } 60 61 // Index indexes FinalizeBlock events for a given block by its height. 62 // The following is indexed: 63 // 64 // primary key: encode(block.height | height) => encode(height) 65 // FinalizeBlock events: encode(eventType.eventAttr|eventValue|height|finalize_block|eventSeq) => encode(height) 66 func (idx *BlockerIndexer) Index(bh types.EventDataNewBlockEvents) error { 67 batch := idx.store.NewBatch() 68 defer batch.Close() 69 70 height := bh.Height 71 72 // 1. index by height 73 key, err := heightKey(height) 74 if err != nil { 75 return fmt.Errorf("failed to create block height index key: %w", err) 76 } 77 if err := batch.Set(key, int64ToBytes(height)); err != nil { 78 return err 79 } 80 81 // 2. index block events 82 if err := idx.indexEvents(batch, bh.Events, height); err != nil { 83 return fmt.Errorf("failed to index FinalizeBlock events: %w", err) 84 } 85 86 return batch.WriteSync() 87 } 88 89 // Search performs a query for block heights that match a given FinalizeBlock 90 // event search criteria. The given query can match against zero, 91 // one or more block heights. In the case of height queries, i.e. block.height=H, 92 // if the height is indexed, that height alone will be returned. An error and 93 // nil slice is returned. Otherwise, a non-nil slice and nil error is returned. 94 func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, error) { 95 results := make([]int64, 0) 96 select { 97 case <-ctx.Done(): 98 return results, nil 99 100 default: 101 } 102 103 conditions := q.Syntax() 104 105 // conditions to skip because they're handled before "everything else" 106 skipIndexes := make([]int, 0) 107 108 var ok bool 109 110 var heightInfo HeightInfo 111 // If we are not matching events and block.height occurs more than once, the later value will 112 // overwrite the first one. 113 conditions, heightInfo, ok = dedupHeight(conditions) 114 115 // Extract ranges. If both upper and lower bounds exist, it's better to get 116 // them in order as to not iterate over kvs that are not within range. 117 ranges, rangeIndexes, heightRange := indexer.LookForRangesWithHeight(conditions) 118 heightInfo.heightRange = heightRange 119 120 // If we have additional constraints and want to query per event 121 // attributes, we cannot simply return all blocks for a height. 122 // But we remember the height we want to find and forward it to 123 // match(). If we only have the height constraint 124 // in the query (the second part of the ||), we don't need to query 125 // per event conditions and return all events within the height range. 126 if ok && heightInfo.onlyHeightEq { 127 ok, err := idx.Has(heightInfo.height) 128 if err != nil { 129 return nil, err 130 } 131 132 if ok { 133 return []int64{heightInfo.height}, nil 134 } 135 136 return results, nil 137 } 138 139 var heightsInitialized bool 140 filteredHeights := make(map[string][]byte) 141 if heightInfo.heightEqIdx != -1 { 142 skipIndexes = append(skipIndexes, heightInfo.heightEqIdx) 143 } 144 145 if len(ranges) > 0 { 146 skipIndexes = append(skipIndexes, rangeIndexes...) 147 148 for _, qr := range ranges { 149 // If we have a query range over height and want to still look for 150 // specific event values we do not want to simply return all 151 // blocks in this height range. We remember the height range info 152 // and pass it on to match() to take into account when processing events. 153 if qr.Key == types.BlockHeightKey && !heightInfo.onlyHeightRange { 154 // If the query contains ranges other than the height then we need to treat the height 155 // range when querying the conditions of the other range. 156 // Otherwise we can just return all the blocks within the height range (as there is no 157 // additional constraint on events) 158 159 continue 160 161 } 162 prefix, err := orderedcode.Append(nil, qr.Key) 163 if err != nil { 164 return nil, fmt.Errorf("failed to create prefix key: %w", err) 165 } 166 167 if !heightsInitialized { 168 filteredHeights, err = idx.matchRange(ctx, qr, prefix, filteredHeights, true, heightInfo) 169 if err != nil { 170 return nil, err 171 } 172 173 heightsInitialized = true 174 175 // Ignore any remaining conditions if the first condition resulted in no 176 // matches (assuming implicit AND operand). 177 if len(filteredHeights) == 0 { 178 break 179 } 180 } else { 181 filteredHeights, err = idx.matchRange(ctx, qr, prefix, filteredHeights, false, heightInfo) 182 if err != nil { 183 return nil, err 184 } 185 } 186 } 187 } 188 189 // for all other conditions 190 for i, c := range conditions { 191 if intInSlice(i, skipIndexes) { 192 continue 193 } 194 195 startKey, err := orderedcode.Append(nil, c.Tag, c.Arg.Value()) 196 if err != nil { 197 return nil, err 198 } 199 200 if !heightsInitialized { 201 filteredHeights, err = idx.match(ctx, c, startKey, filteredHeights, true, heightInfo) 202 if err != nil { 203 return nil, err 204 } 205 206 heightsInitialized = true 207 208 // Ignore any remaining conditions if the first condition resulted in no 209 // matches (assuming implicit AND operand). 210 if len(filteredHeights) == 0 { 211 break 212 } 213 } else { 214 filteredHeights, err = idx.match(ctx, c, startKey, filteredHeights, false, heightInfo) 215 if err != nil { 216 return nil, err 217 } 218 } 219 } 220 221 // fetch matching heights 222 results = make([]int64, 0, len(filteredHeights)) 223 resultMap := make(map[int64]struct{}) 224 for _, hBz := range filteredHeights { 225 h := int64FromBytes(hBz) 226 227 ok, err := idx.Has(h) 228 if err != nil { 229 return nil, err 230 } 231 if ok { 232 if _, ok := resultMap[h]; !ok { 233 resultMap[h] = struct{}{} 234 results = append(results, h) 235 } 236 } 237 238 select { 239 case <-ctx.Done(): 240 break 241 242 default: 243 } 244 } 245 246 sort.Slice(results, func(i, j int) bool { return results[i] < results[j] }) 247 248 return results, nil 249 } 250 251 // matchRange returns all matching block heights that match a given QueryRange 252 // and start key. An already filtered result (filteredHeights) is provided such 253 // that any non-intersecting matches are removed. 254 // 255 // NOTE: The provided filteredHeights may be empty if no previous condition has 256 // matched. 257 func (idx *BlockerIndexer) matchRange( 258 ctx context.Context, 259 qr indexer.QueryRange, 260 startKey []byte, 261 filteredHeights map[string][]byte, 262 firstRun bool, 263 heightInfo HeightInfo, 264 ) (map[string][]byte, error) { 265 // A previous match was attempted but resulted in no matches, so we return 266 // no matches (assuming AND operand). 267 if !firstRun && len(filteredHeights) == 0 { 268 return filteredHeights, nil 269 } 270 271 tmpHeights := make(map[string][]byte) 272 273 it, err := dbm.IteratePrefix(idx.store, startKey) 274 if err != nil { 275 return nil, fmt.Errorf("failed to create prefix iterator: %w", err) 276 } 277 defer it.Close() 278 279 LOOP: 280 for ; it.Valid(); it.Next() { 281 var ( 282 eventValue string 283 err error 284 ) 285 286 if qr.Key == types.BlockHeightKey { 287 eventValue, err = parseValueFromPrimaryKey(it.Key()) 288 } else { 289 eventValue, err = parseValueFromEventKey(it.Key()) 290 } 291 292 if err != nil { 293 continue 294 } 295 296 if _, ok := qr.AnyBound().(*big.Float); ok { 297 v := new(big.Int) 298 v, ok := v.SetString(eventValue, 10) 299 var vF *big.Float 300 if !ok { 301 // The precision here is 125. For numbers bigger than this, the value 302 // will not be parsed properly 303 vF, _, err = big.ParseFloat(eventValue, 10, 125, big.ToNearestEven) 304 if err != nil { 305 continue LOOP 306 } 307 } 308 309 if qr.Key != types.BlockHeightKey { 310 keyHeight, err := parseHeightFromEventKey(it.Key()) 311 if err != nil { 312 idx.log.Error("failure to parse height from key:", err) 313 continue LOOP 314 } 315 withinHeight, err := checkHeightConditions(heightInfo, keyHeight) 316 if err != nil { 317 idx.log.Error("failure checking for height bounds:", err) 318 continue LOOP 319 } 320 if !withinHeight { 321 continue LOOP 322 } 323 } 324 325 var withinBounds bool 326 var err error 327 if !ok { 328 withinBounds, err = idxutil.CheckBounds(qr, vF) 329 } else { 330 withinBounds, err = idxutil.CheckBounds(qr, v) 331 } 332 if err != nil { 333 idx.log.Error("failed to parse bounds:", err) 334 } else { 335 if withinBounds { 336 idx.setTmpHeights(tmpHeights, it) 337 } 338 } 339 } 340 341 select { 342 case <-ctx.Done(): 343 break 344 345 default: 346 } 347 } 348 349 if err := it.Error(); err != nil { 350 return nil, err 351 } 352 353 if len(tmpHeights) == 0 || firstRun { 354 // Either: 355 // 356 // 1. Regardless if a previous match was attempted, which may have had 357 // results, but no match was found for the current condition, then we 358 // return no matches (assuming AND operand). 359 // 360 // 2. A previous match was not attempted, so we return all results. 361 return tmpHeights, nil 362 } 363 364 // Remove/reduce matches in filteredHashes that were not found in this 365 // match (tmpHashes). 366 for k, v := range filteredHeights { 367 tmpHeight := tmpHeights[k] 368 369 // Check whether in this iteration we have not found an overlapping height (tmpHeight == nil) 370 // or whether the events in which the attributed occurred do not match (first part of the condition) 371 if tmpHeight == nil || !bytes.Equal(tmpHeight, v) { 372 delete(filteredHeights, k) 373 374 select { 375 case <-ctx.Done(): 376 break 377 378 default: 379 } 380 } 381 } 382 383 return filteredHeights, nil 384 } 385 386 func (idx *BlockerIndexer) setTmpHeights(tmpHeights map[string][]byte, it dbm.Iterator) { 387 // If we return attributes that occur within the same events, then store the event sequence in the 388 // result map as well 389 eventSeq, _ := parseEventSeqFromEventKey(it.Key()) 390 retVal := it.Value() 391 tmpHeights[string(retVal)+strconv.FormatInt(eventSeq, 10)] = it.Value() 392 393 } 394 395 // match returns all matching heights that meet a given query condition and start 396 // key. An already filtered result (filteredHeights) is provided such that any 397 // non-intersecting matches are removed. 398 // 399 // NOTE: The provided filteredHeights may be empty if no previous condition has 400 // matched. 401 func (idx *BlockerIndexer) match( 402 ctx context.Context, 403 c syntax.Condition, 404 startKeyBz []byte, 405 filteredHeights map[string][]byte, 406 firstRun bool, 407 heightInfo HeightInfo, 408 ) (map[string][]byte, error) { 409 // A previous match was attempted but resulted in no matches, so we return 410 // no matches (assuming AND operand). 411 if !firstRun && len(filteredHeights) == 0 { 412 return filteredHeights, nil 413 } 414 415 tmpHeights := make(map[string][]byte) 416 417 switch { 418 case c.Op == syntax.TEq: 419 it, err := dbm.IteratePrefix(idx.store, startKeyBz) 420 if err != nil { 421 return nil, fmt.Errorf("failed to create prefix iterator: %w", err) 422 } 423 defer it.Close() 424 425 for ; it.Valid(); it.Next() { 426 427 keyHeight, err := parseHeightFromEventKey(it.Key()) 428 if err != nil { 429 idx.log.Error("failure to parse height from key:", err) 430 continue 431 } 432 withinHeight, err := checkHeightConditions(heightInfo, keyHeight) 433 if err != nil { 434 idx.log.Error("failure checking for height bounds:", err) 435 continue 436 } 437 if !withinHeight { 438 continue 439 } 440 441 idx.setTmpHeights(tmpHeights, it) 442 443 if err := ctx.Err(); err != nil { 444 break 445 } 446 } 447 448 if err := it.Error(); err != nil { 449 return nil, err 450 } 451 452 case c.Op == syntax.TExists: 453 prefix, err := orderedcode.Append(nil, c.Tag) 454 if err != nil { 455 return nil, err 456 } 457 458 it, err := dbm.IteratePrefix(idx.store, prefix) 459 if err != nil { 460 return nil, fmt.Errorf("failed to create prefix iterator: %w", err) 461 } 462 defer it.Close() 463 464 for ; it.Valid(); it.Next() { 465 466 keyHeight, err := parseHeightFromEventKey(it.Key()) 467 if err != nil { 468 idx.log.Error("failure to parse height from key:", err) 469 continue 470 } 471 withinHeight, err := checkHeightConditions(heightInfo, keyHeight) 472 if err != nil { 473 idx.log.Error("failure checking for height bounds:", err) 474 continue 475 } 476 if !withinHeight { 477 continue 478 } 479 480 idx.setTmpHeights(tmpHeights, it) 481 482 select { 483 case <-ctx.Done(): 484 break 485 486 default: 487 } 488 } 489 490 if err := it.Error(); err != nil { 491 return nil, err 492 } 493 494 case c.Op == syntax.TContains: 495 prefix, err := orderedcode.Append(nil, c.Tag) 496 if err != nil { 497 return nil, err 498 } 499 500 it, err := dbm.IteratePrefix(idx.store, prefix) 501 if err != nil { 502 return nil, fmt.Errorf("failed to create prefix iterator: %w", err) 503 } 504 defer it.Close() 505 506 for ; it.Valid(); it.Next() { 507 eventValue, err := parseValueFromEventKey(it.Key()) 508 if err != nil { 509 continue 510 } 511 512 if strings.Contains(eventValue, c.Arg.Value()) { 513 keyHeight, err := parseHeightFromEventKey(it.Key()) 514 if err != nil { 515 idx.log.Error("failure to parse height from key:", err) 516 continue 517 } 518 withinHeight, err := checkHeightConditions(heightInfo, keyHeight) 519 if err != nil { 520 idx.log.Error("failure checking for height bounds:", err) 521 continue 522 } 523 if !withinHeight { 524 continue 525 } 526 idx.setTmpHeights(tmpHeights, it) 527 } 528 529 select { 530 case <-ctx.Done(): 531 break 532 533 default: 534 } 535 } 536 if err := it.Error(); err != nil { 537 return nil, err 538 } 539 540 default: 541 return nil, errors.New("other operators should be handled already") 542 } 543 544 if len(tmpHeights) == 0 || firstRun { 545 // Either: 546 // 547 // 1. Regardless if a previous match was attempted, which may have had 548 // results, but no match was found for the current condition, then we 549 // return no matches (assuming AND operand). 550 // 551 // 2. A previous match was not attempted, so we return all results. 552 return tmpHeights, nil 553 } 554 555 // Remove/reduce matches in filteredHeights that were not found in this 556 // match (tmpHeights). 557 for k, v := range filteredHeights { 558 tmpHeight := tmpHeights[k] 559 if tmpHeight == nil || !bytes.Equal(tmpHeight, v) { 560 delete(filteredHeights, k) 561 562 select { 563 case <-ctx.Done(): 564 break 565 566 default: 567 } 568 } 569 } 570 571 return filteredHeights, nil 572 } 573 574 func (idx *BlockerIndexer) indexEvents(batch dbm.Batch, events []abci.Event, height int64) error { 575 heightBz := int64ToBytes(height) 576 577 for _, event := range events { 578 idx.eventSeq = idx.eventSeq + 1 579 // only index events with a non-empty type 580 if len(event.Type) == 0 { 581 continue 582 } 583 584 for _, attr := range event.Attributes { 585 if len(attr.Key) == 0 { 586 continue 587 } 588 589 // index iff the event specified index:true and it's not a reserved event 590 compositeKey := fmt.Sprintf("%s.%s", event.Type, attr.Key) 591 if compositeKey == types.BlockHeightKey { 592 return fmt.Errorf("event type and attribute key \"%s\" is reserved; please use a different key", compositeKey) 593 } 594 595 if attr.GetIndex() { 596 key, err := eventKey(compositeKey, attr.Value, height, idx.eventSeq) 597 if err != nil { 598 return fmt.Errorf("failed to create block index key: %w", err) 599 } 600 601 if err := batch.Set(key, heightBz); err != nil { 602 return err 603 } 604 } 605 } 606 } 607 608 return nil 609 }