gitee.com/liu-zhao234568/cntest@v1.0.0/core/chain_indexer.go (about) 1 // Copyright 2017 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package core 18 19 import ( 20 "context" 21 "encoding/binary" 22 "fmt" 23 "sync" 24 "sync/atomic" 25 "time" 26 27 "gitee.com/liu-zhao234568/cntest/common" 28 "gitee.com/liu-zhao234568/cntest/core/rawdb" 29 "gitee.com/liu-zhao234568/cntest/core/types" 30 "gitee.com/liu-zhao234568/cntest/ethdb" 31 "gitee.com/liu-zhao234568/cntest/event" 32 "gitee.com/liu-zhao234568/cntest/log" 33 ) 34 35 // ChainIndexerBackend defines the methods needed to process chain segments in 36 // the background and write the segment results into the database. These can be 37 // used to create filter blooms or CHTs. 38 type ChainIndexerBackend interface { 39 // Reset initiates the processing of a new chain segment, potentially terminating 40 // any partially completed operations (in case of a reorg). 41 Reset(ctx context.Context, section uint64, prevHead common.Hash) error 42 43 // Process crunches through the next header in the chain segment. The caller 44 // will ensure a sequential order of headers. 45 Process(ctx context.Context, header *types.Header) error 46 47 // Commit finalizes the section metadata and stores it into the database. 48 Commit() error 49 50 // Prune deletes the chain index older than the given threshold. 51 Prune(threshold uint64) error 52 } 53 54 // ChainIndexerChain interface is used for connecting the indexer to a blockchain 55 type ChainIndexerChain interface { 56 // CurrentHeader retrieves the latest locally known header. 57 CurrentHeader() *types.Header 58 59 // SubscribeChainHeadEvent subscribes to new head header notifications. 60 SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription 61 } 62 63 // ChainIndexer does a post-processing job for equally sized sections of the 64 // canonical chain (like BlooomBits and CHT structures). A ChainIndexer is 65 // connected to the blockchain through the event system by starting a 66 // ChainHeadEventLoop in a goroutine. 67 // 68 // Further child ChainIndexers can be added which use the output of the parent 69 // section indexer. These child indexers receive new head notifications only 70 // after an entire section has been finished or in case of rollbacks that might 71 // affect already finished sections. 72 type ChainIndexer struct { 73 chainDb ethdb.Database // Chain database to index the data from 74 indexDb ethdb.Database // Prefixed table-view of the db to write index metadata into 75 backend ChainIndexerBackend // Background processor generating the index data content 76 children []*ChainIndexer // Child indexers to cascade chain updates to 77 78 active uint32 // Flag whether the event loop was started 79 update chan struct{} // Notification channel that headers should be processed 80 quit chan chan error // Quit channel to tear down running goroutines 81 ctx context.Context 82 ctxCancel func() 83 84 sectionSize uint64 // Number of blocks in a single chain segment to process 85 confirmsReq uint64 // Number of confirmations before processing a completed segment 86 87 storedSections uint64 // Number of sections successfully indexed into the database 88 knownSections uint64 // Number of sections known to be complete (block wise) 89 cascadedHead uint64 // Block number of the last completed section cascaded to subindexers 90 91 checkpointSections uint64 // Number of sections covered by the checkpoint 92 checkpointHead common.Hash // Section head belonging to the checkpoint 93 94 throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources 95 96 log log.Logger 97 lock sync.Mutex 98 } 99 100 // NewChainIndexer creates a new chain indexer to do background processing on 101 // chain segments of a given size after certain number of confirmations passed. 102 // The throttling parameter might be used to prevent database thrashing. 103 func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer { 104 c := &ChainIndexer{ 105 chainDb: chainDb, 106 indexDb: indexDb, 107 backend: backend, 108 update: make(chan struct{}, 1), 109 quit: make(chan chan error), 110 sectionSize: section, 111 confirmsReq: confirm, 112 throttling: throttling, 113 log: log.New("type", kind), 114 } 115 // Initialize database dependent fields and start the updater 116 c.loadValidSections() 117 c.ctx, c.ctxCancel = context.WithCancel(context.Background()) 118 119 go c.updateLoop() 120 121 return c 122 } 123 124 // AddCheckpoint adds a checkpoint. Sections are never processed and the chain 125 // is not expected to be available before this point. The indexer assumes that 126 // the backend has sufficient information available to process subsequent sections. 127 // 128 // Note: knownSections == 0 and storedSections == checkpointSections until 129 // syncing reaches the checkpoint 130 func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) { 131 c.lock.Lock() 132 defer c.lock.Unlock() 133 134 // Short circuit if the given checkpoint is below than local's. 135 if c.checkpointSections >= section+1 || section < c.storedSections { 136 return 137 } 138 c.checkpointSections = section + 1 139 c.checkpointHead = shead 140 141 c.setSectionHead(section, shead) 142 c.setValidSections(section + 1) 143 } 144 145 // Start creates a goroutine to feed chain head events into the indexer for 146 // cascading background processing. Children do not need to be started, they 147 // are notified about new events by their parents. 148 func (c *ChainIndexer) Start(chain ChainIndexerChain) { 149 events := make(chan ChainHeadEvent, 10) 150 sub := chain.SubscribeChainHeadEvent(events) 151 152 go c.eventLoop(chain.CurrentHeader(), events, sub) 153 } 154 155 // Close tears down all goroutines belonging to the indexer and returns any error 156 // that might have occurred internally. 157 func (c *ChainIndexer) Close() error { 158 var errs []error 159 160 c.ctxCancel() 161 162 // Tear down the primary update loop 163 errc := make(chan error) 164 c.quit <- errc 165 if err := <-errc; err != nil { 166 errs = append(errs, err) 167 } 168 // If needed, tear down the secondary event loop 169 if atomic.LoadUint32(&c.active) != 0 { 170 c.quit <- errc 171 if err := <-errc; err != nil { 172 errs = append(errs, err) 173 } 174 } 175 // Close all children 176 for _, child := range c.children { 177 if err := child.Close(); err != nil { 178 errs = append(errs, err) 179 } 180 } 181 // Return any failures 182 switch { 183 case len(errs) == 0: 184 return nil 185 186 case len(errs) == 1: 187 return errs[0] 188 189 default: 190 return fmt.Errorf("%v", errs) 191 } 192 } 193 194 // eventLoop is a secondary - optional - event loop of the indexer which is only 195 // started for the outermost indexer to push chain head events into a processing 196 // queue. 197 func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) { 198 // Mark the chain indexer as active, requiring an additional teardown 199 atomic.StoreUint32(&c.active, 1) 200 201 defer sub.Unsubscribe() 202 203 // Fire the initial new head event to start any outstanding processing 204 c.newHead(currentHeader.Number.Uint64(), false) 205 206 var ( 207 prevHeader = currentHeader 208 prevHash = currentHeader.Hash() 209 ) 210 for { 211 select { 212 case errc := <-c.quit: 213 // Chain indexer terminating, report no failure and abort 214 errc <- nil 215 return 216 217 case ev, ok := <-events: 218 // Received a new event, ensure it's not nil (closing) and update 219 if !ok { 220 errc := <-c.quit 221 errc <- nil 222 return 223 } 224 header := ev.Block.Header() 225 if header.ParentHash != prevHash { 226 // Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then) 227 // TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly? 228 229 if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash { 230 if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, header); h != nil { 231 c.newHead(h.Number.Uint64(), true) 232 } 233 } 234 } 235 c.newHead(header.Number.Uint64(), false) 236 237 prevHeader, prevHash = header, header.Hash() 238 } 239 } 240 } 241 242 // newHead notifies the indexer about new chain heads and/or reorgs. 243 func (c *ChainIndexer) newHead(head uint64, reorg bool) { 244 c.lock.Lock() 245 defer c.lock.Unlock() 246 247 // If a reorg happened, invalidate all sections until that point 248 if reorg { 249 // Revert the known section number to the reorg point 250 known := (head + 1) / c.sectionSize 251 stored := known 252 if known < c.checkpointSections { 253 known = 0 254 } 255 if stored < c.checkpointSections { 256 stored = c.checkpointSections 257 } 258 if known < c.knownSections { 259 c.knownSections = known 260 } 261 // Revert the stored sections from the database to the reorg point 262 if stored < c.storedSections { 263 c.setValidSections(stored) 264 } 265 // Update the new head number to the finalized section end and notify children 266 head = known * c.sectionSize 267 268 if head < c.cascadedHead { 269 c.cascadedHead = head 270 for _, child := range c.children { 271 child.newHead(c.cascadedHead, true) 272 } 273 } 274 return 275 } 276 // No reorg, calculate the number of newly known sections and update if high enough 277 var sections uint64 278 if head >= c.confirmsReq { 279 sections = (head + 1 - c.confirmsReq) / c.sectionSize 280 if sections < c.checkpointSections { 281 sections = 0 282 } 283 if sections > c.knownSections { 284 if c.knownSections < c.checkpointSections { 285 // syncing reached the checkpoint, verify section head 286 syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1) 287 if syncedHead != c.checkpointHead { 288 c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead) 289 return 290 } 291 } 292 c.knownSections = sections 293 294 select { 295 case c.update <- struct{}{}: 296 default: 297 } 298 } 299 } 300 } 301 302 // updateLoop is the main event loop of the indexer which pushes chain segments 303 // down into the processing backend. 304 func (c *ChainIndexer) updateLoop() { 305 var ( 306 updating bool 307 updated time.Time 308 ) 309 310 for { 311 select { 312 case errc := <-c.quit: 313 // Chain indexer terminating, report no failure and abort 314 errc <- nil 315 return 316 317 case <-c.update: 318 // Section headers completed (or rolled back), update the index 319 c.lock.Lock() 320 if c.knownSections > c.storedSections { 321 // Periodically print an upgrade log message to the user 322 if time.Since(updated) > 8*time.Second { 323 if c.knownSections > c.storedSections+1 { 324 updating = true 325 c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections) 326 } 327 updated = time.Now() 328 } 329 // Cache the current section count and head to allow unlocking the mutex 330 c.verifyLastHead() 331 section := c.storedSections 332 var oldHead common.Hash 333 if section > 0 { 334 oldHead = c.SectionHead(section - 1) 335 } 336 // Process the newly defined section in the background 337 c.lock.Unlock() 338 newHead, err := c.processSection(section, oldHead) 339 if err != nil { 340 select { 341 case <-c.ctx.Done(): 342 <-c.quit <- nil 343 return 344 default: 345 } 346 c.log.Error("Section processing failed", "error", err) 347 } 348 c.lock.Lock() 349 350 // If processing succeeded and no reorgs occurred, mark the section completed 351 if err == nil && (section == 0 || oldHead == c.SectionHead(section-1)) { 352 c.setSectionHead(section, newHead) 353 c.setValidSections(section + 1) 354 if c.storedSections == c.knownSections && updating { 355 updating = false 356 c.log.Info("Finished upgrading chain index") 357 } 358 c.cascadedHead = c.storedSections*c.sectionSize - 1 359 for _, child := range c.children { 360 c.log.Trace("Cascading chain index update", "head", c.cascadedHead) 361 child.newHead(c.cascadedHead, false) 362 } 363 } else { 364 // If processing failed, don't retry until further notification 365 c.log.Debug("Chain index processing failed", "section", section, "err", err) 366 c.verifyLastHead() 367 c.knownSections = c.storedSections 368 } 369 } 370 // If there are still further sections to process, reschedule 371 if c.knownSections > c.storedSections { 372 time.AfterFunc(c.throttling, func() { 373 select { 374 case c.update <- struct{}{}: 375 default: 376 } 377 }) 378 } 379 c.lock.Unlock() 380 } 381 } 382 } 383 384 // processSection processes an entire section by calling backend functions while 385 // ensuring the continuity of the passed headers. Since the chain mutex is not 386 // held while processing, the continuity can be broken by a long reorg, in which 387 // case the function returns with an error. 388 func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (common.Hash, error) { 389 c.log.Trace("Processing new chain section", "section", section) 390 391 // Reset and partial processing 392 if err := c.backend.Reset(c.ctx, section, lastHead); err != nil { 393 c.setValidSections(0) 394 return common.Hash{}, err 395 } 396 397 for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { 398 hash := rawdb.ReadCanonicalHash(c.chainDb, number) 399 if hash == (common.Hash{}) { 400 return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number) 401 } 402 header := rawdb.ReadHeader(c.chainDb, hash, number) 403 if header == nil { 404 return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4]) 405 } else if header.ParentHash != lastHead { 406 return common.Hash{}, fmt.Errorf("chain reorged during section processing") 407 } 408 if err := c.backend.Process(c.ctx, header); err != nil { 409 return common.Hash{}, err 410 } 411 lastHead = header.Hash() 412 } 413 if err := c.backend.Commit(); err != nil { 414 return common.Hash{}, err 415 } 416 return lastHead, nil 417 } 418 419 // verifyLastHead compares last stored section head with the corresponding block hash in the 420 // actual canonical chain and rolls back reorged sections if necessary to ensure that stored 421 // sections are all valid 422 func (c *ChainIndexer) verifyLastHead() { 423 for c.storedSections > 0 && c.storedSections > c.checkpointSections { 424 if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) { 425 return 426 } 427 c.setValidSections(c.storedSections - 1) 428 } 429 } 430 431 // Sections returns the number of processed sections maintained by the indexer 432 // and also the information about the last header indexed for potential canonical 433 // verifications. 434 func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) { 435 c.lock.Lock() 436 defer c.lock.Unlock() 437 438 c.verifyLastHead() 439 return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1) 440 } 441 442 // AddChildIndexer adds a child ChainIndexer that can use the output of this one 443 func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) { 444 if indexer == c { 445 panic("can't add indexer as a child of itself") 446 } 447 c.lock.Lock() 448 defer c.lock.Unlock() 449 450 c.children = append(c.children, indexer) 451 452 // Cascade any pending updates to new children too 453 sections := c.storedSections 454 if c.knownSections < sections { 455 // if a section is "stored" but not "known" then it is a checkpoint without 456 // available chain data so we should not cascade it yet 457 sections = c.knownSections 458 } 459 if sections > 0 { 460 indexer.newHead(sections*c.sectionSize-1, false) 461 } 462 } 463 464 // Prune deletes all chain data older than given threshold. 465 func (c *ChainIndexer) Prune(threshold uint64) error { 466 return c.backend.Prune(threshold) 467 } 468 469 // loadValidSections reads the number of valid sections from the index database 470 // and caches is into the local state. 471 func (c *ChainIndexer) loadValidSections() { 472 data, _ := c.indexDb.Get([]byte("count")) 473 if len(data) == 8 { 474 c.storedSections = binary.BigEndian.Uint64(data) 475 } 476 } 477 478 // setValidSections writes the number of valid sections to the index database 479 func (c *ChainIndexer) setValidSections(sections uint64) { 480 // Set the current number of valid sections in the database 481 var data [8]byte 482 binary.BigEndian.PutUint64(data[:], sections) 483 c.indexDb.Put([]byte("count"), data[:]) 484 485 // Remove any reorged sections, caching the valids in the mean time 486 for c.storedSections > sections { 487 c.storedSections-- 488 c.removeSectionHead(c.storedSections) 489 } 490 c.storedSections = sections // needed if new > old 491 } 492 493 // SectionHead retrieves the last block hash of a processed section from the 494 // index database. 495 func (c *ChainIndexer) SectionHead(section uint64) common.Hash { 496 var data [8]byte 497 binary.BigEndian.PutUint64(data[:], section) 498 499 hash, _ := c.indexDb.Get(append([]byte("shead"), data[:]...)) 500 if len(hash) == len(common.Hash{}) { 501 return common.BytesToHash(hash) 502 } 503 return common.Hash{} 504 } 505 506 // setSectionHead writes the last block hash of a processed section to the index 507 // database. 508 func (c *ChainIndexer) setSectionHead(section uint64, hash common.Hash) { 509 var data [8]byte 510 binary.BigEndian.PutUint64(data[:], section) 511 512 c.indexDb.Put(append([]byte("shead"), data[:]...), hash.Bytes()) 513 } 514 515 // removeSectionHead removes the reference to a processed section from the index 516 // database. 517 func (c *ChainIndexer) removeSectionHead(section uint64) { 518 var data [8]byte 519 binary.BigEndian.PutUint64(data[:], section) 520 521 c.indexDb.Delete(append([]byte("shead"), data[:]...)) 522 }