github.com/dawnbass68/maddcash@v0.0.0-20201001105353-c91c12cb36e5/db/sync.go (about) 1 package db 2 3 import ( 4 "blockbook/bchain" 5 "blockbook/common" 6 "os" 7 "sync" 8 "sync/atomic" 9 "time" 10 11 "github.com/golang/glog" 12 "github.com/juju/errors" 13 ) 14 15 // SyncWorker is handle to SyncWorker 16 type SyncWorker struct { 17 db *RocksDB 18 chain bchain.BlockChain 19 syncWorkers, syncChunk int 20 dryRun bool 21 startHeight uint32 22 startHash string 23 chanOsSignal chan os.Signal 24 metrics *common.Metrics 25 is *common.InternalState 26 } 27 28 // NewSyncWorker creates new SyncWorker and returns its handle 29 func NewSyncWorker(db *RocksDB, chain bchain.BlockChain, syncWorkers, syncChunk int, minStartHeight int, dryRun bool, chanOsSignal chan os.Signal, metrics *common.Metrics, is *common.InternalState) (*SyncWorker, error) { 30 if minStartHeight < 0 { 31 minStartHeight = 0 32 } 33 return &SyncWorker{ 34 db: db, 35 chain: chain, 36 syncWorkers: syncWorkers, 37 syncChunk: syncChunk, 38 dryRun: dryRun, 39 startHeight: uint32(minStartHeight), 40 chanOsSignal: chanOsSignal, 41 metrics: metrics, 42 is: is, 43 }, nil 44 } 45 46 var errSynced = errors.New("synced") 47 48 // ErrOperationInterrupted is returned when operation is interrupted by OS signal 49 var ErrOperationInterrupted = errors.New("ErrOperationInterrupted") 50 51 // ResyncIndex synchronizes index to the top of the blockchain 52 // onNewBlock is called when new block is connected, but not in initial parallel sync 53 func (w *SyncWorker) ResyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { 54 start := time.Now() 55 w.is.StartedSync() 56 57 err := w.resyncIndex(onNewBlock, initialSync) 58 59 switch err { 60 case nil: 61 d := time.Since(start) 62 glog.Info("resync: finished in ", d) 63 w.metrics.IndexResyncDuration.Observe(float64(d) / 1e6) // in milliseconds 64 w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) 65 bh, _, err := w.db.GetBestBlock() 66 if err == nil { 67 w.is.FinishedSync(bh) 68 } 69 return err 70 case errSynced: 71 // this is not actually error but flag that resync wasn't necessary 72 w.is.FinishedSyncNoChange() 73 w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) 74 if initialSync { 75 d := time.Since(start) 76 glog.Info("resync: finished in ", d) 77 } 78 return nil 79 } 80 81 w.metrics.IndexResyncErrors.With(common.Labels{"error": "failure"}).Inc() 82 83 return err 84 } 85 86 func (w *SyncWorker) resyncIndex(onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { 87 remoteBestHash, err := w.chain.GetBestBlockHash() 88 if err != nil { 89 return err 90 } 91 localBestHeight, localBestHash, err := w.db.GetBestBlock() 92 if err != nil { 93 return err 94 } 95 // If the locally indexed block is the same as the best block on the network, we're done. 96 if localBestHash == remoteBestHash { 97 glog.Infof("resync: synced at %d %s", localBestHeight, localBestHash) 98 return errSynced 99 } 100 if localBestHash != "" { 101 remoteHash, err := w.chain.GetBlockHash(localBestHeight) 102 // for some coins (eth) remote can be at lower best height after rollback 103 if err != nil && err != bchain.ErrBlockNotFound { 104 return err 105 } 106 if remoteHash != localBestHash { 107 // forked - the remote hash differs from the local hash at the same height 108 glog.Info("resync: local is forked at height ", localBestHeight, ", local hash ", localBestHash, ", remote hash", remoteHash) 109 return w.handleFork(localBestHeight, localBestHash, onNewBlock, initialSync) 110 } 111 glog.Info("resync: local at ", localBestHeight, " is behind") 112 w.startHeight = localBestHeight + 1 113 } else { 114 // database is empty, start genesis 115 glog.Info("resync: genesis from block ", w.startHeight) 116 } 117 w.startHash, err = w.chain.GetBlockHash(w.startHeight) 118 if err != nil { 119 return err 120 } 121 // if parallel operation is enabled and the number of blocks to be connected is large, 122 // use parallel routine to load majority of blocks 123 // use parallel sync only in case of initial sync because it puts the db to inconsistent state 124 if w.syncWorkers > 1 && initialSync { 125 remoteBestHeight, err := w.chain.GetBestBlockHeight() 126 if err != nil { 127 return err 128 } 129 if remoteBestHeight < w.startHeight { 130 glog.Error("resync: error - remote best height ", remoteBestHeight, " less than sync start height ", w.startHeight) 131 return errors.New("resync: remote best height error") 132 } 133 if remoteBestHeight-w.startHeight > uint32(w.syncChunk) { 134 glog.Infof("resync: parallel sync of blocks %d-%d, using %d workers", w.startHeight, remoteBestHeight, w.syncWorkers) 135 err = w.ConnectBlocksParallel(w.startHeight, remoteBestHeight) 136 if err != nil { 137 return err 138 } 139 // after parallel load finish the sync using standard way, 140 // new blocks may have been created in the meantime 141 return w.resyncIndex(onNewBlock, initialSync) 142 } 143 } 144 return w.connectBlocks(onNewBlock, initialSync) 145 } 146 147 func (w *SyncWorker) handleFork(localBestHeight uint32, localBestHash string, onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { 148 // find forked blocks, disconnect them and then synchronize again 149 var height uint32 150 hashes := []string{localBestHash} 151 for height = localBestHeight - 1; height >= 0; height-- { 152 local, err := w.db.GetBlockHash(height) 153 if err != nil { 154 return err 155 } 156 if local == "" { 157 break 158 } 159 remote, err := w.chain.GetBlockHash(height) 160 // for some coins (eth) remote can be at lower best height after rollback 161 if err != nil && err != bchain.ErrBlockNotFound { 162 return err 163 } 164 if local == remote { 165 break 166 } 167 hashes = append(hashes, local) 168 } 169 if err := w.DisconnectBlocks(height+1, localBestHeight, hashes); err != nil { 170 return err 171 } 172 return w.resyncIndex(onNewBlock, initialSync) 173 } 174 175 func (w *SyncWorker) connectBlocks(onNewBlock bchain.OnNewBlockFunc, initialSync bool) error { 176 bch := make(chan blockResult, 8) 177 done := make(chan struct{}) 178 defer close(done) 179 180 go w.getBlockChain(bch, done) 181 182 var lastRes, empty blockResult 183 184 connect := func(res blockResult) error { 185 lastRes = res 186 if res.err != nil { 187 return res.err 188 } 189 err := w.db.ConnectBlock(res.block) 190 if err != nil { 191 return err 192 } 193 if onNewBlock != nil { 194 onNewBlock(res.block.Hash, res.block.Height) 195 } 196 if res.block.Height > 0 && res.block.Height%1000 == 0 { 197 glog.Info("connected block ", res.block.Height, " ", res.block.Hash) 198 } 199 200 return nil 201 } 202 203 if initialSync { 204 ConnectLoop: 205 for { 206 select { 207 case <-w.chanOsSignal: 208 glog.Info("connectBlocks interrupted at height ", lastRes.block.Height) 209 return ErrOperationInterrupted 210 case res := <-bch: 211 if res == empty { 212 break ConnectLoop 213 } 214 err := connect(res) 215 if err != nil { 216 return err 217 } 218 } 219 } 220 } else { 221 // while regular sync, OS sig is handled by waitForSignalAndShutdown 222 for res := range bch { 223 err := connect(res) 224 if err != nil { 225 return err 226 } 227 } 228 } 229 230 if lastRes.block != nil { 231 glog.Infof("resync: synced at %d %s", lastRes.block.Height, lastRes.block.Hash) 232 } 233 234 return nil 235 } 236 237 // ConnectBlocksParallel uses parallel goroutines to get data from blockchain daemon 238 func (w *SyncWorker) ConnectBlocksParallel(lower, higher uint32) error { 239 type hashHeight struct { 240 hash string 241 height uint32 242 } 243 var err error 244 var wg sync.WaitGroup 245 bch := make([]chan *bchain.Block, w.syncWorkers) 246 for i := 0; i < w.syncWorkers; i++ { 247 bch[i] = make(chan *bchain.Block) 248 } 249 hch := make(chan hashHeight, w.syncWorkers) 250 hchClosed := atomic.Value{} 251 hchClosed.Store(false) 252 writeBlockDone := make(chan struct{}) 253 terminating := make(chan struct{}) 254 writeBlockWorker := func() { 255 defer close(writeBlockDone) 256 bc, err := w.db.InitBulkConnect() 257 if err != nil { 258 glog.Error("sync: InitBulkConnect error ", err) 259 } 260 lastBlock := lower - 1 261 keep := uint32(w.chain.GetChainParser().KeepBlockAddresses()) 262 WriteBlockLoop: 263 for { 264 select { 265 case b := <-bch[(lastBlock+1)%uint32(w.syncWorkers)]: 266 if b == nil { 267 // channel is closed and empty - work is done 268 break WriteBlockLoop 269 } 270 if b.Height != lastBlock+1 { 271 glog.Fatal("writeBlockWorker skipped block, expected block ", lastBlock+1, ", new block ", b.Height) 272 } 273 err := bc.ConnectBlock(b, b.Height+keep > higher) 274 if err != nil { 275 glog.Fatal("writeBlockWorker ", b.Height, " ", b.Hash, " error ", err) 276 } 277 lastBlock = b.Height 278 case <-terminating: 279 break WriteBlockLoop 280 } 281 } 282 err = bc.Close() 283 if err != nil { 284 glog.Error("sync: bulkconnect.Close error ", err) 285 } 286 glog.Info("WriteBlock exiting...") 287 } 288 getBlockWorker := func(i int) { 289 defer wg.Done() 290 var err error 291 var block *bchain.Block 292 GetBlockLoop: 293 for hh := range hch { 294 for { 295 block, err = w.chain.GetBlock(hh.hash, hh.height) 296 if err != nil { 297 // signal came while looping in the error loop 298 if hchClosed.Load() == true { 299 glog.Error("getBlockWorker ", i, " connect block error ", err, ". Exiting...") 300 return 301 } 302 glog.Error("getBlockWorker ", i, " connect block error ", err, ". Retrying...") 303 w.metrics.IndexResyncErrors.With(common.Labels{"error": "failure"}).Inc() 304 time.Sleep(time.Millisecond * 500) 305 } else { 306 break 307 } 308 } 309 if w.dryRun { 310 continue 311 } 312 select { 313 case bch[hh.height%uint32(w.syncWorkers)] <- block: 314 case <-terminating: 315 break GetBlockLoop 316 } 317 } 318 glog.Info("getBlockWorker ", i, " exiting...") 319 } 320 for i := 0; i < w.syncWorkers; i++ { 321 wg.Add(1) 322 go getBlockWorker(i) 323 } 324 go writeBlockWorker() 325 var hash string 326 start := time.Now() 327 msTime := time.Now().Add(1 * time.Minute) 328 ConnectLoop: 329 for h := lower; h <= higher; { 330 select { 331 case <-w.chanOsSignal: 332 glog.Info("connectBlocksParallel interrupted at height ", h) 333 err = ErrOperationInterrupted 334 // signal all workers to terminate their loops (error loops are interrupted below) 335 close(terminating) 336 break ConnectLoop 337 default: 338 hash, err = w.chain.GetBlockHash(h) 339 if err != nil { 340 glog.Error("GetBlockHash error ", err) 341 w.metrics.IndexResyncErrors.With(common.Labels{"error": "failure"}).Inc() 342 time.Sleep(time.Millisecond * 500) 343 continue 344 } 345 hch <- hashHeight{hash, h} 346 if h > 0 && h%1000 == 0 { 347 glog.Info("connecting block ", h, " ", hash, ", elapsed ", time.Since(start), " ", w.db.GetAndResetConnectBlockStats()) 348 start = time.Now() 349 } 350 if msTime.Before(time.Now()) { 351 glog.Info(w.db.GetMemoryStats()) 352 w.metrics.IndexDBSize.Set(float64(w.db.DatabaseSizeOnDisk())) 353 msTime = time.Now().Add(10 * time.Minute) 354 } 355 h++ 356 } 357 } 358 close(hch) 359 // signal stop to workers that are in a error loop 360 hchClosed.Store(true) 361 // wait for workers and close bch that will stop writer loop 362 wg.Wait() 363 for i := 0; i < w.syncWorkers; i++ { 364 close(bch[i]) 365 } 366 <-writeBlockDone 367 return err 368 } 369 370 type blockResult struct { 371 block *bchain.Block 372 err error 373 } 374 375 func (w *SyncWorker) getBlockChain(out chan blockResult, done chan struct{}) { 376 defer close(out) 377 378 hash := w.startHash 379 height := w.startHeight 380 381 // some coins do not return Next hash 382 // must loop until error 383 for { 384 select { 385 case <-done: 386 return 387 default: 388 } 389 block, err := w.chain.GetBlock(hash, height) 390 if err != nil { 391 if err == bchain.ErrBlockNotFound { 392 break 393 } 394 out <- blockResult{err: err} 395 return 396 } 397 hash = block.Next 398 height++ 399 out <- blockResult{block: block} 400 } 401 } 402 403 // DisconnectBlocks removes all data belonging to blocks in range lower-higher, 404 func (w *SyncWorker) DisconnectBlocks(lower uint32, higher uint32, hashes []string) error { 405 glog.Infof("sync: disconnecting blocks %d-%d", lower, higher) 406 ct := w.chain.GetChainParser().GetChainType() 407 if ct == bchain.ChainBitcoinType { 408 return w.db.DisconnectBlockRangeBitcoinType(lower, higher) 409 } else if ct == bchain.ChainEthereumType { 410 return w.db.DisconnectBlockRangeEthereumType(lower, higher) 411 } 412 return errors.New("Unknown chain type") 413 }