github.com/Unheilbar/quorum@v1.0.0/core/rawdb/freezer.go (about) 1 // Copyright 2019 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package rawdb 18 19 import ( 20 "errors" 21 "fmt" 22 "math" 23 "os" 24 "path/filepath" 25 "sync" 26 "sync/atomic" 27 "time" 28 29 "github.com/ethereum/go-ethereum/common" 30 "github.com/ethereum/go-ethereum/ethdb" 31 "github.com/ethereum/go-ethereum/log" 32 "github.com/ethereum/go-ethereum/metrics" 33 "github.com/ethereum/go-ethereum/params" 34 "github.com/prometheus/tsdb/fileutil" 35 ) 36 37 var ( 38 // errReadOnly is returned if the freezer is opened in read only mode. All the 39 // mutations are disallowed. 40 errReadOnly = errors.New("read only") 41 42 // errUnknownTable is returned if the user attempts to read from a table that is 43 // not tracked by the freezer. 44 errUnknownTable = errors.New("unknown table") 45 46 // errOutOrderInsertion is returned if the user attempts to inject out-of-order 47 // binary blobs into the freezer. 48 errOutOrderInsertion = errors.New("the append operation is out-order") 49 50 // errSymlinkDatadir is returned if the ancient directory specified by user 51 // is a symbolic link. 52 errSymlinkDatadir = errors.New("symbolic link datadir is not supported") 53 ) 54 55 const ( 56 // freezerRecheckInterval is the frequency to check the key-value database for 57 // chain progression that might permit new blocks to be frozen into immutable 58 // storage. 59 freezerRecheckInterval = time.Minute 60 61 // freezerBatchLimit is the maximum number of blocks to freeze in one batch 62 // before doing an fsync and deleting it from the key-value store. 63 freezerBatchLimit = 30000 64 ) 65 66 // freezer is an memory mapped append-only database to store immutable chain data 67 // into flat files: 68 // 69 // - The append only nature ensures that disk writes are minimized. 70 // - The memory mapping ensures we can max out system memory for caching without 71 // reserving it for go-ethereum. This would also reduce the memory requirements 72 // of Geth, and thus also GC overhead. 73 type freezer struct { 74 // WARNING: The `frozen` field is accessed atomically. On 32 bit platforms, only 75 // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, 76 // so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG). 77 frozen uint64 // Number of blocks already frozen 78 threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) 79 80 readonly bool 81 tables map[string]*freezerTable // Data tables for storing everything 82 instanceLock fileutil.Releaser // File-system lock to prevent double opens 83 84 trigger chan chan struct{} // Manual blocking freeze trigger, test determinism 85 86 quit chan struct{} 87 closeOnce sync.Once 88 } 89 90 // newFreezer creates a chain freezer that moves ancient chain data into 91 // append-only flat file containers. 92 func newFreezer(datadir string, namespace string, readonly bool) (*freezer, error) { 93 // Create the initial freezer object 94 var ( 95 readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) 96 writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil) 97 sizeGauge = metrics.NewRegisteredGauge(namespace+"ancient/size", nil) 98 ) 99 // Ensure the datadir is not a symbolic link if it exists. 100 if info, err := os.Lstat(datadir); !os.IsNotExist(err) { 101 if info.Mode()&os.ModeSymlink != 0 { 102 log.Warn("Symbolic link ancient database is not supported", "path", datadir) 103 return nil, errSymlinkDatadir 104 } 105 } 106 // Leveldb uses LOCK as the filelock filename. To prevent the 107 // name collision, we use FLOCK as the lock name. 108 lock, _, err := fileutil.Flock(filepath.Join(datadir, "FLOCK")) 109 if err != nil { 110 return nil, err 111 } 112 // Open all the supported data tables 113 freezer := &freezer{ 114 readonly: readonly, 115 threshold: params.FullImmutabilityThreshold, 116 tables: make(map[string]*freezerTable), 117 instanceLock: lock, 118 trigger: make(chan chan struct{}), 119 quit: make(chan struct{}), 120 } 121 for name, disableSnappy := range FreezerNoSnappy { 122 table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, disableSnappy) 123 if err != nil { 124 for _, table := range freezer.tables { 125 table.Close() 126 } 127 lock.Release() 128 return nil, err 129 } 130 freezer.tables[name] = table 131 } 132 if err := freezer.repair(); err != nil { 133 for _, table := range freezer.tables { 134 table.Close() 135 } 136 lock.Release() 137 return nil, err 138 } 139 log.Info("Opened ancient database", "database", datadir, "readonly", readonly) 140 return freezer, nil 141 } 142 143 // Close terminates the chain freezer, unmapping all the data files. 144 func (f *freezer) Close() error { 145 var errs []error 146 f.closeOnce.Do(func() { 147 close(f.quit) 148 for _, table := range f.tables { 149 if err := table.Close(); err != nil { 150 errs = append(errs, err) 151 } 152 } 153 if err := f.instanceLock.Release(); err != nil { 154 errs = append(errs, err) 155 } 156 }) 157 if errs != nil { 158 return fmt.Errorf("%v", errs) 159 } 160 return nil 161 } 162 163 // HasAncient returns an indicator whether the specified ancient data exists 164 // in the freezer. 165 func (f *freezer) HasAncient(kind string, number uint64) (bool, error) { 166 if table := f.tables[kind]; table != nil { 167 return table.has(number), nil 168 } 169 return false, nil 170 } 171 172 // Ancient retrieves an ancient binary blob from the append-only immutable files. 173 func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) { 174 if table := f.tables[kind]; table != nil { 175 return table.Retrieve(number) 176 } 177 return nil, errUnknownTable 178 } 179 180 // Ancients returns the length of the frozen items. 181 func (f *freezer) Ancients() (uint64, error) { 182 return atomic.LoadUint64(&f.frozen), nil 183 } 184 185 // AncientSize returns the ancient size of the specified category. 186 func (f *freezer) AncientSize(kind string) (uint64, error) { 187 if table := f.tables[kind]; table != nil { 188 return table.size() 189 } 190 return 0, errUnknownTable 191 } 192 193 // AppendAncient injects all binary blobs belong to block at the end of the 194 // append-only immutable table files. 195 // 196 // Notably, this function is lock free but kind of thread-safe. All out-of-order 197 // injection will be rejected. But if two injections with same number happen at 198 // the same time, we can get into the trouble. 199 func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td []byte) (err error) { 200 if f.readonly { 201 return errReadOnly 202 } 203 // Ensure the binary blobs we are appending is continuous with freezer. 204 if atomic.LoadUint64(&f.frozen) != number { 205 return errOutOrderInsertion 206 } 207 // Rollback all inserted data if any insertion below failed to ensure 208 // the tables won't out of sync. 209 defer func() { 210 if err != nil { 211 rerr := f.repair() 212 if rerr != nil { 213 log.Crit("Failed to repair freezer", "err", rerr) 214 } 215 log.Info("Append ancient failed", "number", number, "err", err) 216 } 217 }() 218 // Inject all the components into the relevant data tables 219 if err := f.tables[freezerHashTable].Append(f.frozen, hash[:]); err != nil { 220 log.Error("Failed to append ancient hash", "number", f.frozen, "hash", hash, "err", err) 221 return err 222 } 223 if err := f.tables[freezerHeaderTable].Append(f.frozen, header); err != nil { 224 log.Error("Failed to append ancient header", "number", f.frozen, "hash", hash, "err", err) 225 return err 226 } 227 if err := f.tables[freezerBodiesTable].Append(f.frozen, body); err != nil { 228 log.Error("Failed to append ancient body", "number", f.frozen, "hash", hash, "err", err) 229 return err 230 } 231 if err := f.tables[freezerReceiptTable].Append(f.frozen, receipts); err != nil { 232 log.Error("Failed to append ancient receipts", "number", f.frozen, "hash", hash, "err", err) 233 return err 234 } 235 if err := f.tables[freezerDifficultyTable].Append(f.frozen, td); err != nil { 236 log.Error("Failed to append ancient difficulty", "number", f.frozen, "hash", hash, "err", err) 237 return err 238 } 239 atomic.AddUint64(&f.frozen, 1) // Only modify atomically 240 return nil 241 } 242 243 // TruncateAncients discards any recent data above the provided threshold number. 244 func (f *freezer) TruncateAncients(items uint64) error { 245 if f.readonly { 246 return errReadOnly 247 } 248 if atomic.LoadUint64(&f.frozen) <= items { 249 return nil 250 } 251 for _, table := range f.tables { 252 if err := table.truncate(items); err != nil { 253 return err 254 } 255 } 256 atomic.StoreUint64(&f.frozen, items) 257 return nil 258 } 259 260 // Sync flushes all data tables to disk. 261 func (f *freezer) Sync() error { 262 return f.SyncRetry(1, 1*time.Second) 263 } 264 265 // SyncRetry 266 // Quorum 267 // add retry to sync 268 func (f *freezer) SyncRetry(retry uint8, delay time.Duration) error { 269 var errs []error 270 for _, table := range f.tables { 271 if err := table.Sync(); err != nil { 272 errs = append(errs, err) 273 } 274 } 275 hasError := len(errs) > 0 276 if hasError && retry < 5 { 277 log.Info("sync", "retry", retry, "errors", errs) 278 time.Sleep(delay) 279 return f.SyncRetry(retry+1, delay*2) 280 } else if hasError { 281 return fmt.Errorf("%v", errs) 282 } 283 return nil 284 } 285 286 // End Quorum 287 288 // freeze is a background thread that periodically checks the blockchain for any 289 // import progress and moves ancient data from the fast database into the freezer. 290 // 291 // This functionality is deliberately broken off from block importing to avoid 292 // incurring additional data shuffling delays on block propagation. 293 func (f *freezer) freeze(db ethdb.KeyValueStore) { 294 nfdb := &nofreezedb{KeyValueStore: db} 295 296 var ( 297 backoff bool 298 triggered chan struct{} // Used in tests 299 ) 300 for { 301 select { 302 case <-f.quit: 303 log.Info("Freezer shutting down") 304 return 305 default: 306 } 307 if backoff { 308 // If we were doing a manual trigger, notify it 309 if triggered != nil { 310 triggered <- struct{}{} 311 triggered = nil 312 } 313 select { 314 case <-time.NewTimer(freezerRecheckInterval).C: 315 backoff = false 316 case triggered = <-f.trigger: 317 backoff = false 318 case <-f.quit: 319 return 320 } 321 } 322 // Retrieve the freezing threshold. 323 hash := ReadHeadBlockHash(nfdb) 324 if hash == (common.Hash{}) { 325 log.Debug("Current full block hash unavailable") // new chain, empty database 326 backoff = true 327 continue 328 } 329 number := ReadHeaderNumber(nfdb, hash) 330 threshold := int(atomic.LoadUint64(&f.threshold)) 331 332 switch { 333 case number == nil: 334 log.Error("Current full block number unavailable", "hash", hash) 335 backoff = true 336 continue 337 338 case *number < uint64(params.GetImmutabilityThresholdWithDefault(threshold)): 339 log.Debug("Current full block not old enough", "number", *number, "hash", hash, "delay", params.GetImmutabilityThresholdWithDefault(threshold)) 340 backoff = true 341 continue 342 343 case *number-uint64(params.GetImmutabilityThresholdWithDefault(threshold)) <= f.frozen: 344 log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", f.frozen) 345 backoff = true 346 continue 347 } 348 head := ReadHeader(nfdb, hash, *number) 349 if head == nil { 350 log.Error("Current full block unavailable", "number", *number, "hash", hash) 351 backoff = true 352 continue 353 } 354 // Seems we have data ready to be frozen, process in usable batches 355 limit := *number - uint64(params.GetImmutabilityThresholdWithDefault(threshold)) 356 if limit-f.frozen > freezerBatchLimit { 357 limit = f.frozen + freezerBatchLimit 358 } 359 var ( 360 start = time.Now() 361 first = f.frozen 362 ancients = make([]common.Hash, 0, limit-f.frozen) 363 ) 364 for f.frozen <= limit { 365 // Retrieves all the components of the canonical block 366 hash := ReadCanonicalHash(nfdb, f.frozen) 367 if hash == (common.Hash{}) { 368 log.Error("Canonical hash missing, can't freeze", "number", f.frozen) 369 break 370 } 371 header := ReadHeaderRLP(nfdb, hash, f.frozen) 372 if len(header) == 0 { 373 log.Error("Block header missing, can't freeze", "number", f.frozen, "hash", hash) 374 break 375 } 376 body := ReadBodyRLP(nfdb, hash, f.frozen) 377 if len(body) == 0 { 378 log.Error("Block body missing, can't freeze", "number", f.frozen, "hash", hash) 379 break 380 } 381 receipts := ReadReceiptsRLP(nfdb, hash, f.frozen) 382 if len(receipts) == 0 { 383 log.Error("Block receipts missing, can't freeze", "number", f.frozen, "hash", hash) 384 break 385 } 386 td := ReadTdRLP(nfdb, hash, f.frozen) 387 if len(td) == 0 { 388 log.Error("Total difficulty missing, can't freeze", "number", f.frozen, "hash", hash) 389 break 390 } 391 log.Trace("Deep froze ancient block", "number", f.frozen, "hash", hash) 392 // Inject all the components into the relevant data tables 393 if err := f.AppendAncient(f.frozen, hash[:], header, body, receipts, td); err != nil { 394 break 395 } 396 ancients = append(ancients, hash) 397 } 398 // Batch of blocks have been frozen, flush them before wiping from leveldb 399 if err := f.Sync(); err != nil { 400 log.Crit("Failed to flush frozen tables", "err", err) 401 } 402 // Wipe out all data from the active database 403 batch := db.NewBatch() 404 for i := 0; i < len(ancients); i++ { 405 // Always keep the genesis block in active database 406 if first+uint64(i) != 0 { 407 DeleteBlockWithoutNumber(batch, ancients[i], first+uint64(i)) 408 DeleteCanonicalHash(batch, first+uint64(i)) 409 } 410 } 411 if err := batch.Write(); err != nil { 412 log.Crit("Failed to delete frozen canonical blocks", "err", err) 413 } 414 batch.Reset() 415 416 // Wipe out side chains also and track dangling side chians 417 var dangling []common.Hash 418 for number := first; number < f.frozen; number++ { 419 // Always keep the genesis block in active database 420 if number != 0 { 421 dangling = ReadAllHashes(db, number) 422 for _, hash := range dangling { 423 log.Trace("Deleting side chain", "number", number, "hash", hash) 424 DeleteBlock(batch, hash, number) 425 } 426 } 427 } 428 if err := batch.Write(); err != nil { 429 log.Crit("Failed to delete frozen side blocks", "err", err) 430 } 431 batch.Reset() 432 433 // Step into the future and delete and dangling side chains 434 if f.frozen > 0 { 435 tip := f.frozen 436 for len(dangling) > 0 { 437 drop := make(map[common.Hash]struct{}) 438 for _, hash := range dangling { 439 log.Debug("Dangling parent from freezer", "number", tip-1, "hash", hash) 440 drop[hash] = struct{}{} 441 } 442 children := ReadAllHashes(db, tip) 443 for i := 0; i < len(children); i++ { 444 // Dig up the child and ensure it's dangling 445 child := ReadHeader(nfdb, children[i], tip) 446 if child == nil { 447 log.Error("Missing dangling header", "number", tip, "hash", children[i]) 448 continue 449 } 450 if _, ok := drop[child.ParentHash]; !ok { 451 children = append(children[:i], children[i+1:]...) 452 i-- 453 continue 454 } 455 // Delete all block data associated with the child 456 log.Debug("Deleting dangling block", "number", tip, "hash", children[i], "parent", child.ParentHash) 457 DeleteBlock(batch, children[i], tip) 458 } 459 dangling = children 460 tip++ 461 } 462 if err := batch.Write(); err != nil { 463 log.Crit("Failed to delete dangling side blocks", "err", err) 464 } 465 } 466 // Log something friendly for the user 467 context := []interface{}{ 468 "blocks", f.frozen - first, "elapsed", common.PrettyDuration(time.Since(start)), "number", f.frozen - 1, 469 } 470 if n := len(ancients); n > 0 { 471 context = append(context, []interface{}{"hash", ancients[n-1]}...) 472 } 473 log.Info("Deep froze chain segment", context...) 474 475 // Avoid database thrashing with tiny writes 476 if f.frozen-first < freezerBatchLimit { 477 backoff = true 478 } 479 } 480 } 481 482 // repair truncates all data tables to the same length. 483 func (f *freezer) repair() error { 484 min := uint64(math.MaxUint64) 485 for _, table := range f.tables { 486 items := atomic.LoadUint64(&table.items) 487 if min > items { 488 min = items 489 } 490 } 491 for _, table := range f.tables { 492 if err := table.truncate(min); err != nil { 493 return err 494 } 495 } 496 atomic.StoreUint64(&f.frozen, min) 497 return nil 498 }