github.com/bchainhub/blockbook@v0.3.2/db/rocksdb.go (about) 1 package db 2 3 import ( 4 "blockbook/bchain" 5 "blockbook/common" 6 "bytes" 7 "encoding/binary" 8 "encoding/hex" 9 "encoding/json" 10 "fmt" 11 "math/big" 12 "os" 13 "path/filepath" 14 "sort" 15 "strconv" 16 "time" 17 "unsafe" 18 19 vlq "github.com/bsm/go-vlq" 20 "github.com/golang/glog" 21 "github.com/juju/errors" 22 "github.com/tecbot/gorocksdb" 23 ) 24 25 const dbVersion = 5 26 27 const packedHeightBytes = 4 28 const maxAddrDescLen = 1024 29 30 // iterator creates snapshot, which takes lots of resources 31 // when doing huge scan, it is better to close it and reopen from time to time to free the resources 32 const refreshIterator = 5000000 33 34 // FiatRatesTimeFormat is a format string for storing FiatRates timestamps in rocksdb 35 const FiatRatesTimeFormat = "20060102150405" // YYYYMMDDhhmmss 36 37 // CurrencyRatesTicker contains coin ticker data fetched from API 38 type CurrencyRatesTicker struct { 39 Timestamp *time.Time // return as unix timestamp in API 40 Rates map[string]float64 41 } 42 43 // ResultTickerAsString contains formatted CurrencyRatesTicker data 44 type ResultTickerAsString struct { 45 Timestamp int64 `json:"ts,omitempty"` 46 Rates map[string]float64 `json:"rates"` 47 Error string `json:"error,omitempty"` 48 } 49 50 // ResultTickersAsString contains a formatted CurrencyRatesTicker list 51 type ResultTickersAsString struct { 52 Tickers []ResultTickerAsString `json:"tickers"` 53 } 54 55 // ResultTickerListAsString contains formatted data about available currency tickers 56 type ResultTickerListAsString struct { 57 Timestamp int64 `json:"ts,omitempty"` 58 Tickers []string `json:"available_currencies"` 59 Error string `json:"error,omitempty"` 60 } 61 62 // RepairRocksDB calls RocksDb db repair function 63 func RepairRocksDB(name string) error { 64 glog.Infof("rocksdb: repair") 65 opts := gorocksdb.NewDefaultOptions() 66 return gorocksdb.RepairDb(name, opts) 67 } 68 69 type connectBlockStats struct { 70 txAddressesHit int 71 txAddressesMiss int 72 balancesHit int 73 balancesMiss int 74 } 75 76 // AddressBalanceDetail specifies what data are returned by GetAddressBalance 77 type AddressBalanceDetail int 78 79 const ( 80 // AddressBalanceDetailNoUTXO returns address balance without utxos 81 AddressBalanceDetailNoUTXO = 0 82 // AddressBalanceDetailUTXO returns address balance with utxos 83 AddressBalanceDetailUTXO = 1 84 // addressBalanceDetailUTXOIndexed returns address balance with utxos and index for updates, used only internally 85 addressBalanceDetailUTXOIndexed = 2 86 ) 87 88 // RocksDB handle 89 type RocksDB struct { 90 path string 91 db *gorocksdb.DB 92 wo *gorocksdb.WriteOptions 93 ro *gorocksdb.ReadOptions 94 cfh []*gorocksdb.ColumnFamilyHandle 95 chainParser bchain.BlockChainParser 96 is *common.InternalState 97 metrics *common.Metrics 98 cache *gorocksdb.Cache 99 maxOpenFiles int 100 cbs connectBlockStats 101 } 102 103 const ( 104 cfDefault = iota 105 cfHeight 106 cfAddresses 107 cfBlockTxs 108 cfTransactions 109 cfFiatRates 110 // BitcoinType 111 cfAddressBalance 112 cfTxAddresses 113 // EthereumType 114 cfAddressContracts = cfAddressBalance 115 ) 116 117 // common columns 118 var cfNames []string 119 var cfBaseNames = []string{"default", "height", "addresses", "blockTxs", "transactions", "fiatRates"} 120 121 // type specific columns 122 var cfNamesBitcoinType = []string{"addressBalance", "txAddresses"} 123 var cfNamesEthereumType = []string{"addressContracts"} 124 125 func openDB(path string, c *gorocksdb.Cache, openFiles int) (*gorocksdb.DB, []*gorocksdb.ColumnFamilyHandle, error) { 126 // opts with bloom filter 127 opts := createAndSetDBOptions(10, c, openFiles) 128 // opts for addresses without bloom filter 129 // from documentation: if most of your queries are executed using iterators, you shouldn't set bloom filter 130 optsAddresses := createAndSetDBOptions(0, c, openFiles) 131 // default, height, addresses, blockTxids, transactions 132 cfOptions := []*gorocksdb.Options{opts, opts, optsAddresses, opts, opts, opts} 133 // append type specific options 134 count := len(cfNames) - len(cfOptions) 135 for i := 0; i < count; i++ { 136 cfOptions = append(cfOptions, opts) 137 } 138 db, cfh, err := gorocksdb.OpenDbColumnFamilies(opts, path, cfNames, cfOptions) 139 if err != nil { 140 return nil, nil, err 141 } 142 return db, cfh, nil 143 } 144 145 // NewRocksDB opens an internal handle to RocksDB environment. Close 146 // needs to be called to release it. 147 func NewRocksDB(path string, cacheSize, maxOpenFiles int, parser bchain.BlockChainParser, metrics *common.Metrics) (d *RocksDB, err error) { 148 glog.Infof("rocksdb: opening %s, required data version %v, cache size %v, max open files %v", path, dbVersion, cacheSize, maxOpenFiles) 149 150 cfNames = append([]string{}, cfBaseNames...) 151 chainType := parser.GetChainType() 152 if chainType == bchain.ChainBitcoinType { 153 cfNames = append(cfNames, cfNamesBitcoinType...) 154 } else if chainType == bchain.ChainEthereumType { 155 cfNames = append(cfNames, cfNamesEthereumType...) 156 } else { 157 return nil, errors.New("Unknown chain type") 158 } 159 160 c := gorocksdb.NewLRUCache(cacheSize) 161 db, cfh, err := openDB(path, c, maxOpenFiles) 162 if err != nil { 163 return nil, err 164 } 165 wo := gorocksdb.NewDefaultWriteOptions() 166 ro := gorocksdb.NewDefaultReadOptions() 167 return &RocksDB{path, db, wo, ro, cfh, parser, nil, metrics, c, maxOpenFiles, connectBlockStats{}}, nil 168 } 169 170 func (d *RocksDB) closeDB() error { 171 for _, h := range d.cfh { 172 h.Destroy() 173 } 174 d.db.Close() 175 d.db = nil 176 return nil 177 } 178 179 // FiatRatesConvertDate checks if the date is in correct format and returns the Time object. 180 // Possible formats are: YYYYMMDDhhmmss, YYYYMMDDhhmm, YYYYMMDDhh, YYYYMMDD 181 func FiatRatesConvertDate(date string) (*time.Time, error) { 182 for format := FiatRatesTimeFormat; len(format) >= 8; format = format[:len(format)-2] { 183 convertedDate, err := time.Parse(format, date) 184 if err == nil { 185 return &convertedDate, nil 186 } 187 } 188 msg := "Date \"" + date + "\" does not match any of available formats. " 189 msg += "Possible formats are: YYYYMMDDhhmmss, YYYYMMDDhhmm, YYYYMMDDhh, YYYYMMDD" 190 return nil, errors.New(msg) 191 } 192 193 // FiatRatesStoreTicker stores ticker data at the specified time 194 func (d *RocksDB) FiatRatesStoreTicker(ticker *CurrencyRatesTicker) error { 195 if len(ticker.Rates) == 0 { 196 return errors.New("Error storing ticker: empty rates") 197 } else if ticker.Timestamp == nil { 198 return errors.New("Error storing ticker: empty timestamp") 199 } 200 ratesMarshalled, err := json.Marshal(ticker.Rates) 201 if err != nil { 202 glog.Error("Error marshalling ticker rates: ", err) 203 return err 204 } 205 timeFormatted := ticker.Timestamp.UTC().Format(FiatRatesTimeFormat) 206 err = d.db.PutCF(d.wo, d.cfh[cfFiatRates], []byte(timeFormatted), ratesMarshalled) 207 if err != nil { 208 glog.Error("Error storing ticker: ", err) 209 return err 210 } 211 return nil 212 } 213 214 // FiatRatesFindTicker gets FiatRates data closest to the specified timestamp 215 func (d *RocksDB) FiatRatesFindTicker(tickerTime *time.Time) (*CurrencyRatesTicker, error) { 216 ticker := &CurrencyRatesTicker{} 217 tickerTimeFormatted := tickerTime.UTC().Format(FiatRatesTimeFormat) 218 it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates]) 219 defer it.Close() 220 221 for it.Seek([]byte(tickerTimeFormatted)); it.Valid(); it.Next() { 222 timeObj, err := time.Parse(FiatRatesTimeFormat, string(it.Key().Data())) 223 if err != nil { 224 glog.Error("FiatRatesFindTicker time parse error: ", err) 225 return nil, err 226 } 227 timeObj = timeObj.UTC() 228 ticker.Timestamp = &timeObj 229 err = json.Unmarshal(it.Value().Data(), &ticker.Rates) 230 if err != nil { 231 glog.Error("FiatRatesFindTicker error unpacking rates: ", err) 232 return nil, err 233 } 234 break 235 } 236 if err := it.Err(); err != nil { 237 glog.Error("FiatRatesFindTicker Iterator error: ", err) 238 return nil, err 239 } 240 if !it.Valid() { 241 return nil, nil // ticker not found 242 } 243 return ticker, nil 244 } 245 246 // FiatRatesFindLastTicker gets the last FiatRates record 247 func (d *RocksDB) FiatRatesFindLastTicker() (*CurrencyRatesTicker, error) { 248 ticker := &CurrencyRatesTicker{} 249 it := d.db.NewIteratorCF(d.ro, d.cfh[cfFiatRates]) 250 defer it.Close() 251 252 for it.SeekToLast(); it.Valid(); it.Next() { 253 timeObj, err := time.Parse(FiatRatesTimeFormat, string(it.Key().Data())) 254 if err != nil { 255 glog.Error("FiatRatesFindTicker time parse error: ", err) 256 return nil, err 257 } 258 timeObj = timeObj.UTC() 259 ticker.Timestamp = &timeObj 260 err = json.Unmarshal(it.Value().Data(), &ticker.Rates) 261 if err != nil { 262 glog.Error("FiatRatesFindTicker error unpacking rates: ", err) 263 return nil, err 264 } 265 break 266 } 267 if err := it.Err(); err != nil { 268 glog.Error("FiatRatesFindLastTicker Iterator error: ", err) 269 return ticker, err 270 } 271 if !it.Valid() { 272 return nil, nil // ticker not found 273 } 274 return ticker, nil 275 } 276 277 // Close releases the RocksDB environment opened in NewRocksDB. 278 func (d *RocksDB) Close() error { 279 if d.db != nil { 280 // store the internal state of the app 281 if d.is != nil && d.is.DbState == common.DbStateOpen { 282 d.is.DbState = common.DbStateClosed 283 if err := d.StoreInternalState(d.is); err != nil { 284 glog.Info("internalState: ", err) 285 } 286 } 287 glog.Infof("rocksdb: close") 288 d.closeDB() 289 d.wo.Destroy() 290 d.ro.Destroy() 291 } 292 return nil 293 } 294 295 // Reopen reopens the database 296 // It closes and reopens db, nobody can access the database during the operation! 297 func (d *RocksDB) Reopen() error { 298 err := d.closeDB() 299 if err != nil { 300 return err 301 } 302 d.db = nil 303 db, cfh, err := openDB(d.path, d.cache, d.maxOpenFiles) 304 if err != nil { 305 return err 306 } 307 d.db, d.cfh = db, cfh 308 return nil 309 } 310 311 func atoi(s string) int { 312 i, err := strconv.Atoi(s) 313 if err != nil { 314 return 0 315 } 316 return i 317 } 318 319 // GetMemoryStats returns memory usage statistics as reported by RocksDB 320 func (d *RocksDB) GetMemoryStats() string { 321 var total, indexAndFilter, memtable int 322 type columnStats struct { 323 name string 324 indexAndFilter string 325 memtable string 326 } 327 cs := make([]columnStats, len(cfNames)) 328 for i := 0; i < len(cfNames); i++ { 329 cs[i].name = cfNames[i] 330 cs[i].indexAndFilter = d.db.GetPropertyCF("rocksdb.estimate-table-readers-mem", d.cfh[i]) 331 cs[i].memtable = d.db.GetPropertyCF("rocksdb.cur-size-all-mem-tables", d.cfh[i]) 332 indexAndFilter += atoi(cs[i].indexAndFilter) 333 memtable += atoi(cs[i].memtable) 334 } 335 m := struct { 336 cacheUsage int 337 pinnedCacheUsage int 338 columns []columnStats 339 }{ 340 cacheUsage: d.cache.GetUsage(), 341 pinnedCacheUsage: d.cache.GetPinnedUsage(), 342 columns: cs, 343 } 344 total = m.cacheUsage + indexAndFilter + memtable 345 return fmt.Sprintf("Total %d, indexAndFilter %d, memtable %d, %+v", total, indexAndFilter, memtable, m) 346 } 347 348 // StopIteration is returned by callback function to signal stop of iteration 349 type StopIteration struct{} 350 351 func (e *StopIteration) Error() string { 352 return "" 353 } 354 355 // GetTransactionsCallback is called by GetTransactions/GetAddrDescTransactions for each found tx 356 // indexes contain array of indexes (input negative, output positive) in tx where is given address 357 type GetTransactionsCallback func(txid string, height uint32, indexes []int32) error 358 359 // GetTransactions finds all input/output transactions for address 360 // Transaction are passed to callback function. 361 func (d *RocksDB) GetTransactions(address string, lower uint32, higher uint32, fn GetTransactionsCallback) (err error) { 362 if glog.V(1) { 363 glog.Infof("rocksdb: address get %s %d-%d ", address, lower, higher) 364 } 365 addrDesc, err := d.chainParser.GetAddrDescFromAddress(address) 366 if err != nil { 367 return err 368 } 369 return d.GetAddrDescTransactions(addrDesc, lower, higher, fn) 370 } 371 372 // GetAddrDescTransactions finds all input/output transactions for address descriptor 373 // Transaction are passed to callback function in the order from newest block to the oldest 374 func (d *RocksDB) GetAddrDescTransactions(addrDesc bchain.AddressDescriptor, lower uint32, higher uint32, fn GetTransactionsCallback) (err error) { 375 txidUnpackedLen := d.chainParser.PackedTxidLen() 376 addrDescLen := len(addrDesc) 377 startKey := packAddressKey(addrDesc, higher) 378 stopKey := packAddressKey(addrDesc, lower) 379 indexes := make([]int32, 0, 16) 380 it := d.db.NewIteratorCF(d.ro, d.cfh[cfAddresses]) 381 defer it.Close() 382 for it.Seek(startKey); it.Valid(); it.Next() { 383 key := it.Key().Data() 384 if bytes.Compare(key, stopKey) > 0 { 385 break 386 } 387 if len(key) != addrDescLen+packedHeightBytes { 388 if glog.V(2) { 389 glog.Warningf("rocksdb: addrDesc %s - mixed with %s", addrDesc, hex.EncodeToString(key)) 390 } 391 continue 392 } 393 val := it.Value().Data() 394 if glog.V(2) { 395 glog.Infof("rocksdb: addresses %s: %s", hex.EncodeToString(key), hex.EncodeToString(val)) 396 } 397 _, height, err := unpackAddressKey(key) 398 if err != nil { 399 return err 400 } 401 for len(val) > txidUnpackedLen { 402 tx, err := d.chainParser.UnpackTxid(val[:txidUnpackedLen]) 403 if err != nil { 404 return err 405 } 406 indexes = indexes[:0] 407 val = val[txidUnpackedLen:] 408 for { 409 index, l := unpackVarint32(val) 410 indexes = append(indexes, index>>1) 411 val = val[l:] 412 if index&1 == 1 { 413 break 414 } else if len(val) == 0 { 415 glog.Warningf("rocksdb: addresses contain incorrect data %s: %s", hex.EncodeToString(key), hex.EncodeToString(val)) 416 break 417 } 418 } 419 if err := fn(tx, height, indexes); err != nil { 420 if _, ok := err.(*StopIteration); ok { 421 return nil 422 } 423 return err 424 } 425 } 426 if len(val) != 0 { 427 glog.Warningf("rocksdb: addresses contain incorrect data %s: %s", hex.EncodeToString(key), hex.EncodeToString(val)) 428 } 429 } 430 return nil 431 } 432 433 const ( 434 opInsert = 0 435 opDelete = 1 436 ) 437 438 // ConnectBlock indexes addresses in the block and stores them in db 439 func (d *RocksDB) ConnectBlock(block *bchain.Block) error { 440 wb := gorocksdb.NewWriteBatch() 441 defer wb.Destroy() 442 443 if glog.V(2) { 444 glog.Infof("rocksdb: insert %d %s", block.Height, block.Hash) 445 } 446 447 chainType := d.chainParser.GetChainType() 448 449 if err := d.writeHeightFromBlock(wb, block, opInsert); err != nil { 450 return err 451 } 452 addresses := make(addressesMap) 453 if chainType == bchain.ChainBitcoinType { 454 txAddressesMap := make(map[string]*TxAddresses) 455 balances := make(map[string]*AddrBalance) 456 if err := d.processAddressesBitcoinType(block, addresses, txAddressesMap, balances); err != nil { 457 return err 458 } 459 if err := d.storeTxAddresses(wb, txAddressesMap); err != nil { 460 return err 461 } 462 if err := d.storeBalances(wb, balances); err != nil { 463 return err 464 } 465 if err := d.storeAndCleanupBlockTxs(wb, block); err != nil { 466 return err 467 } 468 } else if chainType == bchain.ChainEthereumType { 469 addressContracts := make(map[string]*AddrContracts) 470 blockTxs, err := d.processAddressesEthereumType(block, addresses, addressContracts) 471 if err != nil { 472 return err 473 } 474 if err := d.storeAddressContracts(wb, addressContracts); err != nil { 475 return err 476 } 477 if err := d.storeAndCleanupBlockTxsEthereumType(wb, block, blockTxs); err != nil { 478 return err 479 } 480 } else { 481 return errors.New("Unknown chain type") 482 } 483 if err := d.storeAddresses(wb, block.Height, addresses); err != nil { 484 return err 485 } 486 if err := d.db.Write(d.wo, wb); err != nil { 487 return err 488 } 489 d.is.AppendBlockTime(uint32(block.Time)) 490 return nil 491 } 492 493 // Addresses index 494 495 type txIndexes struct { 496 btxID []byte 497 indexes []int32 498 } 499 500 // addressesMap is a map of addresses in a block 501 // each address contains a slice of transactions with indexes where the address appears 502 // slice is used instead of map so that order is defined and also search in case of few items 503 type addressesMap map[string][]txIndexes 504 505 type outpoint struct { 506 btxID []byte 507 index int32 508 } 509 510 // TxInput holds input data of the transaction in TxAddresses 511 type TxInput struct { 512 AddrDesc bchain.AddressDescriptor 513 ValueSat big.Int 514 } 515 516 // Addresses converts AddressDescriptor of the input to array of strings 517 func (ti *TxInput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { 518 return p.GetAddressesFromAddrDesc(ti.AddrDesc) 519 } 520 521 // TxOutput holds output data of the transaction in TxAddresses 522 type TxOutput struct { 523 AddrDesc bchain.AddressDescriptor 524 Spent bool 525 ValueSat big.Int 526 } 527 528 // Addresses converts AddressDescriptor of the output to array of strings 529 func (to *TxOutput) Addresses(p bchain.BlockChainParser) ([]string, bool, error) { 530 return p.GetAddressesFromAddrDesc(to.AddrDesc) 531 } 532 533 // TxAddresses stores transaction inputs and outputs with amounts 534 type TxAddresses struct { 535 Height uint32 536 Inputs []TxInput 537 Outputs []TxOutput 538 } 539 540 // Utxo holds information about unspent transaction output 541 type Utxo struct { 542 BtxID []byte 543 Vout int32 544 Height uint32 545 ValueSat big.Int 546 } 547 548 // AddrBalance stores number of transactions and balances of an address 549 type AddrBalance struct { 550 Txs uint32 551 SentSat big.Int 552 BalanceSat big.Int 553 Utxos []Utxo 554 utxosMap map[string]int 555 } 556 557 // ReceivedSat computes received amount from total balance and sent amount 558 func (ab *AddrBalance) ReceivedSat() *big.Int { 559 var r big.Int 560 r.Add(&ab.BalanceSat, &ab.SentSat) 561 return &r 562 } 563 564 // addUtxo 565 func (ab *AddrBalance) addUtxo(u *Utxo) { 566 ab.Utxos = append(ab.Utxos, *u) 567 ab.manageUtxoMap(u) 568 } 569 570 func (ab *AddrBalance) manageUtxoMap(u *Utxo) { 571 l := len(ab.Utxos) 572 if l >= 16 { 573 if len(ab.utxosMap) == 0 { 574 ab.utxosMap = make(map[string]int, 32) 575 for i := 0; i < l; i++ { 576 s := string(ab.Utxos[i].BtxID) 577 if _, e := ab.utxosMap[s]; !e { 578 ab.utxosMap[s] = i 579 } 580 } 581 } else { 582 s := string(u.BtxID) 583 if _, e := ab.utxosMap[s]; !e { 584 ab.utxosMap[s] = l - 1 585 } 586 } 587 } 588 } 589 590 // on disconnect, the added utxos must be inserted in the right position so that utxosMap index works 591 func (ab *AddrBalance) addUtxoInDisconnect(u *Utxo) { 592 insert := -1 593 if len(ab.utxosMap) > 0 { 594 if i, e := ab.utxosMap[string(u.BtxID)]; e { 595 insert = i 596 } 597 } else { 598 for i := range ab.Utxos { 599 utxo := &ab.Utxos[i] 600 if *(*int)(unsafe.Pointer(&utxo.BtxID[0])) == *(*int)(unsafe.Pointer(&u.BtxID[0])) && bytes.Equal(utxo.BtxID, u.BtxID) { 601 insert = i 602 break 603 } 604 } 605 } 606 if insert > -1 { 607 // check if it is necessary to insert the utxo into the array 608 for i := insert; i < len(ab.Utxos); i++ { 609 utxo := &ab.Utxos[i] 610 // either the vout is greater than the inserted vout or it is a different tx 611 if utxo.Vout > u.Vout || *(*int)(unsafe.Pointer(&utxo.BtxID[0])) != *(*int)(unsafe.Pointer(&u.BtxID[0])) || !bytes.Equal(utxo.BtxID, u.BtxID) { 612 // found the right place, insert the utxo 613 ab.Utxos = append(ab.Utxos, *u) 614 copy(ab.Utxos[i+1:], ab.Utxos[i:]) 615 ab.Utxos[i] = *u 616 // reset utxosMap after insert, the index will have to be rebuilt if needed 617 ab.utxosMap = nil 618 return 619 } 620 } 621 } 622 ab.Utxos = append(ab.Utxos, *u) 623 ab.manageUtxoMap(u) 624 } 625 626 // markUtxoAsSpent finds outpoint btxID:vout in utxos and marks it as spent 627 // for small number of utxos the linear search is done, for larger number there is a hashmap index 628 // it is much faster than removing the utxo from the slice as it would cause in memory reallocations 629 func (ab *AddrBalance) markUtxoAsSpent(btxID []byte, vout int32) { 630 if len(ab.utxosMap) == 0 { 631 for i := range ab.Utxos { 632 utxo := &ab.Utxos[i] 633 if utxo.Vout == vout && *(*int)(unsafe.Pointer(&utxo.BtxID[0])) == *(*int)(unsafe.Pointer(&btxID[0])) && bytes.Equal(utxo.BtxID, btxID) { 634 // mark utxo as spent by setting vout=-1 635 utxo.Vout = -1 636 return 637 } 638 } 639 } else { 640 if i, e := ab.utxosMap[string(btxID)]; e { 641 l := len(ab.Utxos) 642 for ; i < l; i++ { 643 utxo := &ab.Utxos[i] 644 if utxo.Vout == vout { 645 if bytes.Equal(utxo.BtxID, btxID) { 646 // mark utxo as spent by setting vout=-1 647 utxo.Vout = -1 648 return 649 } 650 break 651 } 652 } 653 } 654 } 655 glog.Errorf("Utxo %s:%d not found, utxosMap size %d", hex.EncodeToString(btxID), vout, len(ab.utxosMap)) 656 } 657 658 type blockTxs struct { 659 btxID []byte 660 inputs []outpoint 661 } 662 663 func (d *RocksDB) resetValueSatToZero(valueSat *big.Int, addrDesc bchain.AddressDescriptor, logText string) { 664 ad, _, err := d.chainParser.GetAddressesFromAddrDesc(addrDesc) 665 if err != nil { 666 glog.Warningf("rocksdb: unparsable address hex '%v' reached negative %s %v, resetting to 0. Parser error %v", addrDesc, logText, valueSat.String(), err) 667 } else { 668 glog.Warningf("rocksdb: address %v hex '%v' reached negative %s %v, resetting to 0", ad, addrDesc, logText, valueSat.String()) 669 } 670 valueSat.SetInt64(0) 671 } 672 673 // GetAndResetConnectBlockStats gets statistics about cache usage in connect blocks and resets the counters 674 func (d *RocksDB) GetAndResetConnectBlockStats() string { 675 s := fmt.Sprintf("%+v", d.cbs) 676 d.cbs = connectBlockStats{} 677 return s 678 } 679 680 func (d *RocksDB) processAddressesBitcoinType(block *bchain.Block, addresses addressesMap, txAddressesMap map[string]*TxAddresses, balances map[string]*AddrBalance) error { 681 blockTxIDs := make([][]byte, len(block.Txs)) 682 blockTxAddresses := make([]*TxAddresses, len(block.Txs)) 683 // first process all outputs so that inputs can refer to txs in this block 684 for txi := range block.Txs { 685 tx := &block.Txs[txi] 686 btxID, err := d.chainParser.PackTxid(tx.Txid) 687 if err != nil { 688 return err 689 } 690 blockTxIDs[txi] = btxID 691 ta := TxAddresses{Height: block.Height} 692 ta.Outputs = make([]TxOutput, len(tx.Vout)) 693 txAddressesMap[string(btxID)] = &ta 694 blockTxAddresses[txi] = &ta 695 for i, output := range tx.Vout { 696 tao := &ta.Outputs[i] 697 tao.ValueSat = output.ValueSat 698 addrDesc, err := d.chainParser.GetAddrDescFromVout(&output) 699 if err != nil || len(addrDesc) == 0 || len(addrDesc) > maxAddrDescLen { 700 if err != nil { 701 // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) 702 if err != bchain.ErrAddressMissing { 703 glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output %v, error %v", err, block.Height, tx.Txid, output, err) 704 } 705 } else { 706 glog.V(1).Infof("rocksdb: height %d, tx %v, vout %v, skipping addrDesc of length %d", block.Height, tx.Txid, i, len(addrDesc)) 707 } 708 continue 709 } 710 tao.AddrDesc = addrDesc 711 if d.chainParser.IsAddrDescIndexable(addrDesc) { 712 strAddrDesc := string(addrDesc) 713 balance, e := balances[strAddrDesc] 714 if !e { 715 balance, err = d.GetAddrDescBalance(addrDesc, addressBalanceDetailUTXOIndexed) 716 if err != nil { 717 return err 718 } 719 if balance == nil { 720 balance = &AddrBalance{} 721 } 722 balances[strAddrDesc] = balance 723 d.cbs.balancesMiss++ 724 } else { 725 d.cbs.balancesHit++ 726 } 727 balance.BalanceSat.Add(&balance.BalanceSat, &output.ValueSat) 728 balance.addUtxo(&Utxo{ 729 BtxID: btxID, 730 Vout: int32(i), 731 Height: block.Height, 732 ValueSat: output.ValueSat, 733 }) 734 counted := addToAddressesMap(addresses, strAddrDesc, btxID, int32(i)) 735 if !counted { 736 balance.Txs++ 737 } 738 } 739 } 740 } 741 // process inputs 742 for txi := range block.Txs { 743 tx := &block.Txs[txi] 744 spendingTxid := blockTxIDs[txi] 745 ta := blockTxAddresses[txi] 746 ta.Inputs = make([]TxInput, len(tx.Vin)) 747 logged := false 748 for i, input := range tx.Vin { 749 tai := &ta.Inputs[i] 750 btxID, err := d.chainParser.PackTxid(input.Txid) 751 if err != nil { 752 // do not process inputs without input txid 753 if err == bchain.ErrTxidMissing { 754 continue 755 } 756 return err 757 } 758 stxID := string(btxID) 759 ita, e := txAddressesMap[stxID] 760 if !e { 761 ita, err = d.getTxAddresses(btxID) 762 if err != nil { 763 return err 764 } 765 if ita == nil { 766 // allow parser to process unknown input, some coins may implement special handling, default is to log warning 767 tai.AddrDesc = d.chainParser.GetAddrDescForUnknownInput(tx, i) 768 continue 769 } 770 txAddressesMap[stxID] = ita 771 d.cbs.txAddressesMiss++ 772 } else { 773 d.cbs.txAddressesHit++ 774 } 775 if len(ita.Outputs) <= int(input.Vout) { 776 glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is out of bounds of stored tx", block.Height, tx.Txid, input.Txid, input.Vout) 777 continue 778 } 779 spentOutput := &ita.Outputs[int(input.Vout)] 780 if spentOutput.Spent { 781 glog.Warningf("rocksdb: height %d, tx %v, input tx %v vout %v is double spend", block.Height, tx.Txid, input.Txid, input.Vout) 782 } 783 tai.AddrDesc = spentOutput.AddrDesc 784 tai.ValueSat = spentOutput.ValueSat 785 // mark the output as spent in tx 786 spentOutput.Spent = true 787 if len(spentOutput.AddrDesc) == 0 { 788 if !logged { 789 glog.V(1).Infof("rocksdb: height %d, tx %v, input tx %v vout %v skipping empty address", block.Height, tx.Txid, input.Txid, input.Vout) 790 logged = true 791 } 792 continue 793 } 794 if d.chainParser.IsAddrDescIndexable(spentOutput.AddrDesc) { 795 strAddrDesc := string(spentOutput.AddrDesc) 796 balance, e := balances[strAddrDesc] 797 if !e { 798 balance, err = d.GetAddrDescBalance(spentOutput.AddrDesc, addressBalanceDetailUTXOIndexed) 799 if err != nil { 800 return err 801 } 802 if balance == nil { 803 balance = &AddrBalance{} 804 } 805 balances[strAddrDesc] = balance 806 d.cbs.balancesMiss++ 807 } else { 808 d.cbs.balancesHit++ 809 } 810 counted := addToAddressesMap(addresses, strAddrDesc, spendingTxid, ^int32(i)) 811 if !counted { 812 balance.Txs++ 813 } 814 balance.BalanceSat.Sub(&balance.BalanceSat, &spentOutput.ValueSat) 815 balance.markUtxoAsSpent(btxID, int32(input.Vout)) 816 if balance.BalanceSat.Sign() < 0 { 817 d.resetValueSatToZero(&balance.BalanceSat, spentOutput.AddrDesc, "balance") 818 } 819 balance.SentSat.Add(&balance.SentSat, &spentOutput.ValueSat) 820 } 821 } 822 } 823 return nil 824 } 825 826 // addToAddressesMap maintains mapping between addresses and transactions in one block 827 // the method assumes that outputs in the block are processed before the inputs 828 // the return value is true if the tx was processed before, to not to count the tx multiple times 829 func addToAddressesMap(addresses addressesMap, strAddrDesc string, btxID []byte, index int32) bool { 830 // check that the address was already processed in this block 831 // if not found, it has certainly not been counted 832 at, found := addresses[strAddrDesc] 833 if found { 834 // if the tx is already in the slice, append the index to the array of indexes 835 for i, t := range at { 836 if bytes.Equal(btxID, t.btxID) { 837 at[i].indexes = append(t.indexes, index) 838 return true 839 } 840 } 841 } 842 addresses[strAddrDesc] = append(at, txIndexes{ 843 btxID: btxID, 844 indexes: []int32{index}, 845 }) 846 return false 847 } 848 849 func (d *RocksDB) storeAddresses(wb *gorocksdb.WriteBatch, height uint32, addresses addressesMap) error { 850 for addrDesc, txi := range addresses { 851 ba := bchain.AddressDescriptor(addrDesc) 852 key := packAddressKey(ba, height) 853 val := d.packTxIndexes(txi) 854 wb.PutCF(d.cfh[cfAddresses], key, val) 855 } 856 return nil 857 } 858 859 func (d *RocksDB) storeTxAddresses(wb *gorocksdb.WriteBatch, am map[string]*TxAddresses) error { 860 varBuf := make([]byte, maxPackedBigintBytes) 861 buf := make([]byte, 1024) 862 for txID, ta := range am { 863 buf = packTxAddresses(ta, buf, varBuf) 864 wb.PutCF(d.cfh[cfTxAddresses], []byte(txID), buf) 865 } 866 return nil 867 } 868 869 func (d *RocksDB) storeBalances(wb *gorocksdb.WriteBatch, abm map[string]*AddrBalance) error { 870 // allocate buffer initial buffer 871 buf := make([]byte, 1024) 872 varBuf := make([]byte, maxPackedBigintBytes) 873 for addrDesc, ab := range abm { 874 // balance with 0 transactions is removed from db - happens on disconnect 875 if ab == nil || ab.Txs <= 0 { 876 wb.DeleteCF(d.cfh[cfAddressBalance], bchain.AddressDescriptor(addrDesc)) 877 } else { 878 buf = packAddrBalance(ab, buf, varBuf) 879 wb.PutCF(d.cfh[cfAddressBalance], bchain.AddressDescriptor(addrDesc), buf) 880 } 881 } 882 return nil 883 } 884 885 func (d *RocksDB) cleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { 886 keep := d.chainParser.KeepBlockAddresses() 887 // cleanup old block address 888 if block.Height > uint32(keep) { 889 for rh := block.Height - uint32(keep); rh > 0; rh-- { 890 key := packUint(rh) 891 val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], key) 892 if err != nil { 893 return err 894 } 895 // nil data means the key was not found in DB 896 if val.Data() == nil { 897 break 898 } 899 val.Free() 900 d.db.DeleteCF(d.wo, d.cfh[cfBlockTxs], key) 901 } 902 } 903 return nil 904 } 905 906 func (d *RocksDB) storeAndCleanupBlockTxs(wb *gorocksdb.WriteBatch, block *bchain.Block) error { 907 pl := d.chainParser.PackedTxidLen() 908 buf := make([]byte, 0, pl*len(block.Txs)) 909 varBuf := make([]byte, vlq.MaxLen64) 910 zeroTx := make([]byte, pl) 911 for i := range block.Txs { 912 tx := &block.Txs[i] 913 o := make([]outpoint, len(tx.Vin)) 914 for v := range tx.Vin { 915 vin := &tx.Vin[v] 916 btxID, err := d.chainParser.PackTxid(vin.Txid) 917 if err != nil { 918 // do not process inputs without input txid 919 if err == bchain.ErrTxidMissing { 920 btxID = zeroTx 921 } else { 922 return err 923 } 924 } 925 o[v].btxID = btxID 926 o[v].index = int32(vin.Vout) 927 } 928 btxID, err := d.chainParser.PackTxid(tx.Txid) 929 if err != nil { 930 return err 931 } 932 buf = append(buf, btxID...) 933 l := packVaruint(uint(len(o)), varBuf) 934 buf = append(buf, varBuf[:l]...) 935 buf = append(buf, d.packOutpoints(o)...) 936 } 937 key := packUint(block.Height) 938 wb.PutCF(d.cfh[cfBlockTxs], key, buf) 939 return d.cleanupBlockTxs(wb, block) 940 } 941 942 func (d *RocksDB) getBlockTxs(height uint32) ([]blockTxs, error) { 943 pl := d.chainParser.PackedTxidLen() 944 val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) 945 if err != nil { 946 return nil, err 947 } 948 defer val.Free() 949 buf := val.Data() 950 bt := make([]blockTxs, 0, 8) 951 for i := 0; i < len(buf); { 952 if len(buf)-i < pl { 953 glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) 954 return nil, errors.New("Inconsistent data in blockTxs") 955 } 956 txid := append([]byte(nil), buf[i:i+pl]...) 957 i += pl 958 o, ol, err := d.unpackNOutpoints(buf[i:]) 959 if err != nil { 960 glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) 961 return nil, errors.New("Inconsistent data in blockTxs") 962 } 963 bt = append(bt, blockTxs{ 964 btxID: txid, 965 inputs: o, 966 }) 967 i += ol 968 } 969 return bt, nil 970 } 971 972 // GetAddrDescBalance returns AddrBalance for given addrDesc 973 func (d *RocksDB) GetAddrDescBalance(addrDesc bchain.AddressDescriptor, detail AddressBalanceDetail) (*AddrBalance, error) { 974 val, err := d.db.GetCF(d.ro, d.cfh[cfAddressBalance], addrDesc) 975 if err != nil { 976 return nil, err 977 } 978 defer val.Free() 979 buf := val.Data() 980 // 3 is minimum length of addrBalance - 1 byte txs, 1 byte sent, 1 byte balance 981 if len(buf) < 3 { 982 return nil, nil 983 } 984 return unpackAddrBalance(buf, d.chainParser.PackedTxidLen(), detail) 985 } 986 987 // GetAddressBalance returns address balance for an address or nil if address not found 988 func (d *RocksDB) GetAddressBalance(address string, detail AddressBalanceDetail) (*AddrBalance, error) { 989 addrDesc, err := d.chainParser.GetAddrDescFromAddress(address) 990 if err != nil { 991 return nil, err 992 } 993 return d.GetAddrDescBalance(addrDesc, detail) 994 } 995 996 func (d *RocksDB) getTxAddresses(btxID []byte) (*TxAddresses, error) { 997 val, err := d.db.GetCF(d.ro, d.cfh[cfTxAddresses], btxID) 998 if err != nil { 999 return nil, err 1000 } 1001 defer val.Free() 1002 buf := val.Data() 1003 // 2 is minimum length of addrBalance - 1 byte height, 1 byte inputs len, 1 byte outputs len 1004 if len(buf) < 3 { 1005 return nil, nil 1006 } 1007 return unpackTxAddresses(buf) 1008 } 1009 1010 // GetTxAddresses returns TxAddresses for given txid or nil if not found 1011 func (d *RocksDB) GetTxAddresses(txid string) (*TxAddresses, error) { 1012 btxID, err := d.chainParser.PackTxid(txid) 1013 if err != nil { 1014 return nil, err 1015 } 1016 return d.getTxAddresses(btxID) 1017 } 1018 1019 // AddrDescForOutpoint defines function that returns address descriptorfor given outpoint or nil if outpoint not found 1020 func (d *RocksDB) AddrDescForOutpoint(outpoint bchain.Outpoint) bchain.AddressDescriptor { 1021 ta, err := d.GetTxAddresses(outpoint.Txid) 1022 if err != nil || ta == nil { 1023 return nil 1024 } 1025 if outpoint.Vout < 0 { 1026 vin := ^outpoint.Vout 1027 if len(ta.Inputs) <= int(vin) { 1028 return nil 1029 } 1030 return ta.Inputs[vin].AddrDesc 1031 } 1032 if len(ta.Outputs) <= int(outpoint.Vout) { 1033 return nil 1034 } 1035 return ta.Outputs[outpoint.Vout].AddrDesc 1036 } 1037 1038 func packTxAddresses(ta *TxAddresses, buf []byte, varBuf []byte) []byte { 1039 buf = buf[:0] 1040 l := packVaruint(uint(ta.Height), varBuf) 1041 buf = append(buf, varBuf[:l]...) 1042 l = packVaruint(uint(len(ta.Inputs)), varBuf) 1043 buf = append(buf, varBuf[:l]...) 1044 for i := range ta.Inputs { 1045 buf = appendTxInput(&ta.Inputs[i], buf, varBuf) 1046 } 1047 l = packVaruint(uint(len(ta.Outputs)), varBuf) 1048 buf = append(buf, varBuf[:l]...) 1049 for i := range ta.Outputs { 1050 buf = appendTxOutput(&ta.Outputs[i], buf, varBuf) 1051 } 1052 return buf 1053 } 1054 1055 func appendTxInput(txi *TxInput, buf []byte, varBuf []byte) []byte { 1056 la := len(txi.AddrDesc) 1057 l := packVaruint(uint(la), varBuf) 1058 buf = append(buf, varBuf[:l]...) 1059 buf = append(buf, txi.AddrDesc...) 1060 l = packBigint(&txi.ValueSat, varBuf) 1061 buf = append(buf, varBuf[:l]...) 1062 return buf 1063 } 1064 1065 func appendTxOutput(txo *TxOutput, buf []byte, varBuf []byte) []byte { 1066 la := len(txo.AddrDesc) 1067 if txo.Spent { 1068 la = ^la 1069 } 1070 l := packVarint(la, varBuf) 1071 buf = append(buf, varBuf[:l]...) 1072 buf = append(buf, txo.AddrDesc...) 1073 l = packBigint(&txo.ValueSat, varBuf) 1074 buf = append(buf, varBuf[:l]...) 1075 return buf 1076 } 1077 1078 func unpackAddrBalance(buf []byte, txidUnpackedLen int, detail AddressBalanceDetail) (*AddrBalance, error) { 1079 txs, l := unpackVaruint(buf) 1080 sentSat, sl := unpackBigint(buf[l:]) 1081 balanceSat, bl := unpackBigint(buf[l+sl:]) 1082 l = l + sl + bl 1083 ab := &AddrBalance{ 1084 Txs: uint32(txs), 1085 SentSat: sentSat, 1086 BalanceSat: balanceSat, 1087 } 1088 if detail != AddressBalanceDetailNoUTXO { 1089 // estimate the size of utxos to avoid reallocation 1090 ab.Utxos = make([]Utxo, 0, len(buf[l:])/txidUnpackedLen+3) 1091 // ab.utxosMap = make(map[string]int, cap(ab.Utxos)) 1092 for len(buf[l:]) >= txidUnpackedLen+3 { 1093 btxID := append([]byte(nil), buf[l:l+txidUnpackedLen]...) 1094 l += txidUnpackedLen 1095 vout, ll := unpackVaruint(buf[l:]) 1096 l += ll 1097 height, ll := unpackVaruint(buf[l:]) 1098 l += ll 1099 valueSat, ll := unpackBigint(buf[l:]) 1100 l += ll 1101 u := Utxo{ 1102 BtxID: btxID, 1103 Vout: int32(vout), 1104 Height: uint32(height), 1105 ValueSat: valueSat, 1106 } 1107 if detail == AddressBalanceDetailUTXO { 1108 ab.Utxos = append(ab.Utxos, u) 1109 } else { 1110 ab.addUtxo(&u) 1111 } 1112 } 1113 } 1114 return ab, nil 1115 } 1116 1117 func packAddrBalance(ab *AddrBalance, buf, varBuf []byte) []byte { 1118 buf = buf[:0] 1119 l := packVaruint(uint(ab.Txs), varBuf) 1120 buf = append(buf, varBuf[:l]...) 1121 l = packBigint(&ab.SentSat, varBuf) 1122 buf = append(buf, varBuf[:l]...) 1123 l = packBigint(&ab.BalanceSat, varBuf) 1124 buf = append(buf, varBuf[:l]...) 1125 for _, utxo := range ab.Utxos { 1126 // if Vout < 0, utxo is marked as spent 1127 if utxo.Vout >= 0 { 1128 buf = append(buf, utxo.BtxID...) 1129 l = packVaruint(uint(utxo.Vout), varBuf) 1130 buf = append(buf, varBuf[:l]...) 1131 l = packVaruint(uint(utxo.Height), varBuf) 1132 buf = append(buf, varBuf[:l]...) 1133 l = packBigint(&utxo.ValueSat, varBuf) 1134 buf = append(buf, varBuf[:l]...) 1135 } 1136 } 1137 return buf 1138 } 1139 1140 func unpackTxAddresses(buf []byte) (*TxAddresses, error) { 1141 ta := TxAddresses{} 1142 height, l := unpackVaruint(buf) 1143 ta.Height = uint32(height) 1144 inputs, ll := unpackVaruint(buf[l:]) 1145 l += ll 1146 ta.Inputs = make([]TxInput, inputs) 1147 for i := uint(0); i < inputs; i++ { 1148 l += unpackTxInput(&ta.Inputs[i], buf[l:]) 1149 } 1150 outputs, ll := unpackVaruint(buf[l:]) 1151 l += ll 1152 ta.Outputs = make([]TxOutput, outputs) 1153 for i := uint(0); i < outputs; i++ { 1154 l += unpackTxOutput(&ta.Outputs[i], buf[l:]) 1155 } 1156 return &ta, nil 1157 } 1158 1159 func unpackTxInput(ti *TxInput, buf []byte) int { 1160 al, l := unpackVaruint(buf) 1161 ti.AddrDesc = append([]byte(nil), buf[l:l+int(al)]...) 1162 al += uint(l) 1163 ti.ValueSat, l = unpackBigint(buf[al:]) 1164 return l + int(al) 1165 } 1166 1167 func unpackTxOutput(to *TxOutput, buf []byte) int { 1168 al, l := unpackVarint(buf) 1169 if al < 0 { 1170 to.Spent = true 1171 al = ^al 1172 } 1173 to.AddrDesc = append([]byte(nil), buf[l:l+al]...) 1174 al += l 1175 to.ValueSat, l = unpackBigint(buf[al:]) 1176 return l + al 1177 } 1178 1179 func (d *RocksDB) packTxIndexes(txi []txIndexes) []byte { 1180 buf := make([]byte, 0, 32) 1181 bvout := make([]byte, vlq.MaxLen32) 1182 // store the txs in reverse order for ordering from newest to oldest 1183 for j := len(txi) - 1; j >= 0; j-- { 1184 t := &txi[j] 1185 buf = append(buf, []byte(t.btxID)...) 1186 for i, index := range t.indexes { 1187 index <<= 1 1188 if i == len(t.indexes)-1 { 1189 index |= 1 1190 } 1191 l := packVarint32(index, bvout) 1192 buf = append(buf, bvout[:l]...) 1193 } 1194 } 1195 return buf 1196 } 1197 1198 func (d *RocksDB) packOutpoints(outpoints []outpoint) []byte { 1199 buf := make([]byte, 0, 32) 1200 bvout := make([]byte, vlq.MaxLen32) 1201 for _, o := range outpoints { 1202 l := packVarint32(o.index, bvout) 1203 buf = append(buf, []byte(o.btxID)...) 1204 buf = append(buf, bvout[:l]...) 1205 } 1206 return buf 1207 } 1208 1209 func (d *RocksDB) unpackNOutpoints(buf []byte) ([]outpoint, int, error) { 1210 txidUnpackedLen := d.chainParser.PackedTxidLen() 1211 n, p := unpackVaruint(buf) 1212 outpoints := make([]outpoint, n) 1213 for i := uint(0); i < n; i++ { 1214 if p+txidUnpackedLen >= len(buf) { 1215 return nil, 0, errors.New("Inconsistent data in unpackNOutpoints") 1216 } 1217 btxID := append([]byte(nil), buf[p:p+txidUnpackedLen]...) 1218 p += txidUnpackedLen 1219 vout, voutLen := unpackVarint32(buf[p:]) 1220 p += voutLen 1221 outpoints[i] = outpoint{ 1222 btxID: btxID, 1223 index: vout, 1224 } 1225 } 1226 return outpoints, p, nil 1227 } 1228 1229 // Block index 1230 1231 // BlockInfo holds information about blocks kept in column height 1232 type BlockInfo struct { 1233 Hash string 1234 Time int64 1235 Txs uint32 1236 Size uint32 1237 Height uint32 // Height is not packed! 1238 } 1239 1240 func (d *RocksDB) packBlockInfo(block *BlockInfo) ([]byte, error) { 1241 packed := make([]byte, 0, 64) 1242 varBuf := make([]byte, vlq.MaxLen64) 1243 b, err := d.chainParser.PackBlockHash(block.Hash) 1244 if err != nil { 1245 return nil, err 1246 } 1247 pl := d.chainParser.PackedTxidLen() 1248 if len(b) != pl { 1249 glog.Warning("Non standard block hash for height ", block.Height, ", hash [", block.Hash, "]") 1250 if len(b) > pl { 1251 b = b[:pl] 1252 } else { 1253 b = append(b, make([]byte, pl-len(b))...) 1254 } 1255 } 1256 packed = append(packed, b...) 1257 packed = append(packed, packUint(uint32(block.Time))...) 1258 l := packVaruint(uint(block.Txs), varBuf) 1259 packed = append(packed, varBuf[:l]...) 1260 l = packVaruint(uint(block.Size), varBuf) 1261 packed = append(packed, varBuf[:l]...) 1262 return packed, nil 1263 } 1264 1265 func (d *RocksDB) unpackBlockInfo(buf []byte) (*BlockInfo, error) { 1266 pl := d.chainParser.PackedTxidLen() 1267 // minimum length is PackedTxidLen + 4 bytes time + 1 byte txs + 1 byte size 1268 if len(buf) < pl+4+2 { 1269 return nil, nil 1270 } 1271 txid, err := d.chainParser.UnpackBlockHash(buf[:pl]) 1272 if err != nil { 1273 return nil, err 1274 } 1275 t := unpackUint(buf[pl:]) 1276 txs, l := unpackVaruint(buf[pl+4:]) 1277 size, _ := unpackVaruint(buf[pl+4+l:]) 1278 return &BlockInfo{ 1279 Hash: txid, 1280 Time: int64(t), 1281 Txs: uint32(txs), 1282 Size: uint32(size), 1283 }, nil 1284 } 1285 1286 // GetBestBlock returns the block hash of the block with highest height in the db 1287 func (d *RocksDB) GetBestBlock() (uint32, string, error) { 1288 it := d.db.NewIteratorCF(d.ro, d.cfh[cfHeight]) 1289 defer it.Close() 1290 if it.SeekToLast(); it.Valid() { 1291 bestHeight := unpackUint(it.Key().Data()) 1292 info, err := d.unpackBlockInfo(it.Value().Data()) 1293 if info != nil { 1294 if glog.V(1) { 1295 glog.Infof("rocksdb: bestblock %d %+v", bestHeight, info) 1296 } 1297 return bestHeight, info.Hash, err 1298 } 1299 } 1300 return 0, "", nil 1301 } 1302 1303 // GetBlockHash returns block hash at given height or empty string if not found 1304 func (d *RocksDB) GetBlockHash(height uint32) (string, error) { 1305 key := packUint(height) 1306 val, err := d.db.GetCF(d.ro, d.cfh[cfHeight], key) 1307 if err != nil { 1308 return "", err 1309 } 1310 defer val.Free() 1311 info, err := d.unpackBlockInfo(val.Data()) 1312 if info == nil { 1313 return "", err 1314 } 1315 return info.Hash, nil 1316 } 1317 1318 // GetBlockInfo returns block info stored in db 1319 func (d *RocksDB) GetBlockInfo(height uint32) (*BlockInfo, error) { 1320 key := packUint(height) 1321 val, err := d.db.GetCF(d.ro, d.cfh[cfHeight], key) 1322 if err != nil { 1323 return nil, err 1324 } 1325 defer val.Free() 1326 bi, err := d.unpackBlockInfo(val.Data()) 1327 if err != nil || bi == nil { 1328 return nil, err 1329 } 1330 bi.Height = height 1331 return bi, err 1332 } 1333 1334 func (d *RocksDB) writeHeightFromBlock(wb *gorocksdb.WriteBatch, block *bchain.Block, op int) error { 1335 return d.writeHeight(wb, block.Height, &BlockInfo{ 1336 Hash: block.Hash, 1337 Time: block.Time, 1338 Txs: uint32(len(block.Txs)), 1339 Size: uint32(block.Size), 1340 Height: block.Height, 1341 }, op) 1342 } 1343 1344 func (d *RocksDB) writeHeight(wb *gorocksdb.WriteBatch, height uint32, bi *BlockInfo, op int) error { 1345 key := packUint(height) 1346 switch op { 1347 case opInsert: 1348 val, err := d.packBlockInfo(bi) 1349 if err != nil { 1350 return err 1351 } 1352 wb.PutCF(d.cfh[cfHeight], key, val) 1353 d.is.UpdateBestHeight(height) 1354 case opDelete: 1355 wb.DeleteCF(d.cfh[cfHeight], key) 1356 d.is.UpdateBestHeight(height - 1) 1357 } 1358 return nil 1359 } 1360 1361 // Disconnect blocks 1362 1363 func (d *RocksDB) disconnectTxAddressesInputs(wb *gorocksdb.WriteBatch, btxID []byte, inputs []outpoint, txa *TxAddresses, txAddressesToUpdate map[string]*TxAddresses, 1364 getAddressBalance func(addrDesc bchain.AddressDescriptor) (*AddrBalance, error), 1365 addressFoundInTx func(addrDesc bchain.AddressDescriptor, btxID []byte) bool) error { 1366 var err error 1367 var balance *AddrBalance 1368 for i, t := range txa.Inputs { 1369 if len(t.AddrDesc) > 0 { 1370 input := &inputs[i] 1371 exist := addressFoundInTx(t.AddrDesc, btxID) 1372 s := string(input.btxID) 1373 sa, found := txAddressesToUpdate[s] 1374 if !found { 1375 sa, err = d.getTxAddresses(input.btxID) 1376 if err != nil { 1377 return err 1378 } 1379 if sa != nil { 1380 txAddressesToUpdate[s] = sa 1381 } 1382 } 1383 var inputHeight uint32 1384 if sa != nil { 1385 sa.Outputs[input.index].Spent = false 1386 inputHeight = sa.Height 1387 } 1388 if d.chainParser.IsAddrDescIndexable(t.AddrDesc) { 1389 balance, err = getAddressBalance(t.AddrDesc) 1390 if err != nil { 1391 return err 1392 } 1393 if balance != nil { 1394 // subtract number of txs only once 1395 if !exist { 1396 balance.Txs-- 1397 } 1398 balance.SentSat.Sub(&balance.SentSat, &t.ValueSat) 1399 if balance.SentSat.Sign() < 0 { 1400 d.resetValueSatToZero(&balance.SentSat, t.AddrDesc, "sent amount") 1401 } 1402 balance.BalanceSat.Add(&balance.BalanceSat, &t.ValueSat) 1403 balance.addUtxoInDisconnect(&Utxo{ 1404 BtxID: input.btxID, 1405 Vout: input.index, 1406 Height: inputHeight, 1407 ValueSat: t.ValueSat, 1408 }) 1409 } else { 1410 ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) 1411 glog.Warningf("Balance for address %s (%s) not found", ad, t.AddrDesc) 1412 } 1413 } 1414 } 1415 } 1416 return nil 1417 } 1418 1419 func (d *RocksDB) disconnectTxAddressesOutputs(wb *gorocksdb.WriteBatch, btxID []byte, txa *TxAddresses, 1420 getAddressBalance func(addrDesc bchain.AddressDescriptor) (*AddrBalance, error), 1421 addressFoundInTx func(addrDesc bchain.AddressDescriptor, btxID []byte) bool) error { 1422 for i, t := range txa.Outputs { 1423 if len(t.AddrDesc) > 0 { 1424 exist := addressFoundInTx(t.AddrDesc, btxID) 1425 if d.chainParser.IsAddrDescIndexable(t.AddrDesc) { 1426 balance, err := getAddressBalance(t.AddrDesc) 1427 if err != nil { 1428 return err 1429 } 1430 if balance != nil { 1431 // subtract number of txs only once 1432 if !exist { 1433 balance.Txs-- 1434 } 1435 balance.BalanceSat.Sub(&balance.BalanceSat, &t.ValueSat) 1436 if balance.BalanceSat.Sign() < 0 { 1437 d.resetValueSatToZero(&balance.BalanceSat, t.AddrDesc, "balance") 1438 } 1439 balance.markUtxoAsSpent(btxID, int32(i)) 1440 } else { 1441 ad, _, _ := d.chainParser.GetAddressesFromAddrDesc(t.AddrDesc) 1442 glog.Warningf("Balance for address %s (%s) not found", ad, t.AddrDesc) 1443 } 1444 } 1445 } 1446 } 1447 return nil 1448 } 1449 1450 func (d *RocksDB) disconnectBlock(height uint32, blockTxs []blockTxs) error { 1451 wb := gorocksdb.NewWriteBatch() 1452 defer wb.Destroy() 1453 txAddressesToUpdate := make(map[string]*TxAddresses) 1454 txAddresses := make([]*TxAddresses, len(blockTxs)) 1455 txsToDelete := make(map[string]struct{}) 1456 1457 balances := make(map[string]*AddrBalance) 1458 getAddressBalance := func(addrDesc bchain.AddressDescriptor) (*AddrBalance, error) { 1459 var err error 1460 s := string(addrDesc) 1461 b, fb := balances[s] 1462 if !fb { 1463 b, err = d.GetAddrDescBalance(addrDesc, addressBalanceDetailUTXOIndexed) 1464 if err != nil { 1465 return nil, err 1466 } 1467 balances[s] = b 1468 } 1469 return b, nil 1470 } 1471 1472 // all addresses in the block are stored in blockAddressesTxs, together with a map of transactions where they appear 1473 blockAddressesTxs := make(map[string]map[string]struct{}) 1474 // addressFoundInTx handles updates of the blockAddressesTxs map and returns true if the address+tx was already encountered 1475 addressFoundInTx := func(addrDesc bchain.AddressDescriptor, btxID []byte) bool { 1476 sAddrDesc := string(addrDesc) 1477 sBtxID := string(btxID) 1478 a, exist := blockAddressesTxs[sAddrDesc] 1479 if !exist { 1480 blockAddressesTxs[sAddrDesc] = map[string]struct{}{sBtxID: {}} 1481 } else { 1482 _, exist = a[sBtxID] 1483 if !exist { 1484 a[sBtxID] = struct{}{} 1485 } 1486 } 1487 return exist 1488 } 1489 1490 glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") 1491 // when connecting block, outputs are processed first 1492 // when disconnecting, inputs must be reversed first 1493 for i := range blockTxs { 1494 btxID := blockTxs[i].btxID 1495 s := string(btxID) 1496 txsToDelete[s] = struct{}{} 1497 txa, err := d.getTxAddresses(btxID) 1498 if err != nil { 1499 return err 1500 } 1501 if txa == nil { 1502 ut, _ := d.chainParser.UnpackTxid(btxID) 1503 glog.Warning("TxAddress for txid ", ut, " not found") 1504 continue 1505 } 1506 txAddresses[i] = txa 1507 if err := d.disconnectTxAddressesInputs(wb, btxID, blockTxs[i].inputs, txa, txAddressesToUpdate, getAddressBalance, addressFoundInTx); err != nil { 1508 return err 1509 } 1510 } 1511 for i := range blockTxs { 1512 btxID := blockTxs[i].btxID 1513 txa := txAddresses[i] 1514 if txa == nil { 1515 continue 1516 } 1517 if err := d.disconnectTxAddressesOutputs(wb, btxID, txa, getAddressBalance, addressFoundInTx); err != nil { 1518 return err 1519 } 1520 } 1521 for a := range blockAddressesTxs { 1522 key := packAddressKey([]byte(a), height) 1523 wb.DeleteCF(d.cfh[cfAddresses], key) 1524 } 1525 key := packUint(height) 1526 wb.DeleteCF(d.cfh[cfBlockTxs], key) 1527 wb.DeleteCF(d.cfh[cfHeight], key) 1528 d.storeTxAddresses(wb, txAddressesToUpdate) 1529 d.storeBalancesDisconnect(wb, balances) 1530 for s := range txsToDelete { 1531 b := []byte(s) 1532 wb.DeleteCF(d.cfh[cfTransactions], b) 1533 wb.DeleteCF(d.cfh[cfTxAddresses], b) 1534 } 1535 return d.db.Write(d.wo, wb) 1536 } 1537 1538 // DisconnectBlockRangeBitcoinType removes all data belonging to blocks in range lower-higher 1539 // it is able to disconnect only blocks for which there are data in the blockTxs column 1540 func (d *RocksDB) DisconnectBlockRangeBitcoinType(lower uint32, higher uint32) error { 1541 blocks := make([][]blockTxs, higher-lower+1) 1542 for height := lower; height <= higher; height++ { 1543 blockTxs, err := d.getBlockTxs(height) 1544 if err != nil { 1545 return err 1546 } 1547 if len(blockTxs) == 0 { 1548 return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) 1549 } 1550 blocks[height-lower] = blockTxs 1551 } 1552 for height := higher; height >= lower; height-- { 1553 err := d.disconnectBlock(height, blocks[height-lower]) 1554 if err != nil { 1555 return err 1556 } 1557 } 1558 d.is.RemoveLastBlockTimes(int(higher-lower) + 1) 1559 glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) 1560 return nil 1561 } 1562 1563 func (d *RocksDB) storeBalancesDisconnect(wb *gorocksdb.WriteBatch, balances map[string]*AddrBalance) { 1564 for _, b := range balances { 1565 if b != nil { 1566 // remove spent utxos 1567 us := make([]Utxo, 0, len(b.Utxos)) 1568 for _, u := range b.Utxos { 1569 // remove utxos marked as spent 1570 if u.Vout >= 0 { 1571 us = append(us, u) 1572 } 1573 } 1574 b.Utxos = us 1575 // sort utxos by height 1576 sort.SliceStable(b.Utxos, func(i, j int) bool { 1577 return b.Utxos[i].Height < b.Utxos[j].Height 1578 }) 1579 } 1580 } 1581 d.storeBalances(wb, balances) 1582 1583 } 1584 func dirSize(path string) (int64, error) { 1585 var size int64 1586 err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { 1587 if err == nil { 1588 if !info.IsDir() { 1589 size += info.Size() 1590 } 1591 } 1592 return err 1593 }) 1594 return size, err 1595 } 1596 1597 // DatabaseSizeOnDisk returns size of the database in bytes 1598 func (d *RocksDB) DatabaseSizeOnDisk() int64 { 1599 size, err := dirSize(d.path) 1600 if err != nil { 1601 glog.Warning("rocksdb: DatabaseSizeOnDisk: ", err) 1602 return 0 1603 } 1604 return size 1605 } 1606 1607 // GetTx returns transaction stored in db and height of the block containing it 1608 func (d *RocksDB) GetTx(txid string) (*bchain.Tx, uint32, error) { 1609 key, err := d.chainParser.PackTxid(txid) 1610 if err != nil { 1611 return nil, 0, err 1612 } 1613 val, err := d.db.GetCF(d.ro, d.cfh[cfTransactions], key) 1614 if err != nil { 1615 return nil, 0, err 1616 } 1617 defer val.Free() 1618 data := val.Data() 1619 if len(data) > 4 { 1620 return d.chainParser.UnpackTx(data) 1621 } 1622 return nil, 0, nil 1623 } 1624 1625 // PutTx stores transactions in db 1626 func (d *RocksDB) PutTx(tx *bchain.Tx, height uint32, blockTime int64) error { 1627 key, err := d.chainParser.PackTxid(tx.Txid) 1628 if err != nil { 1629 return nil 1630 } 1631 buf, err := d.chainParser.PackTx(tx, height, blockTime) 1632 if err != nil { 1633 return err 1634 } 1635 err = d.db.PutCF(d.wo, d.cfh[cfTransactions], key, buf) 1636 if err == nil { 1637 d.is.AddDBColumnStats(cfTransactions, 1, int64(len(key)), int64(len(buf))) 1638 } 1639 return err 1640 } 1641 1642 // DeleteTx removes transactions from db 1643 func (d *RocksDB) DeleteTx(txid string) error { 1644 key, err := d.chainParser.PackTxid(txid) 1645 if err != nil { 1646 return nil 1647 } 1648 // use write batch so that this delete matches other deletes 1649 wb := gorocksdb.NewWriteBatch() 1650 defer wb.Destroy() 1651 d.internalDeleteTx(wb, key) 1652 return d.db.Write(d.wo, wb) 1653 } 1654 1655 // internalDeleteTx checks if tx is cached and updates internal state accordingly 1656 func (d *RocksDB) internalDeleteTx(wb *gorocksdb.WriteBatch, key []byte) { 1657 val, err := d.db.GetCF(d.ro, d.cfh[cfTransactions], key) 1658 // ignore error, it is only for statistics 1659 if err == nil { 1660 l := len(val.Data()) 1661 if l > 0 { 1662 d.is.AddDBColumnStats(cfTransactions, -1, int64(-len(key)), int64(-l)) 1663 } 1664 defer val.Free() 1665 } 1666 wb.DeleteCF(d.cfh[cfTransactions], key) 1667 } 1668 1669 // internal state 1670 const internalStateKey = "internalState" 1671 1672 func (d *RocksDB) loadBlockTimes() ([]uint32, error) { 1673 var times []uint32 1674 it := d.db.NewIteratorCF(d.ro, d.cfh[cfHeight]) 1675 defer it.Close() 1676 counter := uint32(0) 1677 time := uint32(0) 1678 for it.SeekToFirst(); it.Valid(); it.Next() { 1679 height := unpackUint(it.Key().Data()) 1680 if height > counter { 1681 glog.Warning("gap in cfHeight: expecting ", counter, ", got ", height) 1682 for ; counter < height; counter++ { 1683 times = append(times, time) 1684 } 1685 } 1686 counter++ 1687 info, err := d.unpackBlockInfo(it.Value().Data()) 1688 if err != nil { 1689 return nil, err 1690 } 1691 if info != nil { 1692 time = uint32(info.Time) 1693 } 1694 times = append(times, time) 1695 } 1696 glog.Info("loaded ", len(times), " block times") 1697 return times, nil 1698 } 1699 1700 // LoadInternalState loads from db internal state or initializes a new one if not yet stored 1701 func (d *RocksDB) LoadInternalState(rpcCoin string) (*common.InternalState, error) { 1702 val, err := d.db.GetCF(d.ro, d.cfh[cfDefault], []byte(internalStateKey)) 1703 if err != nil { 1704 return nil, err 1705 } 1706 defer val.Free() 1707 data := val.Data() 1708 var is *common.InternalState 1709 if len(data) == 0 { 1710 is = &common.InternalState{Coin: rpcCoin, UtxoChecked: true} 1711 } else { 1712 is, err = common.UnpackInternalState(data) 1713 if err != nil { 1714 return nil, err 1715 } 1716 // verify that the rpc coin matches DB coin 1717 // running it mismatched would corrupt the database 1718 if is.Coin == "" { 1719 is.Coin = rpcCoin 1720 } else if is.Coin != rpcCoin { 1721 return nil, errors.Errorf("Coins do not match. DB coin %v, RPC coin %v", is.Coin, rpcCoin) 1722 } 1723 } 1724 // make sure that column stats match the columns 1725 sc := is.DbColumns 1726 nc := make([]common.InternalStateColumn, len(cfNames)) 1727 for i := 0; i < len(nc); i++ { 1728 nc[i].Name = cfNames[i] 1729 nc[i].Version = dbVersion 1730 for j := 0; j < len(sc); j++ { 1731 if sc[j].Name == nc[i].Name { 1732 // check the version of the column, if it does not match, the db is not compatible 1733 if sc[j].Version != dbVersion { 1734 return nil, errors.Errorf("DB version %v of column '%v' does not match the required version %v. DB is not compatible.", sc[j].Version, sc[j].Name, dbVersion) 1735 } 1736 nc[i].Rows = sc[j].Rows 1737 nc[i].KeyBytes = sc[j].KeyBytes 1738 nc[i].ValueBytes = sc[j].ValueBytes 1739 nc[i].Updated = sc[j].Updated 1740 break 1741 } 1742 } 1743 } 1744 is.DbColumns = nc 1745 is.BlockTimes, err = d.loadBlockTimes() 1746 if err != nil { 1747 return nil, err 1748 } 1749 // after load, reset the synchronization data 1750 is.IsSynchronized = false 1751 is.IsMempoolSynchronized = false 1752 var t time.Time 1753 is.LastMempoolSync = t 1754 is.SyncMode = false 1755 return is, nil 1756 } 1757 1758 // SetInconsistentState sets the internal state to DbStateInconsistent or DbStateOpen based on inconsistent parameter 1759 // db in left in DbStateInconsistent state cannot be used and must be recreated 1760 func (d *RocksDB) SetInconsistentState(inconsistent bool) error { 1761 if d.is == nil { 1762 return errors.New("Internal state not created") 1763 } 1764 if inconsistent { 1765 d.is.DbState = common.DbStateInconsistent 1766 } else { 1767 d.is.DbState = common.DbStateOpen 1768 } 1769 return d.storeState(d.is) 1770 } 1771 1772 // SetInternalState sets the InternalState to be used by db to collect internal state 1773 func (d *RocksDB) SetInternalState(is *common.InternalState) { 1774 d.is = is 1775 } 1776 1777 // StoreInternalState stores the internal state to db 1778 func (d *RocksDB) StoreInternalState(is *common.InternalState) error { 1779 if d.metrics != nil { 1780 for c := 0; c < len(cfNames); c++ { 1781 rows, keyBytes, valueBytes := d.is.GetDBColumnStatValues(c) 1782 d.metrics.DbColumnRows.With(common.Labels{"column": cfNames[c]}).Set(float64(rows)) 1783 d.metrics.DbColumnSize.With(common.Labels{"column": cfNames[c]}).Set(float64(keyBytes + valueBytes)) 1784 } 1785 } 1786 return d.storeState(is) 1787 } 1788 1789 func (d *RocksDB) storeState(is *common.InternalState) error { 1790 buf, err := is.Pack() 1791 if err != nil { 1792 return err 1793 } 1794 return d.db.PutCF(d.wo, d.cfh[cfDefault], []byte(internalStateKey), buf) 1795 } 1796 1797 func (d *RocksDB) computeColumnSize(col int, stopCompute chan os.Signal) (int64, int64, int64, error) { 1798 var rows, keysSum, valuesSum int64 1799 var seekKey []byte 1800 // do not use cache 1801 ro := gorocksdb.NewDefaultReadOptions() 1802 ro.SetFillCache(false) 1803 for { 1804 var key []byte 1805 it := d.db.NewIteratorCF(ro, d.cfh[col]) 1806 if rows == 0 { 1807 it.SeekToFirst() 1808 } else { 1809 glog.Info("db: Column ", cfNames[col], ": rows ", rows, ", key bytes ", keysSum, ", value bytes ", valuesSum, ", in progress...") 1810 it.Seek(seekKey) 1811 it.Next() 1812 } 1813 for count := 0; it.Valid() && count < refreshIterator; it.Next() { 1814 select { 1815 case <-stopCompute: 1816 return 0, 0, 0, errors.New("Interrupted") 1817 default: 1818 } 1819 key = it.Key().Data() 1820 count++ 1821 rows++ 1822 keysSum += int64(len(key)) 1823 valuesSum += int64(len(it.Value().Data())) 1824 } 1825 seekKey = append([]byte{}, key...) 1826 valid := it.Valid() 1827 it.Close() 1828 if !valid { 1829 break 1830 } 1831 } 1832 return rows, keysSum, valuesSum, nil 1833 } 1834 1835 // ComputeInternalStateColumnStats computes stats of all db columns and sets them to internal state 1836 // can be very slow operation 1837 func (d *RocksDB) ComputeInternalStateColumnStats(stopCompute chan os.Signal) error { 1838 start := time.Now() 1839 glog.Info("db: ComputeInternalStateColumnStats start") 1840 for c := 0; c < len(cfNames); c++ { 1841 rows, keysSum, valuesSum, err := d.computeColumnSize(c, stopCompute) 1842 if err != nil { 1843 return err 1844 } 1845 d.is.SetDBColumnStats(c, rows, keysSum, valuesSum) 1846 glog.Info("db: Column ", cfNames[c], ": rows ", rows, ", key bytes ", keysSum, ", value bytes ", valuesSum) 1847 } 1848 glog.Info("db: ComputeInternalStateColumnStats finished in ", time.Since(start)) 1849 return nil 1850 } 1851 1852 func reorderUtxo(utxos []Utxo, index int) { 1853 var from, to int 1854 for from = index; from >= 0; from-- { 1855 if !bytes.Equal(utxos[from].BtxID, utxos[index].BtxID) { 1856 break 1857 } 1858 } 1859 from++ 1860 for to = index + 1; to < len(utxos); to++ { 1861 if !bytes.Equal(utxos[to].BtxID, utxos[index].BtxID) { 1862 break 1863 } 1864 } 1865 toSort := utxos[from:to] 1866 sort.SliceStable(toSort, func(i, j int) bool { 1867 return toSort[i].Vout < toSort[j].Vout 1868 }) 1869 1870 } 1871 1872 func (d *RocksDB) fixUtxo(addrDesc bchain.AddressDescriptor, ba *AddrBalance) (bool, bool, error) { 1873 reorder := false 1874 var checksum big.Int 1875 var prevUtxo *Utxo 1876 for i := range ba.Utxos { 1877 utxo := &ba.Utxos[i] 1878 checksum.Add(&checksum, &utxo.ValueSat) 1879 if prevUtxo != nil { 1880 if prevUtxo.Vout > utxo.Vout && *(*int)(unsafe.Pointer(&utxo.BtxID[0])) == *(*int)(unsafe.Pointer(&prevUtxo.BtxID[0])) && bytes.Equal(utxo.BtxID, prevUtxo.BtxID) { 1881 reorderUtxo(ba.Utxos, i) 1882 reorder = true 1883 } 1884 } 1885 prevUtxo = utxo 1886 } 1887 if reorder { 1888 // get the checksum again after reorder 1889 checksum.SetInt64(0) 1890 for i := range ba.Utxos { 1891 utxo := &ba.Utxos[i] 1892 checksum.Add(&checksum, &utxo.ValueSat) 1893 } 1894 } 1895 if checksum.Cmp(&ba.BalanceSat) != 0 { 1896 var checksumFromTxs big.Int 1897 var utxos []Utxo 1898 err := d.GetAddrDescTransactions(addrDesc, 0, ^uint32(0), func(txid string, height uint32, indexes []int32) error { 1899 var ta *TxAddresses 1900 var err error 1901 // sort the indexes so that the utxos are appended in the reverse order 1902 sort.Slice(indexes, func(i, j int) bool { 1903 return indexes[i] > indexes[j] 1904 }) 1905 for _, index := range indexes { 1906 // take only outputs 1907 if index < 0 { 1908 break 1909 } 1910 if ta == nil { 1911 ta, err = d.GetTxAddresses(txid) 1912 if err != nil { 1913 return err 1914 } 1915 } 1916 if ta == nil { 1917 return errors.New("DB inconsistency: tx " + txid + ": not found in txAddresses") 1918 } 1919 if len(ta.Outputs) <= int(index) { 1920 glog.Warning("DB inconsistency: txAddresses " + txid + " does not have enough outputs") 1921 } else { 1922 tao := &ta.Outputs[index] 1923 if !tao.Spent { 1924 bTxid, _ := d.chainParser.PackTxid(txid) 1925 checksumFromTxs.Add(&checksumFromTxs, &tao.ValueSat) 1926 utxos = append(utxos, Utxo{BtxID: bTxid, Height: height, Vout: index, ValueSat: tao.ValueSat}) 1927 if checksumFromTxs.Cmp(&ba.BalanceSat) == 0 { 1928 return &StopIteration{} 1929 } 1930 } 1931 } 1932 } 1933 return nil 1934 }) 1935 if err != nil { 1936 return false, false, err 1937 } 1938 fixed := false 1939 if checksumFromTxs.Cmp(&ba.BalanceSat) == 0 { 1940 // reverse the utxos as they are added in descending order by height 1941 for i := len(utxos)/2 - 1; i >= 0; i-- { 1942 opp := len(utxos) - 1 - i 1943 utxos[i], utxos[opp] = utxos[opp], utxos[i] 1944 } 1945 ba.Utxos = utxos 1946 wb := gorocksdb.NewWriteBatch() 1947 err = d.storeBalances(wb, map[string]*AddrBalance{string(addrDesc): ba}) 1948 if err == nil { 1949 err = d.db.Write(d.wo, wb) 1950 } 1951 wb.Destroy() 1952 if err != nil { 1953 return false, false, errors.Errorf("balance %s, checksum %s, from txa %s, txs %d, error storing fixed utxos %v", ba.BalanceSat.String(), checksum.String(), checksumFromTxs.String(), ba.Txs, err) 1954 } 1955 fixed = true 1956 } 1957 return fixed, false, errors.Errorf("balance %s, checksum %s, from txa %s, txs %d", ba.BalanceSat.String(), checksum.String(), checksumFromTxs.String(), ba.Txs) 1958 } else if reorder { 1959 wb := gorocksdb.NewWriteBatch() 1960 err := d.storeBalances(wb, map[string]*AddrBalance{string(addrDesc): ba}) 1961 if err == nil { 1962 err = d.db.Write(d.wo, wb) 1963 } 1964 wb.Destroy() 1965 if err != nil { 1966 return false, false, errors.Errorf("error storing reordered utxos %v", err) 1967 } 1968 } 1969 return false, reorder, nil 1970 } 1971 1972 // FixUtxos checks and fixes possible 1973 func (d *RocksDB) FixUtxos(stop chan os.Signal) error { 1974 if d.chainParser.GetChainType() != bchain.ChainBitcoinType { 1975 glog.Info("FixUtxos: applicable only for bitcoin type coins") 1976 return nil 1977 } 1978 glog.Info("FixUtxos: starting") 1979 var row, errorsCount, fixedCount int64 1980 var seekKey []byte 1981 // do not use cache 1982 ro := gorocksdb.NewDefaultReadOptions() 1983 ro.SetFillCache(false) 1984 for { 1985 var addrDesc bchain.AddressDescriptor 1986 it := d.db.NewIteratorCF(ro, d.cfh[cfAddressBalance]) 1987 if row == 0 { 1988 it.SeekToFirst() 1989 } else { 1990 glog.Info("FixUtxos: row ", row, ", errors ", errorsCount) 1991 it.Seek(seekKey) 1992 it.Next() 1993 } 1994 for count := 0; it.Valid() && count < refreshIterator; it.Next() { 1995 select { 1996 case <-stop: 1997 return errors.New("Interrupted") 1998 default: 1999 } 2000 addrDesc = it.Key().Data() 2001 buf := it.Value().Data() 2002 count++ 2003 row++ 2004 if len(buf) < 3 { 2005 glog.Error("FixUtxos: row ", row, ", addrDesc ", addrDesc, ", empty data") 2006 errorsCount++ 2007 continue 2008 } 2009 ba, err := unpackAddrBalance(buf, d.chainParser.PackedTxidLen(), AddressBalanceDetailUTXO) 2010 if err != nil { 2011 glog.Error("FixUtxos: row ", row, ", addrDesc ", addrDesc, ", unpackAddrBalance error ", err) 2012 errorsCount++ 2013 continue 2014 } 2015 fixed, reordered, err := d.fixUtxo(addrDesc, ba) 2016 if err != nil { 2017 errorsCount++ 2018 glog.Error("FixUtxos: row ", row, ", addrDesc ", addrDesc, ", error ", err, ", fixed ", fixed) 2019 if fixed { 2020 fixedCount++ 2021 } 2022 } else if reordered { 2023 glog.Error("FixUtxos: row ", row, ", addrDesc ", addrDesc, " reordered") 2024 fixedCount++ 2025 } 2026 } 2027 seekKey = append([]byte{}, addrDesc...) 2028 valid := it.Valid() 2029 it.Close() 2030 if !valid { 2031 break 2032 } 2033 } 2034 glog.Info("FixUtxos: finished, scanned ", row, " rows, found ", errorsCount, " errors, fixed ", fixedCount) 2035 return nil 2036 } 2037 2038 // Helpers 2039 2040 func packAddressKey(addrDesc bchain.AddressDescriptor, height uint32) []byte { 2041 buf := make([]byte, len(addrDesc)+packedHeightBytes) 2042 copy(buf, addrDesc) 2043 // pack height as binary complement to achieve ordering from newest to oldest block 2044 binary.BigEndian.PutUint32(buf[len(addrDesc):], ^height) 2045 return buf 2046 } 2047 2048 func unpackAddressKey(key []byte) ([]byte, uint32, error) { 2049 i := len(key) - packedHeightBytes 2050 if i <= 0 { 2051 return nil, 0, errors.New("Invalid address key") 2052 } 2053 // height is packed in binary complement, convert it 2054 return key[:i], ^unpackUint(key[i : i+packedHeightBytes]), nil 2055 } 2056 2057 func packUint(i uint32) []byte { 2058 buf := make([]byte, 4) 2059 binary.BigEndian.PutUint32(buf, i) 2060 return buf 2061 } 2062 2063 func unpackUint(buf []byte) uint32 { 2064 return binary.BigEndian.Uint32(buf) 2065 } 2066 2067 func packVarint32(i int32, buf []byte) int { 2068 return vlq.PutInt(buf, int64(i)) 2069 } 2070 2071 func packVarint(i int, buf []byte) int { 2072 return vlq.PutInt(buf, int64(i)) 2073 } 2074 2075 func packVaruint(i uint, buf []byte) int { 2076 return vlq.PutUint(buf, uint64(i)) 2077 } 2078 2079 func unpackVarint32(buf []byte) (int32, int) { 2080 i, ofs := vlq.Int(buf) 2081 return int32(i), ofs 2082 } 2083 2084 func unpackVarint(buf []byte) (int, int) { 2085 i, ofs := vlq.Int(buf) 2086 return int(i), ofs 2087 } 2088 2089 func unpackVaruint(buf []byte) (uint, int) { 2090 i, ofs := vlq.Uint(buf) 2091 return uint(i), ofs 2092 } 2093 2094 const ( 2095 // number of bits in a big.Word 2096 wordBits = 32 << (uint64(^big.Word(0)) >> 63) 2097 // number of bytes in a big.Word 2098 wordBytes = wordBits / 8 2099 // max packed bigint words 2100 maxPackedBigintWords = (256 - wordBytes) / wordBytes 2101 maxPackedBigintBytes = 249 2102 ) 2103 2104 // big int is packed in BigEndian order without memory allocation as 1 byte length followed by bytes of big int 2105 // number of written bytes is returned 2106 // limitation: bigints longer than 248 bytes are truncated to 248 bytes 2107 // caution: buffer must be big enough to hold the packed big int, buffer 249 bytes big is always safe 2108 func packBigint(bi *big.Int, buf []byte) int { 2109 w := bi.Bits() 2110 lw := len(w) 2111 // zero returns only one byte - zero length 2112 if lw == 0 { 2113 buf[0] = 0 2114 return 1 2115 } 2116 // pack the most significant word in a special way - skip leading zeros 2117 w0 := w[lw-1] 2118 fb := 8 2119 mask := big.Word(0xff) << (wordBits - 8) 2120 for w0&mask == 0 { 2121 fb-- 2122 mask >>= 8 2123 } 2124 for i := fb; i > 0; i-- { 2125 buf[i] = byte(w0) 2126 w0 >>= 8 2127 } 2128 // if the big int is too big (> 2^1984), the number of bytes would not fit to 1 byte 2129 // in this case, truncate the number, it is not expected to work with this big numbers as amounts 2130 s := 0 2131 if lw > maxPackedBigintWords { 2132 s = lw - maxPackedBigintWords 2133 } 2134 // pack the rest of the words in reverse order 2135 for j := lw - 2; j >= s; j-- { 2136 d := w[j] 2137 for i := fb + wordBytes; i > fb; i-- { 2138 buf[i] = byte(d) 2139 d >>= 8 2140 } 2141 fb += wordBytes 2142 } 2143 buf[0] = byte(fb) 2144 return fb + 1 2145 } 2146 2147 func unpackBigint(buf []byte) (big.Int, int) { 2148 var r big.Int 2149 l := int(buf[0]) + 1 2150 r.SetBytes(buf[1:l]) 2151 return r, l 2152 }