github.com/rumhocker/blockbook@v0.3.2/db/rocksdb_ethereumtype.go (about) 1 package db 2 3 import ( 4 "blockbook/bchain" 5 "blockbook/bchain/coins/eth" 6 "bytes" 7 "encoding/hex" 8 9 vlq "github.com/bsm/go-vlq" 10 "github.com/golang/glog" 11 "github.com/juju/errors" 12 "github.com/tecbot/gorocksdb" 13 ) 14 15 // AddrContract is Contract address with number of transactions done by given address 16 type AddrContract struct { 17 Contract bchain.AddressDescriptor 18 Txs uint 19 } 20 21 // AddrContracts contains number of transactions and contracts for an address 22 type AddrContracts struct { 23 TotalTxs uint 24 NonContractTxs uint 25 Contracts []AddrContract 26 } 27 28 func (d *RocksDB) storeAddressContracts(wb *gorocksdb.WriteBatch, acm map[string]*AddrContracts) error { 29 buf := make([]byte, 64) 30 varBuf := make([]byte, vlq.MaxLen64) 31 for addrDesc, acs := range acm { 32 // address with 0 contracts is removed from db - happens on disconnect 33 if acs == nil || (acs.NonContractTxs == 0 && len(acs.Contracts) == 0) { 34 wb.DeleteCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc)) 35 } else { 36 buf = buf[:0] 37 l := packVaruint(acs.TotalTxs, varBuf) 38 buf = append(buf, varBuf[:l]...) 39 l = packVaruint(acs.NonContractTxs, varBuf) 40 buf = append(buf, varBuf[:l]...) 41 for _, ac := range acs.Contracts { 42 buf = append(buf, ac.Contract...) 43 l = packVaruint(ac.Txs, varBuf) 44 buf = append(buf, varBuf[:l]...) 45 } 46 wb.PutCF(d.cfh[cfAddressContracts], bchain.AddressDescriptor(addrDesc), buf) 47 } 48 } 49 return nil 50 } 51 52 // GetAddrDescContracts returns AddrContracts for given addrDesc 53 func (d *RocksDB) GetAddrDescContracts(addrDesc bchain.AddressDescriptor) (*AddrContracts, error) { 54 val, err := d.db.GetCF(d.ro, d.cfh[cfAddressContracts], addrDesc) 55 if err != nil { 56 return nil, err 57 } 58 defer val.Free() 59 buf := val.Data() 60 if len(buf) == 0 { 61 return nil, nil 62 } 63 tt, l := unpackVaruint(buf) 64 buf = buf[l:] 65 nct, l := unpackVaruint(buf) 66 buf = buf[l:] 67 c := make([]AddrContract, 0, 4) 68 for len(buf) > 0 { 69 if len(buf) < eth.EthereumTypeAddressDescriptorLen { 70 return nil, errors.New("Invalid data stored in cfAddressContracts for AddrDesc " + addrDesc.String()) 71 } 72 txs, l := unpackVaruint(buf[eth.EthereumTypeAddressDescriptorLen:]) 73 contract := append(bchain.AddressDescriptor(nil), buf[:eth.EthereumTypeAddressDescriptorLen]...) 74 c = append(c, AddrContract{ 75 Contract: contract, 76 Txs: txs, 77 }) 78 buf = buf[eth.EthereumTypeAddressDescriptorLen+l:] 79 } 80 return &AddrContracts{ 81 TotalTxs: tt, 82 NonContractTxs: nct, 83 Contracts: c, 84 }, nil 85 } 86 87 func findContractInAddressContracts(contract bchain.AddressDescriptor, contracts []AddrContract) (int, bool) { 88 for i := range contracts { 89 if bytes.Equal(contract, contracts[i].Contract) { 90 return i, true 91 } 92 } 93 return 0, false 94 } 95 96 func isZeroAddress(addrDesc bchain.AddressDescriptor) bool { 97 for _, b := range addrDesc { 98 if b != 0 { 99 return false 100 } 101 } 102 return true 103 } 104 105 func (d *RocksDB) addToAddressesAndContractsEthereumType(addrDesc bchain.AddressDescriptor, btxID []byte, index int32, contract bchain.AddressDescriptor, addresses addressesMap, addressContracts map[string]*AddrContracts, addTxCount bool) error { 106 var err error 107 strAddrDesc := string(addrDesc) 108 ac, e := addressContracts[strAddrDesc] 109 if !e { 110 ac, err = d.GetAddrDescContracts(addrDesc) 111 if err != nil { 112 return err 113 } 114 if ac == nil { 115 ac = &AddrContracts{} 116 } 117 addressContracts[strAddrDesc] = ac 118 d.cbs.balancesMiss++ 119 } else { 120 d.cbs.balancesHit++ 121 } 122 if contract == nil { 123 if addTxCount { 124 ac.NonContractTxs++ 125 } 126 } else { 127 // do not store contracts for 0x0000000000000000000000000000000000000000 address 128 if !isZeroAddress(addrDesc) { 129 // locate the contract and set i to the index in the array of contracts 130 i, found := findContractInAddressContracts(contract, ac.Contracts) 131 if !found { 132 i = len(ac.Contracts) 133 ac.Contracts = append(ac.Contracts, AddrContract{Contract: contract}) 134 } 135 // index 0 is for ETH transfers, contract indexes start with 1 136 if index < 0 { 137 index = ^int32(i + 1) 138 } else { 139 index = int32(i + 1) 140 } 141 if addTxCount { 142 ac.Contracts[i].Txs++ 143 } 144 } 145 } 146 counted := addToAddressesMap(addresses, strAddrDesc, btxID, index) 147 if !counted { 148 ac.TotalTxs++ 149 } 150 return nil 151 } 152 153 type ethBlockTxContract struct { 154 addr, contract bchain.AddressDescriptor 155 } 156 157 type ethBlockTx struct { 158 btxID []byte 159 from, to bchain.AddressDescriptor 160 contracts []ethBlockTxContract 161 } 162 163 func (d *RocksDB) processAddressesEthereumType(block *bchain.Block, addresses addressesMap, addressContracts map[string]*AddrContracts) ([]ethBlockTx, error) { 164 blockTxs := make([]ethBlockTx, len(block.Txs)) 165 for txi, tx := range block.Txs { 166 btxID, err := d.chainParser.PackTxid(tx.Txid) 167 if err != nil { 168 return nil, err 169 } 170 blockTx := &blockTxs[txi] 171 blockTx.btxID = btxID 172 var from, to bchain.AddressDescriptor 173 // there is only one output address in EthereumType transaction, store it in format txid 0 174 if len(tx.Vout) == 1 && len(tx.Vout[0].ScriptPubKey.Addresses) == 1 { 175 to, err = d.chainParser.GetAddrDescFromAddress(tx.Vout[0].ScriptPubKey.Addresses[0]) 176 if err != nil { 177 // do not log ErrAddressMissing, transactions can be without to address (for example eth contracts) 178 if err != bchain.ErrAddressMissing { 179 glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, output", err, block.Height, tx.Txid) 180 } 181 continue 182 } 183 if err = d.addToAddressesAndContractsEthereumType(to, btxID, 0, nil, addresses, addressContracts, true); err != nil { 184 return nil, err 185 } 186 blockTx.to = to 187 } 188 // there is only one input address in EthereumType transaction, store it in format txid ^0 189 if len(tx.Vin) == 1 && len(tx.Vin[0].Addresses) == 1 { 190 from, err = d.chainParser.GetAddrDescFromAddress(tx.Vin[0].Addresses[0]) 191 if err != nil { 192 if err != bchain.ErrAddressMissing { 193 glog.Warningf("rocksdb: addrDesc: %v - height %d, tx %v, input", err, block.Height, tx.Txid) 194 } 195 continue 196 } 197 if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(0), nil, addresses, addressContracts, !bytes.Equal(from, to)); err != nil { 198 return nil, err 199 } 200 blockTx.from = from 201 } 202 // store erc20 transfers 203 erc20, err := d.chainParser.EthereumTypeGetErc20FromTx(&tx) 204 if err != nil { 205 glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v", err, block.Height, tx.Txid) 206 } 207 blockTx.contracts = make([]ethBlockTxContract, len(erc20)*2) 208 j := 0 209 for i, t := range erc20 { 210 var contract, from, to bchain.AddressDescriptor 211 contract, err = d.chainParser.GetAddrDescFromAddress(t.Contract) 212 if err == nil { 213 from, err = d.chainParser.GetAddrDescFromAddress(t.From) 214 if err == nil { 215 to, err = d.chainParser.GetAddrDescFromAddress(t.To) 216 } 217 } 218 if err != nil { 219 glog.Warningf("rocksdb: GetErc20FromTx %v - height %d, tx %v, transfer %v", err, block.Height, tx.Txid, t) 220 continue 221 } 222 if err = d.addToAddressesAndContractsEthereumType(to, btxID, int32(i), contract, addresses, addressContracts, true); err != nil { 223 return nil, err 224 } 225 eq := bytes.Equal(from, to) 226 bc := &blockTx.contracts[j] 227 j++ 228 bc.addr = from 229 bc.contract = contract 230 if err = d.addToAddressesAndContractsEthereumType(from, btxID, ^int32(i), contract, addresses, addressContracts, !eq); err != nil { 231 return nil, err 232 } 233 // add to address to blockTx.contracts only if it is different from from address 234 if !eq { 235 bc = &blockTx.contracts[j] 236 j++ 237 bc.addr = to 238 bc.contract = contract 239 } 240 } 241 blockTx.contracts = blockTx.contracts[:j] 242 } 243 return blockTxs, nil 244 } 245 246 func (d *RocksDB) storeAndCleanupBlockTxsEthereumType(wb *gorocksdb.WriteBatch, block *bchain.Block, blockTxs []ethBlockTx) error { 247 pl := d.chainParser.PackedTxidLen() 248 buf := make([]byte, 0, (pl+2*eth.EthereumTypeAddressDescriptorLen)*len(blockTxs)) 249 varBuf := make([]byte, vlq.MaxLen64) 250 zeroAddress := make([]byte, eth.EthereumTypeAddressDescriptorLen) 251 appendAddress := func(a bchain.AddressDescriptor) { 252 if len(a) != eth.EthereumTypeAddressDescriptorLen { 253 buf = append(buf, zeroAddress...) 254 } else { 255 buf = append(buf, a...) 256 } 257 } 258 for i := range blockTxs { 259 blockTx := &blockTxs[i] 260 buf = append(buf, blockTx.btxID...) 261 appendAddress(blockTx.from) 262 appendAddress(blockTx.to) 263 l := packVaruint(uint(len(blockTx.contracts)), varBuf) 264 buf = append(buf, varBuf[:l]...) 265 for j := range blockTx.contracts { 266 c := &blockTx.contracts[j] 267 appendAddress(c.addr) 268 appendAddress(c.contract) 269 } 270 } 271 key := packUint(block.Height) 272 wb.PutCF(d.cfh[cfBlockTxs], key, buf) 273 return d.cleanupBlockTxs(wb, block) 274 } 275 276 func (d *RocksDB) getBlockTxsEthereumType(height uint32) ([]ethBlockTx, error) { 277 pl := d.chainParser.PackedTxidLen() 278 val, err := d.db.GetCF(d.ro, d.cfh[cfBlockTxs], packUint(height)) 279 if err != nil { 280 return nil, err 281 } 282 defer val.Free() 283 buf := val.Data() 284 // nil data means the key was not found in DB 285 if buf == nil { 286 return nil, nil 287 } 288 // buf can be empty slice, this means the block did not contain any transactions 289 bt := make([]ethBlockTx, 0, 8) 290 getAddress := func(i int) (bchain.AddressDescriptor, int, error) { 291 if len(buf)-i < eth.EthereumTypeAddressDescriptorLen { 292 glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) 293 return nil, 0, errors.New("Inconsistent data in blockTxs") 294 } 295 a := append(bchain.AddressDescriptor(nil), buf[i:i+eth.EthereumTypeAddressDescriptorLen]...) 296 // return null addresses as nil 297 for _, b := range a { 298 if b != 0 { 299 return a, i + eth.EthereumTypeAddressDescriptorLen, nil 300 } 301 } 302 return nil, i + eth.EthereumTypeAddressDescriptorLen, nil 303 } 304 var from, to bchain.AddressDescriptor 305 for i := 0; i < len(buf); { 306 if len(buf)-i < pl { 307 glog.Error("rocksdb: Inconsistent data in blockTxs ", hex.EncodeToString(buf)) 308 return nil, errors.New("Inconsistent data in blockTxs") 309 } 310 txid := append([]byte(nil), buf[i:i+pl]...) 311 i += pl 312 from, i, err = getAddress(i) 313 if err != nil { 314 return nil, err 315 } 316 to, i, err = getAddress(i) 317 if err != nil { 318 return nil, err 319 } 320 cc, l := unpackVaruint(buf[i:]) 321 i += l 322 contracts := make([]ethBlockTxContract, cc) 323 for j := range contracts { 324 contracts[j].addr, i, err = getAddress(i) 325 if err != nil { 326 return nil, err 327 } 328 contracts[j].contract, i, err = getAddress(i) 329 if err != nil { 330 return nil, err 331 } 332 } 333 bt = append(bt, ethBlockTx{ 334 btxID: txid, 335 from: from, 336 to: to, 337 contracts: contracts, 338 }) 339 } 340 return bt, nil 341 } 342 343 func (d *RocksDB) disconnectBlockTxsEthereumType(wb *gorocksdb.WriteBatch, height uint32, blockTxs []ethBlockTx, contracts map[string]*AddrContracts) error { 344 glog.Info("Disconnecting block ", height, " containing ", len(blockTxs), " transactions") 345 addresses := make(map[string]map[string]struct{}) 346 disconnectAddress := func(btxID []byte, addrDesc, contract bchain.AddressDescriptor) error { 347 var err error 348 // do not process empty address 349 if len(addrDesc) == 0 { 350 return nil 351 } 352 s := string(addrDesc) 353 txid := string(btxID) 354 // find if tx for this address was already encountered 355 mtx, ftx := addresses[s] 356 if !ftx { 357 mtx = make(map[string]struct{}) 358 mtx[txid] = struct{}{} 359 addresses[s] = mtx 360 } else { 361 _, ftx = mtx[txid] 362 if !ftx { 363 mtx[txid] = struct{}{} 364 } 365 } 366 c, fc := contracts[s] 367 if !fc { 368 c, err = d.GetAddrDescContracts(addrDesc) 369 if err != nil { 370 return err 371 } 372 contracts[s] = c 373 } 374 if c != nil { 375 if !ftx { 376 c.TotalTxs-- 377 } 378 if contract == nil { 379 if c.NonContractTxs > 0 { 380 c.NonContractTxs-- 381 } else { 382 glog.Warning("AddressContracts ", addrDesc, ", EthTxs would be negative, tx ", hex.EncodeToString(btxID)) 383 } 384 } else { 385 i, found := findContractInAddressContracts(contract, c.Contracts) 386 if found { 387 if c.Contracts[i].Txs > 0 { 388 c.Contracts[i].Txs-- 389 if c.Contracts[i].Txs == 0 { 390 c.Contracts = append(c.Contracts[:i], c.Contracts[i+1:]...) 391 } 392 } else { 393 glog.Warning("AddressContracts ", addrDesc, ", contract ", i, " Txs would be negative, tx ", hex.EncodeToString(btxID)) 394 } 395 } else { 396 glog.Warning("AddressContracts ", addrDesc, ", contract ", contract, " not found, tx ", hex.EncodeToString(btxID)) 397 } 398 } 399 } else { 400 glog.Warning("AddressContracts ", addrDesc, " not found, tx ", hex.EncodeToString(btxID)) 401 } 402 return nil 403 } 404 for i := range blockTxs { 405 blockTx := &blockTxs[i] 406 if err := disconnectAddress(blockTx.btxID, blockTx.from, nil); err != nil { 407 return err 408 } 409 // if from==to, tx is counted only once and does not have to be disconnected again 410 if !bytes.Equal(blockTx.from, blockTx.to) { 411 if err := disconnectAddress(blockTx.btxID, blockTx.to, nil); err != nil { 412 return err 413 } 414 } 415 for _, c := range blockTx.contracts { 416 if err := disconnectAddress(blockTx.btxID, c.addr, c.contract); err != nil { 417 return err 418 } 419 } 420 wb.DeleteCF(d.cfh[cfTransactions], blockTx.btxID) 421 } 422 for a := range addresses { 423 key := packAddressKey([]byte(a), height) 424 wb.DeleteCF(d.cfh[cfAddresses], key) 425 } 426 return nil 427 } 428 429 // DisconnectBlockRangeEthereumType removes all data belonging to blocks in range lower-higher 430 // it is able to disconnect only blocks for which there are data in the blockTxs column 431 func (d *RocksDB) DisconnectBlockRangeEthereumType(lower uint32, higher uint32) error { 432 blocks := make([][]ethBlockTx, higher-lower+1) 433 for height := lower; height <= higher; height++ { 434 blockTxs, err := d.getBlockTxsEthereumType(height) 435 if err != nil { 436 return err 437 } 438 // nil blockTxs means blockTxs were not found in db 439 if blockTxs == nil { 440 return errors.Errorf("Cannot disconnect blocks with height %v and lower. It is necessary to rebuild index.", height) 441 } 442 blocks[height-lower] = blockTxs 443 } 444 wb := gorocksdb.NewWriteBatch() 445 defer wb.Destroy() 446 contracts := make(map[string]*AddrContracts) 447 for height := higher; height >= lower; height-- { 448 if err := d.disconnectBlockTxsEthereumType(wb, height, blocks[height-lower], contracts); err != nil { 449 return err 450 } 451 key := packUint(height) 452 wb.DeleteCF(d.cfh[cfBlockTxs], key) 453 wb.DeleteCF(d.cfh[cfHeight], key) 454 } 455 d.storeAddressContracts(wb, contracts) 456 err := d.db.Write(d.wo, wb) 457 if err == nil { 458 d.is.RemoveLastBlockTimes(int(higher-lower) + 1) 459 glog.Infof("rocksdb: blocks %d-%d disconnected", lower, higher) 460 } 461 return err 462 }