github.com/ebceco/ebc@v1.8.19-0.20190309150932-8cb0b9e06484/swarm/storage/ldbstore.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // disk storage layer for the package bzz 18 // DbStore implements the ChunkStore interface and is used by the FileStore as 19 // persistent storage of chunks 20 // it implements purging based on access count allowing for external control of 21 // max capacity 22 23 package storage 24 25 import ( 26 "archive/tar" 27 "bytes" 28 "context" 29 "encoding/binary" 30 "encoding/hex" 31 "errors" 32 "fmt" 33 "io" 34 "io/ioutil" 35 "sync" 36 37 "github.com/ebceco/ebc/metrics" 38 "github.com/ebceco/ebc/rlp" 39 "github.com/ebceco/ebc/swarm/log" 40 "github.com/ebceco/ebc/swarm/storage/mock" 41 "github.com/syndtr/goleveldb/leveldb" 42 ) 43 44 const ( 45 defaultGCRatio = 10 46 defaultMaxGCRound = 10000 47 defaultMaxGCBatch = 5000 48 49 wEntryCnt = 1 << 0 50 wIndexCnt = 1 << 1 51 wAccessCnt = 1 << 2 52 ) 53 54 var ( 55 dbEntryCount = metrics.NewRegisteredCounter("ldbstore.entryCnt", nil) 56 ) 57 58 var ( 59 keyIndex = byte(0) 60 keyAccessCnt = []byte{2} 61 keyEntryCnt = []byte{3} 62 keyDataIdx = []byte{4} 63 keyData = byte(6) 64 keyDistanceCnt = byte(7) 65 keySchema = []byte{8} 66 keyGCIdx = byte(9) // access to chunk data index, used by garbage collection in ascending order from first entry 67 ) 68 69 var ( 70 ErrDBClosed = errors.New("LDBStore closed") 71 ) 72 73 type LDBStoreParams struct { 74 *StoreParams 75 Path string 76 Po func(Address) uint8 77 } 78 79 // NewLDBStoreParams constructs LDBStoreParams with the specified values. 80 func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams { 81 return &LDBStoreParams{ 82 StoreParams: storeparams, 83 Path: path, 84 Po: func(k Address) (ret uint8) { return uint8(Proximity(storeparams.BaseKey, k[:])) }, 85 } 86 } 87 88 type garbage struct { 89 maxRound int // maximum number of chunks to delete in one garbage collection round 90 maxBatch int // maximum number of chunks to delete in one db request batch 91 ratio int // 1/x ratio to calculate the number of chunks to gc on a low capacity db 92 count int // number of chunks deleted in running round 93 target int // number of chunks to delete in running round 94 batch *dbBatch // the delete batch 95 runC chan struct{} // struct in chan means gc is NOT running 96 } 97 98 type LDBStore struct { 99 db *LDBDatabase 100 101 // this should be stored in db, accessed transactionally 102 entryCnt uint64 // number of items in the LevelDB 103 accessCnt uint64 // ever-accumulating number increased every time we read/access an entry 104 dataIdx uint64 // similar to entryCnt, but we only increment it 105 capacity uint64 106 bucketCnt []uint64 107 108 hashfunc SwarmHasher 109 po func(Address) uint8 110 111 batchesC chan struct{} 112 closed bool 113 batch *dbBatch 114 lock sync.RWMutex 115 quit chan struct{} 116 gc *garbage 117 118 // Functions encodeDataFunc is used to bypass 119 // the default functionality of DbStore with 120 // mock.NodeStore for testing purposes. 121 encodeDataFunc func(chunk Chunk) []byte 122 // If getDataFunc is defined, it will be used for 123 // retrieving the chunk data instead from the local 124 // LevelDB database. 125 getDataFunc func(key Address) (data []byte, err error) 126 } 127 128 type dbBatch struct { 129 *leveldb.Batch 130 err error 131 c chan struct{} 132 } 133 134 func newBatch() *dbBatch { 135 return &dbBatch{Batch: new(leveldb.Batch), c: make(chan struct{})} 136 } 137 138 // TODO: Instead of passing the distance function, just pass the address from which distances are calculated 139 // to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing 140 // a function different from the one that is actually used. 141 func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { 142 s = new(LDBStore) 143 s.hashfunc = params.Hash 144 s.quit = make(chan struct{}) 145 146 s.batchesC = make(chan struct{}, 1) 147 go s.writeBatches() 148 s.batch = newBatch() 149 // associate encodeData with default functionality 150 s.encodeDataFunc = encodeData 151 152 s.db, err = NewLDBDatabase(params.Path) 153 if err != nil { 154 return nil, err 155 } 156 157 s.po = params.Po 158 s.setCapacity(params.DbCapacity) 159 160 s.bucketCnt = make([]uint64, 0x100) 161 for i := 0; i < 0x100; i++ { 162 k := make([]byte, 2) 163 k[0] = keyDistanceCnt 164 k[1] = uint8(i) 165 cnt, _ := s.db.Get(k) 166 s.bucketCnt[i] = BytesToU64(cnt) 167 } 168 data, _ := s.db.Get(keyEntryCnt) 169 s.entryCnt = BytesToU64(data) 170 data, _ = s.db.Get(keyAccessCnt) 171 s.accessCnt = BytesToU64(data) 172 data, _ = s.db.Get(keyDataIdx) 173 s.dataIdx = BytesToU64(data) 174 175 // set up garbage collection 176 s.gc = &garbage{ 177 maxBatch: defaultMaxGCBatch, 178 maxRound: defaultMaxGCRound, 179 ratio: defaultGCRatio, 180 } 181 182 s.gc.runC = make(chan struct{}, 1) 183 s.gc.runC <- struct{}{} 184 185 return s, nil 186 } 187 188 // MarkAccessed increments the access counter as a best effort for a chunk, so 189 // the chunk won't get garbage collected. 190 func (s *LDBStore) MarkAccessed(addr Address) { 191 s.lock.Lock() 192 defer s.lock.Unlock() 193 194 if s.closed { 195 return 196 } 197 198 proximity := s.po(addr) 199 s.tryAccessIdx(addr, proximity) 200 } 201 202 // initialize and set values for processing of gc round 203 func (s *LDBStore) startGC(c int) { 204 205 s.gc.count = 0 206 // calculate the target number of deletions 207 if c >= s.gc.maxRound { 208 s.gc.target = s.gc.maxRound 209 } else { 210 s.gc.target = c / s.gc.ratio 211 } 212 s.gc.batch = newBatch() 213 log.Debug("startgc", "requested", c, "target", s.gc.target) 214 } 215 216 // NewMockDbStore creates a new instance of DbStore with 217 // mockStore set to a provided value. If mockStore argument is nil, 218 // this function behaves exactly as NewDbStore. 219 func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error) { 220 s, err = NewLDBStore(params) 221 if err != nil { 222 return nil, err 223 } 224 225 // replace put and get with mock store functionality 226 if mockStore != nil { 227 s.encodeDataFunc = newMockEncodeDataFunc(mockStore) 228 s.getDataFunc = newMockGetDataFunc(mockStore) 229 } 230 return 231 } 232 233 type dpaDBIndex struct { 234 Idx uint64 235 Access uint64 236 } 237 238 func BytesToU64(data []byte) uint64 { 239 if len(data) < 8 { 240 return 0 241 } 242 return binary.BigEndian.Uint64(data) 243 } 244 245 func U64ToBytes(val uint64) []byte { 246 data := make([]byte, 8) 247 binary.BigEndian.PutUint64(data, val) 248 return data 249 } 250 251 func getIndexKey(hash Address) []byte { 252 hashSize := len(hash) 253 key := make([]byte, hashSize+1) 254 key[0] = keyIndex 255 copy(key[1:], hash[:]) 256 return key 257 } 258 259 func getDataKey(idx uint64, po uint8) []byte { 260 key := make([]byte, 10) 261 key[0] = keyData 262 key[1] = po 263 binary.BigEndian.PutUint64(key[2:], idx) 264 265 return key 266 } 267 268 func getGCIdxKey(index *dpaDBIndex) []byte { 269 key := make([]byte, 9) 270 key[0] = keyGCIdx 271 binary.BigEndian.PutUint64(key[1:], index.Access) 272 return key 273 } 274 275 func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte { 276 val := make([]byte, 41) // po = 1, index.Index = 8, Address = 32 277 val[0] = po 278 binary.BigEndian.PutUint64(val[1:], index.Idx) 279 copy(val[9:], addr) 280 return val 281 } 282 283 func parseIdxKey(key []byte) (byte, []byte) { 284 return key[0], key[1:] 285 } 286 287 func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) { 288 index = &dpaDBIndex{ 289 Idx: binary.BigEndian.Uint64(val[1:]), 290 Access: binary.BigEndian.Uint64(accessCnt), 291 } 292 po = val[0] 293 addr = val[9:] 294 return 295 } 296 297 func encodeIndex(index *dpaDBIndex) []byte { 298 data, _ := rlp.EncodeToBytes(index) 299 return data 300 } 301 302 func encodeData(chunk Chunk) []byte { 303 // Always create a new underlying array for the returned byte slice. 304 // The chunk.Address array may be used in the returned slice which 305 // may be changed later in the code or by the LevelDB, resulting 306 // that the Address is changed as well. 307 return append(append([]byte{}, chunk.Address()[:]...), chunk.Data()...) 308 } 309 310 func decodeIndex(data []byte, index *dpaDBIndex) error { 311 dec := rlp.NewStream(bytes.NewReader(data), 0) 312 return dec.Decode(index) 313 } 314 315 func decodeData(addr Address, data []byte) (*chunk, error) { 316 return NewChunk(addr, data[32:]), nil 317 } 318 319 func (s *LDBStore) collectGarbage() error { 320 321 // prevent duplicate gc from starting when one is already running 322 select { 323 case <-s.gc.runC: 324 default: 325 return nil 326 } 327 328 s.lock.Lock() 329 entryCnt := s.entryCnt 330 s.lock.Unlock() 331 332 metrics.GetOrRegisterCounter("ldbstore.collectgarbage", nil).Inc(1) 333 334 // calculate the amount of chunks to collect and reset counter 335 s.startGC(int(entryCnt)) 336 log.Debug("collectGarbage", "target", s.gc.target, "entryCnt", entryCnt) 337 338 var totalDeleted int 339 for s.gc.count < s.gc.target { 340 it := s.db.NewIterator() 341 ok := it.Seek([]byte{keyGCIdx}) 342 var singleIterationCount int 343 344 // every batch needs a lock so we avoid entries changing accessidx in the meantime 345 s.lock.Lock() 346 for ; ok && (singleIterationCount < s.gc.maxBatch); ok = it.Next() { 347 348 // quit if no more access index keys 349 itkey := it.Key() 350 if (itkey == nil) || (itkey[0] != keyGCIdx) { 351 break 352 } 353 354 // get chunk data entry from access index 355 val := it.Value() 356 index, po, hash := parseGCIdxEntry(itkey[1:], val) 357 keyIdx := make([]byte, 33) 358 keyIdx[0] = keyIndex 359 copy(keyIdx[1:], hash) 360 361 // add delete operation to batch 362 s.delete(s.gc.batch.Batch, index, keyIdx, po) 363 singleIterationCount++ 364 s.gc.count++ 365 log.Trace("garbage collect enqueued chunk for deletion", "key", hash) 366 367 // break if target is not on max garbage batch boundary 368 if s.gc.count >= s.gc.target { 369 break 370 } 371 } 372 373 s.writeBatch(s.gc.batch, wEntryCnt) 374 s.lock.Unlock() 375 it.Release() 376 log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count) 377 } 378 379 s.gc.runC <- struct{}{} 380 log.Debug("garbage collect done", "c", s.gc.count) 381 382 metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(totalDeleted)) 383 return nil 384 } 385 386 // Export writes all chunks from the store to a tar archive, returning the 387 // number of chunks written. 388 func (s *LDBStore) Export(out io.Writer) (int64, error) { 389 tw := tar.NewWriter(out) 390 defer tw.Close() 391 392 it := s.db.NewIterator() 393 defer it.Release() 394 var count int64 395 for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() { 396 key := it.Key() 397 if (key == nil) || (key[0] != keyIndex) { 398 break 399 } 400 401 var index dpaDBIndex 402 403 hash := key[1:] 404 decodeIndex(it.Value(), &index) 405 po := s.po(hash) 406 datakey := getDataKey(index.Idx, po) 407 log.Trace("store.export", "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po) 408 data, err := s.db.Get(datakey) 409 if err != nil { 410 log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key, err)) 411 continue 412 } 413 414 hdr := &tar.Header{ 415 Name: hex.EncodeToString(hash), 416 Mode: 0644, 417 Size: int64(len(data)), 418 } 419 if err := tw.WriteHeader(hdr); err != nil { 420 return count, err 421 } 422 if _, err := tw.Write(data); err != nil { 423 return count, err 424 } 425 count++ 426 } 427 428 return count, nil 429 } 430 431 // of chunks read. 432 func (s *LDBStore) Import(in io.Reader) (int64, error) { 433 tr := tar.NewReader(in) 434 435 ctx, cancel := context.WithCancel(context.Background()) 436 defer cancel() 437 438 countC := make(chan int64) 439 errC := make(chan error) 440 var count int64 441 go func() { 442 for { 443 hdr, err := tr.Next() 444 if err == io.EOF { 445 break 446 } else if err != nil { 447 select { 448 case errC <- err: 449 case <-ctx.Done(): 450 } 451 } 452 453 if len(hdr.Name) != 64 { 454 log.Warn("ignoring non-chunk file", "name", hdr.Name) 455 continue 456 } 457 458 keybytes, err := hex.DecodeString(hdr.Name) 459 if err != nil { 460 log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) 461 continue 462 } 463 464 data, err := ioutil.ReadAll(tr) 465 if err != nil { 466 select { 467 case errC <- err: 468 case <-ctx.Done(): 469 } 470 } 471 key := Address(keybytes) 472 chunk := NewChunk(key, data[32:]) 473 474 go func() { 475 select { 476 case errC <- s.Put(ctx, chunk): 477 case <-ctx.Done(): 478 } 479 }() 480 481 count++ 482 } 483 countC <- count 484 }() 485 486 // wait for all chunks to be stored 487 i := int64(0) 488 var total int64 489 for { 490 select { 491 case err := <-errC: 492 if err != nil { 493 return count, err 494 } 495 i++ 496 case total = <-countC: 497 case <-ctx.Done(): 498 return i, ctx.Err() 499 } 500 if total > 0 && i == total { 501 return total, nil 502 } 503 } 504 } 505 506 // Cleanup iterates over the database and deletes chunks if they pass the `f` condition 507 func (s *LDBStore) Cleanup(f func(*chunk) bool) { 508 var errorsFound, removed, total int 509 510 it := s.db.NewIterator() 511 defer it.Release() 512 for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() { 513 key := it.Key() 514 if (key == nil) || (key[0] != keyIndex) { 515 break 516 } 517 total++ 518 var index dpaDBIndex 519 err := decodeIndex(it.Value(), &index) 520 if err != nil { 521 log.Warn("Cannot decode") 522 errorsFound++ 523 continue 524 } 525 hash := key[1:] 526 po := s.po(hash) 527 datakey := getDataKey(index.Idx, po) 528 data, err := s.db.Get(datakey) 529 if err != nil { 530 found := false 531 532 // highest possible proximity is 255 533 for po = 1; po <= 255; po++ { 534 datakey = getDataKey(index.Idx, po) 535 data, err = s.db.Get(datakey) 536 if err == nil { 537 found = true 538 break 539 } 540 } 541 542 if !found { 543 log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key)) 544 errorsFound++ 545 continue 546 } 547 } 548 549 ck := data[:32] 550 c, err := decodeData(ck, data) 551 if err != nil { 552 log.Error("decodeData error", "err", err) 553 continue 554 } 555 556 cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) 557 log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) 558 559 // if chunk is to be removed 560 if f(c) { 561 log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) 562 s.deleteNow(&index, getIndexKey(key[1:]), po) 563 removed++ 564 errorsFound++ 565 } 566 } 567 568 log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) 569 } 570 571 // CleanGCIndex rebuilds the garbage collector index from scratch, while 572 // removing inconsistent elements, e.g., indices with missing data chunks. 573 // WARN: it's a pretty heavy, long running function. 574 func (s *LDBStore) CleanGCIndex() error { 575 s.lock.Lock() 576 defer s.lock.Unlock() 577 578 batch := leveldb.Batch{} 579 580 var okEntryCount uint64 581 var totalEntryCount uint64 582 583 // throw out all gc indices, we will rebuild from cleaned index 584 it := s.db.NewIterator() 585 it.Seek([]byte{keyGCIdx}) 586 var gcDeletes int 587 for it.Valid() { 588 rowType, _ := parseIdxKey(it.Key()) 589 if rowType != keyGCIdx { 590 break 591 } 592 batch.Delete(it.Key()) 593 gcDeletes++ 594 it.Next() 595 } 596 log.Debug("gc", "deletes", gcDeletes) 597 if err := s.db.Write(&batch); err != nil { 598 return err 599 } 600 batch.Reset() 601 602 it.Release() 603 604 // corrected po index pointer values 605 var poPtrs [256]uint64 606 607 // set to true if chunk count not on 4096 iteration boundary 608 var doneIterating bool 609 610 // last key index in previous iteration 611 lastIdxKey := []byte{keyIndex} 612 613 // counter for debug output 614 var cleanBatchCount int 615 616 // go through all key index entries 617 for !doneIterating { 618 cleanBatchCount++ 619 var idxs []dpaDBIndex 620 var chunkHashes [][]byte 621 var pos []uint8 622 it := s.db.NewIterator() 623 624 it.Seek(lastIdxKey) 625 626 // 4096 is just a nice number, don't look for any hidden meaning here... 627 var i int 628 for i = 0; i < 4096; i++ { 629 630 // this really shouldn't happen unless database is empty 631 // but let's keep it to be safe 632 if !it.Valid() { 633 doneIterating = true 634 break 635 } 636 637 // if it's not keyindex anymore we're done iterating 638 rowType, chunkHash := parseIdxKey(it.Key()) 639 if rowType != keyIndex { 640 doneIterating = true 641 break 642 } 643 644 // decode the retrieved index 645 var idx dpaDBIndex 646 err := decodeIndex(it.Value(), &idx) 647 if err != nil { 648 return fmt.Errorf("corrupt index: %v", err) 649 } 650 po := s.po(chunkHash) 651 lastIdxKey = it.Key() 652 653 // if we don't find the data key, remove the entry 654 // if we find it, add to the array of new gc indices to create 655 dataKey := getDataKey(idx.Idx, po) 656 _, err = s.db.Get(dataKey) 657 if err != nil { 658 log.Warn("deleting inconsistent index (missing data)", "key", chunkHash) 659 batch.Delete(it.Key()) 660 } else { 661 idxs = append(idxs, idx) 662 chunkHashes = append(chunkHashes, chunkHash) 663 pos = append(pos, po) 664 okEntryCount++ 665 if idx.Idx > poPtrs[po] { 666 poPtrs[po] = idx.Idx 667 } 668 } 669 totalEntryCount++ 670 it.Next() 671 } 672 it.Release() 673 674 // flush the key index corrections 675 err := s.db.Write(&batch) 676 if err != nil { 677 return err 678 } 679 batch.Reset() 680 681 // add correct gc indices 682 for i, okIdx := range idxs { 683 gcIdxKey := getGCIdxKey(&okIdx) 684 gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i]) 685 batch.Put(gcIdxKey, gcIdxData) 686 log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData) 687 } 688 689 // flush them 690 err = s.db.Write(&batch) 691 if err != nil { 692 return err 693 } 694 batch.Reset() 695 696 log.Debug("clean gc index pass", "batch", cleanBatchCount, "checked", i, "kept", len(idxs)) 697 } 698 699 log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len()) 700 701 // lastly add updated entry count 702 var entryCount [8]byte 703 binary.BigEndian.PutUint64(entryCount[:], okEntryCount) 704 batch.Put(keyEntryCnt, entryCount[:]) 705 706 // and add the new po index pointers 707 var poKey [2]byte 708 poKey[0] = keyDistanceCnt 709 for i, poPtr := range poPtrs { 710 poKey[1] = uint8(i) 711 if poPtr == 0 { 712 batch.Delete(poKey[:]) 713 } else { 714 var idxCount [8]byte 715 binary.BigEndian.PutUint64(idxCount[:], poPtr) 716 batch.Put(poKey[:], idxCount[:]) 717 } 718 } 719 720 // if you made it this far your harddisk has survived. Congratulations 721 return s.db.Write(&batch) 722 } 723 724 // Delete is removes a chunk and updates indices. 725 // Is thread safe 726 func (s *LDBStore) Delete(addr Address) error { 727 s.lock.Lock() 728 defer s.lock.Unlock() 729 730 ikey := getIndexKey(addr) 731 732 idata, err := s.db.Get(ikey) 733 if err != nil { 734 return err 735 } 736 737 var idx dpaDBIndex 738 decodeIndex(idata, &idx) 739 proximity := s.po(addr) 740 return s.deleteNow(&idx, ikey, proximity) 741 } 742 743 // executes one delete operation immediately 744 // see *LDBStore.delete 745 func (s *LDBStore) deleteNow(idx *dpaDBIndex, idxKey []byte, po uint8) error { 746 batch := new(leveldb.Batch) 747 s.delete(batch, idx, idxKey, po) 748 return s.db.Write(batch) 749 } 750 751 // adds a delete chunk operation to the provided batch 752 // if called directly, decrements entrycount regardless if the chunk exists upon deletion. Risk of wrap to max uint64 753 func (s *LDBStore) delete(batch *leveldb.Batch, idx *dpaDBIndex, idxKey []byte, po uint8) { 754 metrics.GetOrRegisterCounter("ldbstore.delete", nil).Inc(1) 755 756 gcIdxKey := getGCIdxKey(idx) 757 batch.Delete(gcIdxKey) 758 dataKey := getDataKey(idx.Idx, po) 759 batch.Delete(dataKey) 760 batch.Delete(idxKey) 761 s.entryCnt-- 762 dbEntryCount.Dec(1) 763 cntKey := make([]byte, 2) 764 cntKey[0] = keyDistanceCnt 765 cntKey[1] = po 766 batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) 767 batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) 768 } 769 770 func (s *LDBStore) BinIndex(po uint8) uint64 { 771 s.lock.RLock() 772 defer s.lock.RUnlock() 773 return s.bucketCnt[po] 774 } 775 776 // Put adds a chunk to the database, adding indices and incrementing global counters. 777 // If it already exists, it merely increments the access count of the existing entry. 778 // Is thread safe 779 func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { 780 metrics.GetOrRegisterCounter("ldbstore.put", nil).Inc(1) 781 log.Trace("ldbstore.put", "key", chunk.Address()) 782 783 ikey := getIndexKey(chunk.Address()) 784 var index dpaDBIndex 785 786 po := s.po(chunk.Address()) 787 788 s.lock.Lock() 789 790 if s.closed { 791 s.lock.Unlock() 792 return ErrDBClosed 793 } 794 batch := s.batch 795 796 log.Trace("ldbstore.put: s.db.Get", "key", chunk.Address(), "ikey", fmt.Sprintf("%x", ikey)) 797 _, err := s.db.Get(ikey) 798 if err != nil { 799 s.doPut(chunk, &index, po) 800 } 801 idata := encodeIndex(&index) 802 s.batch.Put(ikey, idata) 803 804 // add the access-chunkindex index for garbage collection 805 gcIdxKey := getGCIdxKey(&index) 806 gcIdxData := getGCIdxValue(&index, po, chunk.Address()) 807 s.batch.Put(gcIdxKey, gcIdxData) 808 s.lock.Unlock() 809 810 select { 811 case s.batchesC <- struct{}{}: 812 default: 813 } 814 815 select { 816 case <-batch.c: 817 return batch.err 818 case <-ctx.Done(): 819 return ctx.Err() 820 } 821 } 822 823 // force putting into db, does not check or update necessary indices 824 func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { 825 data := s.encodeDataFunc(chunk) 826 dkey := getDataKey(s.dataIdx, po) 827 s.batch.Put(dkey, data) 828 index.Idx = s.dataIdx 829 s.bucketCnt[po] = s.dataIdx 830 s.entryCnt++ 831 dbEntryCount.Inc(1) 832 s.dataIdx++ 833 index.Access = s.accessCnt 834 s.accessCnt++ 835 cntKey := make([]byte, 2) 836 cntKey[0] = keyDistanceCnt 837 cntKey[1] = po 838 s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) 839 } 840 841 func (s *LDBStore) writeBatches() { 842 for { 843 select { 844 case <-s.quit: 845 log.Debug("DbStore: quit batch write loop") 846 return 847 case <-s.batchesC: 848 err := s.writeCurrentBatch() 849 if err != nil { 850 log.Debug("DbStore: quit batch write loop", "err", err.Error()) 851 return 852 } 853 } 854 } 855 856 } 857 858 func (s *LDBStore) writeCurrentBatch() error { 859 s.lock.Lock() 860 defer s.lock.Unlock() 861 b := s.batch 862 l := b.Len() 863 if l == 0 { 864 return nil 865 } 866 s.batch = newBatch() 867 b.err = s.writeBatch(b, wEntryCnt|wAccessCnt|wIndexCnt) 868 close(b.c) 869 if s.entryCnt >= s.capacity { 870 go s.collectGarbage() 871 } 872 return nil 873 } 874 875 // must be called non concurrently 876 func (s *LDBStore) writeBatch(b *dbBatch, wFlag uint8) error { 877 if wFlag&wEntryCnt > 0 { 878 b.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) 879 } 880 if wFlag&wIndexCnt > 0 { 881 b.Put(keyDataIdx, U64ToBytes(s.dataIdx)) 882 } 883 if wFlag&wAccessCnt > 0 { 884 b.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) 885 } 886 l := b.Len() 887 if err := s.db.Write(b.Batch); err != nil { 888 return fmt.Errorf("unable to write batch: %v", err) 889 } 890 log.Trace(fmt.Sprintf("batch write (%d entries)", l)) 891 return nil 892 } 893 894 // newMockEncodeDataFunc returns a function that stores the chunk data 895 // to a mock store to bypass the default functionality encodeData. 896 // The constructed function always returns the nil data, as DbStore does 897 // not need to store the data, but still need to create the index. 898 func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { 899 return func(chunk Chunk) []byte { 900 if err := mockStore.Put(chunk.Address(), encodeData(chunk)); err != nil { 901 log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Address().Log(), err)) 902 } 903 return chunk.Address()[:] 904 } 905 } 906 907 // tryAccessIdx tries to find index entry. If found then increments the access 908 // count for garbage collection and returns the index entry and true for found, 909 // otherwise returns nil and false. 910 func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) { 911 ikey := getIndexKey(addr) 912 idata, err := s.db.Get(ikey) 913 if err != nil { 914 return nil, false 915 } 916 917 index := new(dpaDBIndex) 918 decodeIndex(idata, index) 919 oldGCIdxKey := getGCIdxKey(index) 920 s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) 921 index.Access = s.accessCnt 922 idata = encodeIndex(index) 923 s.accessCnt++ 924 s.batch.Put(ikey, idata) 925 newGCIdxKey := getGCIdxKey(index) 926 newGCIdxData := getGCIdxValue(index, po, ikey[1:]) 927 s.batch.Delete(oldGCIdxKey) 928 s.batch.Put(newGCIdxKey, newGCIdxData) 929 select { 930 case s.batchesC <- struct{}{}: 931 default: 932 } 933 return index, true 934 } 935 936 // GetSchema is returning the current named schema of the datastore as read from LevelDB 937 func (s *LDBStore) GetSchema() (string, error) { 938 s.lock.Lock() 939 defer s.lock.Unlock() 940 941 data, err := s.db.Get(keySchema) 942 if err != nil { 943 if err == leveldb.ErrNotFound { 944 return DbSchemaNone, nil 945 } 946 return "", err 947 } 948 949 return string(data), nil 950 } 951 952 // PutSchema is saving a named schema to the LevelDB datastore 953 func (s *LDBStore) PutSchema(schema string) error { 954 s.lock.Lock() 955 defer s.lock.Unlock() 956 957 return s.db.Put(keySchema, []byte(schema)) 958 } 959 960 // Get retrieves the chunk matching the provided key from the database. 961 // If the chunk entry does not exist, it returns an error 962 // Updates access count and is thread safe 963 func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) { 964 metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1) 965 log.Trace("ldbstore.get", "key", addr) 966 967 s.lock.Lock() 968 defer s.lock.Unlock() 969 return s.get(addr) 970 } 971 972 // TODO: To conform with other private methods of this object indices should not be updated 973 func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { 974 if s.closed { 975 return nil, ErrDBClosed 976 } 977 proximity := s.po(addr) 978 index, found := s.tryAccessIdx(addr, proximity) 979 if found { 980 var data []byte 981 if s.getDataFunc != nil { 982 // if getDataFunc is defined, use it to retrieve the chunk data 983 log.Trace("ldbstore.get retrieve with getDataFunc", "key", addr) 984 data, err = s.getDataFunc(addr) 985 if err != nil { 986 return 987 } 988 } else { 989 // default DbStore functionality to retrieve chunk data 990 datakey := getDataKey(index.Idx, proximity) 991 data, err = s.db.Get(datakey) 992 log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", index.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) 993 if err != nil { 994 log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err) 995 s.deleteNow(index, getIndexKey(addr), s.po(addr)) 996 return 997 } 998 } 999 1000 return decodeData(addr, data) 1001 } else { 1002 err = ErrChunkNotFound 1003 } 1004 1005 return 1006 } 1007 1008 // newMockGetFunc returns a function that reads chunk data from 1009 // the mock database, which is used as the value for DbStore.getFunc 1010 // to bypass the default functionality of DbStore with a mock store. 1011 func newMockGetDataFunc(mockStore *mock.NodeStore) func(addr Address) (data []byte, err error) { 1012 return func(addr Address) (data []byte, err error) { 1013 data, err = mockStore.Get(addr) 1014 if err == mock.ErrNotFound { 1015 // preserve ErrChunkNotFound error 1016 err = ErrChunkNotFound 1017 } 1018 return data, err 1019 } 1020 } 1021 1022 func (s *LDBStore) setCapacity(c uint64) { 1023 s.lock.Lock() 1024 defer s.lock.Unlock() 1025 1026 s.capacity = c 1027 1028 for s.entryCnt > c { 1029 s.collectGarbage() 1030 } 1031 } 1032 1033 func (s *LDBStore) Close() { 1034 close(s.quit) 1035 s.lock.Lock() 1036 s.closed = true 1037 s.lock.Unlock() 1038 // force writing out current batch 1039 s.writeCurrentBatch() 1040 close(s.batchesC) 1041 s.db.Close() 1042 } 1043 1044 // SyncIterator(start, stop, po, f) calls f on each hash of a bin po from start to stop 1045 func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Address, uint64) bool) error { 1046 metrics.GetOrRegisterCounter("ldbstore.synciterator", nil).Inc(1) 1047 1048 sincekey := getDataKey(since, po) 1049 untilkey := getDataKey(until, po) 1050 it := s.db.NewIterator() 1051 defer it.Release() 1052 1053 for ok := it.Seek(sincekey); ok; ok = it.Next() { 1054 metrics.GetOrRegisterCounter("ldbstore.synciterator.seek", nil).Inc(1) 1055 1056 dbkey := it.Key() 1057 if dbkey[0] != keyData || dbkey[1] != po || bytes.Compare(untilkey, dbkey) < 0 { 1058 break 1059 } 1060 key := make([]byte, 32) 1061 val := it.Value() 1062 copy(key, val[:32]) 1063 if !f(Address(key), binary.BigEndian.Uint64(dbkey[2:])) { 1064 break 1065 } 1066 } 1067 return it.Error() 1068 }