github.com/Ethersocial/go-esn@v0.3.7/swarm/storage/ldbstore.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // disk storage layer for the package bzz 18 // DbStore implements the ChunkStore interface and is used by the FileStore as 19 // persistent storage of chunks 20 // it implements purging based on access count allowing for external control of 21 // max capacity 22 23 package storage 24 25 import ( 26 "archive/tar" 27 "bytes" 28 "context" 29 "encoding/binary" 30 "encoding/hex" 31 "errors" 32 "fmt" 33 "io" 34 "io/ioutil" 35 "sort" 36 "sync" 37 38 "github.com/ethersocial/go-esn/metrics" 39 "github.com/ethersocial/go-esn/rlp" 40 "github.com/ethersocial/go-esn/swarm/log" 41 "github.com/ethersocial/go-esn/swarm/storage/mock" 42 "github.com/syndtr/goleveldb/leveldb" 43 "github.com/syndtr/goleveldb/leveldb/opt" 44 ) 45 46 const ( 47 gcArrayFreeRatio = 0.1 48 maxGCitems = 5000 // max number of items to be gc'd per call to collectGarbage() 49 ) 50 51 var ( 52 dbEntryCount = metrics.NewRegisteredCounter("ldbstore.entryCnt", nil) 53 ) 54 55 var ( 56 keyIndex = byte(0) 57 keyOldData = byte(1) 58 keyAccessCnt = []byte{2} 59 keyEntryCnt = []byte{3} 60 keyDataIdx = []byte{4} 61 keyData = byte(6) 62 keyDistanceCnt = byte(7) 63 keySchema = []byte{8} 64 ) 65 66 var ( 67 ErrDBClosed = errors.New("LDBStore closed") 68 ) 69 70 type gcItem struct { 71 idx uint64 72 value uint64 73 idxKey []byte 74 po uint8 75 } 76 77 type LDBStoreParams struct { 78 *StoreParams 79 Path string 80 Po func(Address) uint8 81 } 82 83 // NewLDBStoreParams constructs LDBStoreParams with the specified values. 84 func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams { 85 return &LDBStoreParams{ 86 StoreParams: storeparams, 87 Path: path, 88 Po: func(k Address) (ret uint8) { return uint8(Proximity(storeparams.BaseKey, k[:])) }, 89 } 90 } 91 92 type LDBStore struct { 93 db *LDBDatabase 94 95 // this should be stored in db, accessed transactionally 96 entryCnt uint64 // number of items in the LevelDB 97 accessCnt uint64 // ever-accumulating number increased every time we read/access an entry 98 dataIdx uint64 // similar to entryCnt, but we only increment it 99 capacity uint64 100 bucketCnt []uint64 101 102 hashfunc SwarmHasher 103 po func(Address) uint8 104 105 batchC chan bool 106 batchesC chan struct{} 107 closed bool 108 batch *dbBatch 109 lock sync.RWMutex 110 quit chan struct{} 111 112 // Functions encodeDataFunc is used to bypass 113 // the default functionality of DbStore with 114 // mock.NodeStore for testing purposes. 115 encodeDataFunc func(chunk Chunk) []byte 116 // If getDataFunc is defined, it will be used for 117 // retrieving the chunk data instead from the local 118 // LevelDB database. 119 getDataFunc func(key Address) (data []byte, err error) 120 } 121 122 type dbBatch struct { 123 *leveldb.Batch 124 err error 125 c chan struct{} 126 } 127 128 func newBatch() *dbBatch { 129 return &dbBatch{Batch: new(leveldb.Batch), c: make(chan struct{})} 130 } 131 132 // TODO: Instead of passing the distance function, just pass the address from which distances are calculated 133 // to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing 134 // a function different from the one that is actually used. 135 func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { 136 s = new(LDBStore) 137 s.hashfunc = params.Hash 138 s.quit = make(chan struct{}) 139 140 s.batchesC = make(chan struct{}, 1) 141 go s.writeBatches() 142 s.batch = newBatch() 143 // associate encodeData with default functionality 144 s.encodeDataFunc = encodeData 145 146 s.db, err = NewLDBDatabase(params.Path) 147 if err != nil { 148 return nil, err 149 } 150 151 s.po = params.Po 152 s.setCapacity(params.DbCapacity) 153 154 s.bucketCnt = make([]uint64, 0x100) 155 for i := 0; i < 0x100; i++ { 156 k := make([]byte, 2) 157 k[0] = keyDistanceCnt 158 k[1] = uint8(i) 159 cnt, _ := s.db.Get(k) 160 s.bucketCnt[i] = BytesToU64(cnt) 161 } 162 data, _ := s.db.Get(keyEntryCnt) 163 s.entryCnt = BytesToU64(data) 164 data, _ = s.db.Get(keyAccessCnt) 165 s.accessCnt = BytesToU64(data) 166 data, _ = s.db.Get(keyDataIdx) 167 s.dataIdx = BytesToU64(data) 168 169 return s, nil 170 } 171 172 // NewMockDbStore creates a new instance of DbStore with 173 // mockStore set to a provided value. If mockStore argument is nil, 174 // this function behaves exactly as NewDbStore. 175 func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error) { 176 s, err = NewLDBStore(params) 177 if err != nil { 178 return nil, err 179 } 180 181 // replace put and get with mock store functionality 182 if mockStore != nil { 183 s.encodeDataFunc = newMockEncodeDataFunc(mockStore) 184 s.getDataFunc = newMockGetDataFunc(mockStore) 185 } 186 return 187 } 188 189 type dpaDBIndex struct { 190 Idx uint64 191 Access uint64 192 } 193 194 func BytesToU64(data []byte) uint64 { 195 if len(data) < 8 { 196 return 0 197 } 198 return binary.BigEndian.Uint64(data) 199 } 200 201 func U64ToBytes(val uint64) []byte { 202 data := make([]byte, 8) 203 binary.BigEndian.PutUint64(data, val) 204 return data 205 } 206 207 func (s *LDBStore) updateIndexAccess(index *dpaDBIndex) { 208 index.Access = s.accessCnt 209 } 210 211 func getIndexKey(hash Address) []byte { 212 hashSize := len(hash) 213 key := make([]byte, hashSize+1) 214 key[0] = keyIndex 215 copy(key[1:], hash[:]) 216 return key 217 } 218 219 func getDataKey(idx uint64, po uint8) []byte { 220 key := make([]byte, 10) 221 key[0] = keyData 222 key[1] = po 223 binary.BigEndian.PutUint64(key[2:], idx) 224 225 return key 226 } 227 228 func encodeIndex(index *dpaDBIndex) []byte { 229 data, _ := rlp.EncodeToBytes(index) 230 return data 231 } 232 233 func encodeData(chunk Chunk) []byte { 234 // Always create a new underlying array for the returned byte slice. 235 // The chunk.Address array may be used in the returned slice which 236 // may be changed later in the code or by the LevelDB, resulting 237 // that the Address is changed as well. 238 return append(append([]byte{}, chunk.Address()[:]...), chunk.Data()...) 239 } 240 241 func decodeIndex(data []byte, index *dpaDBIndex) error { 242 dec := rlp.NewStream(bytes.NewReader(data), 0) 243 return dec.Decode(index) 244 } 245 246 func decodeData(addr Address, data []byte) (*chunk, error) { 247 return NewChunk(addr, data[32:]), nil 248 } 249 250 func (s *LDBStore) collectGarbage(ratio float32) { 251 log.Trace("collectGarbage", "ratio", ratio) 252 253 metrics.GetOrRegisterCounter("ldbstore.collectgarbage", nil).Inc(1) 254 255 it := s.db.NewIterator() 256 defer it.Release() 257 258 garbage := []*gcItem{} 259 gcnt := 0 260 261 for ok := it.Seek([]byte{keyIndex}); ok && (gcnt < maxGCitems) && (uint64(gcnt) < s.entryCnt); ok = it.Next() { 262 itkey := it.Key() 263 264 if (itkey == nil) || (itkey[0] != keyIndex) { 265 break 266 } 267 268 // it.Key() contents change on next call to it.Next(), so we must copy it 269 key := make([]byte, len(it.Key())) 270 copy(key, it.Key()) 271 272 val := it.Value() 273 274 var index dpaDBIndex 275 276 hash := key[1:] 277 decodeIndex(val, &index) 278 po := s.po(hash) 279 280 gci := &gcItem{ 281 idxKey: key, 282 idx: index.Idx, 283 value: index.Access, // the smaller, the more likely to be gc'd. see sort comparator below. 284 po: po, 285 } 286 287 garbage = append(garbage, gci) 288 gcnt++ 289 } 290 291 sort.Slice(garbage[:gcnt], func(i, j int) bool { return garbage[i].value < garbage[j].value }) 292 293 cutoff := int(float32(gcnt) * ratio) 294 metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(cutoff)) 295 296 for i := 0; i < cutoff; i++ { 297 s.delete(garbage[i].idx, garbage[i].idxKey, garbage[i].po) 298 } 299 } 300 301 // Export writes all chunks from the store to a tar archive, returning the 302 // number of chunks written. 303 func (s *LDBStore) Export(out io.Writer) (int64, error) { 304 tw := tar.NewWriter(out) 305 defer tw.Close() 306 307 it := s.db.NewIterator() 308 defer it.Release() 309 var count int64 310 for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() { 311 key := it.Key() 312 if (key == nil) || (key[0] != keyIndex) { 313 break 314 } 315 316 var index dpaDBIndex 317 318 hash := key[1:] 319 decodeIndex(it.Value(), &index) 320 po := s.po(hash) 321 datakey := getDataKey(index.Idx, po) 322 log.Trace("store.export", "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po) 323 data, err := s.db.Get(datakey) 324 if err != nil { 325 log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key, err)) 326 continue 327 } 328 329 hdr := &tar.Header{ 330 Name: hex.EncodeToString(hash), 331 Mode: 0644, 332 Size: int64(len(data)), 333 } 334 if err := tw.WriteHeader(hdr); err != nil { 335 return count, err 336 } 337 if _, err := tw.Write(data); err != nil { 338 return count, err 339 } 340 count++ 341 } 342 343 return count, nil 344 } 345 346 // of chunks read. 347 func (s *LDBStore) Import(in io.Reader) (int64, error) { 348 tr := tar.NewReader(in) 349 350 ctx, cancel := context.WithCancel(context.Background()) 351 defer cancel() 352 353 countC := make(chan int64) 354 errC := make(chan error) 355 var count int64 356 go func() { 357 for { 358 hdr, err := tr.Next() 359 if err == io.EOF { 360 break 361 } else if err != nil { 362 select { 363 case errC <- err: 364 case <-ctx.Done(): 365 } 366 } 367 368 if len(hdr.Name) != 64 { 369 log.Warn("ignoring non-chunk file", "name", hdr.Name) 370 continue 371 } 372 373 keybytes, err := hex.DecodeString(hdr.Name) 374 if err != nil { 375 log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) 376 continue 377 } 378 379 data, err := ioutil.ReadAll(tr) 380 if err != nil { 381 select { 382 case errC <- err: 383 case <-ctx.Done(): 384 } 385 } 386 key := Address(keybytes) 387 chunk := NewChunk(key, data[32:]) 388 389 go func() { 390 select { 391 case errC <- s.Put(ctx, chunk): 392 case <-ctx.Done(): 393 } 394 }() 395 396 count++ 397 } 398 countC <- count 399 }() 400 401 // wait for all chunks to be stored 402 i := int64(0) 403 var total int64 404 for { 405 select { 406 case err := <-errC: 407 if err != nil { 408 return count, err 409 } 410 i++ 411 case total = <-countC: 412 case <-ctx.Done(): 413 return i, ctx.Err() 414 } 415 if total > 0 && i == total { 416 return total, nil 417 } 418 } 419 } 420 421 //Cleanup iterates over the database and deletes chunks if they pass the `f` condition 422 func (s *LDBStore) Cleanup(f func(*chunk) bool) { 423 var errorsFound, removed, total int 424 425 it := s.db.NewIterator() 426 defer it.Release() 427 for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() { 428 key := it.Key() 429 if (key == nil) || (key[0] != keyIndex) { 430 break 431 } 432 total++ 433 var index dpaDBIndex 434 err := decodeIndex(it.Value(), &index) 435 if err != nil { 436 log.Warn("Cannot decode") 437 errorsFound++ 438 continue 439 } 440 hash := key[1:] 441 po := s.po(hash) 442 datakey := getDataKey(index.Idx, po) 443 data, err := s.db.Get(datakey) 444 if err != nil { 445 found := false 446 447 // highest possible proximity is 255 448 for po = 1; po <= 255; po++ { 449 datakey = getDataKey(index.Idx, po) 450 data, err = s.db.Get(datakey) 451 if err == nil { 452 found = true 453 break 454 } 455 } 456 457 if !found { 458 log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key)) 459 errorsFound++ 460 continue 461 } 462 } 463 464 ck := data[:32] 465 c, err := decodeData(ck, data) 466 if err != nil { 467 log.Error("decodeData error", "err", err) 468 continue 469 } 470 471 cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) 472 log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) 473 474 // if chunk is to be removed 475 if f(c) { 476 log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) 477 s.delete(index.Idx, getIndexKey(key[1:]), po) 478 removed++ 479 errorsFound++ 480 } 481 } 482 483 log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) 484 } 485 486 func (s *LDBStore) ReIndex() { 487 //Iterates over the database and checks that there are no faulty chunks 488 it := s.db.NewIterator() 489 startPosition := []byte{keyOldData} 490 it.Seek(startPosition) 491 var key []byte 492 var errorsFound, total int 493 for it.Valid() { 494 key = it.Key() 495 if (key == nil) || (key[0] != keyOldData) { 496 break 497 } 498 data := it.Value() 499 hasher := s.hashfunc() 500 hasher.Write(data) 501 hash := hasher.Sum(nil) 502 503 newKey := make([]byte, 10) 504 oldCntKey := make([]byte, 2) 505 newCntKey := make([]byte, 2) 506 oldCntKey[0] = keyDistanceCnt 507 newCntKey[0] = keyDistanceCnt 508 key[0] = keyData 509 key[1] = s.po(Address(key[1:])) 510 oldCntKey[1] = key[1] 511 newCntKey[1] = s.po(Address(newKey[1:])) 512 copy(newKey[2:], key[1:]) 513 newValue := append(hash, data...) 514 515 batch := new(leveldb.Batch) 516 batch.Delete(key) 517 s.bucketCnt[oldCntKey[1]]-- 518 batch.Put(oldCntKey, U64ToBytes(s.bucketCnt[oldCntKey[1]])) 519 batch.Put(newKey, newValue) 520 s.bucketCnt[newCntKey[1]]++ 521 batch.Put(newCntKey, U64ToBytes(s.bucketCnt[newCntKey[1]])) 522 s.db.Write(batch) 523 it.Next() 524 } 525 it.Release() 526 log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) 527 } 528 529 func (s *LDBStore) Delete(addr Address) { 530 s.lock.Lock() 531 defer s.lock.Unlock() 532 533 ikey := getIndexKey(addr) 534 535 var indx dpaDBIndex 536 s.tryAccessIdx(ikey, &indx) 537 538 s.delete(indx.Idx, ikey, s.po(addr)) 539 } 540 541 func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { 542 metrics.GetOrRegisterCounter("ldbstore.delete", nil).Inc(1) 543 544 batch := new(leveldb.Batch) 545 batch.Delete(idxKey) 546 batch.Delete(getDataKey(idx, po)) 547 s.entryCnt-- 548 dbEntryCount.Dec(1) 549 cntKey := make([]byte, 2) 550 cntKey[0] = keyDistanceCnt 551 cntKey[1] = po 552 batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) 553 batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) 554 s.db.Write(batch) 555 } 556 557 func (s *LDBStore) BinIndex(po uint8) uint64 { 558 s.lock.RLock() 559 defer s.lock.RUnlock() 560 return s.bucketCnt[po] 561 } 562 563 func (s *LDBStore) Size() uint64 { 564 s.lock.RLock() 565 defer s.lock.RUnlock() 566 return s.entryCnt 567 } 568 569 func (s *LDBStore) CurrentStorageIndex() uint64 { 570 s.lock.RLock() 571 defer s.lock.RUnlock() 572 return s.dataIdx 573 } 574 575 func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error { 576 metrics.GetOrRegisterCounter("ldbstore.put", nil).Inc(1) 577 log.Trace("ldbstore.put", "key", chunk.Address()) 578 579 ikey := getIndexKey(chunk.Address()) 580 var index dpaDBIndex 581 582 po := s.po(chunk.Address()) 583 584 s.lock.Lock() 585 586 if s.closed { 587 s.lock.Unlock() 588 return ErrDBClosed 589 } 590 batch := s.batch 591 592 log.Trace("ldbstore.put: s.db.Get", "key", chunk.Address(), "ikey", fmt.Sprintf("%x", ikey)) 593 idata, err := s.db.Get(ikey) 594 if err != nil { 595 s.doPut(chunk, &index, po) 596 } else { 597 log.Trace("ldbstore.put: chunk already exists, only update access", "key", chunk.Address) 598 decodeIndex(idata, &index) 599 } 600 index.Access = s.accessCnt 601 s.accessCnt++ 602 idata = encodeIndex(&index) 603 s.batch.Put(ikey, idata) 604 605 s.lock.Unlock() 606 607 select { 608 case s.batchesC <- struct{}{}: 609 default: 610 } 611 612 select { 613 case <-batch.c: 614 return batch.err 615 case <-ctx.Done(): 616 return ctx.Err() 617 } 618 } 619 620 // force putting into db, does not check access index 621 func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) { 622 data := s.encodeDataFunc(chunk) 623 dkey := getDataKey(s.dataIdx, po) 624 s.batch.Put(dkey, data) 625 index.Idx = s.dataIdx 626 s.bucketCnt[po] = s.dataIdx 627 s.entryCnt++ 628 dbEntryCount.Inc(1) 629 s.dataIdx++ 630 631 cntKey := make([]byte, 2) 632 cntKey[0] = keyDistanceCnt 633 cntKey[1] = po 634 s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po])) 635 } 636 637 func (s *LDBStore) writeBatches() { 638 for { 639 select { 640 case <-s.quit: 641 log.Debug("DbStore: quit batch write loop") 642 return 643 case <-s.batchesC: 644 err := s.writeCurrentBatch() 645 if err != nil { 646 log.Debug("DbStore: quit batch write loop", "err", err.Error()) 647 return 648 } 649 } 650 } 651 652 } 653 654 func (s *LDBStore) writeCurrentBatch() error { 655 s.lock.Lock() 656 defer s.lock.Unlock() 657 b := s.batch 658 l := b.Len() 659 if l == 0 { 660 return nil 661 } 662 e := s.entryCnt 663 d := s.dataIdx 664 a := s.accessCnt 665 s.batch = newBatch() 666 b.err = s.writeBatch(b, e, d, a) 667 close(b.c) 668 for e > s.capacity { 669 log.Trace("for >", "e", e, "s.capacity", s.capacity) 670 // Collect garbage in a separate goroutine 671 // to be able to interrupt this loop by s.quit. 672 done := make(chan struct{}) 673 go func() { 674 s.collectGarbage(gcArrayFreeRatio) 675 log.Trace("collectGarbage closing done") 676 close(done) 677 }() 678 679 select { 680 case <-s.quit: 681 return errors.New("CollectGarbage terminated due to quit") 682 case <-done: 683 } 684 e = s.entryCnt 685 } 686 return nil 687 } 688 689 // must be called non concurrently 690 func (s *LDBStore) writeBatch(b *dbBatch, entryCnt, dataIdx, accessCnt uint64) error { 691 b.Put(keyEntryCnt, U64ToBytes(entryCnt)) 692 b.Put(keyDataIdx, U64ToBytes(dataIdx)) 693 b.Put(keyAccessCnt, U64ToBytes(accessCnt)) 694 l := b.Len() 695 if err := s.db.Write(b.Batch); err != nil { 696 return fmt.Errorf("unable to write batch: %v", err) 697 } 698 log.Trace(fmt.Sprintf("batch write (%d entries)", l)) 699 return nil 700 } 701 702 // newMockEncodeDataFunc returns a function that stores the chunk data 703 // to a mock store to bypass the default functionality encodeData. 704 // The constructed function always returns the nil data, as DbStore does 705 // not need to store the data, but still need to create the index. 706 func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte { 707 return func(chunk Chunk) []byte { 708 if err := mockStore.Put(chunk.Address(), encodeData(chunk)); err != nil { 709 log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Address().Log(), err)) 710 } 711 return chunk.Address()[:] 712 } 713 } 714 715 // try to find index; if found, update access cnt and return true 716 func (s *LDBStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { 717 idata, err := s.db.Get(ikey) 718 if err != nil { 719 return false 720 } 721 decodeIndex(idata, index) 722 s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) 723 s.accessCnt++ 724 index.Access = s.accessCnt 725 idata = encodeIndex(index) 726 s.batch.Put(ikey, idata) 727 select { 728 case s.batchesC <- struct{}{}: 729 default: 730 } 731 return true 732 } 733 734 // GetSchema is returning the current named schema of the datastore as read from LevelDB 735 func (s *LDBStore) GetSchema() (string, error) { 736 s.lock.Lock() 737 defer s.lock.Unlock() 738 739 data, err := s.db.Get(keySchema) 740 if err != nil { 741 if err == leveldb.ErrNotFound { 742 return "", nil 743 } 744 return "", err 745 } 746 747 return string(data), nil 748 } 749 750 // PutSchema is saving a named schema to the LevelDB datastore 751 func (s *LDBStore) PutSchema(schema string) error { 752 s.lock.Lock() 753 defer s.lock.Unlock() 754 755 return s.db.Put(keySchema, []byte(schema)) 756 } 757 758 func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) { 759 metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1) 760 log.Trace("ldbstore.get", "key", addr) 761 762 s.lock.Lock() 763 defer s.lock.Unlock() 764 return s.get(addr) 765 } 766 767 func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { 768 var indx dpaDBIndex 769 if s.closed { 770 return nil, ErrDBClosed 771 } 772 if s.tryAccessIdx(getIndexKey(addr), &indx) { 773 var data []byte 774 if s.getDataFunc != nil { 775 // if getDataFunc is defined, use it to retrieve the chunk data 776 log.Trace("ldbstore.get retrieve with getDataFunc", "key", addr) 777 data, err = s.getDataFunc(addr) 778 if err != nil { 779 return 780 } 781 } else { 782 // default DbStore functionality to retrieve chunk data 783 proximity := s.po(addr) 784 datakey := getDataKey(indx.Idx, proximity) 785 data, err = s.db.Get(datakey) 786 log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", indx.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity) 787 if err != nil { 788 log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err) 789 s.delete(indx.Idx, getIndexKey(addr), s.po(addr)) 790 return 791 } 792 } 793 794 return decodeData(addr, data) 795 } else { 796 err = ErrChunkNotFound 797 } 798 799 return 800 } 801 802 // newMockGetFunc returns a function that reads chunk data from 803 // the mock database, which is used as the value for DbStore.getFunc 804 // to bypass the default functionality of DbStore with a mock store. 805 func newMockGetDataFunc(mockStore *mock.NodeStore) func(addr Address) (data []byte, err error) { 806 return func(addr Address) (data []byte, err error) { 807 data, err = mockStore.Get(addr) 808 if err == mock.ErrNotFound { 809 // preserve ErrChunkNotFound error 810 err = ErrChunkNotFound 811 } 812 return data, err 813 } 814 } 815 816 func (s *LDBStore) updateAccessCnt(addr Address) { 817 818 s.lock.Lock() 819 defer s.lock.Unlock() 820 821 var index dpaDBIndex 822 s.tryAccessIdx(getIndexKey(addr), &index) // result_chn == nil, only update access cnt 823 824 } 825 826 func (s *LDBStore) setCapacity(c uint64) { 827 s.lock.Lock() 828 defer s.lock.Unlock() 829 830 s.capacity = c 831 832 if s.entryCnt > c { 833 ratio := float32(1.01) - float32(c)/float32(s.entryCnt) 834 if ratio < gcArrayFreeRatio { 835 ratio = gcArrayFreeRatio 836 } 837 if ratio > 1 { 838 ratio = 1 839 } 840 for s.entryCnt > c { 841 s.collectGarbage(ratio) 842 } 843 } 844 } 845 846 func (s *LDBStore) Close() { 847 close(s.quit) 848 s.lock.Lock() 849 s.closed = true 850 s.lock.Unlock() 851 // force writing out current batch 852 s.writeCurrentBatch() 853 close(s.batchesC) 854 s.db.Close() 855 } 856 857 // SyncIterator(start, stop, po, f) calls f on each hash of a bin po from start to stop 858 func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Address, uint64) bool) error { 859 metrics.GetOrRegisterCounter("ldbstore.synciterator", nil).Inc(1) 860 861 sincekey := getDataKey(since, po) 862 untilkey := getDataKey(until, po) 863 it := s.db.NewIterator() 864 defer it.Release() 865 866 for ok := it.Seek(sincekey); ok; ok = it.Next() { 867 metrics.GetOrRegisterCounter("ldbstore.synciterator.seek", nil).Inc(1) 868 869 dbkey := it.Key() 870 if dbkey[0] != keyData || dbkey[1] != po || bytes.Compare(untilkey, dbkey) < 0 { 871 break 872 } 873 key := make([]byte, 32) 874 val := it.Value() 875 copy(key, val[:32]) 876 if !f(Address(key), binary.BigEndian.Uint64(dbkey[2:])) { 877 break 878 } 879 } 880 return it.Error() 881 } 882 883 func databaseExists(path string) bool { 884 o := &opt.Options{ 885 ErrorIfMissing: true, 886 } 887 tdb, err := leveldb.OpenFile(path, o) 888 if err != nil { 889 return false 890 } 891 defer tdb.Close() 892 return true 893 }