github.com/nitinawathare/ethereumassignment3@v0.0.0-20211021213010-f07344c2b868/go-ethereum/swarm/storage/ldbstore.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  // disk storage layer for the package bzz
    18  // DbStore implements the ChunkStore interface and is used by the FileStore as
    19  // persistent storage of chunks
    20  // it implements purging based on access count allowing for external control of
    21  // max capacity
    22  
    23  package storage
    24  
    25  import (
    26  	"archive/tar"
    27  	"bytes"
    28  	"context"
    29  	"encoding/binary"
    30  	"encoding/hex"
    31  	"errors"
    32  	"fmt"
    33  	"io"
    34  	"io/ioutil"
    35  	"sync"
    36  
    37  	"github.com/ethereum/go-ethereum/metrics"
    38  	"github.com/ethereum/go-ethereum/rlp"
    39  	"github.com/ethereum/go-ethereum/swarm/log"
    40  	"github.com/ethereum/go-ethereum/swarm/storage/mock"
    41  	"github.com/syndtr/goleveldb/leveldb"
    42  )
    43  
    44  const (
    45  	defaultGCRatio    = 10
    46  	defaultMaxGCRound = 10000
    47  	defaultMaxGCBatch = 5000
    48  
    49  	wEntryCnt  = 1 << 0
    50  	wIndexCnt  = 1 << 1
    51  	wAccessCnt = 1 << 2
    52  )
    53  
    54  var (
    55  	dbEntryCount = metrics.NewRegisteredCounter("ldbstore.entryCnt", nil)
    56  )
    57  
    58  var (
    59  	keyIndex       = byte(0)
    60  	keyAccessCnt   = []byte{2}
    61  	keyEntryCnt    = []byte{3}
    62  	keyDataIdx     = []byte{4}
    63  	keyData        = byte(6)
    64  	keyDistanceCnt = byte(7)
    65  	keySchema      = []byte{8}
    66  	keyGCIdx       = byte(9) // access to chunk data index, used by garbage collection in ascending order from first entry
    67  )
    68  
    69  var (
    70  	ErrDBClosed = errors.New("LDBStore closed")
    71  )
    72  
    73  type LDBStoreParams struct {
    74  	*StoreParams
    75  	Path string
    76  	Po   func(Address) uint8
    77  }
    78  
    79  // NewLDBStoreParams constructs LDBStoreParams with the specified values.
    80  func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams {
    81  	return &LDBStoreParams{
    82  		StoreParams: storeparams,
    83  		Path:        path,
    84  		Po:          func(k Address) (ret uint8) { return uint8(Proximity(storeparams.BaseKey, k[:])) },
    85  	}
    86  }
    87  
    88  type garbage struct {
    89  	maxRound int           // maximum number of chunks to delete in one garbage collection round
    90  	maxBatch int           // maximum number of chunks to delete in one db request batch
    91  	ratio    int           // 1/x ratio to calculate the number of chunks to gc on a low capacity db
    92  	count    int           // number of chunks deleted in running round
    93  	target   int           // number of chunks to delete in running round
    94  	batch    *dbBatch      // the delete batch
    95  	runC     chan struct{} // struct in chan means gc is NOT running
    96  }
    97  
    98  type LDBStore struct {
    99  	db *LDBDatabase
   100  
   101  	// this should be stored in db, accessed transactionally
   102  	entryCnt  uint64 // number of items in the LevelDB
   103  	accessCnt uint64 // ever-accumulating number increased every time we read/access an entry
   104  	dataIdx   uint64 // similar to entryCnt, but we only increment it
   105  	capacity  uint64
   106  	bucketCnt []uint64
   107  
   108  	hashfunc SwarmHasher
   109  	po       func(Address) uint8
   110  
   111  	batchesC chan struct{}
   112  	closed   bool
   113  	batch    *dbBatch
   114  	lock     sync.RWMutex
   115  	quit     chan struct{}
   116  	gc       *garbage
   117  
   118  	// Functions encodeDataFunc is used to bypass
   119  	// the default functionality of DbStore with
   120  	// mock.NodeStore for testing purposes.
   121  	encodeDataFunc func(chunk Chunk) []byte
   122  	// If getDataFunc is defined, it will be used for
   123  	// retrieving the chunk data instead from the local
   124  	// LevelDB database.
   125  	getDataFunc func(key Address) (data []byte, err error)
   126  }
   127  
   128  type dbBatch struct {
   129  	*leveldb.Batch
   130  	err error
   131  	c   chan struct{}
   132  }
   133  
   134  func newBatch() *dbBatch {
   135  	return &dbBatch{Batch: new(leveldb.Batch), c: make(chan struct{})}
   136  }
   137  
   138  // TODO: Instead of passing the distance function, just pass the address from which distances are calculated
   139  // to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing
   140  // a function different from the one that is actually used.
   141  func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) {
   142  	s = new(LDBStore)
   143  	s.hashfunc = params.Hash
   144  	s.quit = make(chan struct{})
   145  
   146  	s.batchesC = make(chan struct{}, 1)
   147  	go s.writeBatches()
   148  	s.batch = newBatch()
   149  	// associate encodeData with default functionality
   150  	s.encodeDataFunc = encodeData
   151  
   152  	s.db, err = NewLDBDatabase(params.Path)
   153  	if err != nil {
   154  		return nil, err
   155  	}
   156  
   157  	s.po = params.Po
   158  	s.setCapacity(params.DbCapacity)
   159  
   160  	s.bucketCnt = make([]uint64, 0x100)
   161  	for i := 0; i < 0x100; i++ {
   162  		k := make([]byte, 2)
   163  		k[0] = keyDistanceCnt
   164  		k[1] = uint8(i)
   165  		cnt, _ := s.db.Get(k)
   166  		s.bucketCnt[i] = BytesToU64(cnt)
   167  	}
   168  	data, _ := s.db.Get(keyEntryCnt)
   169  	s.entryCnt = BytesToU64(data)
   170  	data, _ = s.db.Get(keyAccessCnt)
   171  	s.accessCnt = BytesToU64(data)
   172  	data, _ = s.db.Get(keyDataIdx)
   173  	s.dataIdx = BytesToU64(data)
   174  
   175  	// set up garbage collection
   176  	s.gc = &garbage{
   177  		maxBatch: defaultMaxGCBatch,
   178  		maxRound: defaultMaxGCRound,
   179  		ratio:    defaultGCRatio,
   180  	}
   181  
   182  	s.gc.runC = make(chan struct{}, 1)
   183  	s.gc.runC <- struct{}{}
   184  
   185  	return s, nil
   186  }
   187  
   188  // MarkAccessed increments the access counter as a best effort for a chunk, so
   189  // the chunk won't get garbage collected.
   190  func (s *LDBStore) MarkAccessed(addr Address) {
   191  	s.lock.Lock()
   192  	defer s.lock.Unlock()
   193  
   194  	if s.closed {
   195  		return
   196  	}
   197  
   198  	proximity := s.po(addr)
   199  	s.tryAccessIdx(addr, proximity)
   200  }
   201  
   202  // initialize and set values for processing of gc round
   203  func (s *LDBStore) startGC(c int) {
   204  
   205  	s.gc.count = 0
   206  	// calculate the target number of deletions
   207  	if c >= s.gc.maxRound {
   208  		s.gc.target = s.gc.maxRound
   209  	} else {
   210  		s.gc.target = c / s.gc.ratio
   211  	}
   212  	s.gc.batch = newBatch()
   213  	log.Debug("startgc", "requested", c, "target", s.gc.target)
   214  }
   215  
   216  // NewMockDbStore creates a new instance of DbStore with
   217  // mockStore set to a provided value. If mockStore argument is nil,
   218  // this function behaves exactly as NewDbStore.
   219  func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error) {
   220  	s, err = NewLDBStore(params)
   221  	if err != nil {
   222  		return nil, err
   223  	}
   224  
   225  	// replace put and get with mock store functionality
   226  	if mockStore != nil {
   227  		s.encodeDataFunc = newMockEncodeDataFunc(mockStore)
   228  		s.getDataFunc = newMockGetDataFunc(mockStore)
   229  	}
   230  	return
   231  }
   232  
   233  type dpaDBIndex struct {
   234  	Idx    uint64
   235  	Access uint64
   236  }
   237  
   238  func BytesToU64(data []byte) uint64 {
   239  	if len(data) < 8 {
   240  		return 0
   241  	}
   242  	return binary.BigEndian.Uint64(data)
   243  }
   244  
   245  func U64ToBytes(val uint64) []byte {
   246  	data := make([]byte, 8)
   247  	binary.BigEndian.PutUint64(data, val)
   248  	return data
   249  }
   250  
   251  func getIndexKey(hash Address) []byte {
   252  	hashSize := len(hash)
   253  	key := make([]byte, hashSize+1)
   254  	key[0] = keyIndex
   255  	copy(key[1:], hash[:])
   256  	return key
   257  }
   258  
   259  func getDataKey(idx uint64, po uint8) []byte {
   260  	key := make([]byte, 10)
   261  	key[0] = keyData
   262  	key[1] = po
   263  	binary.BigEndian.PutUint64(key[2:], idx)
   264  
   265  	return key
   266  }
   267  
   268  func getGCIdxKey(index *dpaDBIndex) []byte {
   269  	key := make([]byte, 9)
   270  	key[0] = keyGCIdx
   271  	binary.BigEndian.PutUint64(key[1:], index.Access)
   272  	return key
   273  }
   274  
   275  func getGCIdxValue(index *dpaDBIndex, po uint8, addr Address) []byte {
   276  	val := make([]byte, 41) // po = 1, index.Index = 8, Address = 32
   277  	val[0] = po
   278  	binary.BigEndian.PutUint64(val[1:], index.Idx)
   279  	copy(val[9:], addr)
   280  	return val
   281  }
   282  
   283  func parseIdxKey(key []byte) (byte, []byte) {
   284  	return key[0], key[1:]
   285  }
   286  
   287  func parseGCIdxEntry(accessCnt []byte, val []byte) (index *dpaDBIndex, po uint8, addr Address) {
   288  	index = &dpaDBIndex{
   289  		Idx:    binary.BigEndian.Uint64(val[1:]),
   290  		Access: binary.BigEndian.Uint64(accessCnt),
   291  	}
   292  	po = val[0]
   293  	addr = val[9:]
   294  	return
   295  }
   296  
   297  func encodeIndex(index *dpaDBIndex) []byte {
   298  	data, _ := rlp.EncodeToBytes(index)
   299  	return data
   300  }
   301  
   302  func encodeData(chunk Chunk) []byte {
   303  	// Always create a new underlying array for the returned byte slice.
   304  	// The chunk.Address array may be used in the returned slice which
   305  	// may be changed later in the code or by the LevelDB, resulting
   306  	// that the Address is changed as well.
   307  	return append(append([]byte{}, chunk.Address()[:]...), chunk.Data()...)
   308  }
   309  
   310  func decodeIndex(data []byte, index *dpaDBIndex) error {
   311  	dec := rlp.NewStream(bytes.NewReader(data), 0)
   312  	return dec.Decode(index)
   313  }
   314  
   315  func decodeData(addr Address, data []byte) (Chunk, error) {
   316  	return NewChunk(addr, data[32:]), nil
   317  }
   318  
   319  func (s *LDBStore) collectGarbage() error {
   320  	// prevent duplicate gc from starting when one is already running
   321  	select {
   322  	case <-s.gc.runC:
   323  	default:
   324  		return nil
   325  	}
   326  
   327  	s.lock.Lock()
   328  	entryCnt := s.entryCnt
   329  	s.lock.Unlock()
   330  
   331  	metrics.GetOrRegisterCounter("ldbstore.collectgarbage", nil).Inc(1)
   332  
   333  	// calculate the amount of chunks to collect and reset counter
   334  	s.startGC(int(entryCnt))
   335  	log.Debug("collectGarbage", "target", s.gc.target, "entryCnt", entryCnt)
   336  
   337  	for s.gc.count < s.gc.target {
   338  		it := s.db.NewIterator()
   339  		ok := it.Seek([]byte{keyGCIdx})
   340  		var singleIterationCount int
   341  
   342  		// every batch needs a lock so we avoid entries changing accessidx in the meantime
   343  		s.lock.Lock()
   344  		for ; ok && (singleIterationCount < s.gc.maxBatch); ok = it.Next() {
   345  
   346  			// quit if no more access index keys
   347  			itkey := it.Key()
   348  			if (itkey == nil) || (itkey[0] != keyGCIdx) {
   349  				break
   350  			}
   351  
   352  			// get chunk data entry from access index
   353  			val := it.Value()
   354  			index, po, hash := parseGCIdxEntry(itkey[1:], val)
   355  			keyIdx := make([]byte, 33)
   356  			keyIdx[0] = keyIndex
   357  			copy(keyIdx[1:], hash)
   358  
   359  			// add delete operation to batch
   360  			s.delete(s.gc.batch.Batch, index, keyIdx, po)
   361  			singleIterationCount++
   362  			s.gc.count++
   363  			log.Trace("garbage collect enqueued chunk for deletion", "key", hash)
   364  
   365  			// break if target is not on max garbage batch boundary
   366  			if s.gc.count >= s.gc.target {
   367  				break
   368  			}
   369  		}
   370  
   371  		s.writeBatch(s.gc.batch, wEntryCnt)
   372  		log.Trace("garbage collect batch done", "batch", singleIterationCount, "total", s.gc.count)
   373  		s.lock.Unlock()
   374  		it.Release()
   375  	}
   376  
   377  	metrics.GetOrRegisterCounter("ldbstore.collectgarbage.delete", nil).Inc(int64(s.gc.count))
   378  	log.Debug("garbage collect done", "c", s.gc.count)
   379  	s.gc.runC <- struct{}{}
   380  
   381  	return nil
   382  }
   383  
   384  // Export writes all chunks from the store to a tar archive, returning the
   385  // number of chunks written.
   386  func (s *LDBStore) Export(out io.Writer) (int64, error) {
   387  	tw := tar.NewWriter(out)
   388  	defer tw.Close()
   389  
   390  	it := s.db.NewIterator()
   391  	defer it.Release()
   392  	var count int64
   393  	for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() {
   394  		key := it.Key()
   395  		if (key == nil) || (key[0] != keyIndex) {
   396  			break
   397  		}
   398  
   399  		var index dpaDBIndex
   400  
   401  		hash := key[1:]
   402  		decodeIndex(it.Value(), &index)
   403  		po := s.po(hash)
   404  		datakey := getDataKey(index.Idx, po)
   405  		log.Trace("store.export", "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po)
   406  		data, err := s.db.Get(datakey)
   407  		if err != nil {
   408  			log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key, err))
   409  			continue
   410  		}
   411  
   412  		hdr := &tar.Header{
   413  			Name: hex.EncodeToString(hash),
   414  			Mode: 0644,
   415  			Size: int64(len(data)),
   416  		}
   417  		if err := tw.WriteHeader(hdr); err != nil {
   418  			return count, err
   419  		}
   420  		if _, err := tw.Write(data); err != nil {
   421  			return count, err
   422  		}
   423  		count++
   424  	}
   425  
   426  	return count, nil
   427  }
   428  
   429  // of chunks read.
   430  func (s *LDBStore) Import(in io.Reader) (int64, error) {
   431  	tr := tar.NewReader(in)
   432  
   433  	ctx, cancel := context.WithCancel(context.Background())
   434  	defer cancel()
   435  
   436  	countC := make(chan int64)
   437  	errC := make(chan error)
   438  	var count int64
   439  	go func() {
   440  		for {
   441  			hdr, err := tr.Next()
   442  			if err == io.EOF {
   443  				break
   444  			} else if err != nil {
   445  				select {
   446  				case errC <- err:
   447  				case <-ctx.Done():
   448  				}
   449  			}
   450  
   451  			if len(hdr.Name) != 64 {
   452  				log.Warn("ignoring non-chunk file", "name", hdr.Name)
   453  				continue
   454  			}
   455  
   456  			keybytes, err := hex.DecodeString(hdr.Name)
   457  			if err != nil {
   458  				log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err)
   459  				continue
   460  			}
   461  
   462  			data, err := ioutil.ReadAll(tr)
   463  			if err != nil {
   464  				select {
   465  				case errC <- err:
   466  				case <-ctx.Done():
   467  				}
   468  			}
   469  			key := Address(keybytes)
   470  			chunk := NewChunk(key, data[32:])
   471  
   472  			go func() {
   473  				select {
   474  				case errC <- s.Put(ctx, chunk):
   475  				case <-ctx.Done():
   476  				}
   477  			}()
   478  
   479  			count++
   480  		}
   481  		countC <- count
   482  	}()
   483  
   484  	// wait for all chunks to be stored
   485  	i := int64(0)
   486  	var total int64
   487  	for {
   488  		select {
   489  		case err := <-errC:
   490  			if err != nil {
   491  				return count, err
   492  			}
   493  			i++
   494  		case total = <-countC:
   495  		case <-ctx.Done():
   496  			return i, ctx.Err()
   497  		}
   498  		if total > 0 && i == total {
   499  			return total, nil
   500  		}
   501  	}
   502  }
   503  
   504  // Cleanup iterates over the database and deletes chunks if they pass the `f` condition
   505  func (s *LDBStore) Cleanup(f func(Chunk) bool) {
   506  	var errorsFound, removed, total int
   507  
   508  	it := s.db.NewIterator()
   509  	defer it.Release()
   510  	for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() {
   511  		key := it.Key()
   512  		if (key == nil) || (key[0] != keyIndex) {
   513  			break
   514  		}
   515  		total++
   516  		var index dpaDBIndex
   517  		err := decodeIndex(it.Value(), &index)
   518  		if err != nil {
   519  			log.Warn("Cannot decode")
   520  			errorsFound++
   521  			continue
   522  		}
   523  		hash := key[1:]
   524  		po := s.po(hash)
   525  		datakey := getDataKey(index.Idx, po)
   526  		data, err := s.db.Get(datakey)
   527  		if err != nil {
   528  			found := false
   529  
   530  			// The highest possible proximity is 255, so exit loop upon overflow.
   531  			for po = uint8(1); po != 0; po++ {
   532  				datakey = getDataKey(index.Idx, po)
   533  				data, err = s.db.Get(datakey)
   534  				if err == nil {
   535  					found = true
   536  					break
   537  				}
   538  			}
   539  
   540  			if !found {
   541  				log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key))
   542  				errorsFound++
   543  				continue
   544  			}
   545  		}
   546  
   547  		ck := data[:32]
   548  		c, err := decodeData(ck, data)
   549  		if err != nil {
   550  			log.Error("decodeData error", "err", err)
   551  			continue
   552  		}
   553  
   554  		sdata := c.Data()
   555  
   556  		cs := int64(binary.LittleEndian.Uint64(sdata[:8]))
   557  		log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs)
   558  
   559  		// if chunk is to be removed
   560  		if f(c) {
   561  			log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs)
   562  			s.deleteNow(&index, getIndexKey(key[1:]), po)
   563  			removed++
   564  			errorsFound++
   565  		}
   566  	}
   567  
   568  	log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed))
   569  }
   570  
   571  // CleanGCIndex rebuilds the garbage collector index from scratch, while
   572  // removing inconsistent elements, e.g., indices with missing data chunks.
   573  // WARN: it's a pretty heavy, long running function.
   574  func (s *LDBStore) CleanGCIndex() error {
   575  	s.lock.Lock()
   576  	defer s.lock.Unlock()
   577  
   578  	batch := leveldb.Batch{}
   579  
   580  	var okEntryCount uint64
   581  	var totalEntryCount uint64
   582  
   583  	// throw out all gc indices, we will rebuild from cleaned index
   584  	it := s.db.NewIterator()
   585  	it.Seek([]byte{keyGCIdx})
   586  	var gcDeletes int
   587  	for it.Valid() {
   588  		rowType, _ := parseIdxKey(it.Key())
   589  		if rowType != keyGCIdx {
   590  			break
   591  		}
   592  		batch.Delete(it.Key())
   593  		gcDeletes++
   594  		it.Next()
   595  	}
   596  	log.Debug("gc", "deletes", gcDeletes)
   597  	if err := s.db.Write(&batch); err != nil {
   598  		return err
   599  	}
   600  	batch.Reset()
   601  
   602  	it.Release()
   603  
   604  	// corrected po index pointer values
   605  	var poPtrs [256]uint64
   606  
   607  	// set to true if chunk count not on 4096 iteration boundary
   608  	var doneIterating bool
   609  
   610  	// last key index in previous iteration
   611  	lastIdxKey := []byte{keyIndex}
   612  
   613  	// counter for debug output
   614  	var cleanBatchCount int
   615  
   616  	// go through all key index entries
   617  	for !doneIterating {
   618  		cleanBatchCount++
   619  		var idxs []dpaDBIndex
   620  		var chunkHashes [][]byte
   621  		var pos []uint8
   622  		it := s.db.NewIterator()
   623  
   624  		it.Seek(lastIdxKey)
   625  
   626  		// 4096 is just a nice number, don't look for any hidden meaning here...
   627  		var i int
   628  		for i = 0; i < 4096; i++ {
   629  
   630  			// this really shouldn't happen unless database is empty
   631  			// but let's keep it to be safe
   632  			if !it.Valid() {
   633  				doneIterating = true
   634  				break
   635  			}
   636  
   637  			// if it's not keyindex anymore we're done iterating
   638  			rowType, chunkHash := parseIdxKey(it.Key())
   639  			if rowType != keyIndex {
   640  				doneIterating = true
   641  				break
   642  			}
   643  
   644  			// decode the retrieved index
   645  			var idx dpaDBIndex
   646  			err := decodeIndex(it.Value(), &idx)
   647  			if err != nil {
   648  				return fmt.Errorf("corrupt index: %v", err)
   649  			}
   650  			po := s.po(chunkHash)
   651  			lastIdxKey = it.Key()
   652  
   653  			// if we don't find the data key, remove the entry
   654  			// if we find it, add to the array of new gc indices to create
   655  			dataKey := getDataKey(idx.Idx, po)
   656  			_, err = s.db.Get(dataKey)
   657  			if err != nil {
   658  				log.Warn("deleting inconsistent index (missing data)", "key", chunkHash)
   659  				batch.Delete(it.Key())
   660  			} else {
   661  				idxs = append(idxs, idx)
   662  				chunkHashes = append(chunkHashes, chunkHash)
   663  				pos = append(pos, po)
   664  				okEntryCount++
   665  				if idx.Idx > poPtrs[po] {
   666  					poPtrs[po] = idx.Idx
   667  				}
   668  			}
   669  			totalEntryCount++
   670  			it.Next()
   671  		}
   672  		it.Release()
   673  
   674  		// flush the key index corrections
   675  		err := s.db.Write(&batch)
   676  		if err != nil {
   677  			return err
   678  		}
   679  		batch.Reset()
   680  
   681  		// add correct gc indices
   682  		for i, okIdx := range idxs {
   683  			gcIdxKey := getGCIdxKey(&okIdx)
   684  			gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i])
   685  			batch.Put(gcIdxKey, gcIdxData)
   686  			log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData)
   687  		}
   688  
   689  		// flush them
   690  		err = s.db.Write(&batch)
   691  		if err != nil {
   692  			return err
   693  		}
   694  		batch.Reset()
   695  
   696  		log.Debug("clean gc index pass", "batch", cleanBatchCount, "checked", i, "kept", len(idxs))
   697  	}
   698  
   699  	log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len())
   700  
   701  	// lastly add updated entry count
   702  	var entryCount [8]byte
   703  	binary.BigEndian.PutUint64(entryCount[:], okEntryCount)
   704  	batch.Put(keyEntryCnt, entryCount[:])
   705  
   706  	// and add the new po index pointers
   707  	var poKey [2]byte
   708  	poKey[0] = keyDistanceCnt
   709  	for i, poPtr := range poPtrs {
   710  		poKey[1] = uint8(i)
   711  		if poPtr == 0 {
   712  			batch.Delete(poKey[:])
   713  		} else {
   714  			var idxCount [8]byte
   715  			binary.BigEndian.PutUint64(idxCount[:], poPtr)
   716  			batch.Put(poKey[:], idxCount[:])
   717  		}
   718  	}
   719  
   720  	// if you made it this far your harddisk has survived. Congratulations
   721  	return s.db.Write(&batch)
   722  }
   723  
   724  // Delete is removes a chunk and updates indices.
   725  // Is thread safe
   726  func (s *LDBStore) Delete(addr Address) error {
   727  	s.lock.Lock()
   728  	defer s.lock.Unlock()
   729  
   730  	ikey := getIndexKey(addr)
   731  
   732  	idata, err := s.db.Get(ikey)
   733  	if err != nil {
   734  		return err
   735  	}
   736  
   737  	var idx dpaDBIndex
   738  	decodeIndex(idata, &idx)
   739  	proximity := s.po(addr)
   740  	return s.deleteNow(&idx, ikey, proximity)
   741  }
   742  
   743  // executes one delete operation immediately
   744  // see *LDBStore.delete
   745  func (s *LDBStore) deleteNow(idx *dpaDBIndex, idxKey []byte, po uint8) error {
   746  	batch := new(leveldb.Batch)
   747  	s.delete(batch, idx, idxKey, po)
   748  	return s.db.Write(batch)
   749  }
   750  
   751  // adds a delete chunk operation to the provided batch
   752  // if called directly, decrements entrycount regardless if the chunk exists upon deletion. Risk of wrap to max uint64
   753  func (s *LDBStore) delete(batch *leveldb.Batch, idx *dpaDBIndex, idxKey []byte, po uint8) {
   754  	metrics.GetOrRegisterCounter("ldbstore.delete", nil).Inc(1)
   755  
   756  	gcIdxKey := getGCIdxKey(idx)
   757  	batch.Delete(gcIdxKey)
   758  	dataKey := getDataKey(idx.Idx, po)
   759  	batch.Delete(dataKey)
   760  	batch.Delete(idxKey)
   761  	s.entryCnt--
   762  	dbEntryCount.Dec(1)
   763  	cntKey := make([]byte, 2)
   764  	cntKey[0] = keyDistanceCnt
   765  	cntKey[1] = po
   766  	batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt))
   767  	batch.Put(cntKey, U64ToBytes(s.bucketCnt[po]))
   768  }
   769  
   770  func (s *LDBStore) BinIndex(po uint8) uint64 {
   771  	s.lock.RLock()
   772  	defer s.lock.RUnlock()
   773  	return s.bucketCnt[po]
   774  }
   775  
   776  // Put adds a chunk to the database, adding indices and incrementing global counters.
   777  // If it already exists, it merely increments the access count of the existing entry.
   778  // Is thread safe
   779  func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error {
   780  	metrics.GetOrRegisterCounter("ldbstore.put", nil).Inc(1)
   781  	log.Trace("ldbstore.put", "key", chunk.Address())
   782  
   783  	ikey := getIndexKey(chunk.Address())
   784  	var index dpaDBIndex
   785  
   786  	po := s.po(chunk.Address())
   787  
   788  	s.lock.Lock()
   789  
   790  	if s.closed {
   791  		s.lock.Unlock()
   792  		return ErrDBClosed
   793  	}
   794  	batch := s.batch
   795  
   796  	log.Trace("ldbstore.put: s.db.Get", "key", chunk.Address(), "ikey", fmt.Sprintf("%x", ikey))
   797  	_, err := s.db.Get(ikey)
   798  	if err != nil {
   799  		s.doPut(chunk, &index, po)
   800  	}
   801  	idata := encodeIndex(&index)
   802  	s.batch.Put(ikey, idata)
   803  
   804  	// add the access-chunkindex index for garbage collection
   805  	gcIdxKey := getGCIdxKey(&index)
   806  	gcIdxData := getGCIdxValue(&index, po, chunk.Address())
   807  	s.batch.Put(gcIdxKey, gcIdxData)
   808  	s.lock.Unlock()
   809  
   810  	select {
   811  	case s.batchesC <- struct{}{}:
   812  	default:
   813  	}
   814  
   815  	select {
   816  	case <-batch.c:
   817  		return batch.err
   818  	case <-ctx.Done():
   819  		return ctx.Err()
   820  	}
   821  }
   822  
   823  // force putting into db, does not check or update necessary indices
   824  func (s *LDBStore) doPut(chunk Chunk, index *dpaDBIndex, po uint8) {
   825  	data := s.encodeDataFunc(chunk)
   826  	dkey := getDataKey(s.dataIdx, po)
   827  	s.batch.Put(dkey, data)
   828  	index.Idx = s.dataIdx
   829  	s.bucketCnt[po] = s.dataIdx
   830  	s.entryCnt++
   831  	dbEntryCount.Inc(1)
   832  	s.dataIdx++
   833  	index.Access = s.accessCnt
   834  	s.accessCnt++
   835  	cntKey := make([]byte, 2)
   836  	cntKey[0] = keyDistanceCnt
   837  	cntKey[1] = po
   838  	s.batch.Put(cntKey, U64ToBytes(s.bucketCnt[po]))
   839  }
   840  
   841  func (s *LDBStore) writeBatches() {
   842  	for {
   843  		select {
   844  		case <-s.quit:
   845  			log.Debug("DbStore: quit batch write loop")
   846  			return
   847  		case <-s.batchesC:
   848  			err := s.writeCurrentBatch()
   849  			if err != nil {
   850  				log.Debug("DbStore: quit batch write loop", "err", err.Error())
   851  				return
   852  			}
   853  		}
   854  	}
   855  
   856  }
   857  
   858  func (s *LDBStore) writeCurrentBatch() error {
   859  	s.lock.Lock()
   860  	defer s.lock.Unlock()
   861  	b := s.batch
   862  	l := b.Len()
   863  	if l == 0 {
   864  		return nil
   865  	}
   866  	s.batch = newBatch()
   867  	b.err = s.writeBatch(b, wEntryCnt|wAccessCnt|wIndexCnt)
   868  	close(b.c)
   869  	if s.entryCnt >= s.capacity {
   870  		go s.collectGarbage()
   871  	}
   872  	return nil
   873  }
   874  
   875  // must be called non concurrently
   876  func (s *LDBStore) writeBatch(b *dbBatch, wFlag uint8) error {
   877  	if wFlag&wEntryCnt > 0 {
   878  		b.Put(keyEntryCnt, U64ToBytes(s.entryCnt))
   879  	}
   880  	if wFlag&wIndexCnt > 0 {
   881  		b.Put(keyDataIdx, U64ToBytes(s.dataIdx))
   882  	}
   883  	if wFlag&wAccessCnt > 0 {
   884  		b.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
   885  	}
   886  	l := b.Len()
   887  	if err := s.db.Write(b.Batch); err != nil {
   888  		return fmt.Errorf("unable to write batch: %v", err)
   889  	}
   890  	log.Trace(fmt.Sprintf("batch write (%d entries)", l))
   891  	return nil
   892  }
   893  
   894  // newMockEncodeDataFunc returns a function that stores the chunk data
   895  // to a mock store to bypass the default functionality encodeData.
   896  // The constructed function always returns the nil data, as DbStore does
   897  // not need to store the data, but still need to create the index.
   898  func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk Chunk) []byte {
   899  	return func(chunk Chunk) []byte {
   900  		if err := mockStore.Put(chunk.Address(), encodeData(chunk)); err != nil {
   901  			log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Address().Log(), err))
   902  		}
   903  		return chunk.Address()[:]
   904  	}
   905  }
   906  
   907  // tryAccessIdx tries to find index entry. If found then increments the access
   908  // count for garbage collection and returns the index entry and true for found,
   909  // otherwise returns nil and false.
   910  func (s *LDBStore) tryAccessIdx(addr Address, po uint8) (*dpaDBIndex, bool) {
   911  	ikey := getIndexKey(addr)
   912  	idata, err := s.db.Get(ikey)
   913  	if err != nil {
   914  		return nil, false
   915  	}
   916  
   917  	index := new(dpaDBIndex)
   918  	decodeIndex(idata, index)
   919  	oldGCIdxKey := getGCIdxKey(index)
   920  	s.batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt))
   921  	index.Access = s.accessCnt
   922  	idata = encodeIndex(index)
   923  	s.accessCnt++
   924  	s.batch.Put(ikey, idata)
   925  	newGCIdxKey := getGCIdxKey(index)
   926  	newGCIdxData := getGCIdxValue(index, po, ikey[1:])
   927  	s.batch.Delete(oldGCIdxKey)
   928  	s.batch.Put(newGCIdxKey, newGCIdxData)
   929  	select {
   930  	case s.batchesC <- struct{}{}:
   931  	default:
   932  	}
   933  	return index, true
   934  }
   935  
   936  // GetSchema is returning the current named schema of the datastore as read from LevelDB
   937  func (s *LDBStore) GetSchema() (string, error) {
   938  	s.lock.Lock()
   939  	defer s.lock.Unlock()
   940  
   941  	data, err := s.db.Get(keySchema)
   942  	if err != nil {
   943  		if err == leveldb.ErrNotFound {
   944  			return DbSchemaNone, nil
   945  		}
   946  		return "", err
   947  	}
   948  
   949  	return string(data), nil
   950  }
   951  
   952  // PutSchema is saving a named schema to the LevelDB datastore
   953  func (s *LDBStore) PutSchema(schema string) error {
   954  	s.lock.Lock()
   955  	defer s.lock.Unlock()
   956  
   957  	return s.db.Put(keySchema, []byte(schema))
   958  }
   959  
   960  // Get retrieves the chunk matching the provided key from the database.
   961  // If the chunk entry does not exist, it returns an error
   962  // Updates access count and is thread safe
   963  func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) {
   964  	metrics.GetOrRegisterCounter("ldbstore.get", nil).Inc(1)
   965  	log.Trace("ldbstore.get", "key", addr)
   966  
   967  	s.lock.Lock()
   968  	defer s.lock.Unlock()
   969  	return s.get(addr)
   970  }
   971  
   972  // Has queries the underlying DB if a chunk with the given address is stored
   973  // Returns true if the chunk is found, false if not
   974  func (s *LDBStore) Has(_ context.Context, addr Address) bool {
   975  	s.lock.RLock()
   976  	defer s.lock.RUnlock()
   977  
   978  	ikey := getIndexKey(addr)
   979  	_, err := s.db.Get(ikey)
   980  
   981  	return err == nil
   982  }
   983  
   984  // TODO: To conform with other private methods of this object indices should not be updated
   985  func (s *LDBStore) get(addr Address) (chunk Chunk, err error) {
   986  	if s.closed {
   987  		return nil, ErrDBClosed
   988  	}
   989  	proximity := s.po(addr)
   990  	index, found := s.tryAccessIdx(addr, proximity)
   991  	if found {
   992  		var data []byte
   993  		if s.getDataFunc != nil {
   994  			// if getDataFunc is defined, use it to retrieve the chunk data
   995  			log.Trace("ldbstore.get retrieve with getDataFunc", "key", addr)
   996  			data, err = s.getDataFunc(addr)
   997  			if err != nil {
   998  				return
   999  			}
  1000  		} else {
  1001  			// default DbStore functionality to retrieve chunk data
  1002  			datakey := getDataKey(index.Idx, proximity)
  1003  			data, err = s.db.Get(datakey)
  1004  			log.Trace("ldbstore.get retrieve", "key", addr, "indexkey", index.Idx, "datakey", fmt.Sprintf("%x", datakey), "proximity", proximity)
  1005  			if err != nil {
  1006  				log.Trace("ldbstore.get chunk found but could not be accessed", "key", addr, "err", err)
  1007  				s.deleteNow(index, getIndexKey(addr), s.po(addr))
  1008  				if err == leveldb.ErrNotFound {
  1009  					return nil, ErrChunkNotFound
  1010  				}
  1011  				return nil, err
  1012  			}
  1013  		}
  1014  
  1015  		return decodeData(addr, data)
  1016  	} else {
  1017  		err = ErrChunkNotFound
  1018  	}
  1019  
  1020  	return
  1021  }
  1022  
  1023  // newMockGetFunc returns a function that reads chunk data from
  1024  // the mock database, which is used as the value for DbStore.getFunc
  1025  // to bypass the default functionality of DbStore with a mock store.
  1026  func newMockGetDataFunc(mockStore *mock.NodeStore) func(addr Address) (data []byte, err error) {
  1027  	return func(addr Address) (data []byte, err error) {
  1028  		data, err = mockStore.Get(addr)
  1029  		if err == mock.ErrNotFound {
  1030  			// preserve ErrChunkNotFound error
  1031  			err = ErrChunkNotFound
  1032  		}
  1033  		return data, err
  1034  	}
  1035  }
  1036  
  1037  func (s *LDBStore) setCapacity(c uint64) {
  1038  	s.lock.Lock()
  1039  	defer s.lock.Unlock()
  1040  
  1041  	s.capacity = c
  1042  
  1043  	for s.entryCnt > c {
  1044  		s.collectGarbage()
  1045  	}
  1046  }
  1047  
  1048  func (s *LDBStore) Close() {
  1049  	close(s.quit)
  1050  	s.lock.Lock()
  1051  	s.closed = true
  1052  	s.lock.Unlock()
  1053  	// force writing out current batch
  1054  	s.writeCurrentBatch()
  1055  	s.db.Close()
  1056  }
  1057  
  1058  // SyncIterator(start, stop, po, f) calls f on each hash of a bin po from start to stop
  1059  func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Address, uint64) bool) error {
  1060  	metrics.GetOrRegisterCounter("ldbstore.synciterator", nil).Inc(1)
  1061  
  1062  	sincekey := getDataKey(since, po)
  1063  	untilkey := getDataKey(until, po)
  1064  	it := s.db.NewIterator()
  1065  	defer it.Release()
  1066  
  1067  	for ok := it.Seek(sincekey); ok; ok = it.Next() {
  1068  		metrics.GetOrRegisterCounter("ldbstore.synciterator.seek", nil).Inc(1)
  1069  
  1070  		dbkey := it.Key()
  1071  		if dbkey[0] != keyData || dbkey[1] != po || bytes.Compare(untilkey, dbkey) < 0 {
  1072  			break
  1073  		}
  1074  		key := make([]byte, 32)
  1075  		val := it.Value()
  1076  		copy(key, val[:32])
  1077  		if !f(Address(key), binary.BigEndian.Uint64(dbkey[2:])) {
  1078  			break
  1079  		}
  1080  	}
  1081  	return it.Error()
  1082  }