github.com/xxRanger/go-ethereum@v1.8.23/swarm/storage/filestore.go (about)

     1  // Copyright 2016 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package storage
    18  
    19  import (
    20  	"context"
    21  	"io"
    22  	"sort"
    23  	"sync"
    24  )
    25  
    26  /*
    27  FileStore provides the client API entrypoints Store and Retrieve to store and retrieve
    28  It can store anything that has a byte slice representation, so files or serialised objects etc.
    29  
    30  Storage: FileStore calls the Chunker to segment the input datastream of any size to a merkle hashed tree of chunks. The key of the root block is returned to the client.
    31  
    32  Retrieval: given the key of the root block, the FileStore retrieves the block chunks and reconstructs the original data and passes it back as a lazy reader. A lazy reader is a reader with on-demand delayed processing, i.e. the chunks needed to reconstruct a large file are only fetched and processed if that particular part of the document is actually read.
    33  
    34  As the chunker produces chunks, FileStore dispatches them to its own chunk store
    35  implementation for storage or retrieval.
    36  */
    37  
    38  const (
    39  	defaultLDBCapacity                = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB
    40  	defaultCacheCapacity              = 10000   // capacity for in-memory chunks' cache
    41  	defaultChunkRequestsCacheCapacity = 5000000 // capacity for container holding outgoing requests for chunks. should be set to LevelDB capacity
    42  )
    43  
    44  type FileStore struct {
    45  	ChunkStore
    46  	hashFunc SwarmHasher
    47  }
    48  
    49  type FileStoreParams struct {
    50  	Hash string
    51  }
    52  
    53  func NewFileStoreParams() *FileStoreParams {
    54  	return &FileStoreParams{
    55  		Hash: DefaultHash,
    56  	}
    57  }
    58  
    59  // for testing locally
    60  func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error) {
    61  	params := NewDefaultLocalStoreParams()
    62  	params.Init(datadir)
    63  	localStore, err := NewLocalStore(params, nil)
    64  	if err != nil {
    65  		return nil, err
    66  	}
    67  	localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(DefaultHash)))
    68  	return NewFileStore(localStore, NewFileStoreParams()), nil
    69  }
    70  
    71  func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore {
    72  	hashFunc := MakeHashFunc(params.Hash)
    73  	return &FileStore{
    74  		ChunkStore: store,
    75  		hashFunc:   hashFunc,
    76  	}
    77  }
    78  
    79  // Retrieve is a public API. Main entry point for document retrieval directly. Used by the
    80  // FS-aware API and httpaccess
    81  // Chunk retrieval blocks on netStore requests with a timeout so reader will
    82  // report error if retrieval of chunks within requested range time out.
    83  // It returns a reader with the chunk data and whether the content was encrypted
    84  func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool) {
    85  	isEncrypted = len(addr) > f.hashFunc().Size()
    86  	getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted)
    87  	reader = TreeJoin(ctx, addr, getter, 0)
    88  	return
    89  }
    90  
    91  // Store is a public API. Main entry point for document storage directly. Used by the
    92  // FS-aware API and httpaccess
    93  func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) {
    94  	putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt)
    95  	return PyramidSplit(ctx, data, putter, putter)
    96  }
    97  
    98  func (f *FileStore) HashSize() int {
    99  	return f.hashFunc().Size()
   100  }
   101  
   102  // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file
   103  func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) {
   104  	// create a special kind of putter, which only will store the references
   105  	putter := &hashExplorer{
   106  		hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt),
   107  	}
   108  	// do the actual splitting anyway, no way around it
   109  	_, wait, err := PyramidSplit(ctx, data, putter, putter)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  	// wait for splitting to be complete and all chunks processed
   114  	err = wait(ctx)
   115  	if err != nil {
   116  		return nil, err
   117  	}
   118  	// collect all references
   119  	addrs = NewAddressCollection(0)
   120  	for _, ref := range putter.references {
   121  		addrs = append(addrs, Address(ref))
   122  	}
   123  	sort.Sort(addrs)
   124  	return addrs, nil
   125  }
   126  
   127  // hashExplorer is a special kind of putter which will only store chunk references
   128  type hashExplorer struct {
   129  	*hasherStore
   130  	references []Reference
   131  	lock       sync.Mutex
   132  }
   133  
   134  // HashExplorer's Put will add just the chunk hashes to its `References`
   135  func (he *hashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) {
   136  	// Need to do the actual Put, which returns the references
   137  	ref, err := he.hasherStore.Put(ctx, chunkData)
   138  	if err != nil {
   139  		return nil, err
   140  	}
   141  	// internally store the reference
   142  	he.lock.Lock()
   143  	he.references = append(he.references, ref)
   144  	he.lock.Unlock()
   145  	return ref, nil
   146  }