github.com/xxRanger/go-ethereum@v1.8.23/swarm/storage/filestore.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "context" 21 "io" 22 "sort" 23 "sync" 24 ) 25 26 /* 27 FileStore provides the client API entrypoints Store and Retrieve to store and retrieve 28 It can store anything that has a byte slice representation, so files or serialised objects etc. 29 30 Storage: FileStore calls the Chunker to segment the input datastream of any size to a merkle hashed tree of chunks. The key of the root block is returned to the client. 31 32 Retrieval: given the key of the root block, the FileStore retrieves the block chunks and reconstructs the original data and passes it back as a lazy reader. A lazy reader is a reader with on-demand delayed processing, i.e. the chunks needed to reconstruct a large file are only fetched and processed if that particular part of the document is actually read. 33 34 As the chunker produces chunks, FileStore dispatches them to its own chunk store 35 implementation for storage or retrieval. 36 */ 37 38 const ( 39 defaultLDBCapacity = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB 40 defaultCacheCapacity = 10000 // capacity for in-memory chunks' cache 41 defaultChunkRequestsCacheCapacity = 5000000 // capacity for container holding outgoing requests for chunks. should be set to LevelDB capacity 42 ) 43 44 type FileStore struct { 45 ChunkStore 46 hashFunc SwarmHasher 47 } 48 49 type FileStoreParams struct { 50 Hash string 51 } 52 53 func NewFileStoreParams() *FileStoreParams { 54 return &FileStoreParams{ 55 Hash: DefaultHash, 56 } 57 } 58 59 // for testing locally 60 func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error) { 61 params := NewDefaultLocalStoreParams() 62 params.Init(datadir) 63 localStore, err := NewLocalStore(params, nil) 64 if err != nil { 65 return nil, err 66 } 67 localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(DefaultHash))) 68 return NewFileStore(localStore, NewFileStoreParams()), nil 69 } 70 71 func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore { 72 hashFunc := MakeHashFunc(params.Hash) 73 return &FileStore{ 74 ChunkStore: store, 75 hashFunc: hashFunc, 76 } 77 } 78 79 // Retrieve is a public API. Main entry point for document retrieval directly. Used by the 80 // FS-aware API and httpaccess 81 // Chunk retrieval blocks on netStore requests with a timeout so reader will 82 // report error if retrieval of chunks within requested range time out. 83 // It returns a reader with the chunk data and whether the content was encrypted 84 func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool) { 85 isEncrypted = len(addr) > f.hashFunc().Size() 86 getter := NewHasherStore(f.ChunkStore, f.hashFunc, isEncrypted) 87 reader = TreeJoin(ctx, addr, getter, 0) 88 return 89 } 90 91 // Store is a public API. Main entry point for document storage directly. Used by the 92 // FS-aware API and httpaccess 93 func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error) { 94 putter := NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt) 95 return PyramidSplit(ctx, data, putter, putter) 96 } 97 98 func (f *FileStore) HashSize() int { 99 return f.hashFunc().Size() 100 } 101 102 // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file 103 func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) { 104 // create a special kind of putter, which only will store the references 105 putter := &hashExplorer{ 106 hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt), 107 } 108 // do the actual splitting anyway, no way around it 109 _, wait, err := PyramidSplit(ctx, data, putter, putter) 110 if err != nil { 111 return nil, err 112 } 113 // wait for splitting to be complete and all chunks processed 114 err = wait(ctx) 115 if err != nil { 116 return nil, err 117 } 118 // collect all references 119 addrs = NewAddressCollection(0) 120 for _, ref := range putter.references { 121 addrs = append(addrs, Address(ref)) 122 } 123 sort.Sort(addrs) 124 return addrs, nil 125 } 126 127 // hashExplorer is a special kind of putter which will only store chunk references 128 type hashExplorer struct { 129 *hasherStore 130 references []Reference 131 lock sync.Mutex 132 } 133 134 // HashExplorer's Put will add just the chunk hashes to its `References` 135 func (he *hashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) { 136 // Need to do the actual Put, which returns the references 137 ref, err := he.hasherStore.Put(ctx, chunkData) 138 if err != nil { 139 return nil, err 140 } 141 // internally store the reference 142 he.lock.Lock() 143 he.references = append(he.references, ref) 144 he.lock.Unlock() 145 return ref, nil 146 }