github.com/sberex/go-sberex@v1.8.2-0.20181113200658-ed96ac38f7d7/swarm/storage/types.go (about) 1 // This file is part of the go-sberex library. The go-sberex library is 2 // free software: you can redistribute it and/or modify it under the terms 3 // of the GNU Lesser General Public License as published by the Free 4 // Software Foundation, either version 3 of the License, or (at your option) 5 // any later version. 6 // 7 // The go-sberex library is distributed in the hope that it will be useful, 8 // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 // General Public License <http://www.gnu.org/licenses/> for more details. 11 12 package storage 13 14 import ( 15 "bytes" 16 "crypto" 17 "fmt" 18 "hash" 19 "io" 20 "sync" 21 22 "github.com/Sberex/go-sberex/bmt" 23 "github.com/Sberex/go-sberex/common" 24 "github.com/Sberex/go-sberex/crypto/sha3" 25 ) 26 27 type Hasher func() hash.Hash 28 type SwarmHasher func() SwarmHash 29 30 // Peer is the recorded as Source on the chunk 31 // should probably not be here? but network should wrap chunk object 32 type Peer interface{} 33 34 type Key []byte 35 36 func (x Key) Size() uint { 37 return uint(len(x)) 38 } 39 40 func (x Key) isEqual(y Key) bool { 41 return bytes.Equal(x, y) 42 } 43 44 func (h Key) bits(i, j uint) uint { 45 ii := i >> 3 46 jj := i & 7 47 if ii >= h.Size() { 48 return 0 49 } 50 51 if jj+j <= 8 { 52 return uint((h[ii] >> jj) & ((1 << j) - 1)) 53 } 54 55 res := uint(h[ii] >> jj) 56 jj = 8 - jj 57 j -= jj 58 for j != 0 { 59 ii++ 60 if j < 8 { 61 res += uint(h[ii]&((1<<j)-1)) << jj 62 return res 63 } 64 res += uint(h[ii]) << jj 65 jj += 8 66 j -= 8 67 } 68 return res 69 } 70 71 func IsZeroKey(key Key) bool { 72 return len(key) == 0 || bytes.Equal(key, ZeroKey) 73 } 74 75 var ZeroKey = Key(common.Hash{}.Bytes()) 76 77 func MakeHashFunc(hash string) SwarmHasher { 78 switch hash { 79 case "SHA256": 80 return func() SwarmHash { return &HashWithLength{crypto.SHA256.New()} } 81 case "SHA3": 82 return func() SwarmHash { return &HashWithLength{sha3.NewKeccak256()} } 83 case "BMT": 84 return func() SwarmHash { 85 hasher := sha3.NewKeccak256 86 pool := bmt.NewTreePool(hasher, bmt.DefaultSegmentCount, bmt.DefaultPoolSize) 87 return bmt.New(pool) 88 } 89 } 90 return nil 91 } 92 93 func (key Key) Hex() string { 94 return fmt.Sprintf("%064x", []byte(key[:])) 95 } 96 97 func (key Key) Log() string { 98 if len(key[:]) < 4 { 99 return fmt.Sprintf("%x", []byte(key[:])) 100 } 101 return fmt.Sprintf("%08x", []byte(key[:4])) 102 } 103 104 func (key Key) String() string { 105 return fmt.Sprintf("%064x", []byte(key)[:]) 106 } 107 108 func (key Key) MarshalJSON() (out []byte, err error) { 109 return []byte(`"` + key.String() + `"`), nil 110 } 111 112 func (key *Key) UnmarshalJSON(value []byte) error { 113 s := string(value) 114 *key = make([]byte, 32) 115 h := common.Hex2Bytes(s[1 : len(s)-1]) 116 copy(*key, h) 117 return nil 118 } 119 120 // each chunk when first requested opens a record associated with the request 121 // next time a request for the same chunk arrives, this record is updated 122 // this request status keeps track of the request ID-s as well as the requesting 123 // peers and has a channel that is closed when the chunk is retrieved. Multiple 124 // local callers can wait on this channel (or combined with a timeout, block with a 125 // select). 126 type RequestStatus struct { 127 Key Key 128 Source Peer 129 C chan bool 130 Requesters map[uint64][]interface{} 131 } 132 133 func newRequestStatus(key Key) *RequestStatus { 134 return &RequestStatus{ 135 Key: key, 136 Requesters: make(map[uint64][]interface{}), 137 C: make(chan bool), 138 } 139 } 140 141 // Chunk also serves as a request object passed to ChunkStores 142 // in case it is a retrieval request, Data is nil and Size is 0 143 // Note that Size is not the size of the data chunk, which is Data.Size() 144 // but the size of the subtree encoded in the chunk 145 // 0 if request, to be supplied by the dpa 146 type Chunk struct { 147 Key Key // always 148 SData []byte // nil if request, to be supplied by dpa 149 Size int64 // size of the data covered by the subtree encoded in this chunk 150 Source Peer // peer 151 C chan bool // to signal data delivery by the dpa 152 Req *RequestStatus // request Status needed by netStore 153 wg *sync.WaitGroup // wg to synchronize 154 dbStored chan bool // never remove a chunk from memStore before it is written to dbStore 155 } 156 157 func NewChunk(key Key, rs *RequestStatus) *Chunk { 158 return &Chunk{Key: key, Req: rs} 159 } 160 161 /* 162 The ChunkStore interface is implemented by : 163 164 - MemStore: a memory cache 165 - DbStore: local disk/db store 166 - LocalStore: a combination (sequence of) memStore and dbStore 167 - NetStore: cloud storage abstraction layer 168 - DPA: local requests for swarm storage and retrieval 169 */ 170 type ChunkStore interface { 171 Put(*Chunk) // effectively there is no error even if there is an error 172 Get(Key) (*Chunk, error) 173 Close() 174 } 175 176 /* 177 Chunker is the interface to a component that is responsible for disassembling and assembling larger data and indended to be the dependency of a DPA storage system with fixed maximum chunksize. 178 179 It relies on the underlying chunking model. 180 181 When calling Split, the caller provides a channel (chan *Chunk) on which it receives chunks to store. The DPA delegates to storage layers (implementing ChunkStore interface). 182 183 Split returns an error channel, which the caller can monitor. 184 After getting notified that all the data has been split (the error channel is closed), the caller can safely read or save the root key. Optionally it times out if not all chunks get stored or not the entire stream of data has been processed. By inspecting the errc channel the caller can check if any explicit errors (typically IO read/write failures) occurred during splitting. 185 186 When calling Join with a root key, the caller gets returned a seekable lazy reader. The caller again provides a channel on which the caller receives placeholder chunks with missing data. The DPA is supposed to forward this to the chunk stores and notify the chunker if the data has been delivered (i.e. retrieved from memory cache, disk-persisted db or cloud based swarm delivery). As the seekable reader is used, the chunker then puts these together the relevant parts on demand. 187 */ 188 type Splitter interface { 189 /* 190 When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes. 191 New chunks to store are coming to caller via the chunk storage channel, which the caller provides. 192 wg is a Waitgroup (can be nil) that can be used to block until the local storage finishes 193 The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel. 194 A closed error signals process completion at which point the key can be considered final if there were no errors. 195 */ 196 Split(io.Reader, int64, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error) 197 198 /* This is the first step in making files mutable (not chunks).. 199 Append allows adding more data chunks to the end of the already existsing file. 200 The key for the root chunk is supplied to load the respective tree. 201 Rest of the parameters behave like Split. 202 */ 203 Append(Key, io.Reader, chan *Chunk, *sync.WaitGroup, *sync.WaitGroup) (Key, error) 204 } 205 206 type Joiner interface { 207 /* 208 Join reconstructs original content based on a root key. 209 When joining, the caller gets returned a Lazy SectionReader, which is 210 seekable and implements on-demand fetching of chunks as and where it is read. 211 New chunks to retrieve are coming to caller via the Chunk channel, which the caller provides. 212 If an error is encountered during joining, it appears as a reader error. 213 The SectionReader. 214 As a result, partial reads from a document are possible even if other parts 215 are corrupt or lost. 216 The chunks are not meant to be validated by the chunker when joining. This 217 is because it is left to the DPA to decide which sources are trusted. 218 */ 219 Join(key Key, chunkC chan *Chunk) LazySectionReader 220 } 221 222 type Chunker interface { 223 Joiner 224 Splitter 225 // returns the key length 226 // KeySize() int64 227 } 228 229 // Size, Seek, Read, ReadAt 230 type LazySectionReader interface { 231 Size(chan bool) (int64, error) 232 io.Seeker 233 io.Reader 234 io.ReaderAt 235 } 236 237 type LazyTestSectionReader struct { 238 *io.SectionReader 239 } 240 241 func (self *LazyTestSectionReader) Size(chan bool) (int64, error) { 242 return self.SectionReader.Size(), nil 243 }