github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/storage/types.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "bytes" 21 "context" 22 "crypto" 23 "crypto/rand" 24 "encoding/binary" 25 "fmt" 26 "hash" 27 "io" 28 "io/ioutil" 29 30 "github.com/ethereum/go-ethereum/common" 31 "github.com/ethereum/go-ethereum/crypto/sha3" 32 "github.com/ethereum/go-ethereum/swarm/bmt" 33 ch "github.com/ethereum/go-ethereum/swarm/chunk" 34 ) 35 36 const MaxPO = 16 37 const AddressLength = 32 38 39 type Hasher func() hash.Hash 40 type SwarmHasher func() SwarmHash 41 42 // Peer is the recorded as Source on the chunk 43 // should probably not be here? but network should wrap chunk object 44 type Peer interface{} 45 46 type Address []byte 47 48 func (a Address) Size() uint { 49 return uint(len(a)) 50 } 51 52 func (a Address) isEqual(y Address) bool { 53 return bytes.Equal(a, y) 54 } 55 56 func (a Address) bits(i, j uint) uint { 57 ii := i >> 3 58 jj := i & 7 59 if ii >= a.Size() { 60 return 0 61 } 62 63 if jj+j <= 8 { 64 return uint((a[ii] >> jj) & ((1 << j) - 1)) 65 } 66 67 res := uint(a[ii] >> jj) 68 jj = 8 - jj 69 j -= jj 70 for j != 0 { 71 ii++ 72 if j < 8 { 73 res += uint(a[ii]&((1<<j)-1)) << jj 74 return res 75 } 76 res += uint(a[ii]) << jj 77 jj += 8 78 j -= 8 79 } 80 return res 81 } 82 83 // Proximity(x, y) returns the proximity order of the MSB distance between x and y 84 // 85 // The distance metric MSB(x, y) of two equal length byte sequences x an y is the 86 // value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. 87 // the binary cast is big endian: most significant bit first (=MSB). 88 // 89 // Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. 90 // It is defined as the reverse rank of the integer part of the base 2 91 // logarithm of the distance. 92 // It is calculated by counting the number of common leading zeros in the (MSB) 93 // binary representation of the x^y. 94 // 95 // (0 farthest, 255 closest, 256 self) 96 func Proximity(one, other []byte) (ret int) { 97 b := (MaxPO-1)/8 + 1 98 if b > len(one) { 99 b = len(one) 100 } 101 m := 8 102 for i := 0; i < b; i++ { 103 oxo := one[i] ^ other[i] 104 if i == b-1 { 105 m = MaxPO % 8 106 } 107 for j := 0; j < m; j++ { 108 if (oxo>>uint8(7-j))&0x01 != 0 { 109 return i*8 + j 110 } 111 } 112 } 113 return MaxPO 114 } 115 116 func IsZeroAddr(addr Address) bool { 117 return len(addr) == 0 || bytes.Equal(addr, ZeroAddr) 118 } 119 120 var ZeroAddr = Address(common.Hash{}.Bytes()) 121 122 func MakeHashFunc(hash string) SwarmHasher { 123 switch hash { 124 case "SHA256": 125 return func() SwarmHash { return &HashWithLength{crypto.SHA256.New()} } 126 case "SHA3": 127 return func() SwarmHash { return &HashWithLength{sha3.NewKeccak256()} } 128 case "BMT": 129 return func() SwarmHash { 130 hasher := sha3.NewKeccak256 131 hasherSize := hasher().Size() 132 segmentCount := ch.DefaultSize / hasherSize 133 pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize) 134 return bmt.New(pool) 135 } 136 } 137 return nil 138 } 139 140 func (a Address) Hex() string { 141 return fmt.Sprintf("%064x", []byte(a[:])) 142 } 143 144 func (a Address) Log() string { 145 if len(a[:]) < 8 { 146 return fmt.Sprintf("%x", []byte(a[:])) 147 } 148 return fmt.Sprintf("%016x", []byte(a[:8])) 149 } 150 151 func (a Address) String() string { 152 return fmt.Sprintf("%064x", []byte(a)) 153 } 154 155 func (a Address) MarshalJSON() (out []byte, err error) { 156 return []byte(`"` + a.String() + `"`), nil 157 } 158 159 func (a *Address) UnmarshalJSON(value []byte) error { 160 s := string(value) 161 *a = make([]byte, 32) 162 h := common.Hex2Bytes(s[1 : len(s)-1]) 163 copy(*a, h) 164 return nil 165 } 166 167 type AddressCollection []Address 168 169 func NewAddressCollection(l int) AddressCollection { 170 return make(AddressCollection, l) 171 } 172 173 func (c AddressCollection) Len() int { 174 return len(c) 175 } 176 177 func (c AddressCollection) Less(i, j int) bool { 178 return bytes.Compare(c[i], c[j]) == -1 179 } 180 181 func (c AddressCollection) Swap(i, j int) { 182 c[i], c[j] = c[j], c[i] 183 } 184 185 // Chunk interface implemented by context.Contexts and data chunks 186 type Chunk interface { 187 Address() Address 188 Payload() []byte 189 SpanBytes() []byte 190 Span() int64 191 Data() []byte 192 } 193 194 type chunk struct { 195 addr Address 196 sdata []byte 197 span int64 198 } 199 200 func NewChunk(addr Address, data []byte) *chunk { 201 return &chunk{ 202 addr: addr, 203 sdata: data, 204 span: -1, 205 } 206 } 207 208 func (c *chunk) Address() Address { 209 return c.addr 210 } 211 212 func (c *chunk) SpanBytes() []byte { 213 return c.sdata[:8] 214 } 215 216 func (c *chunk) Span() int64 { 217 if c.span == -1 { 218 c.span = int64(binary.LittleEndian.Uint64(c.sdata[:8])) 219 } 220 return c.span 221 } 222 223 func (c *chunk) Data() []byte { 224 return c.sdata 225 } 226 227 func (c *chunk) Payload() []byte { 228 return c.sdata[8:] 229 } 230 231 // String() for pretty printing 232 func (self *chunk) String() string { 233 return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata)) 234 } 235 236 func GenerateRandomChunk(dataSize int64) Chunk { 237 hasher := MakeHashFunc(DefaultHash)() 238 sdata := make([]byte, dataSize+8) 239 rand.Read(sdata[8:]) 240 binary.LittleEndian.PutUint64(sdata[:8], uint64(dataSize)) 241 hasher.ResetWithLength(sdata[:8]) 242 hasher.Write(sdata[8:]) 243 return NewChunk(hasher.Sum(nil), sdata) 244 } 245 246 func GenerateRandomChunks(dataSize int64, count int) (chunks []Chunk) { 247 if dataSize > ch.DefaultSize { 248 dataSize = ch.DefaultSize 249 } 250 for i := 0; i < count; i++ { 251 ch := GenerateRandomChunk(ch.DefaultSize) 252 chunks = append(chunks, ch) 253 } 254 return chunks 255 } 256 257 func GenerateRandomData(l int) (r io.Reader, slice []byte) { 258 slice, err := ioutil.ReadAll(io.LimitReader(rand.Reader, int64(l))) 259 if err != nil { 260 panic("rand error") 261 } 262 // log.Warn("generate random data", "len", len(slice), "data", common.Bytes2Hex(slice)) 263 r = io.LimitReader(bytes.NewReader(slice), int64(l)) 264 return r, slice 265 } 266 267 // Size, Seek, Read, ReadAt 268 type LazySectionReader interface { 269 Context() context.Context 270 Size(context.Context, chan bool) (int64, error) 271 io.Seeker 272 io.Reader 273 io.ReaderAt 274 } 275 276 type LazyTestSectionReader struct { 277 *io.SectionReader 278 } 279 280 func (r *LazyTestSectionReader) Size(context.Context, chan bool) (int64, error) { 281 return r.SectionReader.Size(), nil 282 } 283 284 func (r *LazyTestSectionReader) Context() context.Context { 285 return context.TODO() 286 } 287 288 type StoreParams struct { 289 Hash SwarmHasher `toml:"-"` 290 DbCapacity uint64 291 CacheCapacity uint 292 BaseKey []byte 293 } 294 295 func NewDefaultStoreParams() *StoreParams { 296 return NewStoreParams(defaultLDBCapacity, defaultCacheCapacity, nil, nil) 297 } 298 299 func NewStoreParams(ldbCap uint64, cacheCap uint, hash SwarmHasher, basekey []byte) *StoreParams { 300 if basekey == nil { 301 basekey = make([]byte, 32) 302 } 303 if hash == nil { 304 hash = MakeHashFunc(DefaultHash) 305 } 306 return &StoreParams{ 307 Hash: hash, 308 DbCapacity: ldbCap, 309 CacheCapacity: cacheCap, 310 BaseKey: basekey, 311 } 312 } 313 314 type ChunkData []byte 315 316 type Reference []byte 317 318 // Putter is responsible to store data and create a reference for it 319 type Putter interface { 320 Put(context.Context, ChunkData) (Reference, error) 321 // RefSize returns the length of the Reference created by this Putter 322 RefSize() int64 323 // Close is to indicate that no more chunk data will be Put on this Putter 324 Close() 325 // Wait returns if all data has been store and the Close() was called. 326 Wait(context.Context) error 327 } 328 329 // Getter is an interface to retrieve a chunk's data by its reference 330 type Getter interface { 331 Get(context.Context, Reference) (ChunkData, error) 332 } 333 334 // NOTE: this returns invalid data if chunk is encrypted 335 func (c ChunkData) Size() uint64 { 336 return binary.LittleEndian.Uint64(c[:8]) 337 } 338 339 func (c ChunkData) Data() []byte { 340 return c[8:] 341 } 342 343 type ChunkValidator interface { 344 Validate(addr Address, data []byte) bool 345 } 346 347 // Provides method for validation of content address in chunks 348 // Holds the corresponding hasher to create the address 349 type ContentAddressValidator struct { 350 Hasher SwarmHasher 351 } 352 353 // Constructor 354 func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { 355 return &ContentAddressValidator{ 356 Hasher: hasher, 357 } 358 } 359 360 // Validate that the given key is a valid content address for the given data 361 func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { 362 if l := len(data); l < 9 || l > ch.DefaultSize+8 { 363 // log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l) 364 return false 365 } 366 367 hasher := v.Hasher() 368 hasher.ResetWithLength(data[:8]) 369 hasher.Write(data[8:]) 370 hash := hasher.Sum(nil) 371 372 return bytes.Equal(hash, addr[:]) 373 } 374 375 type ChunkStore interface { 376 Put(ctx context.Context, ch Chunk) (err error) 377 Get(rctx context.Context, ref Address) (ch Chunk, err error) 378 Close() 379 } 380 381 // SyncChunkStore is a ChunkStore which supports syncing 382 type SyncChunkStore interface { 383 ChunkStore 384 BinIndex(po uint8) uint64 385 Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error 386 FetchFunc(ctx context.Context, ref Address) func(context.Context) error 387 } 388 389 // FakeChunkStore doesn't store anything, just implements the ChunkStore interface 390 // It can be used to inject into a hasherStore if you don't want to actually store data just do the 391 // hashing 392 type FakeChunkStore struct { 393 } 394 395 // Put doesn't store anything it is just here to implement ChunkStore 396 func (f *FakeChunkStore) Put(_ context.Context, ch Chunk) error { 397 return nil 398 } 399 400 // Gut doesn't store anything it is just here to implement ChunkStore 401 func (f *FakeChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { 402 panic("FakeChunkStore doesn't support Get") 403 } 404 405 // Close doesn't store anything it is just here to implement ChunkStore 406 func (f *FakeChunkStore) Close() { 407 }