github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/storage/common_test.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/rand" 23 "flag" 24 "fmt" 25 "io" 26 "io/ioutil" 27 "os" 28 "sync" 29 "testing" 30 "time" 31 32 "github.com/ethereum/go-ethereum/log" 33 ch "github.com/ethereum/go-ethereum/swarm/chunk" 34 colorable "github.com/mattn/go-colorable" 35 ) 36 37 var ( 38 loglevel = flag.Int("loglevel", 3, "verbosity of logs") 39 getTimeout = 30 * time.Second 40 ) 41 42 func init() { 43 flag.Parse() 44 log.PrintOrigins(true) 45 log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) 46 } 47 48 type brokenLimitedReader struct { 49 lr io.Reader 50 errAt int 51 off int 52 size int 53 } 54 55 func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader { 56 return &brokenLimitedReader{ 57 lr: data, 58 errAt: errAt, 59 size: size, 60 } 61 } 62 63 func newLDBStore(t *testing.T) (*LDBStore, func()) { 64 dir, err := ioutil.TempDir("", "bzz-storage-test") 65 if err != nil { 66 t.Fatal(err) 67 } 68 log.Trace("memstore.tempdir", "dir", dir) 69 70 ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) 71 db, err := NewLDBStore(ldbparams) 72 if err != nil { 73 t.Fatal(err) 74 } 75 76 cleanup := func() { 77 db.Close() 78 err := os.RemoveAll(dir) 79 if err != nil { 80 t.Fatal(err) 81 } 82 } 83 84 return db, cleanup 85 } 86 87 func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) { 88 return mput(store, n, GenerateRandomChunk) 89 } 90 91 func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error) { 92 // put to localstore and wait for stored channel 93 // does not check delivery error state 94 errc := make(chan error) 95 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 96 defer cancel() 97 for i := int64(0); i < int64(n); i++ { 98 chunk := f(ch.DefaultSize) 99 go func() { 100 select { 101 case errc <- store.Put(ctx, chunk): 102 case <-ctx.Done(): 103 } 104 }() 105 hs = append(hs, chunk) 106 } 107 108 // wait for all chunks to be stored 109 for i := 0; i < n; i++ { 110 err := <-errc 111 if err != nil { 112 return nil, err 113 } 114 } 115 return hs, nil 116 } 117 118 func mget(store ChunkStore, hs []Address, f func(h Address, chunk Chunk) error) error { 119 wg := sync.WaitGroup{} 120 wg.Add(len(hs)) 121 errc := make(chan error) 122 123 for _, k := range hs { 124 go func(h Address) { 125 defer wg.Done() 126 // TODO: write timeout with context 127 chunk, err := store.Get(context.TODO(), h) 128 if err != nil { 129 errc <- err 130 return 131 } 132 if f != nil { 133 err = f(h, chunk) 134 if err != nil { 135 errc <- err 136 return 137 } 138 } 139 }(k) 140 } 141 go func() { 142 wg.Wait() 143 close(errc) 144 }() 145 var err error 146 select { 147 case err = <-errc: 148 case <-time.NewTimer(5 * time.Second).C: 149 err = fmt.Errorf("timed out after 5 seconds") 150 } 151 return err 152 } 153 154 func testDataReader(l int) (r io.Reader) { 155 return io.LimitReader(rand.Reader, int64(l)) 156 } 157 158 func (r *brokenLimitedReader) Read(buf []byte) (int, error) { 159 if r.off+len(buf) > r.errAt { 160 return 0, fmt.Errorf("Broken reader") 161 } 162 r.off += len(buf) 163 return r.lr.Read(buf) 164 } 165 166 func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) { 167 chunks, err := mputRandomChunks(m, n, chunksize) 168 if err != nil { 169 t.Fatalf("expected no error, got %v", err) 170 } 171 err = mget(m, chunkAddresses(chunks), nil) 172 if err != nil { 173 t.Fatalf("testStore failed: %v", err) 174 } 175 } 176 177 func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { 178 chunks, err := mputRandomChunks(m, n, chunksize) 179 if err != nil { 180 t.Fatalf("expected no error, got %v", err) 181 } 182 f := func(h Address, chunk Chunk) error { 183 if !bytes.Equal(h, chunk.Address()) { 184 return fmt.Errorf("key does not match retrieved chunk Address") 185 } 186 hasher := MakeHashFunc(DefaultHash)() 187 hasher.ResetWithLength(chunk.SpanBytes()) 188 hasher.Write(chunk.Payload()) 189 exp := hasher.Sum(nil) 190 if !bytes.Equal(h, exp) { 191 return fmt.Errorf("key is not hash of chunk data") 192 } 193 return nil 194 } 195 err = mget(m, chunkAddresses(chunks), f) 196 if err != nil { 197 t.Fatalf("testStore failed: %v", err) 198 } 199 } 200 201 func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) { 202 chunks := make([]Chunk, n) 203 i := 0 204 f := func(dataSize int64) Chunk { 205 chunk := GenerateRandomChunk(dataSize) 206 chunks[i] = chunk 207 i++ 208 return chunk 209 } 210 211 mput(store, n, f) 212 213 f = func(dataSize int64) Chunk { 214 chunk := chunks[i] 215 i++ 216 return chunk 217 } 218 219 b.ReportAllocs() 220 b.ResetTimer() 221 222 for j := 0; j < b.N; j++ { 223 i = 0 224 mput(store, n, f) 225 } 226 } 227 228 func benchmarkStoreGet(store ChunkStore, n int, chunksize int64, b *testing.B) { 229 chunks, err := mputRandomChunks(store, n, chunksize) 230 if err != nil { 231 b.Fatalf("expected no error, got %v", err) 232 } 233 b.ReportAllocs() 234 b.ResetTimer() 235 addrs := chunkAddresses(chunks) 236 for i := 0; i < b.N; i++ { 237 err := mget(store, addrs, nil) 238 if err != nil { 239 b.Fatalf("mget failed: %v", err) 240 } 241 } 242 } 243 244 // MapChunkStore is a very simple ChunkStore implementation to store chunks in a map in memory. 245 type MapChunkStore struct { 246 chunks map[string]Chunk 247 mu sync.RWMutex 248 } 249 250 func NewMapChunkStore() *MapChunkStore { 251 return &MapChunkStore{ 252 chunks: make(map[string]Chunk), 253 } 254 } 255 256 func (m *MapChunkStore) Put(_ context.Context, ch Chunk) error { 257 m.mu.Lock() 258 defer m.mu.Unlock() 259 m.chunks[ch.Address().Hex()] = ch 260 return nil 261 } 262 263 func (m *MapChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { 264 m.mu.RLock() 265 defer m.mu.RUnlock() 266 chunk := m.chunks[ref.Hex()] 267 if chunk == nil { 268 return nil, ErrChunkNotFound 269 } 270 return chunk, nil 271 } 272 273 func (m *MapChunkStore) Close() { 274 } 275 276 func chunkAddresses(chunks []Chunk) []Address { 277 addrs := make([]Address, len(chunks)) 278 for i, ch := range chunks { 279 addrs[i] = ch.Address() 280 } 281 return addrs 282 }