github.com/nitinawathare/ethereumassignment3@v0.0.0-20211021213010-f07344c2b868/go-ethereum/swarm/network/stream/common_test.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package stream 18 19 import ( 20 "context" 21 "errors" 22 "flag" 23 "fmt" 24 "io" 25 "io/ioutil" 26 "math/rand" 27 "os" 28 "strings" 29 "sync" 30 "sync/atomic" 31 "time" 32 33 "github.com/ethereum/go-ethereum/crypto" 34 "github.com/ethereum/go-ethereum/log" 35 "github.com/ethereum/go-ethereum/p2p/enode" 36 "github.com/ethereum/go-ethereum/p2p/simulations/adapters" 37 p2ptest "github.com/ethereum/go-ethereum/p2p/testing" 38 "github.com/ethereum/go-ethereum/swarm/network" 39 "github.com/ethereum/go-ethereum/swarm/network/simulation" 40 "github.com/ethereum/go-ethereum/swarm/state" 41 "github.com/ethereum/go-ethereum/swarm/storage" 42 mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" 43 "github.com/ethereum/go-ethereum/swarm/testutil" 44 colorable "github.com/mattn/go-colorable" 45 ) 46 47 var ( 48 loglevel = flag.Int("loglevel", 2, "verbosity of logs") 49 nodes = flag.Int("nodes", 0, "number of nodes") 50 chunks = flag.Int("chunks", 0, "number of chunks") 51 useMockStore = flag.Bool("mockstore", false, "disabled mock store (default: enabled)") 52 longrunning = flag.Bool("longrunning", false, "do run long-running tests") 53 54 bucketKeyDB = simulation.BucketKey("db") 55 bucketKeyStore = simulation.BucketKey("store") 56 bucketKeyFileStore = simulation.BucketKey("filestore") 57 bucketKeyNetStore = simulation.BucketKey("netstore") 58 bucketKeyDelivery = simulation.BucketKey("delivery") 59 bucketKeyRegistry = simulation.BucketKey("registry") 60 61 chunkSize = 4096 62 pof = network.Pof 63 ) 64 65 func init() { 66 flag.Parse() 67 rand.Seed(time.Now().UnixNano()) 68 69 log.PrintOrigins(true) 70 log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) 71 } 72 73 // newNetStoreAndDelivery is a default constructor for BzzAddr, NetStore and Delivery, used in Simulations 74 func newNetStoreAndDelivery(ctx *adapters.ServiceContext, bucket *sync.Map) (*network.BzzAddr, *storage.NetStore, *Delivery, func(), error) { 75 addr := network.NewAddr(ctx.Config.Node()) 76 77 netStore, delivery, cleanup, err := netStoreAndDeliveryWithAddr(ctx, bucket, addr) 78 if err != nil { 79 return nil, nil, nil, nil, err 80 } 81 82 netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New 83 84 return addr, netStore, delivery, cleanup, nil 85 } 86 87 // newNetStoreAndDeliveryWithBzzAddr is a constructor for NetStore and Delivery, used in Simulations, accepting any BzzAddr 88 func newNetStoreAndDeliveryWithBzzAddr(ctx *adapters.ServiceContext, bucket *sync.Map, addr *network.BzzAddr) (*storage.NetStore, *Delivery, func(), error) { 89 netStore, delivery, cleanup, err := netStoreAndDeliveryWithAddr(ctx, bucket, addr) 90 if err != nil { 91 return nil, nil, nil, err 92 } 93 94 netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New 95 96 return netStore, delivery, cleanup, nil 97 } 98 99 // newNetStoreAndDeliveryWithRequestFunc is a constructor for NetStore and Delivery, used in Simulations, accepting any NetStore.RequestFunc 100 func newNetStoreAndDeliveryWithRequestFunc(ctx *adapters.ServiceContext, bucket *sync.Map, rf network.RequestFunc) (*network.BzzAddr, *storage.NetStore, *Delivery, func(), error) { 101 addr := network.NewAddr(ctx.Config.Node()) 102 103 netStore, delivery, cleanup, err := netStoreAndDeliveryWithAddr(ctx, bucket, addr) 104 if err != nil { 105 return nil, nil, nil, nil, err 106 } 107 108 netStore.NewNetFetcherFunc = network.NewFetcherFactory(rf, true).New 109 110 return addr, netStore, delivery, cleanup, nil 111 } 112 113 func netStoreAndDeliveryWithAddr(ctx *adapters.ServiceContext, bucket *sync.Map, addr *network.BzzAddr) (*storage.NetStore, *Delivery, func(), error) { 114 n := ctx.Config.Node() 115 116 store, datadir, err := createTestLocalStorageForID(n.ID(), addr) 117 if *useMockStore { 118 store, datadir, err = createMockStore(mockmem.NewGlobalStore(), n.ID(), addr) 119 } 120 if err != nil { 121 return nil, nil, nil, err 122 } 123 localStore := store.(*storage.LocalStore) 124 netStore, err := storage.NewNetStore(localStore, nil) 125 if err != nil { 126 return nil, nil, nil, err 127 } 128 129 fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams()) 130 131 kad := network.NewKademlia(addr.Over(), network.NewKadParams()) 132 delivery := NewDelivery(kad, netStore) 133 134 bucket.Store(bucketKeyStore, store) 135 bucket.Store(bucketKeyDB, netStore) 136 bucket.Store(bucketKeyDelivery, delivery) 137 bucket.Store(bucketKeyFileStore, fileStore) 138 // for the kademlia object, we use the global key from the simulation package, 139 // as the simulation will try to access it in the WaitTillHealthy with that key 140 bucket.Store(simulation.BucketKeyKademlia, kad) 141 142 cleanup := func() { 143 netStore.Close() 144 os.RemoveAll(datadir) 145 } 146 147 return netStore, delivery, cleanup, nil 148 } 149 150 func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { 151 // setup 152 addr := network.RandomAddr() // tested peers peer address 153 to := network.NewKademlia(addr.OAddr, network.NewKadParams()) 154 155 // temp datadir 156 datadir, err := ioutil.TempDir("", "streamer") 157 if err != nil { 158 return nil, nil, nil, nil, err 159 } 160 removeDataDir := func() { 161 os.RemoveAll(datadir) 162 } 163 164 params := storage.NewDefaultLocalStoreParams() 165 params.Init(datadir) 166 params.BaseKey = addr.Over() 167 168 localStore, err := storage.NewTestLocalStoreForAddr(params) 169 if err != nil { 170 removeDataDir() 171 return nil, nil, nil, nil, err 172 } 173 174 netStore, err := storage.NewNetStore(localStore, nil) 175 if err != nil { 176 removeDataDir() 177 return nil, nil, nil, nil, err 178 } 179 180 delivery := NewDelivery(to, netStore) 181 netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New 182 streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil) 183 teardown := func() { 184 streamer.Close() 185 removeDataDir() 186 } 187 prvkey, err := crypto.GenerateKey() 188 if err != nil { 189 removeDataDir() 190 return nil, nil, nil, nil, err 191 } 192 193 protocolTester := p2ptest.NewProtocolTester(prvkey, 1, streamer.runProtocol) 194 195 err = waitForPeers(streamer, 10*time.Second, 1) 196 if err != nil { 197 teardown() 198 return nil, nil, nil, nil, errors.New("timeout: peer is not created") 199 } 200 201 return protocolTester, streamer, localStore, teardown, nil 202 } 203 204 func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error { 205 ticker := time.NewTicker(10 * time.Millisecond) 206 timeoutTimer := time.NewTimer(timeout) 207 for { 208 select { 209 case <-ticker.C: 210 if streamer.peersCount() >= expectedPeers { 211 return nil 212 } 213 case <-timeoutTimer.C: 214 return errors.New("timeout") 215 } 216 } 217 } 218 219 type roundRobinStore struct { 220 index uint32 221 stores []storage.ChunkStore 222 } 223 224 func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { 225 return &roundRobinStore{ 226 stores: stores, 227 } 228 } 229 230 // not used in this context, only to fulfill ChunkStore interface 231 func (rrs *roundRobinStore) Has(ctx context.Context, addr storage.Address) bool { 232 panic("RoundRobinStor doesn't support HasChunk") 233 } 234 235 func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (storage.Chunk, error) { 236 return nil, errors.New("get not well defined on round robin store") 237 } 238 239 func (rrs *roundRobinStore) Put(ctx context.Context, chunk storage.Chunk) error { 240 i := atomic.AddUint32(&rrs.index, 1) 241 idx := int(i) % len(rrs.stores) 242 return rrs.stores[idx].Put(ctx, chunk) 243 } 244 245 func (rrs *roundRobinStore) Close() { 246 for _, store := range rrs.stores { 247 store.Close() 248 } 249 } 250 251 func readAll(fileStore *storage.FileStore, hash []byte) (int64, error) { 252 r, _ := fileStore.Retrieve(context.TODO(), hash) 253 buf := make([]byte, 1024) 254 var n int 255 var total int64 256 var err error 257 for (total == 0 || n > 0) && err == nil { 258 n, err = r.ReadAt(buf, total) 259 total += int64(n) 260 } 261 if err != nil && err != io.EOF { 262 return total, err 263 } 264 return total, nil 265 } 266 267 func uploadFilesToNodes(sim *simulation.Simulation) ([]storage.Address, []string, error) { 268 nodes := sim.UpNodeIDs() 269 nodeCnt := len(nodes) 270 log.Debug(fmt.Sprintf("Uploading %d files to nodes", nodeCnt)) 271 //array holding generated files 272 rfiles := make([]string, nodeCnt) 273 //array holding the root hashes of the files 274 rootAddrs := make([]storage.Address, nodeCnt) 275 276 var err error 277 //for every node, generate a file and upload 278 for i, id := range nodes { 279 item, ok := sim.NodeItem(id, bucketKeyFileStore) 280 if !ok { 281 return nil, nil, fmt.Errorf("Error accessing localstore") 282 } 283 fileStore := item.(*storage.FileStore) 284 //generate a file 285 rfiles[i], err = generateRandomFile() 286 if err != nil { 287 return nil, nil, err 288 } 289 //store it (upload it) on the FileStore 290 ctx := context.TODO() 291 rk, wait, err := fileStore.Store(ctx, strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false) 292 log.Debug("Uploaded random string file to node") 293 if err != nil { 294 return nil, nil, err 295 } 296 err = wait(ctx) 297 if err != nil { 298 return nil, nil, err 299 } 300 rootAddrs[i] = rk 301 } 302 return rootAddrs, rfiles, nil 303 } 304 305 //generate a random file (string) 306 func generateRandomFile() (string, error) { 307 //generate a random file size between minFileSize and maxFileSize 308 fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize 309 log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize)) 310 b := testutil.RandomBytes(1, fileSize*1024) 311 return string(b), nil 312 } 313 314 //create a local store for the given node 315 func createTestLocalStorageForID(id enode.ID, addr *network.BzzAddr) (storage.ChunkStore, string, error) { 316 var datadir string 317 var err error 318 datadir, err = ioutil.TempDir("", fmt.Sprintf("syncer-test-%s", id.TerminalString())) 319 if err != nil { 320 return nil, "", err 321 } 322 var store storage.ChunkStore 323 params := storage.NewDefaultLocalStoreParams() 324 params.ChunkDbPath = datadir 325 params.BaseKey = addr.Over() 326 store, err = storage.NewTestLocalStoreForAddr(params) 327 if err != nil { 328 os.RemoveAll(datadir) 329 return nil, "", err 330 } 331 return store, datadir, nil 332 } 333 334 // watchDisconnections receives simulation peer events in a new goroutine and sets atomic value 335 // disconnected to true in case of a disconnect event. 336 func watchDisconnections(ctx context.Context, sim *simulation.Simulation) (disconnected *boolean) { 337 log.Debug("Watching for disconnections") 338 disconnections := sim.PeerEvents( 339 ctx, 340 sim.NodeIDs(), 341 simulation.NewPeerEventsFilter().Drop(), 342 ) 343 disconnected = new(boolean) 344 go func() { 345 for { 346 select { 347 case <-ctx.Done(): 348 return 349 case d := <-disconnections: 350 if d.Error != nil { 351 log.Error("peer drop event error", "node", d.NodeID, "peer", d.PeerID, "err", d.Error) 352 } else { 353 log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) 354 } 355 disconnected.set(true) 356 } 357 } 358 }() 359 return disconnected 360 } 361 362 // boolean is used to concurrently set 363 // and read a boolean value. 364 type boolean struct { 365 v bool 366 mu sync.RWMutex 367 } 368 369 // set sets the value. 370 func (b *boolean) set(v bool) { 371 b.mu.Lock() 372 defer b.mu.Unlock() 373 374 b.v = v 375 } 376 377 // bool reads the value. 378 func (b *boolean) bool() bool { 379 b.mu.RLock() 380 defer b.mu.RUnlock() 381 382 return b.v 383 }