github.com/nitinawathare/ethereumassignment3@v0.0.0-20211021213010-f07344c2b868/go-ethereum/swarm/network/stream/common_test.go (about)

     1  // Copyright 2018 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package stream
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"flag"
    23  	"fmt"
    24  	"io"
    25  	"io/ioutil"
    26  	"math/rand"
    27  	"os"
    28  	"strings"
    29  	"sync"
    30  	"sync/atomic"
    31  	"time"
    32  
    33  	"github.com/ethereum/go-ethereum/crypto"
    34  	"github.com/ethereum/go-ethereum/log"
    35  	"github.com/ethereum/go-ethereum/p2p/enode"
    36  	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
    37  	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
    38  	"github.com/ethereum/go-ethereum/swarm/network"
    39  	"github.com/ethereum/go-ethereum/swarm/network/simulation"
    40  	"github.com/ethereum/go-ethereum/swarm/state"
    41  	"github.com/ethereum/go-ethereum/swarm/storage"
    42  	mockmem "github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
    43  	"github.com/ethereum/go-ethereum/swarm/testutil"
    44  	colorable "github.com/mattn/go-colorable"
    45  )
    46  
    47  var (
    48  	loglevel     = flag.Int("loglevel", 2, "verbosity of logs")
    49  	nodes        = flag.Int("nodes", 0, "number of nodes")
    50  	chunks       = flag.Int("chunks", 0, "number of chunks")
    51  	useMockStore = flag.Bool("mockstore", false, "disabled mock store (default: enabled)")
    52  	longrunning  = flag.Bool("longrunning", false, "do run long-running tests")
    53  
    54  	bucketKeyDB        = simulation.BucketKey("db")
    55  	bucketKeyStore     = simulation.BucketKey("store")
    56  	bucketKeyFileStore = simulation.BucketKey("filestore")
    57  	bucketKeyNetStore  = simulation.BucketKey("netstore")
    58  	bucketKeyDelivery  = simulation.BucketKey("delivery")
    59  	bucketKeyRegistry  = simulation.BucketKey("registry")
    60  
    61  	chunkSize = 4096
    62  	pof       = network.Pof
    63  )
    64  
    65  func init() {
    66  	flag.Parse()
    67  	rand.Seed(time.Now().UnixNano())
    68  
    69  	log.PrintOrigins(true)
    70  	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
    71  }
    72  
    73  // newNetStoreAndDelivery is a default constructor for BzzAddr, NetStore and Delivery, used in Simulations
    74  func newNetStoreAndDelivery(ctx *adapters.ServiceContext, bucket *sync.Map) (*network.BzzAddr, *storage.NetStore, *Delivery, func(), error) {
    75  	addr := network.NewAddr(ctx.Config.Node())
    76  
    77  	netStore, delivery, cleanup, err := netStoreAndDeliveryWithAddr(ctx, bucket, addr)
    78  	if err != nil {
    79  		return nil, nil, nil, nil, err
    80  	}
    81  
    82  	netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
    83  
    84  	return addr, netStore, delivery, cleanup, nil
    85  }
    86  
    87  // newNetStoreAndDeliveryWithBzzAddr is a constructor for NetStore and Delivery, used in Simulations, accepting any BzzAddr
    88  func newNetStoreAndDeliveryWithBzzAddr(ctx *adapters.ServiceContext, bucket *sync.Map, addr *network.BzzAddr) (*storage.NetStore, *Delivery, func(), error) {
    89  	netStore, delivery, cleanup, err := netStoreAndDeliveryWithAddr(ctx, bucket, addr)
    90  	if err != nil {
    91  		return nil, nil, nil, err
    92  	}
    93  
    94  	netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
    95  
    96  	return netStore, delivery, cleanup, nil
    97  }
    98  
    99  // newNetStoreAndDeliveryWithRequestFunc is a constructor for NetStore and Delivery, used in Simulations, accepting any NetStore.RequestFunc
   100  func newNetStoreAndDeliveryWithRequestFunc(ctx *adapters.ServiceContext, bucket *sync.Map, rf network.RequestFunc) (*network.BzzAddr, *storage.NetStore, *Delivery, func(), error) {
   101  	addr := network.NewAddr(ctx.Config.Node())
   102  
   103  	netStore, delivery, cleanup, err := netStoreAndDeliveryWithAddr(ctx, bucket, addr)
   104  	if err != nil {
   105  		return nil, nil, nil, nil, err
   106  	}
   107  
   108  	netStore.NewNetFetcherFunc = network.NewFetcherFactory(rf, true).New
   109  
   110  	return addr, netStore, delivery, cleanup, nil
   111  }
   112  
   113  func netStoreAndDeliveryWithAddr(ctx *adapters.ServiceContext, bucket *sync.Map, addr *network.BzzAddr) (*storage.NetStore, *Delivery, func(), error) {
   114  	n := ctx.Config.Node()
   115  
   116  	store, datadir, err := createTestLocalStorageForID(n.ID(), addr)
   117  	if *useMockStore {
   118  		store, datadir, err = createMockStore(mockmem.NewGlobalStore(), n.ID(), addr)
   119  	}
   120  	if err != nil {
   121  		return nil, nil, nil, err
   122  	}
   123  	localStore := store.(*storage.LocalStore)
   124  	netStore, err := storage.NewNetStore(localStore, nil)
   125  	if err != nil {
   126  		return nil, nil, nil, err
   127  	}
   128  
   129  	fileStore := storage.NewFileStore(netStore, storage.NewFileStoreParams())
   130  
   131  	kad := network.NewKademlia(addr.Over(), network.NewKadParams())
   132  	delivery := NewDelivery(kad, netStore)
   133  
   134  	bucket.Store(bucketKeyStore, store)
   135  	bucket.Store(bucketKeyDB, netStore)
   136  	bucket.Store(bucketKeyDelivery, delivery)
   137  	bucket.Store(bucketKeyFileStore, fileStore)
   138  	// for the kademlia object, we use the global key from the simulation package,
   139  	// as the simulation will try to access it in the WaitTillHealthy with that key
   140  	bucket.Store(simulation.BucketKeyKademlia, kad)
   141  
   142  	cleanup := func() {
   143  		netStore.Close()
   144  		os.RemoveAll(datadir)
   145  	}
   146  
   147  	return netStore, delivery, cleanup, nil
   148  }
   149  
   150  func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
   151  	// setup
   152  	addr := network.RandomAddr() // tested peers peer address
   153  	to := network.NewKademlia(addr.OAddr, network.NewKadParams())
   154  
   155  	// temp datadir
   156  	datadir, err := ioutil.TempDir("", "streamer")
   157  	if err != nil {
   158  		return nil, nil, nil, nil, err
   159  	}
   160  	removeDataDir := func() {
   161  		os.RemoveAll(datadir)
   162  	}
   163  
   164  	params := storage.NewDefaultLocalStoreParams()
   165  	params.Init(datadir)
   166  	params.BaseKey = addr.Over()
   167  
   168  	localStore, err := storage.NewTestLocalStoreForAddr(params)
   169  	if err != nil {
   170  		removeDataDir()
   171  		return nil, nil, nil, nil, err
   172  	}
   173  
   174  	netStore, err := storage.NewNetStore(localStore, nil)
   175  	if err != nil {
   176  		removeDataDir()
   177  		return nil, nil, nil, nil, err
   178  	}
   179  
   180  	delivery := NewDelivery(to, netStore)
   181  	netStore.NewNetFetcherFunc = network.NewFetcherFactory(delivery.RequestFromPeers, true).New
   182  	streamer := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), registryOptions, nil)
   183  	teardown := func() {
   184  		streamer.Close()
   185  		removeDataDir()
   186  	}
   187  	prvkey, err := crypto.GenerateKey()
   188  	if err != nil {
   189  		removeDataDir()
   190  		return nil, nil, nil, nil, err
   191  	}
   192  
   193  	protocolTester := p2ptest.NewProtocolTester(prvkey, 1, streamer.runProtocol)
   194  
   195  	err = waitForPeers(streamer, 10*time.Second, 1)
   196  	if err != nil {
   197  		teardown()
   198  		return nil, nil, nil, nil, errors.New("timeout: peer is not created")
   199  	}
   200  
   201  	return protocolTester, streamer, localStore, teardown, nil
   202  }
   203  
   204  func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error {
   205  	ticker := time.NewTicker(10 * time.Millisecond)
   206  	timeoutTimer := time.NewTimer(timeout)
   207  	for {
   208  		select {
   209  		case <-ticker.C:
   210  			if streamer.peersCount() >= expectedPeers {
   211  				return nil
   212  			}
   213  		case <-timeoutTimer.C:
   214  			return errors.New("timeout")
   215  		}
   216  	}
   217  }
   218  
   219  type roundRobinStore struct {
   220  	index  uint32
   221  	stores []storage.ChunkStore
   222  }
   223  
   224  func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore {
   225  	return &roundRobinStore{
   226  		stores: stores,
   227  	}
   228  }
   229  
   230  // not used in this context, only to fulfill ChunkStore interface
   231  func (rrs *roundRobinStore) Has(ctx context.Context, addr storage.Address) bool {
   232  	panic("RoundRobinStor doesn't support HasChunk")
   233  }
   234  
   235  func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (storage.Chunk, error) {
   236  	return nil, errors.New("get not well defined on round robin store")
   237  }
   238  
   239  func (rrs *roundRobinStore) Put(ctx context.Context, chunk storage.Chunk) error {
   240  	i := atomic.AddUint32(&rrs.index, 1)
   241  	idx := int(i) % len(rrs.stores)
   242  	return rrs.stores[idx].Put(ctx, chunk)
   243  }
   244  
   245  func (rrs *roundRobinStore) Close() {
   246  	for _, store := range rrs.stores {
   247  		store.Close()
   248  	}
   249  }
   250  
   251  func readAll(fileStore *storage.FileStore, hash []byte) (int64, error) {
   252  	r, _ := fileStore.Retrieve(context.TODO(), hash)
   253  	buf := make([]byte, 1024)
   254  	var n int
   255  	var total int64
   256  	var err error
   257  	for (total == 0 || n > 0) && err == nil {
   258  		n, err = r.ReadAt(buf, total)
   259  		total += int64(n)
   260  	}
   261  	if err != nil && err != io.EOF {
   262  		return total, err
   263  	}
   264  	return total, nil
   265  }
   266  
   267  func uploadFilesToNodes(sim *simulation.Simulation) ([]storage.Address, []string, error) {
   268  	nodes := sim.UpNodeIDs()
   269  	nodeCnt := len(nodes)
   270  	log.Debug(fmt.Sprintf("Uploading %d files to nodes", nodeCnt))
   271  	//array holding generated files
   272  	rfiles := make([]string, nodeCnt)
   273  	//array holding the root hashes of the files
   274  	rootAddrs := make([]storage.Address, nodeCnt)
   275  
   276  	var err error
   277  	//for every node, generate a file and upload
   278  	for i, id := range nodes {
   279  		item, ok := sim.NodeItem(id, bucketKeyFileStore)
   280  		if !ok {
   281  			return nil, nil, fmt.Errorf("Error accessing localstore")
   282  		}
   283  		fileStore := item.(*storage.FileStore)
   284  		//generate a file
   285  		rfiles[i], err = generateRandomFile()
   286  		if err != nil {
   287  			return nil, nil, err
   288  		}
   289  		//store it (upload it) on the FileStore
   290  		ctx := context.TODO()
   291  		rk, wait, err := fileStore.Store(ctx, strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false)
   292  		log.Debug("Uploaded random string file to node")
   293  		if err != nil {
   294  			return nil, nil, err
   295  		}
   296  		err = wait(ctx)
   297  		if err != nil {
   298  			return nil, nil, err
   299  		}
   300  		rootAddrs[i] = rk
   301  	}
   302  	return rootAddrs, rfiles, nil
   303  }
   304  
   305  //generate a random file (string)
   306  func generateRandomFile() (string, error) {
   307  	//generate a random file size between minFileSize and maxFileSize
   308  	fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
   309  	log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
   310  	b := testutil.RandomBytes(1, fileSize*1024)
   311  	return string(b), nil
   312  }
   313  
   314  //create a local store for the given node
   315  func createTestLocalStorageForID(id enode.ID, addr *network.BzzAddr) (storage.ChunkStore, string, error) {
   316  	var datadir string
   317  	var err error
   318  	datadir, err = ioutil.TempDir("", fmt.Sprintf("syncer-test-%s", id.TerminalString()))
   319  	if err != nil {
   320  		return nil, "", err
   321  	}
   322  	var store storage.ChunkStore
   323  	params := storage.NewDefaultLocalStoreParams()
   324  	params.ChunkDbPath = datadir
   325  	params.BaseKey = addr.Over()
   326  	store, err = storage.NewTestLocalStoreForAddr(params)
   327  	if err != nil {
   328  		os.RemoveAll(datadir)
   329  		return nil, "", err
   330  	}
   331  	return store, datadir, nil
   332  }
   333  
   334  // watchDisconnections receives simulation peer events in a new goroutine and sets atomic value
   335  // disconnected to true in case of a disconnect event.
   336  func watchDisconnections(ctx context.Context, sim *simulation.Simulation) (disconnected *boolean) {
   337  	log.Debug("Watching for disconnections")
   338  	disconnections := sim.PeerEvents(
   339  		ctx,
   340  		sim.NodeIDs(),
   341  		simulation.NewPeerEventsFilter().Drop(),
   342  	)
   343  	disconnected = new(boolean)
   344  	go func() {
   345  		for {
   346  			select {
   347  			case <-ctx.Done():
   348  				return
   349  			case d := <-disconnections:
   350  				if d.Error != nil {
   351  					log.Error("peer drop event error", "node", d.NodeID, "peer", d.PeerID, "err", d.Error)
   352  				} else {
   353  					log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID)
   354  				}
   355  				disconnected.set(true)
   356  			}
   357  		}
   358  	}()
   359  	return disconnected
   360  }
   361  
   362  // boolean is used to concurrently set
   363  // and read a boolean value.
   364  type boolean struct {
   365  	v  bool
   366  	mu sync.RWMutex
   367  }
   368  
   369  // set sets the value.
   370  func (b *boolean) set(v bool) {
   371  	b.mu.Lock()
   372  	defer b.mu.Unlock()
   373  
   374  	b.v = v
   375  }
   376  
   377  // bool reads the value.
   378  func (b *boolean) bool() bool {
   379  	b.mu.RLock()
   380  	defer b.mu.RUnlock()
   381  
   382  	return b.v
   383  }