github.com/daragao/go-ethereum@v1.8.14-0.20180809141559-45eaef243198/swarm/network/stream/common_test.go (about)

     1  // Copyright 2018 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package stream
    18  
    19  import (
    20  	"context"
    21  	crand "crypto/rand"
    22  	"errors"
    23  	"flag"
    24  	"fmt"
    25  	"io"
    26  	"io/ioutil"
    27  	"math/rand"
    28  	"os"
    29  	"strings"
    30  	"sync/atomic"
    31  	"testing"
    32  	"time"
    33  
    34  	"github.com/ethereum/go-ethereum/log"
    35  	"github.com/ethereum/go-ethereum/p2p/discover"
    36  	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
    37  	"github.com/ethereum/go-ethereum/swarm/network"
    38  	"github.com/ethereum/go-ethereum/swarm/network/simulation"
    39  	"github.com/ethereum/go-ethereum/swarm/pot"
    40  	"github.com/ethereum/go-ethereum/swarm/state"
    41  	"github.com/ethereum/go-ethereum/swarm/storage"
    42  	mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
    43  	colorable "github.com/mattn/go-colorable"
    44  )
    45  
    46  var (
    47  	loglevel     = flag.Int("loglevel", 2, "verbosity of logs")
    48  	nodes        = flag.Int("nodes", 0, "number of nodes")
    49  	chunks       = flag.Int("chunks", 0, "number of chunks")
    50  	useMockStore = flag.Bool("mockstore", false, "disabled mock store (default: enabled)")
    51  	longrunning  = flag.Bool("longrunning", false, "do run long-running tests")
    52  
    53  	bucketKeyDB        = simulation.BucketKey("db")
    54  	bucketKeyStore     = simulation.BucketKey("store")
    55  	bucketKeyFileStore = simulation.BucketKey("filestore")
    56  	bucketKeyNetStore  = simulation.BucketKey("netstore")
    57  	bucketKeyDelivery  = simulation.BucketKey("delivery")
    58  	bucketKeyRegistry  = simulation.BucketKey("registry")
    59  
    60  	chunkSize = 4096
    61  	pof       = pot.DefaultPof(256)
    62  )
    63  
    64  func init() {
    65  	flag.Parse()
    66  	rand.Seed(time.Now().UnixNano())
    67  
    68  	log.PrintOrigins(true)
    69  	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
    70  }
    71  
    72  func createGlobalStore() (string, *mockdb.GlobalStore, error) {
    73  	var globalStore *mockdb.GlobalStore
    74  	globalStoreDir, err := ioutil.TempDir("", "global.store")
    75  	if err != nil {
    76  		log.Error("Error initiating global store temp directory!", "err", err)
    77  		return "", nil, err
    78  	}
    79  	globalStore, err = mockdb.NewGlobalStore(globalStoreDir)
    80  	if err != nil {
    81  		log.Error("Error initiating global store!", "err", err)
    82  		return "", nil, err
    83  	}
    84  	return globalStoreDir, globalStore, nil
    85  }
    86  
    87  func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
    88  	// setup
    89  	addr := network.RandomAddr() // tested peers peer address
    90  	to := network.NewKademlia(addr.OAddr, network.NewKadParams())
    91  
    92  	// temp datadir
    93  	datadir, err := ioutil.TempDir("", "streamer")
    94  	if err != nil {
    95  		return nil, nil, nil, func() {}, err
    96  	}
    97  	removeDataDir := func() {
    98  		os.RemoveAll(datadir)
    99  	}
   100  
   101  	params := storage.NewDefaultLocalStoreParams()
   102  	params.Init(datadir)
   103  	params.BaseKey = addr.Over()
   104  
   105  	localStore, err := storage.NewTestLocalStoreForAddr(params)
   106  	if err != nil {
   107  		return nil, nil, nil, removeDataDir, err
   108  	}
   109  
   110  	db := storage.NewDBAPI(localStore)
   111  	delivery := NewDelivery(to, db)
   112  	streamer := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), nil)
   113  	teardown := func() {
   114  		streamer.Close()
   115  		removeDataDir()
   116  	}
   117  	protocolTester := p2ptest.NewProtocolTester(t, network.NewNodeIDFromAddr(addr), 1, streamer.runProtocol)
   118  
   119  	err = waitForPeers(streamer, 1*time.Second, 1)
   120  	if err != nil {
   121  		return nil, nil, nil, nil, errors.New("timeout: peer is not created")
   122  	}
   123  
   124  	return protocolTester, streamer, localStore, teardown, nil
   125  }
   126  
   127  func waitForPeers(streamer *Registry, timeout time.Duration, expectedPeers int) error {
   128  	ticker := time.NewTicker(10 * time.Millisecond)
   129  	timeoutTimer := time.NewTimer(timeout)
   130  	for {
   131  		select {
   132  		case <-ticker.C:
   133  			if streamer.peersCount() >= expectedPeers {
   134  				return nil
   135  			}
   136  		case <-timeoutTimer.C:
   137  			return errors.New("timeout")
   138  		}
   139  	}
   140  }
   141  
   142  type roundRobinStore struct {
   143  	index  uint32
   144  	stores []storage.ChunkStore
   145  }
   146  
   147  func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore {
   148  	return &roundRobinStore{
   149  		stores: stores,
   150  	}
   151  }
   152  
   153  func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (*storage.Chunk, error) {
   154  	return nil, errors.New("get not well defined on round robin store")
   155  }
   156  
   157  func (rrs *roundRobinStore) Put(ctx context.Context, chunk *storage.Chunk) {
   158  	i := atomic.AddUint32(&rrs.index, 1)
   159  	idx := int(i) % len(rrs.stores)
   160  	rrs.stores[idx].Put(ctx, chunk)
   161  }
   162  
   163  func (rrs *roundRobinStore) Close() {
   164  	for _, store := range rrs.stores {
   165  		store.Close()
   166  	}
   167  }
   168  
   169  func readAll(fileStore *storage.FileStore, hash []byte) (int64, error) {
   170  	r, _ := fileStore.Retrieve(context.TODO(), hash)
   171  	buf := make([]byte, 1024)
   172  	var n int
   173  	var total int64
   174  	var err error
   175  	for (total == 0 || n > 0) && err == nil {
   176  		n, err = r.ReadAt(buf, total)
   177  		total += int64(n)
   178  	}
   179  	if err != nil && err != io.EOF {
   180  		return total, err
   181  	}
   182  	return total, nil
   183  }
   184  
   185  func uploadFilesToNodes(sim *simulation.Simulation) ([]storage.Address, []string, error) {
   186  	nodes := sim.UpNodeIDs()
   187  	nodeCnt := len(nodes)
   188  	log.Debug(fmt.Sprintf("Uploading %d files to nodes", nodeCnt))
   189  	//array holding generated files
   190  	rfiles := make([]string, nodeCnt)
   191  	//array holding the root hashes of the files
   192  	rootAddrs := make([]storage.Address, nodeCnt)
   193  
   194  	var err error
   195  	//for every node, generate a file and upload
   196  	for i, id := range nodes {
   197  		item, ok := sim.NodeItem(id, bucketKeyFileStore)
   198  		if !ok {
   199  			return nil, nil, fmt.Errorf("Error accessing localstore")
   200  		}
   201  		fileStore := item.(*storage.FileStore)
   202  		//generate a file
   203  		rfiles[i], err = generateRandomFile()
   204  		if err != nil {
   205  			return nil, nil, err
   206  		}
   207  		//store it (upload it) on the FileStore
   208  		ctx := context.TODO()
   209  		rk, wait, err := fileStore.Store(ctx, strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false)
   210  		log.Debug("Uploaded random string file to node")
   211  		if err != nil {
   212  			return nil, nil, err
   213  		}
   214  		err = wait(ctx)
   215  		if err != nil {
   216  			return nil, nil, err
   217  		}
   218  		rootAddrs[i] = rk
   219  	}
   220  	return rootAddrs, rfiles, nil
   221  }
   222  
   223  //generate a random file (string)
   224  func generateRandomFile() (string, error) {
   225  	//generate a random file size between minFileSize and maxFileSize
   226  	fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
   227  	log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
   228  	b := make([]byte, fileSize*1024)
   229  	_, err := crand.Read(b)
   230  	if err != nil {
   231  		log.Error("Error generating random file.", "err", err)
   232  		return "", err
   233  	}
   234  	return string(b), nil
   235  }
   236  
   237  //create a local store for the given node
   238  func createTestLocalStorageForID(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, string, error) {
   239  	var datadir string
   240  	var err error
   241  	datadir, err = ioutil.TempDir("", fmt.Sprintf("syncer-test-%s", id.TerminalString()))
   242  	if err != nil {
   243  		return nil, "", err
   244  	}
   245  	var store storage.ChunkStore
   246  	params := storage.NewDefaultLocalStoreParams()
   247  	params.ChunkDbPath = datadir
   248  	params.BaseKey = addr.Over()
   249  	store, err = storage.NewTestLocalStoreForAddr(params)
   250  	if err != nil {
   251  		os.RemoveAll(datadir)
   252  		return nil, "", err
   253  	}
   254  	return store, datadir, nil
   255  }