github.com/aaa256/atlantis@v0.0.0-20210707112435-42ee889287a2/swarm/network/stream/snapshot_retrieval_test.go (about)

     1  // Copyright 2018 The go-athereum Authors
     2  // This file is part of the go-athereum library.
     3  //
     4  // The go-athereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-athereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-athereum library. If not, see <http://www.gnu.org/licenses/>.
    16  package stream
    17  
    18  import (
    19  	"context"
    20  	crand "crypto/rand"
    21  	"fmt"
    22  	"math/rand"
    23  	"strings"
    24  	"sync"
    25  	"testing"
    26  	"time"
    27  
    28  	"github.com/athereum/go-athereum/common"
    29  	"github.com/athereum/go-athereum/p2p/discover"
    30  	"github.com/athereum/go-athereum/p2p/simulations"
    31  	"github.com/athereum/go-athereum/swarm/log"
    32  	"github.com/athereum/go-athereum/swarm/network"
    33  	streamTesting "github.com/athereum/go-athereum/swarm/network/stream/testing"
    34  	"github.com/athereum/go-athereum/swarm/storage"
    35  )
    36  
    37  //constants for random file generation
    38  const (
    39  	minFileSize = 2
    40  	maxFileSize = 40
    41  )
    42  
    43  func initRetrievalTest() {
    44  	//global func to get overlay address from discover ID
    45  	toAddr = func(id discover.NodeID) *network.BzzAddr {
    46  		addr := network.NewAddrFromNodeID(id)
    47  		return addr
    48  	}
    49  	//global func to create local store
    50  	createStoreFunc = createTestLocalStorageForId
    51  	//local stores
    52  	stores = make(map[discover.NodeID]storage.ChunkStore)
    53  	//data directories for each node and store
    54  	datadirs = make(map[discover.NodeID]string)
    55  	//deliveries for each node
    56  	deliveries = make(map[discover.NodeID]*Delivery)
    57  	//global retrieve func
    58  	getRetrieveFunc = func(id discover.NodeID) func(chunk *storage.Chunk) error {
    59  		return func(chunk *storage.Chunk) error {
    60  			skipCheck := true
    61  			return deliveries[id].RequestFromPeers(chunk.Addr[:], skipCheck)
    62  		}
    63  	}
    64  	//registries, map of discover.NodeID to its streamer
    65  	registries = make(map[discover.NodeID]*TestRegistry)
    66  	//not needed for this test but required from common_test for NewStreamService
    67  	waitPeerErrC = make(chan error)
    68  	//also not needed for this test but required for NewStreamService
    69  	peerCount = func(id discover.NodeID) int {
    70  		if ids[0] == id || ids[len(ids)-1] == id {
    71  			return 1
    72  		}
    73  		return 2
    74  	}
    75  }
    76  
    77  //This test is a retrieval test for nodes.
    78  //A configurable number of nodes can be
    79  //provided to the test.
    80  //Files are uploaded to nodes, other nodes try to retrieve the file
    81  //Number of nodes can be provided via commandline too.
    82  func TestFileRetrieval(t *testing.T) {
    83  	if *nodes != 0 {
    84  		fileRetrievalTest(t, *nodes)
    85  	} else {
    86  		nodeCnt := []int{16}
    87  		//if the `longrunning` flag has been provided
    88  		//run more test combinations
    89  		if *longrunning {
    90  			nodeCnt = append(nodeCnt, 32, 64, 128)
    91  		}
    92  		for _, n := range nodeCnt {
    93  			fileRetrievalTest(t, n)
    94  		}
    95  	}
    96  }
    97  
    98  //This test is a retrieval test for nodes.
    99  //One node is randomly selected to be the pivot node.
   100  //A configurable number of chunks and nodes can be
   101  //provided to the test, the number of chunks is uploaded
   102  //to the pivot node and other nodes try to retrieve the chunk(s).
   103  //Number of chunks and nodes can be provided via commandline too.
   104  func TestRetrieval(t *testing.T) {
   105  	//if nodes/chunks have been provided via commandline,
   106  	//run the tests with these values
   107  	if *nodes != 0 && *chunks != 0 {
   108  		retrievalTest(t, *chunks, *nodes)
   109  	} else {
   110  		var nodeCnt []int
   111  		var chnkCnt []int
   112  		//if the `longrunning` flag has been provided
   113  		//run more test combinations
   114  		if *longrunning {
   115  			nodeCnt = []int{16, 32, 128}
   116  			chnkCnt = []int{4, 32, 256}
   117  		} else {
   118  			//default test
   119  			nodeCnt = []int{16}
   120  			chnkCnt = []int{32}
   121  		}
   122  		for _, n := range nodeCnt {
   123  			for _, c := range chnkCnt {
   124  				retrievalTest(t, c, n)
   125  			}
   126  		}
   127  	}
   128  }
   129  
   130  //Every test runs 3 times, a live, a history, and a live AND history
   131  func fileRetrievalTest(t *testing.T, nodeCount int) {
   132  	//test live and NO history
   133  	log.Info("Testing live and no history", "nodeCount", nodeCount)
   134  	live = true
   135  	history = false
   136  	err := runFileRetrievalTest(nodeCount)
   137  	if err != nil {
   138  		t.Fatal(err)
   139  	}
   140  	//test history only
   141  	log.Info("Testing history only", "nodeCount", nodeCount)
   142  	live = false
   143  	history = true
   144  	err = runFileRetrievalTest(nodeCount)
   145  	if err != nil {
   146  		t.Fatal(err)
   147  	}
   148  	//finally test live and history
   149  	log.Info("Testing live and history", "nodeCount", nodeCount)
   150  	live = true
   151  	err = runFileRetrievalTest(nodeCount)
   152  	if err != nil {
   153  		t.Fatal(err)
   154  	}
   155  }
   156  
   157  //Every test runs 3 times, a live, a history, and a live AND history
   158  func retrievalTest(t *testing.T, chunkCount int, nodeCount int) {
   159  	//test live and NO history
   160  	log.Info("Testing live and no history", "chunkCount", chunkCount, "nodeCount", nodeCount)
   161  	live = true
   162  	history = false
   163  	err := runRetrievalTest(chunkCount, nodeCount)
   164  	if err != nil {
   165  		t.Fatal(err)
   166  	}
   167  	//test history only
   168  	log.Info("Testing history only", "chunkCount", chunkCount, "nodeCount", nodeCount)
   169  	live = false
   170  	history = true
   171  	err = runRetrievalTest(chunkCount, nodeCount)
   172  	if err != nil {
   173  		t.Fatal(err)
   174  	}
   175  	//finally test live and history
   176  	log.Info("Testing live and history", "chunkCount", chunkCount, "nodeCount", nodeCount)
   177  	live = true
   178  	err = runRetrievalTest(chunkCount, nodeCount)
   179  	if err != nil {
   180  		t.Fatal(err)
   181  	}
   182  }
   183  
   184  /*
   185  
   186  The upload is done by dependency to the global
   187  `live` and `history` variables;
   188  
   189  If `live` is set, first stream subscriptions are established,
   190  then files are uploaded to nodes.
   191  
   192  If `history` is enabled, first upload files, then build up subscriptions.
   193  
   194  The test loads a snapshot file to construct the swarm network,
   195  assuming that the snapshot file identifies a healthy
   196  kademlia network. Nevertheless a health check runs in the
   197  simulation's `action` function.
   198  
   199  The snapshot should have 'streamer' in its service list.
   200  */
   201  func runFileRetrievalTest(nodeCount int) error {
   202  	//for every run (live, history), int the variables
   203  	initRetrievalTest()
   204  	//the ids of the snapshot nodes, initiate only now as we need nodeCount
   205  	ids = make([]discover.NodeID, nodeCount)
   206  	//channel to check for disconnection errors
   207  	disconnectC := make(chan error)
   208  	//channel to close disconnection watcher routine
   209  	quitC := make(chan struct{})
   210  	//the test conf (using same as in `snapshot_sync_test`
   211  	conf = &synctestConfig{}
   212  	//map of overlay address to discover ID
   213  	conf.addrToIdMap = make(map[string]discover.NodeID)
   214  	//array where the generated chunk hashes will be stored
   215  	conf.hashes = make([]storage.Address, 0)
   216  	//load nodes from the snapshot file
   217  	net, err := initNetWithSnapshot(nodeCount)
   218  	if err != nil {
   219  		return err
   220  	}
   221  	var rpcSubscriptionsWg sync.WaitGroup
   222  	//do cleanup after test is terminated
   223  	defer func() {
   224  		//shutdown the snapshot network
   225  		net.Shutdown()
   226  		//after the test, clean up local stores initialized with createLocalStoreForId
   227  		localStoreCleanup()
   228  		//finally clear all data directories
   229  		datadirsCleanup()
   230  	}()
   231  	//get the nodes of the network
   232  	nodes := net.GetNodes()
   233  	//iterate over all nodes...
   234  	for c := 0; c < len(nodes); c++ {
   235  		//create an array of discovery nodeIDS
   236  		ids[c] = nodes[c].ID()
   237  		a := network.ToOverlayAddr(ids[c].Bytes())
   238  		//append it to the array of all overlay addresses
   239  		conf.addrs = append(conf.addrs, a)
   240  		conf.addrToIdMap[string(a)] = ids[c]
   241  	}
   242  
   243  	//needed for healthy call
   244  	ppmap = network.NewPeerPotMap(testMinProxBinSize, conf.addrs)
   245  
   246  	//an array for the random files
   247  	var randomFiles []string
   248  	//channel to signal when the upload has finished
   249  	uploadFinished := make(chan struct{})
   250  	//channel to trigger new node checks
   251  	trigger := make(chan discover.NodeID)
   252  	//simulation action
   253  	action := func(ctx context.Context) error {
   254  		//first run the health check on all nodes,
   255  		//wait until nodes are all healthy
   256  		ticker := time.NewTicker(200 * time.Millisecond)
   257  		defer ticker.Stop()
   258  		for range ticker.C {
   259  			healthy := true
   260  			for _, id := range ids {
   261  				r := registries[id]
   262  				//PeerPot for this node
   263  				addr := common.Bytes2Hex(r.addr.OAddr)
   264  				pp := ppmap[addr]
   265  				//call Healthy RPC
   266  				h := r.delivery.overlay.Healthy(pp)
   267  				//print info
   268  				log.Debug(r.delivery.overlay.String())
   269  				log.Debug(fmt.Sprintf("IS HEALTHY: %t", h.GotNN && h.KnowNN && h.Full))
   270  				if !h.GotNN || !h.Full {
   271  					healthy = false
   272  					break
   273  				}
   274  			}
   275  			if healthy {
   276  				break
   277  			}
   278  		}
   279  
   280  		if history {
   281  			log.Info("Uploading for history")
   282  			//If testing only history, we upload the chunk(s) first
   283  			conf.hashes, randomFiles, err = uploadFilesToNodes(nodes)
   284  			if err != nil {
   285  				return err
   286  			}
   287  		}
   288  
   289  		//variables needed to wait for all subscriptions established before uploading
   290  		errc := make(chan error)
   291  
   292  		//now setup and start event watching in order to know when we can upload
   293  		ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout*time.Second)
   294  		defer watchCancel()
   295  
   296  		log.Info("Setting up stream subscription")
   297  		//We need two iterations, one to subscribe to the subscription events
   298  		//(so we know when setup phase is finished), and one to
   299  		//actually run the stream subscriptions. We can't do it in the same iteration,
   300  		//because while the first nodes in the loop are setting up subscriptions,
   301  		//the latter ones have not subscribed to listen to peer events yet,
   302  		//and then we miss events.
   303  
   304  		//first iteration: setup disconnection watcher and subscribe to peer events
   305  		for j, id := range ids {
   306  			log.Trace(fmt.Sprintf("Subscribe to subscription events: %d", j))
   307  			client, err := net.GetNode(id).Client()
   308  			if err != nil {
   309  				return err
   310  			}
   311  			wsDoneC := watchSubscriptionEvents(ctx, id, client, errc, quitC)
   312  			// doneC is nil, the error happened which is sent to errc channel, already
   313  			if wsDoneC == nil {
   314  				continue
   315  			}
   316  			rpcSubscriptionsWg.Add(1)
   317  			go func() {
   318  				<-wsDoneC
   319  				rpcSubscriptionsWg.Done()
   320  			}()
   321  
   322  			//watch for peers disconnecting
   323  			wdDoneC, err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC)
   324  			if err != nil {
   325  				return err
   326  			}
   327  			rpcSubscriptionsWg.Add(1)
   328  			go func() {
   329  				<-wdDoneC
   330  				rpcSubscriptionsWg.Done()
   331  			}()
   332  		}
   333  
   334  		//second iteration: start syncing and setup stream subscriptions
   335  		for j, id := range ids {
   336  			log.Trace(fmt.Sprintf("Start syncing and stream subscriptions: %d", j))
   337  			client, err := net.GetNode(id).Client()
   338  			if err != nil {
   339  				return err
   340  			}
   341  			//start syncing!
   342  			var cnt int
   343  			err = client.CallContext(ctx, &cnt, "stream_startSyncing")
   344  			if err != nil {
   345  				return err
   346  			}
   347  			//increment the number of subscriptions we need to wait for
   348  			//by the count returned from startSyncing (SYNC subscriptions)
   349  			subscriptionCount += cnt
   350  			//now also add the number of RETRIEVAL_REQUEST subscriptions
   351  			for snid := range registries[id].peers {
   352  				subscriptionCount++
   353  				err = client.CallContext(ctx, nil, "stream_subscribeStream", snid, NewStream(swarmChunkServerStreamName, "", false), nil, Top)
   354  				if err != nil {
   355  					return err
   356  				}
   357  			}
   358  		}
   359  
   360  		//now wait until the number of expected subscriptions has been finished
   361  		//`watchSubscriptionEvents` will write with a `nil` value to errc
   362  		//every time a `SubscriptionMsg` has been received
   363  		for err := range errc {
   364  			if err != nil {
   365  				return err
   366  			}
   367  			//`nil` received, decrement count
   368  			subscriptionCount--
   369  			//all subscriptions received
   370  			if subscriptionCount == 0 {
   371  				break
   372  			}
   373  		}
   374  
   375  		log.Info("Stream subscriptions successfully requested, action terminated")
   376  
   377  		if live {
   378  			//upload generated files to nodes
   379  			var hashes []storage.Address
   380  			var rfiles []string
   381  			hashes, rfiles, err = uploadFilesToNodes(nodes)
   382  			if err != nil {
   383  				return err
   384  			}
   385  			conf.hashes = append(conf.hashes, hashes...)
   386  			randomFiles = append(randomFiles, rfiles...)
   387  			//signal to the trigger loop that the upload has finished
   388  			uploadFinished <- struct{}{}
   389  		}
   390  
   391  		return nil
   392  	}
   393  
   394  	//check defines what will be checked during the test
   395  	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
   396  
   397  		select {
   398  		case <-ctx.Done():
   399  			return false, ctx.Err()
   400  		case e := <-disconnectC:
   401  			log.Error(e.Error())
   402  			return false, fmt.Errorf("Disconnect event detected, network unhealthy")
   403  		default:
   404  		}
   405  		log.Trace(fmt.Sprintf("Checking node: %s", id))
   406  		//if there are more than one chunk, test only succeeds if all expected chunks are found
   407  		allSuccess := true
   408  
   409  		//check on the node's FileStore (netstore)
   410  		fileStore := registries[id].fileStore
   411  		//check all chunks
   412  		for i, hash := range conf.hashes {
   413  			reader, _ := fileStore.Retrieve(hash)
   414  			//check that we can read the file size and that it corresponds to the generated file size
   415  			if s, err := reader.Size(nil); err != nil || s != int64(len(randomFiles[i])) {
   416  				allSuccess = false
   417  				log.Warn("Retrieve error", "err", err, "hash", hash, "nodeId", id)
   418  			} else {
   419  				log.Debug(fmt.Sprintf("File with root hash %x successfully retrieved", hash))
   420  			}
   421  		}
   422  
   423  		return allSuccess, nil
   424  	}
   425  
   426  	//for each tick, run the checks on all nodes
   427  	timingTicker := time.NewTicker(5 * time.Second)
   428  	defer timingTicker.Stop()
   429  	go func() {
   430  		//for live upload, we should wait for uploads to have finished
   431  		//before starting to trigger the checks, due to file size
   432  		if live {
   433  			<-uploadFinished
   434  		}
   435  		for range timingTicker.C {
   436  			for i := 0; i < len(ids); i++ {
   437  				log.Trace(fmt.Sprintf("triggering step %d, id %s", i, ids[i]))
   438  				trigger <- ids[i]
   439  			}
   440  		}
   441  	}()
   442  
   443  	log.Info("Starting simulation run...")
   444  
   445  	timeout := MaxTimeout * time.Second
   446  	ctx, cancel := context.WithTimeout(context.Background(), timeout)
   447  	defer cancel()
   448  
   449  	//run the simulation
   450  	result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
   451  		Action:  action,
   452  		Trigger: trigger,
   453  		Expect: &simulations.Expectation{
   454  			Nodes: ids,
   455  			Check: check,
   456  		},
   457  	})
   458  
   459  	if result.Error != nil {
   460  		return result.Error
   461  	}
   462  
   463  	return nil
   464  }
   465  
   466  /*
   467  The test generates the given number of chunks.
   468  
   469  The upload is done by dependency to the global
   470  `live` and `history` variables;
   471  
   472  If `live` is set, first stream subscriptions are established, then
   473  upload to a random node.
   474  
   475  If `history` is enabled, first upload then build up subscriptions.
   476  
   477  The test loads a snapshot file to construct the swarm network,
   478  assuming that the snapshot file identifies a healthy
   479  kademlia network. Nevertheless a health check runs in the
   480  simulation's `action` function.
   481  
   482  The snapshot should have 'streamer' in its service list.
   483  */
   484  func runRetrievalTest(chunkCount int, nodeCount int) error {
   485  	//for every run (live, history), int the variables
   486  	initRetrievalTest()
   487  	//the ids of the snapshot nodes, initiate only now as we need nodeCount
   488  	ids = make([]discover.NodeID, nodeCount)
   489  	//channel to check for disconnection errors
   490  	disconnectC := make(chan error)
   491  	//channel to close disconnection watcher routine
   492  	quitC := make(chan struct{})
   493  	//the test conf (using same as in `snapshot_sync_test`
   494  	conf = &synctestConfig{}
   495  	//map of overlay address to discover ID
   496  	conf.addrToIdMap = make(map[string]discover.NodeID)
   497  	//array where the generated chunk hashes will be stored
   498  	conf.hashes = make([]storage.Address, 0)
   499  	//load nodes from the snapshot file
   500  	net, err := initNetWithSnapshot(nodeCount)
   501  	if err != nil {
   502  		return err
   503  	}
   504  	var rpcSubscriptionsWg sync.WaitGroup
   505  	//do cleanup after test is terminated
   506  	defer func() {
   507  		//shutdown the snapshot network
   508  		net.Shutdown()
   509  		//after the test, clean up local stores initialized with createLocalStoreForId
   510  		localStoreCleanup()
   511  		//finally clear all data directories
   512  		datadirsCleanup()
   513  	}()
   514  	//get the nodes of the network
   515  	nodes := net.GetNodes()
   516  	//select one index at random...
   517  	idx := rand.Intn(len(nodes))
   518  	//...and get the the node at that index
   519  	//this is the node selected for upload
   520  	uploadNode := nodes[idx]
   521  	//iterate over all nodes...
   522  	for c := 0; c < len(nodes); c++ {
   523  		//create an array of discovery nodeIDS
   524  		ids[c] = nodes[c].ID()
   525  		a := network.ToOverlayAddr(ids[c].Bytes())
   526  		//append it to the array of all overlay addresses
   527  		conf.addrs = append(conf.addrs, a)
   528  		conf.addrToIdMap[string(a)] = ids[c]
   529  	}
   530  
   531  	//needed for healthy call
   532  	ppmap = network.NewPeerPotMap(testMinProxBinSize, conf.addrs)
   533  
   534  	trigger := make(chan discover.NodeID)
   535  	//simulation action
   536  	action := func(ctx context.Context) error {
   537  		//first run the health check on all nodes,
   538  		//wait until nodes are all healthy
   539  		ticker := time.NewTicker(200 * time.Millisecond)
   540  		defer ticker.Stop()
   541  		for range ticker.C {
   542  			healthy := true
   543  			for _, id := range ids {
   544  				r := registries[id]
   545  				//PeerPot for this node
   546  				addr := common.Bytes2Hex(network.ToOverlayAddr(id.Bytes()))
   547  				pp := ppmap[addr]
   548  				//call Healthy RPC
   549  				h := r.delivery.overlay.Healthy(pp)
   550  				//print info
   551  				log.Debug(r.delivery.overlay.String())
   552  				log.Debug(fmt.Sprintf("IS HEALTHY: %t", h.GotNN && h.KnowNN && h.Full))
   553  				if !h.GotNN || !h.Full {
   554  					healthy = false
   555  					break
   556  				}
   557  			}
   558  			if healthy {
   559  				break
   560  			}
   561  		}
   562  
   563  		if history {
   564  			log.Info("Uploading for history")
   565  			//If testing only history, we upload the chunk(s) first
   566  			conf.hashes, err = uploadFileToSingleNodeStore(uploadNode.ID(), chunkCount)
   567  			if err != nil {
   568  				return err
   569  			}
   570  		}
   571  
   572  		//variables needed to wait for all subscriptions established before uploading
   573  		errc := make(chan error)
   574  
   575  		//now setup and start event watching in order to know when we can upload
   576  		ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout*time.Second)
   577  		defer watchCancel()
   578  
   579  		log.Info("Setting up stream subscription")
   580  		//We need two iterations, one to subscribe to the subscription events
   581  		//(so we know when setup phase is finished), and one to
   582  		//actually run the stream subscriptions. We can't do it in the same iteration,
   583  		//because while the first nodes in the loop are setting up subscriptions,
   584  		//the latter ones have not subscribed to listen to peer events yet,
   585  		//and then we miss events.
   586  
   587  		//first iteration: setup disconnection watcher and subscribe to peer events
   588  		for j, id := range ids {
   589  			log.Trace(fmt.Sprintf("Subscribe to subscription events: %d", j))
   590  			client, err := net.GetNode(id).Client()
   591  			if err != nil {
   592  				return err
   593  			}
   594  
   595  			//check for `SubscribeMsg` events to know when setup phase is complete
   596  			wsDoneC := watchSubscriptionEvents(ctx, id, client, errc, quitC)
   597  			// doneC is nil, the error happened which is sent to errc channel, already
   598  			if wsDoneC == nil {
   599  				continue
   600  			}
   601  			rpcSubscriptionsWg.Add(1)
   602  			go func() {
   603  				<-wsDoneC
   604  				rpcSubscriptionsWg.Done()
   605  			}()
   606  
   607  			//watch for peers disconnecting
   608  			wdDoneC, err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC)
   609  			if err != nil {
   610  				return err
   611  			}
   612  			rpcSubscriptionsWg.Add(1)
   613  			go func() {
   614  				<-wdDoneC
   615  				rpcSubscriptionsWg.Done()
   616  			}()
   617  		}
   618  
   619  		//second iteration: start syncing and setup stream subscriptions
   620  		for j, id := range ids {
   621  			log.Trace(fmt.Sprintf("Start syncing and stream subscriptions: %d", j))
   622  			client, err := net.GetNode(id).Client()
   623  			if err != nil {
   624  				return err
   625  			}
   626  			//start syncing!
   627  			var cnt int
   628  			err = client.CallContext(ctx, &cnt, "stream_startSyncing")
   629  			if err != nil {
   630  				return err
   631  			}
   632  			//increment the number of subscriptions we need to wait for
   633  			//by the count returned from startSyncing (SYNC subscriptions)
   634  			subscriptionCount += cnt
   635  			//now also add the number of RETRIEVAL_REQUEST subscriptions
   636  			for snid := range registries[id].peers {
   637  				subscriptionCount++
   638  				err = client.CallContext(ctx, nil, "stream_subscribeStream", snid, NewStream(swarmChunkServerStreamName, "", false), nil, Top)
   639  				if err != nil {
   640  					return err
   641  				}
   642  			}
   643  		}
   644  
   645  		//now wait until the number of expected subscriptions has been finished
   646  		//`watchSubscriptionEvents` will write with a `nil` value to errc
   647  		//every time a `SubscriptionMsg` has been received
   648  		for err := range errc {
   649  			if err != nil {
   650  				return err
   651  			}
   652  			//`nil` received, decrement count
   653  			subscriptionCount--
   654  			//all subscriptions received
   655  			if subscriptionCount == 0 {
   656  				break
   657  			}
   658  		}
   659  
   660  		log.Info("Stream subscriptions successfully requested, action terminated")
   661  
   662  		if live {
   663  			//now upload the chunks to the selected random single node
   664  			chnks, err := uploadFileToSingleNodeStore(uploadNode.ID(), chunkCount)
   665  			if err != nil {
   666  				return err
   667  			}
   668  			conf.hashes = append(conf.hashes, chnks...)
   669  		}
   670  
   671  		return nil
   672  	}
   673  
   674  	chunkSize := storage.DefaultChunkSize
   675  
   676  	//check defines what will be checked during the test
   677  	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
   678  
   679  		//don't check the uploader node
   680  		if id == uploadNode.ID() {
   681  			return true, nil
   682  		}
   683  
   684  		select {
   685  		case <-ctx.Done():
   686  			return false, ctx.Err()
   687  		case e := <-disconnectC:
   688  			log.Error(e.Error())
   689  			return false, fmt.Errorf("Disconnect event detected, network unhealthy")
   690  		default:
   691  		}
   692  		log.Trace(fmt.Sprintf("Checking node: %s", id))
   693  		//if there are more than one chunk, test only succeeds if all expected chunks are found
   694  		allSuccess := true
   695  
   696  		//check on the node's FileStore (netstore)
   697  		fileStore := registries[id].fileStore
   698  		//check all chunks
   699  		for _, chnk := range conf.hashes {
   700  			reader, _ := fileStore.Retrieve(chnk)
   701  			//assuming that reading the Size of the chunk is enough to know we found it
   702  			if s, err := reader.Size(nil); err != nil || s != chunkSize {
   703  				allSuccess = false
   704  				log.Warn("Retrieve error", "err", err, "chunk", chnk, "nodeId", id)
   705  			} else {
   706  				log.Debug(fmt.Sprintf("Chunk %x found", chnk))
   707  			}
   708  		}
   709  		return allSuccess, nil
   710  	}
   711  
   712  	//for each tick, run the checks on all nodes
   713  	timingTicker := time.NewTicker(5 * time.Second)
   714  	defer timingTicker.Stop()
   715  	go func() {
   716  		for range timingTicker.C {
   717  			for i := 0; i < len(ids); i++ {
   718  				log.Trace(fmt.Sprintf("triggering step %d, id %s", i, ids[i]))
   719  				trigger <- ids[i]
   720  			}
   721  		}
   722  	}()
   723  
   724  	log.Info("Starting simulation run...")
   725  
   726  	timeout := MaxTimeout * time.Second
   727  	ctx, cancel := context.WithTimeout(context.Background(), timeout)
   728  	defer cancel()
   729  
   730  	//run the simulation
   731  	result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
   732  		Action:  action,
   733  		Trigger: trigger,
   734  		Expect: &simulations.Expectation{
   735  			Nodes: ids,
   736  			Check: check,
   737  		},
   738  	})
   739  
   740  	if result.Error != nil {
   741  		return result.Error
   742  	}
   743  
   744  	return nil
   745  }
   746  
   747  //upload generated files to nodes
   748  //every node gets one file uploaded
   749  func uploadFilesToNodes(nodes []*simulations.Node) ([]storage.Address, []string, error) {
   750  	nodeCnt := len(nodes)
   751  	log.Debug(fmt.Sprintf("Uploading %d files to nodes", nodeCnt))
   752  	//array holding generated files
   753  	rfiles := make([]string, nodeCnt)
   754  	//array holding the root hashes of the files
   755  	rootAddrs := make([]storage.Address, nodeCnt)
   756  
   757  	var err error
   758  	//for every node, generate a file and upload
   759  	for i, n := range nodes {
   760  		id := n.ID()
   761  		fileStore := registries[id].fileStore
   762  		//generate a file
   763  		rfiles[i], err = generateRandomFile()
   764  		if err != nil {
   765  			return nil, nil, err
   766  		}
   767  		//store it (upload it) on the FileStore
   768  		rk, wait, err := fileStore.Store(strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false)
   769  		log.Debug("Uploaded random string file to node")
   770  		wait()
   771  		if err != nil {
   772  			return nil, nil, err
   773  		}
   774  		rootAddrs[i] = rk
   775  	}
   776  	return rootAddrs, rfiles, nil
   777  }
   778  
   779  //generate a random file (string)
   780  func generateRandomFile() (string, error) {
   781  	//generate a random file size between minFileSize and maxFileSize
   782  	fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
   783  	log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
   784  	b := make([]byte, fileSize*1024)
   785  	_, err := crand.Read(b)
   786  	if err != nil {
   787  		log.Error("Error generating random file.", "err", err)
   788  		return "", err
   789  	}
   790  	return string(b), nil
   791  }