github.com/muhammedhassanm/blockchain@v0.0.0-20200120143007-697261defd4d/go-ethereum-master/swarm/network_test.go (about)

     1  // Copyright 2018 The go-ethereum Authors
     2  // This file is part of the go-ethereum library.
     3  //
     4  // The go-ethereum library is free software: you can redistribute it and/or modify
     5  // it under the terms of the GNU Lesser General Public License as published by
     6  // the Free Software Foundation, either version 3 of the License, or
     7  // (at your option) any later version.
     8  //
     9  // The go-ethereum library is distributed in the hope that it will be useful,
    10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    12  // GNU Lesser General Public License for more details.
    13  //
    14  // You should have received a copy of the GNU Lesser General Public License
    15  // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
    16  
    17  package swarm
    18  
    19  import (
    20  	"context"
    21  	"flag"
    22  	"fmt"
    23  	"io/ioutil"
    24  	"math/rand"
    25  	"os"
    26  	"sync"
    27  	"sync/atomic"
    28  	"testing"
    29  	"time"
    30  
    31  	"github.com/ethereum/go-ethereum/common"
    32  	"github.com/ethereum/go-ethereum/crypto"
    33  	"github.com/ethereum/go-ethereum/log"
    34  	"github.com/ethereum/go-ethereum/node"
    35  	"github.com/ethereum/go-ethereum/p2p/discover"
    36  	"github.com/ethereum/go-ethereum/p2p/simulations"
    37  	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
    38  	"github.com/ethereum/go-ethereum/swarm/api"
    39  	"github.com/ethereum/go-ethereum/swarm/network"
    40  	"github.com/ethereum/go-ethereum/swarm/storage"
    41  	colorable "github.com/mattn/go-colorable"
    42  )
    43  
    44  var (
    45  	loglevel     = flag.Int("loglevel", 2, "verbosity of logs")
    46  	longrunning  = flag.Bool("longrunning", false, "do run long-running tests")
    47  	waitKademlia = flag.Bool("waitkademlia", false, "wait for healthy kademlia before checking files availability")
    48  )
    49  
    50  func init() {
    51  	rand.Seed(time.Now().UnixNano())
    52  
    53  	flag.Parse()
    54  
    55  	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
    56  }
    57  
    58  // TestSwarmNetwork runs a series of test simulations with
    59  // static and dynamic Swarm nodes in network simulation, by
    60  // uploading files to every node and retrieving them.
    61  func TestSwarmNetwork(t *testing.T) {
    62  	for _, tc := range []struct {
    63  		name     string
    64  		steps    []testSwarmNetworkStep
    65  		options  *testSwarmNetworkOptions
    66  		disabled bool
    67  	}{
    68  		{
    69  			name: "10_nodes",
    70  			steps: []testSwarmNetworkStep{
    71  				{
    72  					nodeCount: 10,
    73  				},
    74  			},
    75  			options: &testSwarmNetworkOptions{
    76  				Timeout: 45 * time.Second,
    77  			},
    78  		},
    79  		{
    80  			name: "10_nodes_skip_check",
    81  			steps: []testSwarmNetworkStep{
    82  				{
    83  					nodeCount: 10,
    84  				},
    85  			},
    86  			options: &testSwarmNetworkOptions{
    87  				Timeout:   45 * time.Second,
    88  				SkipCheck: true,
    89  			},
    90  		},
    91  		{
    92  			name: "100_nodes",
    93  			steps: []testSwarmNetworkStep{
    94  				{
    95  					nodeCount: 100,
    96  				},
    97  			},
    98  			options: &testSwarmNetworkOptions{
    99  				Timeout: 3 * time.Minute,
   100  			},
   101  			disabled: !*longrunning,
   102  		},
   103  		{
   104  			name: "100_nodes_skip_check",
   105  			steps: []testSwarmNetworkStep{
   106  				{
   107  					nodeCount: 100,
   108  				},
   109  			},
   110  			options: &testSwarmNetworkOptions{
   111  				Timeout:   3 * time.Minute,
   112  				SkipCheck: true,
   113  			},
   114  			disabled: !*longrunning,
   115  		},
   116  		{
   117  			name: "inc_node_count",
   118  			steps: []testSwarmNetworkStep{
   119  				{
   120  					nodeCount: 2,
   121  				},
   122  				{
   123  					nodeCount: 5,
   124  				},
   125  				{
   126  					nodeCount: 10,
   127  				},
   128  			},
   129  			options: &testSwarmNetworkOptions{
   130  				Timeout: 90 * time.Second,
   131  			},
   132  			disabled: !*longrunning,
   133  		},
   134  		{
   135  			name: "dec_node_count",
   136  			steps: []testSwarmNetworkStep{
   137  				{
   138  					nodeCount: 10,
   139  				},
   140  				{
   141  					nodeCount: 6,
   142  				},
   143  				{
   144  					nodeCount: 3,
   145  				},
   146  			},
   147  			options: &testSwarmNetworkOptions{
   148  				Timeout: 90 * time.Second,
   149  			},
   150  			disabled: !*longrunning,
   151  		},
   152  		{
   153  			name: "dec_inc_node_count",
   154  			steps: []testSwarmNetworkStep{
   155  				{
   156  					nodeCount: 5,
   157  				},
   158  				{
   159  					nodeCount: 3,
   160  				},
   161  				{
   162  					nodeCount: 10,
   163  				},
   164  			},
   165  			options: &testSwarmNetworkOptions{
   166  				Timeout: 90 * time.Second,
   167  			},
   168  		},
   169  		{
   170  			name: "inc_dec_node_count",
   171  			steps: []testSwarmNetworkStep{
   172  				{
   173  					nodeCount: 3,
   174  				},
   175  				{
   176  					nodeCount: 5,
   177  				},
   178  				{
   179  					nodeCount: 25,
   180  				},
   181  				{
   182  					nodeCount: 10,
   183  				},
   184  				{
   185  					nodeCount: 4,
   186  				},
   187  			},
   188  			options: &testSwarmNetworkOptions{
   189  				Timeout: 5 * time.Minute,
   190  			},
   191  			disabled: !*longrunning,
   192  		},
   193  		{
   194  			name: "inc_dec_node_count_skip_check",
   195  			steps: []testSwarmNetworkStep{
   196  				{
   197  					nodeCount: 3,
   198  				},
   199  				{
   200  					nodeCount: 5,
   201  				},
   202  				{
   203  					nodeCount: 25,
   204  				},
   205  				{
   206  					nodeCount: 10,
   207  				},
   208  				{
   209  					nodeCount: 4,
   210  				},
   211  			},
   212  			options: &testSwarmNetworkOptions{
   213  				Timeout:   5 * time.Minute,
   214  				SkipCheck: true,
   215  			},
   216  			disabled: !*longrunning,
   217  		},
   218  	} {
   219  		if tc.disabled {
   220  			continue
   221  		}
   222  		t.Run(tc.name, func(t *testing.T) {
   223  			testSwarmNetwork(t, tc.options, tc.steps...)
   224  		})
   225  	}
   226  }
   227  
   228  // testSwarmNetworkStep is the configuration
   229  // for the state of the simulation network.
   230  type testSwarmNetworkStep struct {
   231  	// number of swarm nodes that must be in the Up state
   232  	nodeCount int
   233  }
   234  
   235  // file represents the file uploaded on a particular node.
   236  type file struct {
   237  	addr   storage.Address
   238  	data   string
   239  	nodeID discover.NodeID
   240  }
   241  
   242  // check represents a reference to a file that is retrieved
   243  // from a particular node.
   244  type check struct {
   245  	key    string
   246  	nodeID discover.NodeID
   247  }
   248  
   249  // testSwarmNetworkOptions contains optional parameters for running
   250  // testSwarmNetwork.
   251  type testSwarmNetworkOptions struct {
   252  	Timeout   time.Duration
   253  	SkipCheck bool
   254  }
   255  
   256  // testSwarmNetwork is a helper function used for testing different
   257  // static and dynamic Swarm network simulations.
   258  // It is responsible for:
   259  //  - Setting up a Swarm network simulation, and updates the number of nodes within the network on every step according to steps.
   260  //  - Uploading a unique file to every node on every step.
   261  //  - May wait for Kademlia on every node to be healthy.
   262  //  - Checking if a file is retrievable from all nodes.
   263  func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwarmNetworkStep) {
   264  	dir, err := ioutil.TempDir("", "swarm-network-test")
   265  	if err != nil {
   266  		t.Fatal(err)
   267  	}
   268  	defer os.RemoveAll(dir)
   269  
   270  	if o == nil {
   271  		o = new(testSwarmNetworkOptions)
   272  	}
   273  
   274  	ctx := context.Background()
   275  	if o.Timeout > 0 {
   276  		var cancel context.CancelFunc
   277  		ctx, cancel = context.WithTimeout(ctx, o.Timeout)
   278  		defer cancel()
   279  	}
   280  
   281  	swarms := make(map[discover.NodeID]*Swarm)
   282  	files := make([]file, 0)
   283  
   284  	services := map[string]adapters.ServiceFunc{
   285  		"swarm": func(ctx *adapters.ServiceContext) (node.Service, error) {
   286  			config := api.NewConfig()
   287  
   288  			dir, err := ioutil.TempDir(dir, "node")
   289  			if err != nil {
   290  				return nil, err
   291  			}
   292  
   293  			config.Path = dir
   294  
   295  			privkey, err := crypto.GenerateKey()
   296  			if err != nil {
   297  				return nil, err
   298  			}
   299  
   300  			config.Init(privkey)
   301  			config.DeliverySkipCheck = o.SkipCheck
   302  
   303  			s, err := NewSwarm(config, nil)
   304  			if err != nil {
   305  				return nil, err
   306  			}
   307  			log.Info("new swarm", "bzzKey", config.BzzKey, "baseAddr", fmt.Sprintf("%x", s.bzz.BaseAddr()))
   308  			swarms[ctx.Config.ID] = s
   309  			return s, nil
   310  		},
   311  	}
   312  
   313  	a := adapters.NewSimAdapter(services)
   314  	net := simulations.NewNetwork(a, &simulations.NetworkConfig{
   315  		ID:             "0",
   316  		DefaultService: "swarm",
   317  	})
   318  	defer net.Shutdown()
   319  
   320  	trigger := make(chan discover.NodeID)
   321  
   322  	sim := simulations.NewSimulation(net)
   323  
   324  	for i, step := range steps {
   325  		log.Debug("test sync step", "n", i+1, "nodes", step.nodeCount)
   326  
   327  		change := step.nodeCount - len(allNodeIDs(net))
   328  
   329  		if change > 0 {
   330  			_, err := addNodes(change, net)
   331  			if err != nil {
   332  				t.Fatal(err)
   333  			}
   334  		} else if change < 0 {
   335  			err := removeNodes(-change, net)
   336  			if err != nil {
   337  				t.Fatal(err)
   338  			}
   339  		} else {
   340  			t.Logf("step %v: no change in nodes", i)
   341  			continue
   342  		}
   343  
   344  		nodeIDs := allNodeIDs(net)
   345  		shuffle(len(nodeIDs), func(i, j int) {
   346  			nodeIDs[i], nodeIDs[j] = nodeIDs[j], nodeIDs[i]
   347  		})
   348  		for _, id := range nodeIDs {
   349  			key, data, err := uploadFile(swarms[id])
   350  			if err != nil {
   351  				t.Fatal(err)
   352  			}
   353  			log.Trace("file uploaded", "node", id, "key", key.String())
   354  			files = append(files, file{
   355  				addr:   key,
   356  				data:   data,
   357  				nodeID: id,
   358  			})
   359  		}
   360  
   361  		// Prepare PeerPot map for checking Kademlia health
   362  		var ppmap map[string]*network.PeerPot
   363  		nIDs := allNodeIDs(net)
   364  		addrs := make([][]byte, len(nIDs))
   365  		if *waitKademlia {
   366  			for i, id := range nIDs {
   367  				addrs[i] = swarms[id].bzz.BaseAddr()
   368  			}
   369  			ppmap = network.NewPeerPotMap(2, addrs)
   370  		}
   371  
   372  		var checkStatusM sync.Map
   373  		var nodeStatusM sync.Map
   374  		var totalFoundCount uint64
   375  
   376  		result := sim.Run(ctx, &simulations.Step{
   377  			Action: func(ctx context.Context) error {
   378  				if *waitKademlia {
   379  					// Wait for healthy Kademlia on every node before checking files
   380  					ticker := time.NewTicker(200 * time.Millisecond)
   381  					defer ticker.Stop()
   382  
   383  					for range ticker.C {
   384  						healthy := true
   385  						log.Debug("kademlia health check", "node count", len(nIDs), "addr count", len(addrs))
   386  						for i, id := range nIDs {
   387  							swarm := swarms[id]
   388  							//PeerPot for this node
   389  							addr := common.Bytes2Hex(swarm.bzz.BaseAddr())
   390  							pp := ppmap[addr]
   391  							//call Healthy RPC
   392  							h := swarm.bzz.Healthy(pp)
   393  							//print info
   394  							log.Debug(swarm.bzz.String())
   395  							log.Debug("kademlia", "empty bins", pp.EmptyBins, "gotNN", h.GotNN, "knowNN", h.KnowNN, "full", h.Full)
   396  							log.Debug("kademlia", "health", h.GotNN && h.KnowNN && h.Full, "addr", fmt.Sprintf("%x", swarm.bzz.BaseAddr()), "id", id, "i", i)
   397  							log.Debug("kademlia", "ill condition", !h.GotNN || !h.Full, "addr", fmt.Sprintf("%x", swarm.bzz.BaseAddr()), "id", id, "i", i)
   398  							if !h.GotNN || !h.Full {
   399  								healthy = false
   400  								break
   401  							}
   402  						}
   403  						if healthy {
   404  							break
   405  						}
   406  					}
   407  				}
   408  
   409  				go func() {
   410  					// File retrieval check is repeated until all uploaded files are retrieved from all nodes
   411  					// or until the timeout is reached.
   412  					for {
   413  						if retrieve(net, files, swarms, trigger, &checkStatusM, &nodeStatusM, &totalFoundCount) == 0 {
   414  							return
   415  						}
   416  					}
   417  				}()
   418  				return nil
   419  			},
   420  			Trigger: trigger,
   421  			Expect: &simulations.Expectation{
   422  				Nodes: allNodeIDs(net),
   423  				Check: func(ctx context.Context, id discover.NodeID) (bool, error) {
   424  					// The check is done by a goroutine in the action function.
   425  					return true, nil
   426  				},
   427  			},
   428  		})
   429  		if result.Error != nil {
   430  			t.Fatal(result.Error)
   431  		}
   432  		log.Debug("done: test sync step", "n", i+1, "nodes", step.nodeCount)
   433  	}
   434  }
   435  
   436  // allNodeIDs is returning NodeID for every node that is Up.
   437  func allNodeIDs(net *simulations.Network) (nodes []discover.NodeID) {
   438  	for _, n := range net.GetNodes() {
   439  		if n.Up {
   440  			nodes = append(nodes, n.ID())
   441  		}
   442  	}
   443  	return
   444  }
   445  
   446  // addNodes adds a number of nodes to the network.
   447  func addNodes(count int, net *simulations.Network) (ids []discover.NodeID, err error) {
   448  	for i := 0; i < count; i++ {
   449  		nodeIDs := allNodeIDs(net)
   450  		l := len(nodeIDs)
   451  		nodeconf := adapters.RandomNodeConfig()
   452  		node, err := net.NewNodeWithConfig(nodeconf)
   453  		if err != nil {
   454  			return nil, fmt.Errorf("create node: %v", err)
   455  		}
   456  		err = net.Start(node.ID())
   457  		if err != nil {
   458  			return nil, fmt.Errorf("start node: %v", err)
   459  		}
   460  
   461  		log.Debug("created node", "id", node.ID())
   462  
   463  		// connect nodes in a chain
   464  		if l > 0 {
   465  			var otherNodeID discover.NodeID
   466  			for i := l - 1; i >= 0; i-- {
   467  				n := net.GetNode(nodeIDs[i])
   468  				if n.Up {
   469  					otherNodeID = n.ID()
   470  					break
   471  				}
   472  			}
   473  			log.Debug("connect nodes", "one", node.ID(), "other", otherNodeID)
   474  			if err := net.Connect(node.ID(), otherNodeID); err != nil {
   475  				return nil, err
   476  			}
   477  		}
   478  		ids = append(ids, node.ID())
   479  	}
   480  	return ids, nil
   481  }
   482  
   483  // removeNodes stops a random nodes in the network.
   484  func removeNodes(count int, net *simulations.Network) error {
   485  	for i := 0; i < count; i++ {
   486  		// allNodeIDs are returning only the Up nodes.
   487  		nodeIDs := allNodeIDs(net)
   488  		if len(nodeIDs) == 0 {
   489  			break
   490  		}
   491  		node := net.GetNode(nodeIDs[rand.Intn(len(nodeIDs))])
   492  		if err := node.Stop(); err != nil {
   493  			return err
   494  		}
   495  		log.Debug("removed node", "id", node.ID())
   496  	}
   497  	return nil
   498  }
   499  
   500  // uploadFile, uploads a short file to the swarm instance
   501  // using the api.Put method.
   502  func uploadFile(swarm *Swarm) (storage.Address, string, error) {
   503  	b := make([]byte, 8)
   504  	_, err := rand.Read(b)
   505  	if err != nil {
   506  		return nil, "", err
   507  	}
   508  	// File data is very short, but it is ensured that its
   509  	// uniqueness is very certain.
   510  	data := fmt.Sprintf("test content %s %x", time.Now().Round(0), b)
   511  	ctx := context.TODO()
   512  	k, wait, err := swarm.api.Put(ctx, data, "text/plain", false)
   513  	if err != nil {
   514  		return nil, "", err
   515  	}
   516  	if wait != nil {
   517  		err = wait(ctx)
   518  	}
   519  	return k, data, err
   520  }
   521  
   522  // retrieve is the function that is used for checking the availability of
   523  // uploaded files in testSwarmNetwork test helper function.
   524  func retrieve(
   525  	net *simulations.Network,
   526  	files []file,
   527  	swarms map[discover.NodeID]*Swarm,
   528  	trigger chan discover.NodeID,
   529  	checkStatusM *sync.Map,
   530  	nodeStatusM *sync.Map,
   531  	totalFoundCount *uint64,
   532  ) (missing uint64) {
   533  	shuffle(len(files), func(i, j int) {
   534  		files[i], files[j] = files[j], files[i]
   535  	})
   536  
   537  	var totalWg sync.WaitGroup
   538  	errc := make(chan error)
   539  
   540  	nodeIDs := allNodeIDs(net)
   541  
   542  	totalCheckCount := len(nodeIDs) * len(files)
   543  
   544  	for _, id := range nodeIDs {
   545  		if _, ok := nodeStatusM.Load(id); ok {
   546  			continue
   547  		}
   548  		start := time.Now()
   549  		var checkCount uint64
   550  		var foundCount uint64
   551  
   552  		totalWg.Add(1)
   553  
   554  		var wg sync.WaitGroup
   555  
   556  		for _, f := range files {
   557  			swarm := swarms[id]
   558  
   559  			checkKey := check{
   560  				key:    f.addr.String(),
   561  				nodeID: id,
   562  			}
   563  			if n, ok := checkStatusM.Load(checkKey); ok && n.(int) == 0 {
   564  				continue
   565  			}
   566  
   567  			checkCount++
   568  			wg.Add(1)
   569  			go func(f file, id discover.NodeID) {
   570  				defer wg.Done()
   571  
   572  				log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount))
   573  
   574  				r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/")
   575  				if err != nil {
   576  					errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err)
   577  					return
   578  				}
   579  				d, err := ioutil.ReadAll(r)
   580  				if err != nil {
   581  					errc <- fmt.Errorf("api get: read response: node %s, key %s: kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err)
   582  					return
   583  				}
   584  				data := string(d)
   585  				if data != f.data {
   586  					errc <- fmt.Errorf("file contend missmatch: node %s, key %s, expected %q, got %q", id, f.addr, f.data, data)
   587  					return
   588  				}
   589  				checkStatusM.Store(checkKey, 0)
   590  				atomic.AddUint64(&foundCount, 1)
   591  				log.Info("api get: file found", "node", id.String(), "key", f.addr.String(), "content", data, "files found", atomic.LoadUint64(&foundCount))
   592  			}(f, id)
   593  		}
   594  
   595  		go func(id discover.NodeID) {
   596  			defer totalWg.Done()
   597  			wg.Wait()
   598  
   599  			atomic.AddUint64(totalFoundCount, foundCount)
   600  
   601  			if foundCount == checkCount {
   602  				log.Info("all files are found for node", "id", id.String(), "duration", time.Since(start))
   603  				nodeStatusM.Store(id, 0)
   604  				trigger <- id
   605  				return
   606  			}
   607  			log.Debug("files missing for node", "id", id.String(), "check", checkCount, "found", foundCount)
   608  		}(id)
   609  
   610  	}
   611  
   612  	go func() {
   613  		totalWg.Wait()
   614  		close(errc)
   615  	}()
   616  
   617  	var errCount int
   618  	for err := range errc {
   619  		if err != nil {
   620  			errCount++
   621  		}
   622  		log.Warn(err.Error())
   623  	}
   624  
   625  	log.Info("check stats", "total check count", totalCheckCount, "total files found", atomic.LoadUint64(totalFoundCount), "total errors", errCount)
   626  
   627  	return uint64(totalCheckCount) - atomic.LoadUint64(totalFoundCount)
   628  }
   629  
   630  // Backported from stdlib https://golang.org/src/math/rand/rand.go?s=11175:11215#L333
   631  //
   632  // Replace with rand.Shuffle from go 1.10 when go 1.9 support is dropped.
   633  //
   634  // shuffle pseudo-randomizes the order of elements.
   635  // n is the number of elements. Shuffle panics if n < 0.
   636  // swap swaps the elements with indexes i and j.
   637  func shuffle(n int, swap func(i, j int)) {
   638  	if n < 0 {
   639  		panic("invalid argument to Shuffle")
   640  	}
   641  
   642  	// Fisher-Yates shuffle: https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
   643  	// Shuffle really ought not be called with n that doesn't fit in 32 bits.
   644  	// Not only will it take a very long time, but with 2³¹! possible permutations,
   645  	// there's no way that any PRNG can have a big enough internal state to
   646  	// generate even a minuscule percentage of the possible permutations.
   647  	// Nevertheless, the right API signature accepts an int n, so handle it as best we can.
   648  	i := n - 1
   649  	for ; i > 1<<31-1-1; i-- {
   650  		j := int(rand.Int63n(int64(i + 1)))
   651  		swap(i, j)
   652  	}
   653  	for ; i > 0; i-- {
   654  		j := int(rand.Int31n(int32(i + 1)))
   655  		swap(i, j)
   656  	}
   657  }