bitbucket.org/number571/tendermint@v0.8.14/test/e2e/runner/perturb.go (about)

     1  package main
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	rpctypes "bitbucket.org/number571/tendermint/rpc/core/types"
     8  	e2e "bitbucket.org/number571/tendermint/test/e2e/pkg"
     9  )
    10  
    11  // Perturbs a running testnet.
    12  func Perturb(testnet *e2e.Testnet) error {
    13  	for _, node := range testnet.Nodes {
    14  		for _, perturbation := range node.Perturbations {
    15  			_, err := PerturbNode(node, perturbation)
    16  			if err != nil {
    17  				return err
    18  			}
    19  			time.Sleep(20 * time.Second) // give network some time to recover between each
    20  		}
    21  	}
    22  	return nil
    23  }
    24  
    25  // PerturbNode perturbs a node with a given perturbation, returning its status
    26  // after recovering.
    27  func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
    28  	testnet := node.Testnet
    29  	switch perturbation {
    30  	case e2e.PerturbationDisconnect:
    31  		logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name))
    32  		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
    33  			return nil, err
    34  		}
    35  		time.Sleep(10 * time.Second)
    36  		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
    37  			return nil, err
    38  		}
    39  
    40  	case e2e.PerturbationKill:
    41  		logger.Info(fmt.Sprintf("Killing node %v...", node.Name))
    42  		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
    43  			return nil, err
    44  		}
    45  		time.Sleep(10 * time.Second)
    46  		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
    47  			return nil, err
    48  		}
    49  
    50  	case e2e.PerturbationPause:
    51  		logger.Info(fmt.Sprintf("Pausing node %v...", node.Name))
    52  		if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
    53  			return nil, err
    54  		}
    55  		time.Sleep(10 * time.Second)
    56  		if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
    57  			return nil, err
    58  		}
    59  
    60  	case e2e.PerturbationRestart:
    61  		logger.Info(fmt.Sprintf("Restarting node %v...", node.Name))
    62  		if err := execCompose(testnet.Dir, "restart", node.Name); err != nil {
    63  			return nil, err
    64  		}
    65  
    66  	default:
    67  		return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
    68  	}
    69  
    70  	// Seed nodes do not have an RPC endpoint exposed so we cannot assert that
    71  	// the node recovered. All we can do is hope.
    72  	if node.Mode == e2e.ModeSeed {
    73  		return nil, nil
    74  	}
    75  
    76  	status, err := waitForNode(node, 0, 3*time.Minute)
    77  	if err != nil {
    78  		return nil, err
    79  	}
    80  	logger.Info(fmt.Sprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight))
    81  	return status, nil
    82  }