github.com/lazyledger/lazyledger-core@v0.35.0-dev.0.20210613111200-4c651f053571/test/e2e/runner/perturb.go (about)

     1  package main
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	rpctypes "github.com/lazyledger/lazyledger-core/rpc/core/types"
     8  	e2e "github.com/lazyledger/lazyledger-core/test/e2e/pkg"
     9  )
    10  
    11  // Perturbs a running testnet.
    12  func Perturb(testnet *e2e.Testnet) error {
    13  	for _, node := range testnet.Nodes {
    14  		for _, perturbation := range node.Perturbations {
    15  			_, err := PerturbNode(node, perturbation)
    16  			if err != nil {
    17  				return err
    18  			}
    19  			time.Sleep(3 * time.Second) // give network some time to recover between each
    20  		}
    21  	}
    22  	return nil
    23  }
    24  
    25  // PerturbNode perturbs a node with a given perturbation, returning its status
    26  // after recovering.
    27  func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
    28  	testnet := node.Testnet
    29  	switch perturbation {
    30  	case e2e.PerturbationDisconnect:
    31  		logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name))
    32  		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
    33  			return nil, err
    34  		}
    35  		time.Sleep(10 * time.Second)
    36  		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
    37  			return nil, err
    38  		}
    39  
    40  	case e2e.PerturbationKill:
    41  		logger.Info(fmt.Sprintf("Killing node %v...", node.Name))
    42  		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
    43  			return nil, err
    44  		}
    45  		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
    46  			return nil, err
    47  		}
    48  
    49  	case e2e.PerturbationPause:
    50  		logger.Info(fmt.Sprintf("Pausing node %v...", node.Name))
    51  		if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
    52  			return nil, err
    53  		}
    54  		time.Sleep(10 * time.Second)
    55  		if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
    56  			return nil, err
    57  		}
    58  
    59  	case e2e.PerturbationRestart:
    60  		logger.Info(fmt.Sprintf("Restarting node %v...", node.Name))
    61  		if err := execCompose(testnet.Dir, "restart", node.Name); err != nil {
    62  			return nil, err
    63  		}
    64  
    65  	default:
    66  		return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
    67  	}
    68  
    69  	status, err := waitForNode(node, 0, 20*time.Second)
    70  	if err != nil {
    71  		return nil, err
    72  	}
    73  	logger.Info(fmt.Sprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight))
    74  	return status, nil
    75  }