github.com/badrootd/nibiru-cometbft@v0.37.5-0.20240307173500-2a75559eee9b/test/e2e/runner/perturb.go (about)

     1  package main
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	"github.com/badrootd/nibiru-cometbft/libs/log"
     8  	rpctypes "github.com/badrootd/nibiru-cometbft/rpc/core/types"
     9  	e2e "github.com/badrootd/nibiru-cometbft/test/e2e/pkg"
    10  )
    11  
    12  // Perturbs a running testnet.
    13  func Perturb(testnet *e2e.Testnet) error {
    14  	for _, node := range testnet.Nodes {
    15  		for _, perturbation := range node.Perturbations {
    16  			_, err := PerturbNode(node, perturbation)
    17  			if err != nil {
    18  				return err
    19  			}
    20  			time.Sleep(3 * time.Second) // give network some time to recover between each
    21  		}
    22  	}
    23  	return nil
    24  }
    25  
    26  // PerturbNode perturbs a node with a given perturbation, returning its status
    27  // after recovering.
    28  func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
    29  	testnet := node.Testnet
    30  	out, err := execComposeOutput(testnet.Dir, "ps", "-q", node.Name)
    31  	if err != nil {
    32  		return nil, err
    33  	}
    34  	name := node.Name
    35  	upgraded := false
    36  	if len(out) == 0 {
    37  		name = name + "_u"
    38  		upgraded = true
    39  		logger.Info("perturb node", "msg",
    40  			log.NewLazySprintf("Node %v already upgraded, operating on alternate container %v",
    41  				node.Name, name))
    42  	}
    43  
    44  	switch perturbation {
    45  	case e2e.PerturbationDisconnect:
    46  		logger.Info("perturb node", "msg", log.NewLazySprintf("Disconnecting node %v...", node.Name))
    47  		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, name); err != nil {
    48  			return nil, err
    49  		}
    50  		time.Sleep(10 * time.Second)
    51  		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, name); err != nil {
    52  			return nil, err
    53  		}
    54  
    55  	case e2e.PerturbationKill:
    56  		logger.Info("perturb node", "msg", log.NewLazySprintf("Killing node %v...", node.Name))
    57  		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", name); err != nil {
    58  			return nil, err
    59  		}
    60  		if err := execCompose(testnet.Dir, "start", name); err != nil {
    61  			return nil, err
    62  		}
    63  
    64  	case e2e.PerturbationPause:
    65  		logger.Info("perturb node", "msg", log.NewLazySprintf("Pausing node %v...", node.Name))
    66  		if err := execCompose(testnet.Dir, "pause", name); err != nil {
    67  			return nil, err
    68  		}
    69  		time.Sleep(10 * time.Second)
    70  		if err := execCompose(testnet.Dir, "unpause", name); err != nil {
    71  			return nil, err
    72  		}
    73  
    74  	case e2e.PerturbationRestart:
    75  		logger.Info("perturb node", "msg", log.NewLazySprintf("Restarting node %v...", node.Name))
    76  		if err := execCompose(testnet.Dir, "restart", name); err != nil {
    77  			return nil, err
    78  		}
    79  
    80  	case e2e.PerturbationUpgrade:
    81  		oldV := node.Version
    82  		newV := node.Testnet.UpgradeVersion
    83  		if upgraded {
    84  			return nil, fmt.Errorf("node %v can't be upgraded twice from version '%v' to version '%v'",
    85  				node.Name, oldV, newV)
    86  		}
    87  		if oldV == newV {
    88  			logger.Info("perturb node", "msg",
    89  				log.NewLazySprintf("Skipping upgrade of node %v to version '%v'; versions are equal.",
    90  					node.Name, newV))
    91  			break
    92  		}
    93  		logger.Info("perturb node", "msg",
    94  			log.NewLazySprintf("Upgrading node %v from version '%v' to version '%v'...",
    95  				node.Name, oldV, newV))
    96  
    97  		if err := execCompose(testnet.Dir, "stop", name); err != nil {
    98  			return nil, err
    99  		}
   100  		time.Sleep(10 * time.Second)
   101  		if err := execCompose(testnet.Dir, "up", "-d", name+"_u"); err != nil {
   102  			return nil, err
   103  		}
   104  
   105  	default:
   106  		return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
   107  	}
   108  
   109  	status, err := waitForNode(node, 0, 20*time.Second)
   110  	if err != nil {
   111  		return nil, err
   112  	}
   113  	logger.Info("perturb node",
   114  		"msg",
   115  		log.NewLazySprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight))
   116  	return status, nil
   117  }