github.com/ari-anchor/sei-tendermint@v0.0.0-20230519144642-dc826b7b56bb/test/e2e/runner/perturb.go (about)

     1  package main
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/ari-anchor/sei-tendermint/libs/log"
     9  	rpctypes "github.com/ari-anchor/sei-tendermint/rpc/coretypes"
    10  	e2e "github.com/ari-anchor/sei-tendermint/test/e2e/pkg"
    11  )
    12  
    13  // Perturbs a running testnet.
    14  func Perturb(ctx context.Context, logger log.Logger, testnet *e2e.Testnet) error {
    15  	timer := time.NewTimer(0) // first tick fires immediately; reset below
    16  	defer timer.Stop()
    17  
    18  	for _, node := range testnet.Nodes {
    19  		for _, perturbation := range node.Perturbations {
    20  			select {
    21  			case <-ctx.Done():
    22  				return ctx.Err()
    23  			case <-timer.C:
    24  				_, err := PerturbNode(ctx, logger, node, perturbation)
    25  				if err != nil {
    26  					return err
    27  				}
    28  
    29  				// give network some time to recover between each
    30  				timer.Reset(20 * time.Second)
    31  			}
    32  		}
    33  	}
    34  	return nil
    35  }
    36  
    37  // PerturbNode perturbs a node with a given perturbation, returning its status
    38  // after recovering.
    39  func PerturbNode(ctx context.Context, logger log.Logger, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
    40  	testnet := node.Testnet
    41  	switch perturbation {
    42  	case e2e.PerturbationDisconnect:
    43  		logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name))
    44  		if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
    45  			return nil, err
    46  		}
    47  		time.Sleep(10 * time.Second)
    48  		if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
    49  			return nil, err
    50  		}
    51  
    52  	case e2e.PerturbationKill:
    53  		logger.Info(fmt.Sprintf("Killing node %v...", node.Name))
    54  		if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
    55  			return nil, err
    56  		}
    57  		time.Sleep(10 * time.Second)
    58  		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
    59  			return nil, err
    60  		}
    61  
    62  	case e2e.PerturbationPause:
    63  		logger.Info(fmt.Sprintf("Pausing node %v...", node.Name))
    64  		if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
    65  			return nil, err
    66  		}
    67  		time.Sleep(10 * time.Second)
    68  		if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
    69  			return nil, err
    70  		}
    71  
    72  	case e2e.PerturbationRestart:
    73  		logger.Info(fmt.Sprintf("Restarting node %v...", node.Name))
    74  		if err := execCompose(testnet.Dir, "kill", "-s", "SIGTERM", node.Name); err != nil {
    75  			return nil, err
    76  		}
    77  		time.Sleep(10 * time.Second)
    78  		if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
    79  			return nil, err
    80  		}
    81  
    82  	default:
    83  		return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
    84  	}
    85  
    86  	// Seed nodes do not have an RPC endpoint exposed so we cannot assert that
    87  	// the node recovered. All we can do is hope.
    88  	if node.Mode == e2e.ModeSeed {
    89  		return nil, nil
    90  	}
    91  
    92  	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
    93  	defer cancel()
    94  	status, err := waitForNode(ctx, logger, node, 0)
    95  	if err != nil {
    96  		return nil, err
    97  	}
    98  	logger.Info(fmt.Sprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight))
    99  	return status, nil
   100  }