github.com/smartcontractkit/chainlink-testing-framework/libs@v0.0.0-20240227141906-ec710b4eb1a3/k8s/client/chaos.go (about)

     1  package client
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/cdk8s-team/cdk8s-core-go/cdk8s/v2"
     9  	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
    10  	"github.com/rs/zerolog/log"
    11  	v1 "k8s.io/api/core/v1"
    12  	"k8s.io/apimachinery/pkg/util/json"
    13  	"k8s.io/apimachinery/pkg/util/wait"
    14  
    15  	"github.com/smartcontractkit/chainlink-testing-framework/libs/k8s/config"
    16  )
    17  
    18  // Chaos is controller that manages Chaosmesh CRD instances to run experiments
    19  type Chaos struct {
    20  	Client         *K8sClient
    21  	ResourceByName map[string]string
    22  	Namespace      string
    23  }
    24  
    25  type ChaosState struct {
    26  	ChaosDetails v1alpha1.ChaosStatus `json:"status"`
    27  }
    28  
    29  // NewChaos creates controller to run and stop chaos experiments
    30  func NewChaos(client *K8sClient, namespace string) *Chaos {
    31  	return &Chaos{
    32  		Client:         client,
    33  		ResourceByName: make(map[string]string),
    34  		Namespace:      namespace,
    35  	}
    36  }
    37  
    38  // Run runs experiment and saves its ID
    39  func (c *Chaos) Run(app cdk8s.App, id string, resource string) (string, error) {
    40  	log.Info().Msg("Applying chaos experiment")
    41  	config.JSIIGlobalMu.Lock()
    42  	manifest := *app.SynthYaml()
    43  	config.JSIIGlobalMu.Unlock()
    44  	log.Trace().Str("Raw", manifest).Msg("Manifest")
    45  	c.ResourceByName[id] = resource
    46  	if err := c.Client.Apply(context.Background(), manifest, c.Namespace, false); err != nil {
    47  		return id, err
    48  	}
    49  	if err := c.checkForPodsExistence(app); err != nil {
    50  		return id, err
    51  	}
    52  	err := c.waitForChaosStatus(id, v1alpha1.ConditionAllInjected, 5*time.Minute)
    53  	if err != nil {
    54  		return id, err
    55  	}
    56  	return id, nil
    57  }
    58  
    59  func (c *Chaos) waitForChaosStatus(id string, condition v1alpha1.ChaosConditionType, timeout time.Duration) error {
    60  	var result ChaosState
    61  	log.Info().Msgf("waiting for chaos experiment state %s", condition)
    62  	if timeout < time.Minute {
    63  		log.Info().Msg("timeout is less than 1 minute, setting to 1 minute")
    64  		timeout = time.Minute
    65  	}
    66  	ctx, cancel := context.WithTimeout(context.Background(), timeout)
    67  	defer cancel()
    68  	return wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
    69  		data, err := c.Client.ClientSet.
    70  			RESTClient().
    71  			Get().
    72  			RequestURI(fmt.Sprintf("/apis/chaos-mesh.org/v1alpha1/namespaces/%s/%s/%s", c.Namespace, c.ResourceByName[id], id)).
    73  			Do(ctx).
    74  			Raw()
    75  		if err == nil {
    76  			err = json.Unmarshal(data, &result)
    77  			if err != nil {
    78  				return false, err
    79  			}
    80  			for _, c := range result.ChaosDetails.Conditions {
    81  				if c.Type == condition && c.Status == v1.ConditionTrue {
    82  					return true, err
    83  				}
    84  			}
    85  		}
    86  		return false, nil
    87  	})
    88  }
    89  
    90  func (c *Chaos) WaitForAllRecovered(id string, timeout time.Duration) error {
    91  	return c.waitForChaosStatus(id, v1alpha1.ConditionAllRecovered, timeout)
    92  }
    93  
    94  // Stop removes a chaos experiment
    95  func (c *Chaos) Stop(id string) error {
    96  	defer delete(c.ResourceByName, id)
    97  	return c.Client.DeleteResource(c.Namespace, c.ResourceByName[id], id)
    98  }
    99  
   100  func (c *Chaos) checkForPodsExistence(app cdk8s.App) error {
   101  	charts := app.Charts()
   102  	var selectors []string
   103  	for _, chart := range *charts {
   104  		json := chart.ToJson()
   105  		for _, j := range *json {
   106  			m := j.(map[string]interface{})
   107  			fmt.Println(m)
   108  			kind := m["kind"].(string)
   109  			if kind == "PodChaos" || kind == "NetworkChaos" {
   110  				selectors = append(selectors, getLabelSelectors(m["spec"].(map[string]interface{})))
   111  			}
   112  			if kind == "NetworkChaos" {
   113  				target := m["spec"].(map[string]interface{})["target"].(map[string]interface{})
   114  				selectors = append(selectors, getLabelSelectors(target))
   115  			}
   116  		}
   117  	}
   118  	for _, selector := range selectors {
   119  		podList, err := c.Client.ListPods(c.Namespace, selector)
   120  		if err != nil {
   121  			return err
   122  		}
   123  		if podList == nil || len(podList.Items) == 0 {
   124  			return fmt.Errorf("no pods found for selector %s", selector)
   125  		}
   126  		log.Info().
   127  			Int("podsCount", len(podList.Items)).
   128  			Str("selector", selector).
   129  			Msgf("found pods for chaos experiment")
   130  	}
   131  	return nil
   132  }
   133  
   134  func getLabelSelectors(spec map[string]interface{}) string {
   135  	if spec == nil {
   136  		return ""
   137  	}
   138  	s := spec["selector"].(map[string]interface{})
   139  	if s == nil {
   140  		return ""
   141  	}
   142  	m := s["labelSelectors"].(map[string]interface{})
   143  	selector := ""
   144  	for key, value := range m {
   145  		if selector == "" {
   146  			selector = fmt.Sprintf("%s=%s", key, value)
   147  		} else {
   148  			selector = fmt.Sprintf("%s, %s=%s", selector, key, value)
   149  		}
   150  	}
   151  	return selector
   152  }