github.com/percona/percona-xtradb-cluster-operator@v1.14.0/e2e-tests/self-healing-advanced-chaos/run (about)

     1  #!/bin/bash
     2  
     3  set -o errexit
     4  
     5  test_dir=$(realpath $(dirname $0))
     6  . "${test_dir}"/../functions
     7  cluster="self-healing-advanced"
     8  
     9  set_debug
    10  
    11  crash_cluster_with_kubectl() {
    12  	# run chaos for Pods
    13  	for _ in $(seq 1 5); do
    14  		kubectl delete pod $cluster-pxc-0 $cluster-pxc-1 $cluster-pxc-2 --grace-period=0 --force || :
    15  	done
    16  
    17  	sleep 60
    18  
    19  	for i in $(seq 0 2); do
    20  		wait_crash_pod $cluster-pxc-$i
    21  	done
    22  }
    23  
    24  crash_cluster_with_chaos_mesh() {
    25  	yq eval '
    26  		.metadata.name = "chaos-cluster-kill" |
    27  		.spec.mode = "all" |
    28  		del(.spec.selector.pods.test-namespace) |
    29  		.spec.selector.pods.'$namespace'[0] = "'$cluster'-pxc-0" |
    30  		.spec.selector.pods.'$namespace'[1] = "'$cluster'-pxc-1" |
    31  		.spec.selector.pods.'$namespace'[2] = "'$cluster'-pxc-2"' $conf_dir/chaos-pod-kill.yml \
    32  		| kubectl apply -f -
    33  
    34  	sleep 60
    35  
    36  	for i in $(seq 0 2); do
    37  		wait_crash_pod $cluster-pxc-$i
    38  	done
    39  }
    40  
    41  check_if_cluster_restored() {
    42  	desc 'check data consistency'
    43  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password"
    44  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password"
    45  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password"
    46  }
    47  
    48  main() {
    49  	create_infra $namespace
    50  	deploy_chaos_mesh $namespace
    51  
    52  	desc 'start cluster'
    53  	spinup_pxc "$cluster" "$test_dir/conf/$cluster.yml"
    54  
    55  	desc 'delete all PXC pods and check full crash'
    56  	crash_cluster_with_kubectl
    57  
    58  	cat_config "$test_dir/conf/$cluster.yml" \
    59  		| sed -e 's/autoRecovery: false/autoRecovery: true/' \
    60  		| kubectl apply -f-
    61  
    62  	sleep 240
    63  	wait_for_running $cluster-pxc 3
    64  	wait_cluster_consistency "$cluster" 3 2
    65  
    66  	desc 'check if full cluster crash repaired'
    67  	check_if_cluster_restored
    68  
    69  	desc 'crash all pxc pods with chaos mesh'
    70  	crash_cluster_with_chaos_mesh
    71  
    72  	sleep 240
    73  	wait_for_running $cluster-pxc 3
    74  	wait_cluster_consistency "$cluster" 3 2
    75  	desc 'check if full cluster crash repaired'
    76  	check_if_cluster_restored
    77  
    78  	destroy_chaos_mesh
    79  	destroy "$namespace"
    80  	desc "test passed"
    81  }
    82  
    83  main