github.com/percona/percona-xtradb-cluster-operator@v1.14.0/e2e-tests/self-healing-chaos/run (about)

     1  #!/bin/bash
     2  
     3  set -o errexit
     4  
     5  test_dir=$(realpath $(dirname $0))
     6  . ${test_dir}/../functions
     7  cluster="some-name"
     8  
     9  set_debug
    10  
    11  check_pod_restarted() {
    12  	local pod=$1
    13  	local nr_restarts=$(kubectl get pod $pod -ojson | jq '.status.containerStatuses[] | select(.name == "pxc").restartCount')
    14  
    15  	if [ $nr_restarts -eq 0 ]; then
    16  		echo "Chaos mesh didn't work for some reason. Please check!!!"
    17  		echo "Pod $pod restarts: $nr_restarts, but it should have been more!"
    18  		exit 1
    19  	fi
    20  }
    21  
    22  kill_pod() {
    23  	local pod=$1
    24  
    25  	yq eval '
    26  		.metadata.name = "chaos-pod-kill" |
    27  		del(.spec.selector.pods.test-namespace) |
    28  		.spec.selector.pods.'$namespace'[0] = "'$pod'"' $conf_dir/chaos-pod-kill.yml \
    29  		| kubectl apply -f -
    30  	sleep 5
    31  
    32  	# check if all 3 Pods started
    33  	wait_for_running $cluster-pxc 3
    34  	wait_cluster_consistency "$cluster" 3 2
    35  
    36  	desc 'check data consistency for chaosed Pod'
    37  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $pod.$cluster-pxc -uroot -proot_password"
    38  }
    39  
    40  fail_pod() {
    41  	local pod=$1
    42  
    43  	# run chaos for Pod
    44  	yq eval '
    45  		.metadata.name = "chaos-pod-failure" |
    46  		del(.spec.selector.pods.test-namespace) |
    47  		.spec.selector.pods.'$namespace'[0] = "'$pod'"' $conf_dir/chaos-pod-failure.yml \
    48  		| kubectl apply -f -
    49  	sleep 10
    50  
    51  	# write data
    52  	run_mysql \
    53  		'INSERT myApp.myApp (id) VALUES (100501)' \
    54  		"-h $cluster-proxysql -uroot -proot_password"
    55  
    56  	# wait until the chaos stops
    57  	sleep 60
    58  
    59  	# check if all 3 Pods started
    60  	wait_for_running $cluster-pxc 3
    61  	wait_cluster_consistency "$cluster" 3 2
    62  
    63  	check_pod_restarted $pod
    64  
    65  	desc 'check data consistency for chaosed Pod'
    66  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $pod.$cluster-pxc -uroot -proot_password" "-2nd"
    67  }
    68  
    69  network_loss() {
    70  	local pod=$1
    71  
    72  	# run chaos for Pod
    73  	yq eval '
    74  		.metadata.name = "chaos-pod-network-loss" |
    75  		del(.spec.selector.pods.test-namespace) |
    76  		.spec.selector.pods.'$namespace'[0] = "'$pod'"' $conf_dir/chaos-network-loss.yml \
    77  		| kubectl apply -f -
    78  	sleep 10
    79  
    80  	# write data
    81  	run_mysql \
    82  		'INSERT myApp.myApp (id) VALUES (100502)' \
    83  		"-h $cluster-pxc -uroot -proot_password"
    84  
    85  	# wait until the chaos stops
    86  	sleep 60
    87  
    88  	# check if all 3 Pods started
    89  	wait_for_running $cluster-pxc 3
    90  	wait_cluster_consistency "$cluster" 3 2
    91  
    92  	desc 'check data consistency for chaosed Pod'
    93  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $pod.$cluster-pxc -uroot -proot_password" "-3rd"
    94  }
    95  
    96  recreate() {
    97  	run_mysql_local \
    98  		'
    99              SET mysql-connect_timeout_server_max = 100500;
   100              LOAD MYSQL VARIABLES TO RUNTIME;
   101              SAVE MYSQL VARIABLES TO MEMORY;
   102              SAVE MYSQL VARIABLES TO DISK;
   103          ' \
   104  		"-h127.0.0.1 -P6032 -uproxyadmin -padmin_password" "$cluster-proxysql-0"
   105  
   106  	desc "delete cluster"
   107  	kubectl delete \
   108  		-f $conf_dir/$cluster.yml
   109  	wait_for_delete pod/$cluster-pxc-2
   110  	wait_for_delete pod/$cluster-pxc-1
   111  	wait_for_delete pod/$cluster-pxc-0
   112  
   113  	desc "recreate cluster"
   114  	apply_config "$conf_dir/$cluster.yml"
   115  
   116  	wait_for_running $cluster-pxc 3
   117  	wait_cluster_consistency "$cluster" 3 2
   118  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password" "-3rd"
   119  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password" "-3rd"
   120  	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password" "-3rd"
   121  }
   122  
   123  main() {
   124  	create_infra $namespace
   125  	deploy_chaos_mesh $namespace
   126  
   127  	desc 'start cluster'
   128  	spinup_pxc "$cluster" "$conf_dir/$cluster.yml"
   129  
   130  	desc 'kill pod-0 pod'
   131  	kill_pod "$cluster-pxc-0"
   132  
   133  	desc 'fail pod-0 pod for 60s'
   134  	fail_pod "$cluster-pxc-0"
   135  
   136  	desc 'emulate bad network pod-0 for 60s'
   137  	network_loss "$cluster-pxc-0"
   138  
   139  	desc 'recreate cluster'
   140  	recreate
   141  
   142  	destroy_chaos_mesh
   143  	destroy $namespace
   144  	desc "test passed"
   145  }
   146  
   147  main