github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/plugins/plugin_test.sh (about) 1 #!/bin/bash 2 set -e 3 set -o pipefail 4 5 plugin=$1 6 plugin_config=$2 7 secondary_plugin_config=$3 8 MINIMUM_API_VERSION="0.3.0" 9 10 # ---------------------------------------------- 11 # Test suite setup 12 # This will put small amounts of data in the 13 # plugin destination location 14 # ---------------------------------------------- 15 if [ $# -lt 2 ] || [ $# -gt 3 ] 16 then 17 echo "Usage: plugin_test.sh [path_to_executable] [plugin_config] [optional_config_for_secondary_destination]" 18 exit 1 19 fi 20 21 if [[ "$plugin_config" != /* ]] ; then 22 echo "Must provide an absolute path to the plugin config" 23 exit 1 24 fi 25 26 # This should be time_second=$(date +"%Y%m%d%H%M%S") but concurrent 27 # runs require randomness to ensure they don't collide with each other 28 # in writing/deleting backups. We ensure there are 14 characters 29 # through the expression below. 30 time_second=$(expr 99999999999999 - $(od -vAn -N5 -tu < /dev/urandom | tr -d ' \n')) 31 current_date=$(echo $time_second | cut -c 1-8) 32 33 testdir="/tmp/testseg/backups/${current_date}/${time_second}" 34 testfile="$testdir/testfile_$time_second.txt" 35 testdata="$testdir/testdata_$time_second.txt" 36 test_no_data="$testdir/test_no_data_$time_second.txt" 37 testdatasmall="$testdir/testdatasmall_$time_second.txt" 38 testdatalarge="$testdir/testdatalarge_$time_second.txt" 39 40 logdir="/tmp/test_bench_logs" 41 42 text="this is some text" 43 data=`LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 1000 ; echo` 44 data_large=`LC_ALL=C tr -dc 'A-Za-z0-9' </dev/urandom | head -c 1000000 ; echo` 45 mkdir -p $testdir 46 mkdir -p $logdir 47 echo $text > $testfile 48 49 # ---------------------------------------------- 50 # Cleanup functions 51 # ---------------------------------------------- 52 cleanup_test_dir() { 53 if [ $# -ne 1 ] 54 then 55 echo "Must call cleanup_test_dir with only 1 argument" 56 exit 1 57 fi 58 59 testdir_to_clean=$1 60 61 $plugin cleanup_plugin_for_backup $plugin_config $testdir_to_clean coordinator \"-1\" 62 $plugin cleanup_plugin_for_backup $plugin_config $testdir_to_clean segment_host 63 $plugin cleanup_plugin_for_backup $plugin_config $testdir_to_clean segment \"0\" 64 echo "[PASSED - CLEANUP] cleanup_plugin_for_backup" 65 66 $plugin cleanup_plugin_for_restore $plugin_config $testdir_to_clean coordinator \"-1\" 67 $plugin cleanup_plugin_for_restore $plugin_config $testdir_to_clean segment_host 68 $plugin cleanup_plugin_for_restore $plugin_config $testdir_to_clean segment \"0\" 69 echo "[PASSED - CLEANUP] cleanup_plugin_for_restore" 70 } 71 72 echo "# ----------------------------------------------" 73 echo "# Starting gpbackup plugin tests" 74 echo "# ----------------------------------------------" 75 76 # ---------------------------------------------- 77 # Check API version 78 # ---------------------------------------------- 79 80 echo "[RUNNING] plugin_api_version" 81 api_version=`$plugin plugin_api_version` 82 # `awk` call returns 1 for true, 0 for false (contrary to bash logic) 83 if (( 0 == $(echo "$MINIMUM_API_VERSION $api_version" | awk '{print ($1 <= $2)}') )) ; then 84 echo "Plugin API version is less than the minimum supported version $MINIMUM_API_VERSION" 85 exit 1 86 fi 87 echo "[PASSED] plugin_api_version" 88 89 echo "[RUNNING] --version" 90 native_version=`$plugin --version` 91 echo "$native_version" | grep --regexp '.* version .*' > /dev/null 2>&1 92 if [[ ! $? -eq 0 ]]; then 93 echo "Plugin --version is not in expected format of <plugin name> version <version>" 94 exit 1 95 fi 96 echo "[PASSED] --version" 97 98 # ---------------------------------------------- 99 # Setup and Backup/Restore file functions 100 # ---------------------------------------------- 101 102 echo "[RUNNING] setup_plugin_for_backup on coordinator" 103 $plugin setup_plugin_for_backup $plugin_config $testdir coordinator \"-1\" 104 echo "[RUNNING] setup_plugin_for_backup on segment_host" 105 $plugin setup_plugin_for_backup $plugin_config $testdir segment_host 106 echo "[RUNNING] setup_plugin_for_backup on segment 0" 107 $plugin setup_plugin_for_backup $plugin_config $testdir segment \"0\" 108 109 echo "[RUNNING] backup_file" 110 $plugin backup_file $plugin_config $testfile 111 # plugins should leave copies of the files locally when they run backup_file 112 test -f $testfile 113 114 echo "[RUNNING] setup_plugin_for_restore on coordinator" 115 $plugin setup_plugin_for_restore $plugin_config $testdir coordinator \"-1\" 116 echo "[RUNNING] setup_plugin_for_restore on segment_host" 117 $plugin setup_plugin_for_restore $plugin_config $testdir segment_host 118 echo "[RUNNING] setup_plugin_for_restore on segment 0" 119 $plugin setup_plugin_for_restore $plugin_config $testdir segment \"0\" 120 121 echo "[RUNNING] restore_file" 122 rm $testfile 123 $plugin restore_file $plugin_config $testfile 124 output=`cat $testfile` 125 if [ "$output" != "$text" ]; then 126 echo "Failed to backup and restore file using plugin" 127 exit 1 128 fi 129 130 echo "[RUNNING] attempting to restore_file of non-existent file should fail" 131 set +e 132 $plugin restore_file $plugin_config "$testdir/there_is_no_file_to_restore" > /dev/null 2>&1 133 nonexist_file_restore=$(echo $?) 134 set -e 135 if [ "$nonexist_file_restore" == "0" ]; then 136 echo "Failed, when trying to restore a file that does not exist, should have error" 137 exit 1 138 fi 139 140 141 if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then 142 rm $testfile 143 echo "[RUNNING] restore_file (from secondary destination)" 144 $plugin restore_file $secondary_plugin_config $testfile 145 output=`cat $testfile` 146 if [ "$output" != "$text" ]; then 147 echo "Failed to backup and restore file using plugin from secondary destination" 148 exit 1 149 fi 150 fi 151 echo "[PASSED] setup_plugin_for_backup" 152 echo "[PASSED] backup_file" 153 echo "[PASSED] setup_plugin_for_restore" 154 echo "[PASSED] restore_file" 155 cleanup_test_dir $testdir 156 157 # ---------------------------------------------- 158 # Backup/Restore data functions 159 # ---------------------------------------------- 160 161 echo "[RUNNING] backup_data" 162 echo $data | $plugin backup_data $plugin_config $testdata 163 echo "[RUNNING] restore_data" 164 output=`$plugin restore_data $plugin_config $testdata` 165 166 if [ "$output" != "$data" ]; then 167 echo "Failed to backup and restore data using plugin" 168 exit 1 169 fi 170 171 if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then 172 echo "[RUNNING] restore_data (from secondary destination)" 173 output=`$plugin restore_data $secondary_plugin_config $testdata` 174 175 if [ "$output" != "$data" ]; then 176 echo "Failed to backup and restore data using plugin" 177 exit 1 178 fi 179 fi 180 echo "[PASSED] backup_data" 181 echo "[PASSED] restore_data" 182 cleanup_test_dir $testdir 183 184 echo "[RUNNING] backup_data with no data" 185 echo -n "" | $plugin backup_data $plugin_config $test_no_data 186 echo "[RUNNING] restore_data with no data" 187 output=`$plugin restore_data $plugin_config $test_no_data` 188 189 if [ "$output" != "" ]; then 190 echo "Failed to backup and restore data using plugin" 191 exit 1 192 fi 193 194 if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then 195 echo "[RUNNING] restore_data with no data (from secondary destination)" 196 output=`$plugin restore_data $secondary_plugin_config $test_no_data` 197 198 if [ "$output" != "" ]; then 199 echo "Failed to backup and restore data using plugin" 200 exit 1 201 fi 202 fi 203 echo "[PASSED] backup_data with no data" 204 echo "[PASSED] restore_data with no data" 205 cleanup_test_dir $testdir 206 207 # ---------------------------------------------- 208 # Restore subset data functions 209 # ---------------------------------------------- 210 if [[ "$plugin" == *gpbackup_ddboost_plugin ]]; then 211 echo "[RUNNING] backup_data of small data for subset restore" 212 echo $data | $plugin backup_data $plugin_config $testdatasmall 213 echo "1 3 10" > "$testdir/offsets" 214 echo "[RUNNING] restore_data_subset" 215 echo $plugin restore_data_subset $plugin_config $testdatasmall "$testdir/offsets" 216 output=`$plugin restore_data_subset $plugin_config $testdatasmall "$testdir/offsets"` 217 data_subset=$(echo $data | cut -c4-10) 218 if [ "$output" != "$data_subset" ]; then 219 echo "Failure in restore_data_subset of small data using plugin" 220 exit 1 221 fi 222 echo "[PASSED] restore_data_subset with small data" 223 224 echo "[RUNNING] backup_data of large data for subset restore" 225 echo $data_large | $plugin backup_data $plugin_config $testdatalarge 226 echo "1 900000 900001" > "$testdir/offsets" 227 echo "[RUNNING] restore_data_subset" 228 output=`$plugin restore_data_subset $plugin_config $testdatalarge "$testdir/offsets"` 229 data_part=$(echo $data_large | cut -c900001-900001) 230 if [ "$output" != "$data_part" ]; then 231 echo "Failure restore_data_subset of one partition from large data" 232 exit 1 233 fi 234 echo "[PASSED] restore_data_subset of one partition from large data" 235 236 echo "2 0 700000 900000 900001" > "$testdir/offsets" 237 echo "[RUNNING] restore_data_subset" 238 output=`$plugin restore_data_subset $plugin_config $testdatalarge "$testdir/offsets"` 239 data_part1=$(echo $data_large | cut -c1-700000) 240 data_part2=$(echo $data_large | cut -c900001-900001) 241 if [ "$output" != "$data_part1$data_part2" ]; then 242 echo "Failure restore_data_subset of two partitions from large data" 243 exit 1 244 fi 245 echo "[PASSED] restore_data_subset of two partitions from large data" 246 cleanup_test_dir $testdir 247 fi 248 249 # ---------------------------------------------- 250 # Delete backup directory function 251 # ---------------------------------------------- 252 time_second_for_del=$(expr 99999999999999 - $(od -vAn -N5 -tu < /dev/urandom | tr -d ' \n')) 253 current_date_for_del=$(echo $time_second_for_del | cut -c 1-8) 254 255 time_second_for_del2=$(expr $time_second_for_del + 1) 256 current_date_for_del2=$(echo $time_second_for_del2 | cut -c 1-8) 257 258 testdir_for_del="/tmp/testseg/backups/${current_date_for_del}/${time_second_for_del}" 259 testdata_for_del="$testdir_for_del/testdata_$time_second_for_del.txt" 260 testfile_for_del="$testdir_for_del/testfile_$time_second_for_del.txt" 261 262 testdir_for_del2="/tmp/testseg/backups/${current_date_for_del2}/${time_second_for_del2}" 263 testdata_for_del2="$testdir_for_del2/testdata_$time_second_for_del2.txt" 264 testfile_for_del2="$testdir_for_del2/testfile_$time_second_for_del2.txt" 265 266 mkdir -p $testdir_for_del 267 mkdir -p $testdir_for_del2 268 269 echo $text > $testfile_for_del 270 echo $text > $testfile_for_del2 271 272 echo "[RUNNING] delete_backup" 273 274 # first backup 275 $plugin setup_plugin_for_backup $plugin_config $testdir_for_del coordinator \"-1\" 276 $plugin setup_plugin_for_backup $plugin_config $testdir_for_del segment_host 277 $plugin setup_plugin_for_backup $plugin_config $testdir_for_del segment \"0\" 278 279 echo $data | $plugin backup_data $plugin_config $testdata_for_del 280 $plugin backup_file $plugin_config $testfile_for_del 281 282 # second backup 283 $plugin setup_plugin_for_backup $plugin_config $testdir_for_del2 coordinator \"-1\" 284 $plugin setup_plugin_for_backup $plugin_config $testdir_for_del2 segment_host 285 $plugin setup_plugin_for_backup $plugin_config $testdir_for_del2 segment \"0\" 286 287 echo $data | $plugin backup_data $plugin_config $testdata_for_del2 288 $plugin backup_file $plugin_config $testfile_for_del2 289 290 # here's the real test: can we delete and confirm deletion, while sibling backup remains? 291 $plugin delete_backup $plugin_config $time_second_for_del 292 293 set +e 294 # test deletion from local server 295 output_data_restore=$($plugin restore_data $plugin_config $testdata_for_del 2>/dev/null) 296 retval_data_restore=$(echo $?) 297 if [ "${output_data_restore}" = "${data}" ] || [ "$retval_data_restore" = "0" ] ; then 298 echo "Failed to delete backup data from local server using plugin" 299 exit 1 300 fi 301 $plugin restore_file $plugin_config $testfile_for_del 2>/dev/null 302 retval_file_restore=$(echo $?) 303 if [ "$retval_file_restore" = "0" ] ; then 304 echo "Failed to delete backup file from local server using plugin" 305 exit 1 306 fi 307 308 # test deletion from remote server 309 if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then 310 output_data_restore=$($plugin restore_data $secondary_plugin_config $testdata_for_del 2>/dev/null) 311 retval_data_restore=$(echo $?) 312 if [ "${output_data_restore}" = "${data}" ] || [ "$retval_data_restore" = "0" ] ; then 313 echo "Failed to delete backup data from remote server using plugin" 314 exit 1 315 fi 316 $plugin restore_file $secondary_plugin_config $testfile_for_del 2>/dev/null 317 retval_file_restore=$(echo $?) 318 if [ "$retval_file_restore" = "0" ] ; then 319 echo "Failed to delete backup file from remote server using plugin" 320 exit 1 321 fi 322 fi 323 324 # confirm sibling backup remains 325 log_sibling_restore="$logdir/sib_restore" 326 output_data_restore=$($plugin restore_data $plugin_config $testdata_for_del2 2>$log_sibling_restore) 327 retval_data_restore=$(echo $?) 328 if [ "${output_data_restore}" != "${data}" ] || [ "$retval_data_restore" != "0" ] ; then 329 echo 330 cat $log_sibling_restore 331 echo 332 echo "Failed to leave behind a sibling backup after a sibling delete from local server using plugin" 333 exit 1 334 fi 335 336 set -e 337 echo "[PASSED] delete_backup" 338 cleanup_test_dir $testdir_for_del 339 340 # ---------------------------------------------- 341 # Replicate backup function 342 # ---------------------------------------------- 343 if [[ "$plugin" == *gpbackup_ddboost_plugin ]] && [[ "$plugin_config" == *ddboost_config.yaml ]]; then 344 time_second_for_repl=$(expr 99999999999999 - $(od -vAn -N5 -tu < /dev/urandom | tr -d ' \n')) 345 current_date_for_repl=$(echo $time_second_for_repl | cut -c 1-8) 346 347 testdir_for_repl="/tmp/testseg/backups/${current_date_for_repl}/${time_second_for_repl}" 348 testdata_for_repl="$testdir_for_repl/testdata_$time_second_for_repl.txt" 349 350 mkdir -p $testdir_for_repl 351 352 echo "[RUNNING] backup_data without replication to replicate later" 353 $plugin setup_plugin_for_backup $plugin_config $testdir_for_repl coordinator \"-1\" 354 $plugin setup_plugin_for_backup $plugin_config $testdir_for_repl segment_host 355 $plugin setup_plugin_for_backup $plugin_config $testdir_for_repl segment \"0\" 356 357 echo $data | $plugin backup_data $plugin_config $testdata_for_repl 358 echo "[PASSED] backup_data without replication to replicate later" 359 360 echo "[RUNNING] replicate_backup" 361 $plugin replicate_backup $plugin_config $time_second_for_repl 362 363 set +e 364 echo "[RUNNING] replicate_backup second time to verify error" 365 output=`$plugin replicate_backup $plugin_config $time_second_for_repl 2>&1` 366 echo "replicate backup second time, output: $output" 367 if [[ ! "$output" == *"DDBoost open file in exclusive mode failed on remote"* ]]; then 368 echo "Error doesn't match when trying to replicate already replicated backup" 369 exit 1 370 fi 371 echo "[PASSED] replicate_backup second time to verify error" 372 set -e 373 374 if [ -n "$secondary_plugin_config" ]; then 375 echo "[RUNNING] restore_data (from secondary destination)" 376 output=`$plugin restore_data $secondary_plugin_config $testdata_for_repl` 377 378 if [ "$output" != "$data" ]; then 379 echo "Failed to replicate backup using plugin" 380 exit 1 381 fi 382 fi 383 echo "[PASSED] replicate backup" 384 cleanup_test_dir $testdir_for_repl 385 fi 386 387 set +e 388 echo "[RUNNING] fails with unknown command" 389 $plugin unknown_command &> /dev/null 390 if [ $? -eq 0 ] ; then 391 echo "Plugin should exit non-zero when provided an unknown command" 392 exit 1 393 fi 394 echo "[PASSED] fails with unknown command" 395 set -e 396 397 # ---------------------------------------------- 398 # Run test gpbackup and gprestore with plugin 399 # ---------------------------------------------- 400 #gpbackup --dbname $test_db --plugin-config $plugin_config $further_options > $log_file 401 402 test_backup_and_restore_with_plugin() { 403 flags=$1 404 restore_filter=$2 405 test_db=plugin_test_db 406 log_file="$logdir/plugin_test_log_file" 407 408 psql -X -d postgres -qc "DROP DATABASE IF EXISTS $test_db" 2>/dev/null 409 createdb $test_db 410 psql -X -d $test_db -qc "CREATE TABLE test1(i int) DISTRIBUTED RANDOMLY; INSERT INTO test1 select generate_series(1,50000)" 411 if [ "$restore_filter" == "restore-filter" ] ; then 412 psql -X -d $test_db -qc "CREATE TABLE test2(i int) DISTRIBUTED RANDOMLY; INSERT INTO test2 VALUES(3333)" 413 flags_restore="--include-table public.test2" 414 fi 415 416 set +e 417 # save the encrypt key file, if it exists 418 if [ -f "$COORDINATOR_DATA_DIRECTORY/.encrypt" ] ; then 419 mv $COORDINATOR_DATA_DIRECTORY/.encrypt /tmp/.encrypt_saved 420 fi 421 echo "gpbackup_ddboost_plugin: 66706c6c6e677a6965796f68343365303133336f6c73366b316868326764" > $COORDINATOR_DATA_DIRECTORY/.encrypt 422 423 echo "[RUNNING] gpbackup with test database (using ${flags} ${flags_restore})" 424 gpbackup --dbname $test_db --plugin-config $plugin_config $flags &> $log_file 425 if [ ! $? -eq 0 ]; then 426 echo 427 cat $log_file 428 echo 429 echo "gpbackup failed. Check gpbackup log file in ~/gpAdminLogs for details." 430 exit 1 431 fi 432 timestamp=`head -10 $log_file | grep "Backup Timestamp " | grep -Eo "[[:digit:]]{14}"` 433 dropdb $test_db 434 435 echo "[RUNNING] gprestore with test database" 436 gprestore --timestamp $timestamp --plugin-config $plugin_config --create-db $flags_restore &> $log_file 437 if [ ! $? -eq 0 ]; then 438 echo 439 cat $log_file 440 echo 441 echo "gprestore failed. Check gprestore log file in ~/gpAdminLogs for details." 442 exit 1 443 fi 444 445 if [ "$restore_filter" == "restore-filter" ] ; then 446 result=`psql -X -d $test_db -tc "SELECT table_name FROM information_schema.tables WHERE table_schema='public'" | xargs` 447 if [ "$result" == *"test1"* ]; then 448 echo "Expected relation test1 to not exist" 449 exit 1 450 fi 451 result=`psql -X -d $test_db -tc "SELECT * FROM test2" | xargs` 452 if [ "$flags" != "--metadata-only" ] && [ "$result" != "3333" ]; then 453 echo "Expected relation test2 value: 3333, got %result" 454 exit 1 455 fi 456 else 457 result=`psql -X -d $test_db -tc "SELECT count(*) FROM test1" | xargs` 458 if [ "$flags" != "--metadata-only" ] && [ "$result" != "50000" ]; then 459 echo "Expected to restore 50000 rows, got $result" 460 exit 1 461 fi 462 fi 463 464 if [[ "$plugin_config" == *ddboost_config_replication.yaml ]] && [[ -n "$secondary_plugin_config" ]]; then 465 dropdb $test_db 466 echo "[RUNNING] gprestore with test database from secondary destination" 467 gprestore --timestamp $timestamp --plugin-config $secondary_plugin_config --create-db &> $log_file 468 if [ ! $? -eq 0 ]; then 469 echo 470 cat $log_file 471 echo 472 echo "gprestore from secondary destination failed. Check gprestore log file in ~/gpAdminLogs for details." 473 exit 1 474 fi 475 result=`psql -X -d $test_db -tc "SELECT count(*) FROM test1" | xargs` 476 if [ "$flags" != "--metadata-only" ] && [ "$result" != "50000" ]; then 477 echo "Expected to restore 50000 rows, got $result" 478 exit 1 479 fi 480 fi 481 # replace the encrypt key file to its proper location 482 if [ -f "/tmp/.encrypt_saved" ] ; then 483 mv /tmp/.encrypt_saved $COORDINATOR_DATA_DIRECTORY/.encrypt 484 fi 485 set -e 486 echo "[PASSED] gpbackup and gprestore (using ${flags})" 487 } 488 489 test_backup_and_restore_with_plugin "--single-data-file --no-compression --copy-queue-size 4" "--copy-queue-size 4" 490 test_backup_and_restore_with_plugin "--no-compression --single-data-file" 491 test_backup_and_restore_with_plugin "--no-compression" 492 test_backup_and_restore_with_plugin "--metadata-only" 493 test_backup_and_restore_with_plugin "--no-compression --single-data-file" "restore-filter" 494 495 496 # ---------------------------------------------- 497 # Cleanup test artifacts 498 # ---------------------------------------------- 499 echo "Cleaning up leftover test artifacts" 500 501 dropdb $test_db 502 rm -r $logdir 503 rm -r /tmp/testseg 504 505 if (( 1 == $(echo "0.4.0 $api_version" | awk '{print ($1 > $2)}') )) ; then 506 echo "[SKIPPING] cleanup of uploaded test artifacts using plugins (only compatible with version >= 0.4.0)" 507 else 508 $plugin delete_backup $plugin_config $time_second 509 fi 510 511 echo "# ----------------------------------------------" 512 echo "# Finished gpbackup plugin tests" 513 echo "# ----------------------------------------------"