storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/mint/run/core/s3cmd/test.sh (about) 1 #!/bin/bash 2 # 3 # Mint (C) 2017-2020 Minio, Inc. 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 18 19 20 if [ -n "$MINT_MODE" ]; then 21 if [ -z "${MINT_DATA_DIR+x}" ]; then 22 echo "MINT_DATA_DIR not defined" 23 exit 1 24 fi 25 if [ -z "${SERVER_ENDPOINT+x}" ]; then 26 echo "SERVER_ENDPOINT not defined" 27 exit 1 28 fi 29 if [ -z "${ACCESS_KEY+x}" ]; then 30 echo "ACCESS_KEY not defined" 31 exit 1 32 fi 33 if [ -z "${SECRET_KEY+x}" ]; then 34 echo "SECRET_KEY not defined" 35 exit 1 36 fi 37 fi 38 39 if [ -z "${SERVER_ENDPOINT+x}" ]; then 40 SERVER_ENDPOINT="play.minio.io:9000" 41 ACCESS_KEY="Q3AM3UQ867SPQQA43P2F" 42 SECRET_KEY="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" 43 ENABLE_HTTPS=1 44 SERVER_REGION="us-east-1" 45 fi 46 47 WORK_DIR="$PWD" 48 DATA_DIR="$MINT_DATA_DIR" 49 if [ -z "$MINT_MODE" ]; then 50 WORK_DIR="$PWD/.run-$RANDOM" 51 DATA_DIR="$WORK_DIR/data" 52 fi 53 54 FILE_1_MB="$DATA_DIR/datafile-1-MB" 55 FILE_65_MB="$DATA_DIR/datafile-65-MB" 56 declare FILE_1_MB_MD5SUM 57 declare FILE_65_MB_MD5SUM 58 59 BUCKET_NAME="s3cmd-test-bucket-$RANDOM" 60 S3CMD=$(command -v s3cmd) 61 declare -a S3CMD_CMD 62 63 function get_md5sum() 64 { 65 filename="$FILE_1_MB" 66 out=$(md5sum "$filename" 2>/dev/null) 67 rv=$? 68 if [ "$rv" -eq 0 ]; then 69 awk '{ print $1 }' <<< "$out" 70 fi 71 72 return "$rv" 73 } 74 75 function get_time() 76 { 77 date +%s%N 78 } 79 80 function get_duration() 81 { 82 start_time=$1 83 end_time=$(get_time) 84 85 echo $(( (end_time - start_time) / 1000000 )) 86 } 87 88 function log_success() 89 { 90 if [ -n "$MINT_MODE" ]; then 91 printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$(get_duration "$1")" "$2" 92 fi 93 } 94 95 function show() 96 { 97 if [ -z "$MINT_MODE" ]; then 98 func_name="$1" 99 echo "Running $func_name()" 100 fi 101 } 102 103 function fail() 104 { 105 rv="$1" 106 shift 107 108 if [ "$rv" -ne 0 ]; then 109 echo "$@" 110 fi 111 112 return "$rv" 113 } 114 115 function assert() 116 { 117 expected_rv="$1" 118 shift 119 start_time="$1" 120 shift 121 func_name="$1" 122 shift 123 124 err=$("$@" 2>&1) 125 rv=$? 126 if [ "$rv" -ne 0 ] && [ "$expected_rv" -eq 0 ]; then 127 if [ -n "$MINT_MODE" ]; then 128 err=$(printf '%s' "$err" | python -c 'import sys,json; print(json.dumps(sys.stdin.read()))') 129 ## err is already JSON string, no need to double quote 130 printf '{"name": "s3cmd", "duration": "%d", "function": "%s", "status": "FAIL", "error": %s}\n' "$(get_duration "$start_time")" "$func_name" "$err" 131 else 132 echo "s3cmd: $func_name: $err" 133 fi 134 135 exit "$rv" 136 fi 137 138 return 0 139 } 140 141 function assert_success() { 142 assert 0 "$@" 143 } 144 145 function assert_failure() { 146 assert 1 "$@" 147 } 148 149 function s3cmd_cmd() 150 { 151 cmd=( "${S3CMD_CMD[@]}" "$@" ) 152 "${cmd[@]}" 153 rv=$? 154 return "$rv" 155 } 156 157 function check_md5sum() 158 { 159 expected_checksum="$1" 160 shift 161 filename="$*" 162 163 checksum="$(get_md5sum "$filename")" 164 rv=$? 165 if [ "$rv" -ne 0 ]; then 166 echo "unable to get md5sum for $filename" 167 return "$rv" 168 fi 169 170 if [ "$checksum" != "$expected_checksum" ]; then 171 echo "$filename: md5sum mismatch" 172 return 1 173 fi 174 175 return 0 176 } 177 178 function test_make_bucket() 179 { 180 show "${FUNCNAME[0]}" 181 182 start_time=$(get_time) 183 bucket_name="s3cmd-test-bucket-$RANDOM" 184 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" 185 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb "s3://${bucket_name}" 186 187 log_success "$start_time" "${FUNCNAME[0]}" 188 } 189 190 function test_make_bucket_error() { 191 show "${FUNCNAME[0]}" 192 193 start_time=$(get_time) 194 bucket_name="S3CMD-test%bucket%$RANDOM" 195 assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" 196 197 log_success "$start_time" "${FUNCNAME[0]}" 198 } 199 200 function setup() 201 { 202 start_time=$(get_time) 203 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${BUCKET_NAME}" 204 } 205 206 function teardown() 207 { 208 start_time=$(get_time) 209 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm --force --recursive "s3://${BUCKET_NAME}" 210 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force "s3://${BUCKET_NAME}" 211 } 212 213 function test_put_object() 214 { 215 show "${FUNCNAME[0]}" 216 217 start_time=$(get_time) 218 object_name="s3cmd-test-object-$RANDOM" 219 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" 220 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 221 222 log_success "$start_time" "${FUNCNAME[0]}" 223 } 224 225 function test_put_object_error() 226 { 227 show "${FUNCNAME[0]}" 228 start_time=$(get_time) 229 230 object_long_name=$(printf "s3cmd-test-object-%01100d" 1) 231 assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_long_name}" 232 233 log_success "$start_time" "${FUNCNAME[0]}" 234 } 235 236 function test_put_object_multipart() 237 { 238 show "${FUNCNAME[0]}" 239 240 start_time=$(get_time) 241 object_name="s3cmd-test-object-$RANDOM" 242 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" 243 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 244 245 log_success "$start_time" "${FUNCNAME[0]}" 246 } 247 248 function test_get_object() 249 { 250 show "${FUNCNAME[0]}" 251 252 start_time=$(get_time) 253 object_name="s3cmd-test-object-$RANDOM" 254 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" 255 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" 256 assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_1_MB_MD5SUM" "${object_name}.downloaded" 257 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 258 assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" 259 260 log_success "$start_time" "${FUNCNAME[0]}" 261 } 262 263 function test_get_object_error() 264 { 265 show "${FUNCNAME[0]}" 266 267 start_time=$(get_time) 268 object_name="s3cmd-test-object-$RANDOM" 269 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_1_MB}" "s3://${BUCKET_NAME}/${object_name}" 270 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 271 assert_failure "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" 272 assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" 273 274 log_success "$start_time" "${FUNCNAME[0]}" 275 } 276 277 function test_get_object_multipart() 278 { 279 show "${FUNCNAME[0]}" 280 281 start_time=$(get_time) 282 object_name="s3cmd-test-object-$RANDOM" 283 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd put "${FILE_65_MB}" "s3://${BUCKET_NAME}/${object_name}" 284 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd get "s3://${BUCKET_NAME}/${object_name}" "${object_name}.downloaded" 285 assert_success "$start_time" "${FUNCNAME[0]}" check_md5sum "$FILE_65_MB_MD5SUM" "${object_name}.downloaded" 286 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rm "s3://${BUCKET_NAME}/${object_name}" 287 assert_success "$start_time" "${FUNCNAME[0]}" rm -f "${object_name}.downloaded" 288 289 log_success "$start_time" "${FUNCNAME[0]}" 290 } 291 292 function test_sync_list_objects() 293 { 294 show "${FUNCNAME[0]}" 295 296 start_time=$(get_time) 297 bucket_name="s3cmd-test-bucket-$RANDOM" 298 object_name="s3cmd-test-object-$RANDOM" 299 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd mb "s3://${bucket_name}" 300 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd sync "$DATA_DIR/" "s3://${bucket_name}" 301 302 diff -bB <(ls "$DATA_DIR") <("${S3CMD_CMD[@]}" ls "s3://${bucket_name}" | awk '{print $4}' | sed "s/s3:*..${bucket_name}.//g") >/dev/null 2>&1 303 assert_success "$start_time" "${FUNCNAME[0]}" fail $? "sync and list differs" 304 assert_success "$start_time" "${FUNCNAME[0]}" s3cmd_cmd rb --force --recursive "s3://${bucket_name}" 305 306 log_success "$start_time" "${FUNCNAME[0]}" 307 } 308 309 function run_test() 310 { 311 test_make_bucket 312 test_make_bucket_error 313 314 setup 315 316 test_put_object 317 test_put_object_error 318 test_put_object_multipart 319 test_get_object 320 test_get_object_multipart 321 test_sync_list_objects 322 323 teardown 324 } 325 326 function __init__() 327 { 328 set -e 329 330 S3CMD_CONFIG_DIR="/tmp/.s3cmd-$RANDOM" 331 mkdir -p $S3CMD_CONFIG_DIR 332 S3CMD_CONFIG_FILE="$S3CMD_CONFIG_DIR/s3cfg" 333 334 # configure s3cmd 335 cat > $S3CMD_CONFIG_FILE <<EOF 336 signature_v2 = False 337 host_base = $SERVER_ENDPOINT 338 host_bucket = $SERVER_ENDPOINT 339 bucket_location = $SERVER_REGION 340 use_https = $ENABLE_HTTPS 341 access_key = $ACCESS_KEY 342 secret_key = $SECRET_KEY 343 EOF 344 345 # For Mint, setup is already done. For others, setup the environment 346 if [ -z "$MINT_MODE" ]; then 347 mkdir -p "$WORK_DIR" 348 mkdir -p "$DATA_DIR" 349 350 # If s3cmd executable binary is not available in current directory, use it in the path. 351 if [ ! -x "$S3CMD" ]; then 352 echo "'s3cmd' executable binary not found in current directory and in path" 353 exit 1 354 fi 355 fi 356 357 if [ ! -x "$S3CMD" ]; then 358 echo "$S3CMD executable binary not found" 359 exit 1 360 fi 361 362 S3CMD_CMD=( "${S3CMD}" --config "$S3CMD_CONFIG_FILE" ) 363 364 if [ ! -e "$FILE_1_MB" ]; then 365 shred -n 1 -s 1MB - >"$FILE_1_MB" 366 fi 367 368 if [ ! -e "$FILE_65_MB" ]; then 369 shred -n 1 -s 65MB - >"$FILE_65_MB" 370 fi 371 372 set -E 373 set -o pipefail 374 375 FILE_1_MB_MD5SUM="$(get_md5sum "$FILE_1_MB")" 376 rv=$? 377 if [ $rv -ne 0 ]; then 378 echo "unable to get md5sum of $FILE_1_MB" 379 exit 1 380 fi 381 382 FILE_65_MB_MD5SUM="$(get_md5sum "$FILE_65_MB")" 383 rv=$? 384 if [ $rv -ne 0 ]; then 385 echo "unable to get md5sum of $FILE_65_MB" 386 exit 1 387 fi 388 389 set +e 390 } 391 392 function main() 393 { 394 ( run_test ) 395 rv=$? 396 397 rm -fr "$S3CMD_CONFIG_FILE" 398 if [ -z "$MINT_MODE" ]; then 399 rm -fr "$WORK_DIR" "$DATA_DIR" 400 fi 401 402 exit "$rv" 403 } 404 405 __init__ "$@" 406 main "$@"