github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/docs/site-replication/run-multi-site-oidc.sh (about) 1 #!/usr/bin/env bash 2 3 # shellcheck disable=SC2120 4 exit_1() { 5 cleanup 6 7 echo "minio1 ============" 8 cat /tmp/minio1_1.log 9 echo "minio2 ============" 10 cat /tmp/minio2_1.log 11 echo "minio3 ============" 12 cat /tmp/minio3_1.log 13 14 exit 1 15 } 16 17 cleanup() { 18 echo "Cleaning up instances of MinIO" 19 pkill minio 20 pkill -9 minio 21 rm -rf /tmp/minio{1,2,3} 22 } 23 24 cleanup 25 26 unset MINIO_KMS_KES_CERT_FILE 27 unset MINIO_KMS_KES_KEY_FILE 28 unset MINIO_KMS_KES_ENDPOINT 29 unset MINIO_KMS_KES_KEY_NAME 30 31 export MINIO_CI_CD=1 32 export MINIO_BROWSER=off 33 export MINIO_ROOT_USER="minio" 34 export MINIO_ROOT_PASSWORD="minio123" 35 export MINIO_KMS_AUTO_ENCRYPTION=off 36 export MINIO_PROMETHEUS_AUTH_TYPE=public 37 export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= 38 export MINIO_IDENTITY_OPENID_CONFIG_URL="http://localhost:5556/dex/.well-known/openid-configuration" 39 export MINIO_IDENTITY_OPENID_CLIENT_ID="minio-client-app" 40 export MINIO_IDENTITY_OPENID_CLIENT_SECRET="minio-client-app-secret" 41 export MINIO_IDENTITY_OPENID_CLAIM_NAME="groups" 42 export MINIO_IDENTITY_OPENID_SCOPES="openid,groups" 43 44 export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:10000/oauth_callback" 45 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 & 46 site1_pid=$! 47 export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:11000/oauth_callback" 48 minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4} >/tmp/minio2_1.log 2>&1 & 49 site2_pid=$! 50 51 export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:12000/oauth_callback" 52 minio server --address ":9003" --console-address ":12000" /tmp/minio3/{1...4} >/tmp/minio3_1.log 2>&1 & 53 site3_pid=$! 54 55 if [ ! -f ./mc ]; then 56 wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && 57 chmod +x mc 58 fi 59 60 sleep 10 61 62 export MC_HOST_minio1=http://minio:minio123@localhost:9001 63 export MC_HOST_minio2=http://minio:minio123@localhost:9002 64 export MC_HOST_minio3=http://minio:minio123@localhost:9003 65 66 ./mc admin replicate add minio1 minio2 minio3 67 68 ./mc admin policy create minio1 projecta ./docs/site-replication/rw.json 69 sleep 5 70 71 ./mc admin policy info minio2 projecta >/dev/null 2>&1 72 if [ $? -ne 0 ]; then 73 echo "expecting the command to succeed, exiting.." 74 exit_1 75 fi 76 ./mc admin policy info minio3 projecta >/dev/null 2>&1 77 if [ $? -ne 0 ]; then 78 echo "expecting the command to succeed, exiting.." 79 exit_1 80 fi 81 82 ./mc admin policy remove minio3 projecta 83 84 sleep 10 85 ./mc admin policy info minio1 projecta 86 if [ $? -eq 0 ]; then 87 echo "expecting the command to fail, exiting.." 88 exit_1 89 fi 90 91 ./mc admin policy info minio2 projecta 92 if [ $? -eq 0 ]; then 93 echo "expecting the command to fail, exiting.." 94 exit_1 95 fi 96 97 ./mc admin policy create minio1 projecta ./docs/site-replication/rw.json 98 sleep 5 99 100 # Generate STS credential with STS call to minio1 101 STS_CRED=$(MINIO_ENDPOINT=http://localhost:9001 go run ./docs/site-replication/gen-oidc-sts-cred.go) 102 103 MC_HOST_foo=http://${STS_CRED}@localhost:9001 ./mc ls foo 104 if [ $? -ne 0 ]; then 105 echo "Expected sts credential to work, exiting.." 106 exit_1 107 fi 108 109 sleep 2 110 111 # Check that the STS credential works on minio2 and minio3. 112 MC_HOST_foo=http://${STS_CRED}@localhost:9002 ./mc ls foo 113 if [ $? -ne 0 ]; then 114 echo "Expected sts credential to work, exiting.." 115 exit_1 116 fi 117 118 MC_HOST_foo=http://${STS_CRED}@localhost:9003 ./mc ls foo 119 if [ $? -ne 0 ]; then 120 echo "Expected sts credential to work, exiting.." 121 exit_1 122 fi 123 124 STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1) 125 126 # Create service account for STS user 127 ./mc admin user svcacct add minio2 $STS_ACCESS_KEY --access-key testsvc --secret-key testsvc123 128 if [ $? -ne 0 ]; then 129 echo "adding svc account failed, exiting.." 130 exit_1 131 fi 132 133 sleep 10 134 135 ./mc admin user svcacct info minio1 testsvc 136 if [ $? -ne 0 ]; then 137 echo "svc account not mirrored, exiting.." 138 exit_1 139 fi 140 141 ./mc admin user svcacct info minio2 testsvc 142 if [ $? -ne 0 ]; then 143 echo "svc account not mirrored, exiting.." 144 exit_1 145 fi 146 147 ./mc admin user svcacct rm minio1 testsvc 148 if [ $? -ne 0 ]; then 149 echo "removing svc account failed, exiting.." 150 exit_1 151 fi 152 153 sleep 10 154 ./mc admin user svcacct info minio2 testsvc 155 if [ $? -eq 0 ]; then 156 echo "svc account found after delete, exiting.." 157 exit_1 158 fi 159 160 ./mc admin user svcacct info minio3 testsvc 161 if [ $? -eq 0 ]; then 162 echo "svc account found after delete, exiting.." 163 exit_1 164 fi 165 166 # create a bucket bucket2 on minio1. 167 ./mc mb minio1/bucket2 168 169 ./mc mb minio1/newbucket 170 171 # copy large upload to newbucket on minio1 172 truncate -s 17M lrgfile 173 expected_checksum=$(cat ./lrgfile | md5sum) 174 175 ./mc cp ./lrgfile minio1/newbucket 176 sleep 5 177 ./mc stat minio2/newbucket 178 if [ $? -ne 0 ]; then 179 echo "expecting bucket to be present. exiting.." 180 exit_1 181 fi 182 183 ./mc stat minio3/newbucket 184 if [ $? -ne 0 ]; then 185 echo "expecting bucket to be present. exiting.." 186 exit_1 187 fi 188 189 ./mc cp README.md minio2/newbucket/ 190 191 sleep 5 192 ./mc stat minio1/newbucket/README.md 193 if [ $? -ne 0 ]; then 194 echo "expecting object to be present. exiting.." 195 exit_1 196 fi 197 198 ./mc stat minio3/newbucket/README.md 199 if [ $? -ne 0 ]; then 200 echo "expecting object to be present. exiting.." 201 exit_1 202 fi 203 204 ./mc rm minio3/newbucket/README.md 205 sleep 5 206 207 ./mc stat minio2/newbucket/README.md 208 if [ $? -eq 0 ]; then 209 echo "expected file to be deleted, exiting.." 210 exit_1 211 fi 212 213 ./mc stat minio1/newbucket/README.md 214 if [ $? -eq 0 ]; then 215 echo "expected file to be deleted, exiting.." 216 exit_1 217 fi 218 219 sleep 10 220 ./mc stat minio3/newbucket/lrgfile 221 if [ $? -ne 0 ]; then 222 echo "expected object to be present, exiting.." 223 exit_1 224 fi 225 actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) 226 if [ "${expected_checksum}" != "${actual_checksum}" ]; then 227 echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" 228 exit 229 fi 230 rm ./lrgfile 231 232 ./mc rm -r --versions --force minio1/newbucket/lrgfile 233 if [ $? -ne 0 ]; then 234 echo "expected object to be present, exiting.." 235 exit_1 236 fi 237 238 sleep 5 239 ./mc stat minio1/newbucket/lrgfile 240 if [ $? -eq 0 ]; then 241 echo "expected object to be deleted permanently after replication, exiting.." 242 exit_1 243 fi 244 245 ./mc mb --with-lock minio3/newbucket-olock 246 sleep 5 247 248 enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) 249 if [ $? -ne 0 ]; then 250 echo "expected bucket to be mirrored with object-lock but not present, exiting..." 251 exit_1 252 fi 253 254 if [ "${enabled_minio2}" != "Enabled" ]; then 255 echo "expected bucket to be mirrored with object-lock enabled, exiting..." 256 exit_1 257 fi 258 259 enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) 260 if [ $? -ne 0 ]; then 261 echo "expected bucket to be mirrored with object-lock but not present, exiting..." 262 exit_1 263 fi 264 265 if [ "${enabled_minio1}" != "Enabled" ]; then 266 echo "expected bucket to be mirrored with object-lock enabled, exiting..." 267 exit_1 268 fi 269 270 # "Test if most recent tag update is replicated" 271 ./mc tag set minio2/newbucket "key=val1" 272 if [ $? -ne 0 ]; then 273 echo "expecting tag set to be successful. exiting.." 274 exit_1 275 fi 276 277 sleep 10 278 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) 279 if [ "${val}" != "val1" ]; then 280 echo "expected bucket tag to have replicated, exiting..." 281 exit_1 282 fi 283 # stop minio1 instance 284 kill -9 ${site1_pid} 285 # Update tag on minio2/newbucket when minio1 is down 286 ./mc tag set minio2/newbucket "key=val2" 287 # create a new bucket on minio2. This should replicate to minio1 after it comes online. 288 ./mc mb minio2/newbucket2 289 # delete bucket2 on minio2. This should replicate to minio1 after it comes online. 290 ./mc rb minio2/bucket2 291 292 # Restart minio1 instance 293 minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 & 294 sleep 200 295 296 # Test whether most recent tag update on minio2 is replicated to minio1 297 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) 298 if [ "${val}" != "val2" ]; then 299 echo "expected bucket tag to have replicated, exiting..." 300 exit_1 301 fi 302 303 # Test if bucket created/deleted when minio1 is down healed 304 diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null 305 if [ $? -ne 0 ]; then 306 echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." 307 exit_1 308 fi 309 310 # force a resync after removing all site replication 311 ./mc admin replicate rm --all --force minio1 312 ./mc rb minio2 --force --dangerous 313 ./mc admin replicate add minio1 minio2 314 ./mc admin replicate resync start minio1 minio2 315 sleep 30 316 317 ./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt 318 ./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt 319 320 out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) 321 ret=$? 322 if [ $ret -ne 0 ]; then 323 echo "BUG: expected no missing entries after replication resync: $out" 324 exit 1 325 fi