github.com/extrame/fabric-ca@v2.0.0-alpha+incompatible/scripts/fvt/cluster_test.sh (about) 1 #!/bin/bash 2 # 3 # Copyright IBM Corp. All Rights Reserved. 4 # 5 # SPDX-License-Identifier: Apache-2.0 6 # 7 8 : ${TESTCASE="ca_cluster"} 9 FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca" 10 SCRIPTDIR="$FABRIC_CA/scripts/fvt" 11 . $SCRIPTDIR/fabric-ca_utils 12 ROOTDIR=/tmp/cluster 13 INTDIR=$ROOTDIR/int 14 ROOTUSERDIR=$ROOTDIR/users 15 INTUSERDIR=$ROOTDIR/int/users 16 ENROLLCERT=msp/signcerts/cert.pem 17 DEFAULT_CA_CONFIG=fabric-ca-config.yaml 18 RC=0 19 DBNAME=fabric_ca 20 INTDBNAME=intfabric_ca 21 NUMSERVER="$1" # Number of CA instances behind the proxy 22 NUMINTERMD="$2" # Number of intermediate CAs 23 NUMCAS="$3" # cacount; as a simplifying assumption, 24 # if NUMSERVER > NUMCAS, 25 # then NUMSERVER % NUMCAS = 0 26 # else NUMCAS % NUMSERVER = 0 27 ITERATIONS="$4" # num of commands to run in parallel 28 # As a simplifying assumption, ITERATIONS % 4 = 0 29 NUMJOBS="$5" # num of concurrent jobs 30 31 # Using the default, the test should complete in ≈ 1 min 32 : ${NUMSERVER:=2} 33 : ${NUMINTERMD:=2} 34 : ${NUMCAS:=2} 35 : ${ITERATIONS:=8} 36 : ${NUMJOBS:=1024} # spawn as many jobs as there are potential requests 37 NUMUSERS=$((NUMCAS*ITERATIONS)) # NUMUSERS % NUMSERVER should = 0 38 USERNAME="testuser" 39 ROOT_CA_ADDR=localhost 40 INTUSER="intermediateCa16" 41 INTPSWD="intermediateCa16pw" 42 SHELL=/bin/bash 43 44 setTLS 45 export SHELL PROXY_PORT ROOTDIR ITERATIONS USERNAME PROTO TLSOPT FABRIC_CA_CLIENTEXEC ENROLLCERT 46 47 function enrollAdmin() { 48 local port="$1" 49 local ca="$2" 50 local dir="$3" 51 : ${port:=$PROXY_PORT} 52 53 mkdir -p $dir/admin$ca 54 touch $dir/admin$ca/log.txt 55 FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ 56 $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \ 57 -u ${PROTO}admin:adminpw@localhost:$port \ 58 --csr.hosts admin@fab-client.raleigh.ibm.com \ 59 --csr.hosts admin.fabric.raleigh.ibm.com,127.0.0.2 \ 60 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 61 } 62 63 function revokeUsers() { 64 local port="$1" 65 local ca="$2" 66 local user="$3" 67 local dir="$4" 68 : ${port:=$PROXY_PORT} 69 70 FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ 71 $FABRIC_CA_CLIENTEXEC revoke --gencrl --debug $TLSOPT \ 72 -u ${PROTO}admin:adminpw@localhost:$port \ 73 --revoke.name ${USERNAME}${ca}-${user} \ 74 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 75 } 76 77 function enrollUsers() { 78 local port="$1" 79 local ca="$2" 80 local user="$3" 81 local dir="$4" 82 : ${port:=$PROXY_PORT} 83 mkdir -p $dir/admin$ca 84 touch $dir/admin$ca/log.txt 85 FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \ 86 $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \ 87 -u ${PROTO}${USERNAME}${ca}-${user}:${USERNAME}${ca}-${user}@localhost:$port \ 88 --csr.hosts ${USERNAME}${ca}-${user}@fab-client.raleigh.ibm.com \ 89 --csr.hosts ${USERNAME}${ca}-${user}.fabric.raleigh.ibm.com \ 90 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 91 test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'=' '{print $NF}')" 92 } 93 94 function reenrollUsers() { 95 local port="$1" 96 local ca="$2" 97 local user="$3" 98 local dir="$4" 99 : ${port:=$PROXY_PORT} 100 mkdir -p $dir/admin$ca 101 touch $dir/admin$ca/log.txt 102 FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \ 103 $FABRIC_CA_CLIENTEXEC reenroll --debug $TLSOPT \ 104 -u ${PROTO}@localhost:$port \ 105 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 106 test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'=' '{print $NF}')" 107 } 108 109 function register() { 110 local port="$1" 111 local ca="$2" 112 local user="$3" 113 local dir="$4" 114 : ${port:=$PROXY_PORT} 115 FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ 116 $FABRIC_CA_CLIENTEXEC register --debug -u ${PROTO}localhost:$port $TLSOPT \ 117 --id.name ${USERNAME}${ca}-${user} \ 118 --id.secret ${USERNAME}${ca}-${user} \ 119 --id.type client \ 120 --id.maxenrollments $ITERATIONS \ 121 --id.affiliation bank_a \ 122 --id.attrs test=testValue \ 123 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 124 } 125 126 function DBvalidateUsers() { 127 # Query the DB and verify the user state: 128 # 0 - registered, but not enrolled 129 # 1 - enrolled 130 local state="$1" 131 local dbname="$2" 132 local StatusField=6 133 local fsopt="" 134 135 case $DRIVER in 136 postgres) StatusField=11 ;; 137 esac 138 139 DBNAME=$dbname $SCRIPTDIR/fabric-ca_setup.sh -L -d $DRIVER \ 140 -n $NUMSERVER -u $NUMCAS 2>/dev/null | 141 awk -v u="$USERNAME" $fsopt \ 142 -v s="$state" \ 143 -v n="$NUMUSERS" \ 144 -v f="$StatusField" \ 145 -v t=0 ' 146 $1~u && $f==s {t++} 147 END { if (t!=n) exit 1 }' 148 } 149 150 function showUsers() { 151 $SCRIPTDIR/fabric-ca_setup.sh -L -d $DRIVER \ 152 -n $NUMSERVER -u $NUMCAS 2>/dev/null | 153 awk -v u="$USERNAME" '$1~u' 154 } 155 156 function verifyDistribution() { 157 case "$FABRIC_TLS" in 158 1|y|yes|true) return 0 ;; 159 esac 160 161 local numCluster="$1" 162 local backendName="$2" 163 local expectedCount="$3" 164 local percentage="$4" 165 166 for s in $(seq $numCluster); do 167 verifyServerTraffic "" "/$backendName$((s-1))""" $expectedCount $percentage 168 test $? -eq 0 && 169 echo " >>>>>>>>>> VerifyServerTraffic for $backendName$s...PASSED" || 170 ErrorMsg " >>>>>>>>>> VerifyServerTraffic for $backendName$s...FAILED" 171 done 172 } 173 174 export -f enrollAdmin register enrollUsers revokeUsers reenrollUsers 175 176 function checkStatus() { 177 # Parse the joblog exitstatus (column 7) for all jobs 178 # 0 - success 179 # ^0 - failed 180 # Success is measured by the number of successful jobs, i.e., 181 # there should be one successful job for each request sent: 182 # Number of exit '0' entries == NUMUSERS 183 local log="$1" 184 local number="$2" 185 : ${number:="$NUMUSERS"} 186 awk -v u=$number ' 187 NR!=1 && $7==0 {rc+=1} 188 END {if (rc!=u) exit 1}' $log 189 test $? -ne 0 && ErrorMsg "FAILED" || echo "PASSED" 190 } 191 192 193 for DRIVER in mysql postgres; do 194 echo "Testing $DRIVER >>>>>>>>>>>>>" 195 # Delete all of the DBs 196 echo -e " >>>>>>>>>> Deleting all databases" 197 $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 198 DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 199 rm -rf $ROOTDIR 200 201 # Initialize all of the configs, certs, keys and directories 202 mkdir -p $ROOTUSERDIR 203 mkdir -p $INTUSERDIR 204 echo -e " >>>>>>>>>> Initializing Root CAs" 205 $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -I -n 1 -u $NUMCAS \ 206 -n $NUMSERVER -D -d $DRIVER > $ROOTDIR/log.txt 2>&1 207 208 echo -e " >>>>>>>>>> Initializing Intermediate CAs" 209 210 # Copy root CA config file to int CA home and change the database name 211 cp "$ROOTDIR/$DEFAULT_RUN_CONFIG_FILE_NAME" "$INTDIR/$DEFAULT_RUN_CONFIG_FILE_NAME" 212 sed -i "/datasource:/ s/datasource:\(.*\)fabric_ca\(.*\)/datasource:\1intfabric_ca\2/" "$INTDIR/$DEFAULT_RUN_CONFIG_FILE_NAME" 213 214 ################################################################## 215 ## Customize enrollment for each CA 216 ################################################################## 217 ca=0 218 rootCafiles="" 219 intermediateCafiles="" 220 rootDBconfig="" 221 intermediateDBconfig="" 222 # append the customized DB config to each CA's config file 223 while test $((ca++)) -lt $NUMCAS; do 224 # Copy CA config files of root CA server to int CA home and change the database name 225 mkdir -p "$INTDIR/ca/ca$ca" || true 226 cp "$ROOTDIR/ca/ca$ca/fabric-ca-config.yaml" "$INTDIR/ca/ca$ca/fabric-ca-config.yaml" 227 sed -i "/datasource:/ s/datasource:\(.*\)fabric_ca_ca$ca\(.*\)/datasource:\1intfabric_ca_ca$ca\2/" "$INTDIR/ca/ca$ca/fabric-ca-config.yaml" 228 229 # build the list of cafiles to be passed to server start 230 rootCafiles="$rootCafiles,$ROOTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}" 231 intermediateCafiles="$intermediateCafiles,$INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}" 232 233 # each intermediate CA needs an enrollment identity and parentserver.url 234 # each also needs a unique caname, or the sever start will fail 235 enrollment=" 236 intermediate: 237 parentserver: 238 url: ${PROTO}intermediateCa${ca}:intermediateCa${ca}pw@127.0.0.1:$CA_DEFAULT_PORT 239 caname: ca${ca} 240 enrollment: 241 name: intermediateCa${ca} 242 secret: intermediateCa${ca}pw 243 hosts: 244 - localhost 245 " 246 # append the intermediate CA config to each CA's config 247 cat >> $INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG} <<EOF 248 $enrollment 249 EOF 250 done 251 252 # strip the leading comma from the files list 253 rootCafiles=${rootCafiles#,} 254 intermediateCafiles=${intermediateCafiles#,} 255 # remove the pathlength restriction 256 sed -i 's/cacount:.*/cacount:/g 257 s/maxpathlen:.*/maxpathlen:/g 258 s/pathlength:.*/pathlength:/g' $ROOTDIR/runFabricCaFvt.yaml $INTDIR/runFabricCaFvt.yaml 259 # Remove all of the CSR.CN data from intermediate CAs -- 260 # otherwise, server startup will fail 261 find $INTDIR/ca -name $DEFAULT_CA_CONFIG -exec sed -i "s/cn:.*/cn:/g" {} \; 262 263 # Start all Root and intermediate CAs 264 echo -e " >>>>>>>>>> Starting $NUMSERVER Root CA instances with $NUMCAS servers each" 265 $SCRIPTDIR/fabric-ca_setup.sh -N -X -x $ROOTDIR -S -n $NUMSERVER -D -d $DRIVER \ 266 -- "--cafiles" "$rootCafiles" >> $ROOTDIR/log.txt 2>&1 || 267 ErrorExit "Failed to start root servers" 268 echo -e " >>>>>>>>>> Starting $NUMSERVER Intermediate CA instances with $NUMCAS servers each" 269 270 $SCRIPTDIR/fabric-ca_setup.sh -n $NUMSERVER -S -r $INTERMEDIATE_CA_DEFAULT_PORT -x $INTDIR \ 271 -U "https://$INTUSER:$INTPSWD@$ROOT_CA_ADDR:$PROXY_PORT" \ 272 -- "--cafiles" "$intermediateCafiles" >> $INTDIR/log.txt 2>&1 || 273 ErrorExit "Failed to start intermediate servers" 274 275 ######################################################### 276 # The bulk of the work comes here -- 277 # register and enroll users, in parallel, $NUMJOBS at a time 278 ######################################################### 279 for SERVER in $PROXY_PORT $INTERMEDIATE_PROXY_PORT ; do 280 # The intermediate CAs do not share the root CA's DBs; 281 # each has a unique DB (in a unique directory, if file-based 282 # If the CA is root, the haproxy backend name is 'server' 283 # If the CA is intermediate, the haproxy backend name is 'intserver' 284 if test "$SERVER" = "$INTERMEDIATE_PROXY_PORT"; then 285 dbname=$INTDBNAME 286 userdir=$INTUSERDIR 287 stype=intermediate 288 backend=intserver 289 else 290 dbname=$DBNAME 291 userdir=$ROOTUSERDIR 292 stype=root 293 backend=server 294 fi 295 296 count=0 297 # for each block of requests, we will validate round-robin 298 # distribution to all participating CAs 299 verifyDistribution $NUMSERVER $backend $count 0 300 301 echo -e " >>>>>>>>>> Testing $stype CA using DB name $dbname" 302 303 # Enroll the admins -- the total number of enrollment requests 304 # sent is calulated to be the larger of NUMSERVER | NUMCAS 305 test $NUMCAS -ge $NUMSERVER && numReq=1 || numReq=$((NUMSERVER/NUMCAS)) 306 printf " >>>>>>>>>> Enrolling ${NUMCAS} admins, $numReq times..." 307 parallel -k --no-notice --jobs $NUMJOBS --joblog $userdir/adminEnroll.log \ 308 enrollAdmin $SERVER {1} $userdir ::: $(seq $NUMCAS) ::: $(seq $numReq) 309 checkStatus $userdir/adminEnroll.log $((numReq*NUMCAS)) || ErrorExit "Enroll of admins failed" 310 # Register $NUMUSERS users and validate their status in the DB 311 test $NUMCAS -lt $NUMSERVER && count=1 || count=$((NUMCAS/NUMSERVER)) 312 verifyDistribution $NUMSERVER $backend $count 313 314 # Register $NUMUSERS users and validate their status in the DB 315 printf " >>>>>>>>>> Registering $NUMUSERS users (${NUMCAS}x${ITERATIONS})..." 316 parallel --no-notice --jobs $NUMJOBS --joblog $userdir/register.log \ 317 register $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS) 318 checkStatus $userdir/register.log 319 DBvalidateUsers 0 $dbname && 320 echo -e " >>>>>>>>>> Validating user status in DB...PASSED" || 321 ErrorMsg " >>>>>>>>>> Validating user status in DB...FAILED" 322 count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) )) 323 verifyDistribution $NUMSERVER $backend $count 324 325 # Enroll $NUMUSERS users and validate their status in the DB 326 printf " >>>>>>>>>> Enrolling $NUMUSERS users (${NUMCAS}x${ITERATIONS})..." 327 parallel --no-notice --jobs $NUMJOBS --joblog $userdir/userEnroll.log \ 328 enrollUsers $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS) 329 checkStatus $userdir/userEnroll.log 330 DBvalidateUsers 1 $dbname && 331 echo -e " >>>>>>>>>> Validating user status in DB...PASSED" || 332 ErrorMsg " >>>>>>>>>> Validating user status in DB...FAILED" 333 count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) )) 334 verifyDistribution $NUMSERVER $backend $count 335 336 # Do all of the following simultaneously 337 # enroll Enroll an identity 338 # getcacert Get CA certificate chain 339 # reenroll Reenroll an identity 340 # register Register an identity 341 # revoke Revoke an identity 342 # gencrl Generate a CRL 343 344 > /tmp/cmd.lst 345 for ca in $(seq $NUMCAS); do 346 # Create the register cmd list of brand new users where 347 # the previous register task left off 348 echo " >>>>>>> generating register command list ($((ITERATIONS*NUMCAS)))" 349 for user in $(seq $((ITERATIONS+1)) $((ITERATIONS+ITERATIONS)) ); do 350 echo register $SERVER $ca $user $userdir >> /tmp/cmd.lst 351 done 352 # Create the enroll cmd list - 353 # issue enroll for the first 1/2 of the previously enrolled users 354 echo " >>>>>>> generating enroll command list ($((ITERATIONS/2*NUMCAS)))" 355 for user in $(seq $((ITERATIONS/2)) ); do 356 echo enrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst 357 done 358 # Create the renroll cmd list - 359 # issue renroll for the third 1/4 of the previously enrolled users 360 echo " >>>>>>> generating renroll command list ($((ITERATIONS/4*NUMCAS)))" 361 for user in $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do 362 echo reenrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst 363 done 364 # Create the revoke cmd list - 365 # issue renroll for the last 1/4 of the previously enrolled users 366 echo " >>>>>>> generating revoke command list ($((ITERATIONS/4*NUMCAS)))" 367 for user in $(seq $((ITERATIONS/4*3+1)) $ITERATIONS ); do 368 echo revokeUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst 369 done 370 # Create the getcacert cmd list - 371 echo " >>>>>>> generating getcacert command list ($((ITERATIONS*NUMCAS/2)))" 372 for user in $(seq $((ITERATIONS/2)) ); do 373 echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC getcacert --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/cacert.txt 2>&1" >> /tmp/cmd.lst 374 done 375 # Create the gencrl cmd list - 376 echo " >>>>>>> generating gencrl command list ($((ITERATIONS*NUMCAS/2)))" 377 for user in $(seq $((ITERATIONS/2+1)) $ITERATIONS); do 378 echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1" >> /tmp/cmd.lst 379 done 380 done 381 382 shuf --output=$userdir/cmd.lst /tmp/cmd.lst 383 # OK, here goes... 384 printf " >>>>>>>>>> Executing all $((ITERATIONS*3*NUMCAS)) jobs..." 385 parallel --no-notice --jobs $NUMJOBS --joblog $userdir/cmd.log < $userdir/cmd.lst 386 checkStatus $userdir/cmd.log $((ITERATIONS*3*NUMCAS)) 387 count=$((count+$((ITERATIONS*3*NUMCAS/NUMSERVER)))) 388 sleep 1 389 verifyDistribution $NUMSERVER $backend $count 390 391 sleep 1 392 # Lastly, 1/4 of user certs should be revoked 393 echo " >>>>>>>>>> Checking crl; expect $((ITERATIONS/4)) revoked certificates..." 394 for ca in $(seq $NUMCAS); do 395 FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1 396 revoked=$(openssl crl -in $userdir/admin$ca/msp/crls/crl.pem -text -noout | grep -c 'Serial Number:') 397 test $revoked -eq $((ITERATIONS/4)) && 398 echo -e " >>>>>>>>>> crl check for ca$ca ...PASSED" || 399 ErrorMsg " >>>>>>>>>> crl check for ca$ca...FAILED got ('$revoked') revoked certs on localhost:$SERVER" 400 done 401 402 # issue revoke for the third 1/4 of the previously enrolled users; 403 # count the number of entries in the base crl 404 for ca in $(seq $NUMCAS); do 405 #prev_revoked=$(openssl crl -noout -text -in $userdir/admin$ca/msp/crls/crl.pem | grep -c 'Serial Number:') 406 ### @TODO Work-around for FAB-7223: CRL pem file should wrap at 64 characters ### 407 prev_revoked="$(fold -w 64 $userdir/admin$ca/msp/crls/crl.pem | openssl crl -noout -text | grep -c 'Serial Number:')" 408 for user in $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do 409 # delete the current crl 410 rm $userdir/admin$ca/msp/crls/crl.pem 411 revokeUsers $SERVER $ca $user $userdir 412 # 2 entries should be added to the base crl for each revocation 413 # since this group of users has re-enrolled (have two e-certs) 414 #curr_revoked=$(openssl crl -noout -text -in $userdir/admin$ca/msp/crls/crl.pem | grep -c 'Serial Number:') 415 curr_revoked="$(fold -w 64 $userdir/admin$ca/msp/crls/crl.pem | openssl crl -noout -text | grep -c 'Serial Number:')" 416 test "$((curr_revoked-prev_revoked))" -eq 2 && 417 echo -e " >>>>>>>>>> revoke/gencrl check for ${stype}CA${ca}...PASSED" || 418 ErrorMsg " >>>>>>>>>> wrong number of certs in CRL for ${stype}CA${ca}...FAILED got ('$curr_revoked') revoked certs on localhost:$SERVER" 419 prev_revoked=$curr_revoked 420 done 421 done 422 done 423 echo "" 424 echo "" 425 done 426 427 # Delete all of the DBs 428 echo -e " >>>>>>>>>> Deleting all databases" 429 $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 430 DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 431 CleanUp $RC 432 exit $RC