github.com/silveraid/fabric-ca@v1.1.0-preview.0.20180127000700-71974f53ab08/scripts/fvt/cluster_test.sh (about) 1 #!/bin/bash 2 3 : ${TESTCASE="ca_cluster"} 4 FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca" 5 SCRIPTDIR="$FABRIC_CA/scripts/fvt" 6 . $SCRIPTDIR/fabric-ca_utils 7 ROOTDIR=/tmp/cluster 8 INTDIR=$ROOTDIR/int 9 ROOTUSERDIR=$ROOTDIR/users 10 INTUSERDIR=$ROOTDIR/int/users 11 ENROLLCERT=msp/signcerts/cert.pem 12 DEFAULT_CA_CONFIG=fabric-ca-config.yaml 13 RC=0 14 DBNAME=fabric_ca 15 INTDBNAME=intfabric_ca 16 NUMSERVER="$1" # Number of CA instances behind the proxy 17 NUMINTERMD="$2" # Number of intermediate CAs 18 NUMCAS="$3" # cacount; as a simplifying assumption, 19 # if NUMSERVER > NUMCAS, 20 # then NUMSERVER % NUMCAS = 0 21 # else NUMCAS % NUMSERVER = 0 22 ITERATIONS="$4" # num of commands to run in parallel 23 # As a simplifying assumption, ITERATIONS % 4 = 0 24 NUMJOBS="$5" # num of concurrent jobs 25 26 # Using the default, the test should complete in ≈ 1 min 27 : ${NUMSERVER:=2} 28 : ${NUMINTERMD:=2} 29 : ${NUMCAS:=2} 30 : ${ITERATIONS:=8} 31 : ${NUMJOBS:=1024} # spawn as many jobs as there are potential requests 32 NUMUSERS=$((NUMCAS*ITERATIONS)) # NUMUSERS % NUMSERVER should = 0 33 USERNAME="testuser" 34 ROOT_CA_ADDR=localhost 35 INTUSER="intermediateCa16" 36 INTPSWD="intermediateCa16pw" 37 SHELL=/bin/bash 38 39 setTLS 40 export SHELL PROXY_PORT ROOTDIR ITERATIONS USERNAME PROTO TLSOPT FABRIC_CA_CLIENTEXEC ENROLLCERT 41 42 function enrollAdmin() { 43 local port="$1" 44 local ca="$2" 45 local dir="$3" 46 : ${port:=$PROXY_PORT} 47 48 mkdir -p $dir/admin$ca 49 touch $dir/admin$ca/log.txt 50 FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ 51 $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \ 52 -u ${PROTO}admin:adminpw@localhost:$port \ 53 --csr.hosts admin@fab-client.raleigh.ibm.com \ 54 --csr.hosts admin.fabric.raleigh.ibm.com,127.0.0.2 \ 55 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 56 } 57 58 function revokeUsers() { 59 local port="$1" 60 local ca="$2" 61 local user="$3" 62 local dir="$4" 63 : ${port:=$PROXY_PORT} 64 65 FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ 66 $FABRIC_CA_CLIENTEXEC revoke --gencrl --debug $TLSOPT \ 67 -u ${PROTO}admin:adminpw@localhost:$port \ 68 --revoke.name ${USERNAME}${ca}-${user} \ 69 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 70 } 71 72 function enrollUsers() { 73 local port="$1" 74 local ca="$2" 75 local user="$3" 76 local dir="$4" 77 : ${port:=$PROXY_PORT} 78 mkdir -p $dir/admin$ca 79 touch $dir/admin$ca/log.txt 80 FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \ 81 $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \ 82 -u ${PROTO}${USERNAME}${ca}-${user}:${USERNAME}${ca}-${user}@localhost:$port \ 83 --csr.hosts ${USERNAME}${ca}-${user}@fab-client.raleigh.ibm.com \ 84 --csr.hosts ${USERNAME}${ca}-${user}.fabric.raleigh.ibm.com \ 85 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 86 test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'=' '{print $NF}')" 87 } 88 89 function reenrollUsers() { 90 local port="$1" 91 local ca="$2" 92 local user="$3" 93 local dir="$4" 94 : ${port:=$PROXY_PORT} 95 mkdir -p $dir/admin$ca 96 touch $dir/admin$ca/log.txt 97 FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \ 98 $FABRIC_CA_CLIENTEXEC reenroll --debug $TLSOPT \ 99 -u ${PROTO}@localhost:$port \ 100 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 101 test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'=' '{print $NF}')" 102 } 103 104 function register() { 105 local port="$1" 106 local ca="$2" 107 local user="$3" 108 local dir="$4" 109 : ${port:=$PROXY_PORT} 110 FABRIC_CA_CLIENT_HOME=$dir/admin$ca \ 111 $FABRIC_CA_CLIENTEXEC register --debug -u ${PROTO}localhost:$port $TLSOPT \ 112 --id.name ${USERNAME}${ca}-${user} \ 113 --id.secret ${USERNAME}${ca}-${user} \ 114 --id.type client \ 115 --id.maxenrollments $ITERATIONS \ 116 --id.affiliation bank_a \ 117 --id.attrs test=testValue \ 118 --caname ca$ca >> $dir/admin$ca/log.txt 2>&1 119 } 120 121 function DBvalidateUsers() { 122 # Query the DB and verify the user state: 123 # 0 - registered, but not enrolled 124 # 1 - enrolled 125 local state="$1" 126 local dbname="$2" 127 local StatusField=6 128 local fsopt="" 129 130 case $DRIVER in 131 postgres) StatusField=11 ;; 132 esac 133 134 DBNAME=$dbname $SCRIPTDIR/fabric-ca_setup.sh -L -d $DRIVER \ 135 -n $NUMSERVER -u $NUMCAS 2>/dev/null | 136 awk -v u="$USERNAME" $fsopt \ 137 -v s="$state" \ 138 -v n="$NUMUSERS" \ 139 -v f="$StatusField" \ 140 -v t=0 ' 141 $1~u && $f==s {t++} 142 END { if (t!=n) exit 1 }' 143 } 144 145 function showUsers() { 146 $SCRIPTDIR/fabric-ca_setup.sh -L -d $DRIVER \ 147 -n $NUMSERVER -u $NUMCAS 2>/dev/null | 148 awk -v u="$USERNAME" '$1~u' 149 } 150 151 function verifyDistribution() { 152 case "$FABRIC_TLS" in 153 1|y|yes|true) return 0 ;; 154 esac 155 156 local numCluster="$1" 157 local backendName="$2" 158 local expectedCount="$3" 159 local percentage="$4" 160 161 for s in $(seq $numCluster); do 162 verifyServerTraffic "" "/$backendName$((s-1))""" $expectedCount $percentage 163 test $? -eq 0 && 164 echo " >>>>>>>>>> VerifyServerTraffic for $backendName$s...PASSED" || 165 ErrorMsg " >>>>>>>>>> VerifyServerTraffic for $backendName$s...FAILED" 166 done 167 } 168 169 export -f enrollAdmin register enrollUsers revokeUsers reenrollUsers 170 171 function checkStatus() { 172 # Parse the joblog exitstatus (column 7) for all jobs 173 # 0 - success 174 # ^0 - failed 175 # Success is measured by the number of successful jobs, i.e., 176 # there should be one successful job for each request sent: 177 # Number of exit '0' entries == NUMUSERS 178 local log="$1" 179 local number="$2" 180 : ${number:="$NUMUSERS"} 181 awk -v u=$number ' 182 NR!=1 && $7==0 {rc+=1} 183 END {if (rc!=u) exit 1}' $log 184 test $? -ne 0 && ErrorMsg "FAILED" || echo "PASSED" 185 } 186 187 188 for DRIVER in mysql postgres; do 189 echo "Testing $DRIVER >>>>>>>>>>>>>" 190 # Delete all of the DBs 191 echo -e " >>>>>>>>>> Deleting all databases" 192 $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 193 DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 194 rm -rf $ROOTDIR 195 196 # Initialize all of the configs, certs, keys and directories 197 mkdir -p $ROOTUSERDIR 198 mkdir -p $INTUSERDIR 199 echo -e " >>>>>>>>>> Initializing Root CAs" 200 $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -I -n 1 -u $NUMCAS \ 201 -n $NUMSERVER -D -d $DRIVER > $ROOTDIR/log.txt 2>&1 202 echo -e " >>>>>>>>>> Initializing Intermediate CAs" 203 DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -D -d $DRIVER -u $NUMCAS -I -r $INTERMEDIATE_CA_DEFAULT_PORT -x $INTDIR \ 204 -U "${PROTO}$INTUSER:$INTPSWD@$ROOT_CA_ADDR:$CA_DEFAULT_PORT" > $INTDIR/log.txt 2>&1 205 206 ################################################################## 207 ## Customize enrollment for each CA 208 ################################################################## 209 ca=0 210 rootCafiles="" 211 intermediateCafiles="" 212 rootDBconfig="" 213 intermediateDBconfig="" 214 # append the customized DB config to each CA's config file 215 while test $((ca++)) -lt $NUMCAS; do 216 # build the list of cafiles to be passed to server start 217 rootCafiles="$rootCafiles,$ROOTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}" 218 intermediateCafiles="$intermediateCafiles,$INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}" 219 220 # each intermediate CA needs an enrollment identity and parentserver.url 221 # each also needs a unique caname, or the sever start will fail 222 enrollment=" 223 intermediate: 224 parentserver: 225 url: ${PROTO}intermediateCa${ca}:intermediateCa${ca}pw@127.0.0.1:$CA_DEFAULT_PORT 226 caname: ca${ca} 227 enrollment: 228 name: intermediateCa${ca} 229 secret: intermediateCa${ca}pw 230 hosts: 231 - localhost 232 " 233 # append the intermediate CA config to each CA's config 234 cat >> $INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG} <<EOF 235 $enrollment 236 EOF 237 done 238 239 # strip the leading comma from the files list 240 rootCafiles=${rootCafiles#,} 241 intermediateCafiles=${intermediateCafiles#,} 242 # remove the pathlength restriction 243 sed -i 's/cacount:.*/cacount:/g 244 s/maxpathlen:.*/maxpathlen:/g 245 s/pathlength:.*/pathlength:/g' $ROOTDIR/runFabricCaFvt.yaml $INTDIR/runFabricCaFvt.yaml 246 # Remove all of the CSR.CN data from intermediate CAs -- 247 # otherwise, server startup will fail 248 find $INTDIR/ca -name $DEFAULT_CA_CONFIG -exec sed -i "s/cn:.*/cn:/g" {} \; 249 250 # Start all Root and intermediate CAs 251 echo -e " >>>>>>>>>> Starting $NUMSERVER Root CA instances with $NUMCAS servers each" 252 $SCRIPTDIR/fabric-ca_setup.sh -N -X -x $ROOTDIR -S -n $NUMSERVER -D -d $DRIVER \ 253 -- "--cafiles" "$rootCafiles" >> $ROOTDIR/log.txt 2>&1 || 254 ErrorExit "Failed to start root servers" 255 echo -e " >>>>>>>>>> Starting $NUMSERVER Intermediate CA instances with $NUMCAS servers each" 256 $SCRIPTDIR/fabric-ca_setup.sh -n $NUMSERVER -S -r $INTERMEDIATE_CA_DEFAULT_PORT -x $INTDIR \ 257 -U "https://$INTUSER:$INTPSWD@$ROOT_CA_ADDR:$PROXY_PORT" \ 258 -- "--cafiles" "$intermediateCafiles" >> $INTDIR/log.txt 2>&1 || 259 ErrorExit "Failed to intermediate servers" 260 261 ######################################################### 262 # The bulk of the work comes here -- 263 # register and enroll users, in parallel, $NUMJOBS at a time 264 ######################################################### 265 for SERVER in $PROXY_PORT $INTERMEDIATE_PROXY_PORT ; do 266 # The intermediate CAs do not share the root CA's DBs; 267 # each has a unique DB (in a unique directory, if file-based 268 # If the CA is root, the haproxy backend name is 'server' 269 # If the CA is intermediate, the haproxy backend name is 'intserver' 270 if test "$SERVER" = "$INTERMEDIATE_PROXY_PORT"; then 271 dbname=$INTDBNAME 272 userdir=$INTUSERDIR 273 stype=intermediate 274 backend=intserver 275 else 276 dbname=$DBNAME 277 userdir=$ROOTUSERDIR 278 stype=root 279 backend=server 280 fi 281 282 count=0 283 # for each block of requests, we will validate round-robin 284 # distribution to all participating CAs 285 verifyDistribution $NUMSERVER $backend $count 0 286 287 echo -e " >>>>>>>>>> Testing $stype CA using DB name $dbname" 288 289 # Enroll the admins -- the total number of enrollment requests 290 # sent is calulated to be the larger of NUMSERVER | NUMCAS 291 test $NUMCAS -ge $NUMSERVER && numReq=1 || numReq=$((NUMSERVER/NUMCAS)) 292 printf " >>>>>>>>>> Enrolling ${NUMCAS} admins, $numReq times..." 293 parallel -k --no-notice --jobs $NUMJOBS --joblog $userdir/adminEnroll.log \ 294 enrollAdmin $SERVER {1} $userdir ::: $(seq $NUMCAS) ::: $(seq $numReq) 295 checkStatus $userdir/adminEnroll.log $((numReq*NUMCAS)) || ErrorExit "Enroll of admins failed" 296 # Register $NUMUSERS users and validate their status in the DB 297 test $NUMCAS -lt $NUMSERVER && count=1 || count=$((NUMCAS/NUMSERVER)) 298 verifyDistribution $NUMSERVER $backend $count 299 300 # Register $NUMUSERS users and validate their status in the DB 301 printf " >>>>>>>>>> Registering $NUMUSERS users (${NUMCAS}x${ITERATIONS})..." 302 parallel --no-notice --jobs $NUMJOBS --joblog $userdir/register.log \ 303 register $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS) 304 checkStatus $userdir/register.log 305 DBvalidateUsers 0 $dbname && 306 echo -e " >>>>>>>>>> Validating user status in DB...PASSED" || 307 ErrorMsg " >>>>>>>>>> Validating user status in DB...FAILED" 308 count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) )) 309 verifyDistribution $NUMSERVER $backend $count 310 311 # Enroll $NUMUSERS users and validate their status in the DB 312 printf " >>>>>>>>>> Enrolling $NUMUSERS users (${NUMCAS}x${ITERATIONS})..." 313 parallel --no-notice --jobs $NUMJOBS --joblog $userdir/userEnroll.log \ 314 enrollUsers $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS) 315 checkStatus $userdir/userEnroll.log 316 DBvalidateUsers 1 $dbname && 317 echo -e " >>>>>>>>>> Validating user status in DB...PASSED" || 318 ErrorMsg " >>>>>>>>>> Validating user status in DB...FAILED" 319 count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) )) 320 verifyDistribution $NUMSERVER $backend $count 321 322 # Do all of the following simultaneously 323 # enroll Enroll an identity 324 # getcacert Get CA certificate chain 325 # reenroll Reenroll an identity 326 # register Register an identity 327 # revoke Revoke an identity 328 # gencrl Generate a CRL 329 330 > /tmp/cmd.lst 331 for ca in $(seq $NUMCAS); do 332 # Create the register cmd list of brand new users where 333 # the previous register task left off 334 echo " >>>>>>> generating register command list ($((ITERATIONS*NUMCAS)))" 335 for user in $(seq $((ITERATIONS+1)) $((ITERATIONS+ITERATIONS)) ); do 336 echo register $SERVER $ca $user $userdir >> /tmp/cmd.lst 337 done 338 # Create the enroll cmd list - 339 # issue enroll for the first 1/2 of the previously enrolled users 340 echo " >>>>>>> generating enroll command list ($((ITERATIONS/2*NUMCAS)))" 341 for user in $(seq $((ITERATIONS/2)) ); do 342 echo enrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst 343 done 344 # Create the renroll cmd list - 345 # issue renroll for the third 1/4 of the previously enrolled users 346 echo " >>>>>>> generating renroll command list ($((ITERATIONS/4*NUMCAS)))" 347 for user in $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do 348 echo reenrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst 349 done 350 # Create the revoke cmd list - 351 # issue renroll for the last 1/4 of the previously enrolled users 352 echo " >>>>>>> generating revoke command list ($((ITERATIONS/4*NUMCAS)))" 353 for user in $(seq $((ITERATIONS/4*3+1)) $ITERATIONS ); do 354 echo revokeUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst 355 done 356 # Create the getcacert cmd list - 357 echo " >>>>>>> generating getcacert command list ($((ITERATIONS*NUMCAS/2)))" 358 for user in $(seq $((ITERATIONS/2)) ); do 359 echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC getcacert --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/cacert.txt 2>&1" >> /tmp/cmd.lst 360 done 361 # Create the gencrl cmd list - 362 echo " >>>>>>> generating gencrl command list ($((ITERATIONS*NUMCAS/2)))" 363 for user in $(seq $((ITERATIONS/2+1)) $ITERATIONS); do 364 echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1" >> /tmp/cmd.lst 365 done 366 done 367 368 shuf --output=$userdir/cmd.lst /tmp/cmd.lst 369 # OK, here goes... 370 printf " >>>>>>>>>> Executing all $((ITERATIONS*3*NUMCAS)) jobs..." 371 parallel --no-notice --jobs $NUMJOBS --joblog $userdir/cmd.log < $userdir/cmd.lst 372 checkStatus $userdir/cmd.log $((ITERATIONS*3*NUMCAS)) 373 count=$((count+$((ITERATIONS*3*NUMCAS/NUMSERVER)))) 374 sleep 1 375 verifyDistribution $NUMSERVER $backend $count 376 377 sleep 1 378 # Lastly, 1/4 of user certs should be revoked 379 echo " >>>>>>>>>> Checking crl; expect $((ITERATIONS/4)) revoked certificates..." 380 for ca in $(seq $NUMCAS); do 381 FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1 382 revoked=$(openssl crl -in $userdir/admin$ca/msp/crls/crl.pem -text -noout | grep -c 'Serial Number:') 383 test $revoked -eq $((ITERATIONS/4)) && 384 echo -e " >>>>>>>>>> crl check for ca$ca ...PASSED" || 385 ErrorMsg " >>>>>>>>>> crl check for ca$ca...FAILED got ('$revoked') revoked certs on localhost:$SERVER" 386 done 387 388 # issue revoke for the third 1/4 of the previously enrolled users; 389 # count the number of entries in the base crl 390 for ca in $(seq $NUMCAS); do 391 #prev_revoked=$(openssl crl -noout -text -in $userdir/admin$ca/msp/crls/crl.pem | grep -c 'Serial Number:') 392 ### @TODO Work-around for FAB-7223: CRL pem file should wrap at 64 characters ### 393 prev_revoked="$(fold -w 64 $userdir/admin$ca/msp/crls/crl.pem | openssl crl -noout -text | grep -c 'Serial Number:')" 394 for user in $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do 395 # delete the current crl 396 rm $userdir/admin$ca/msp/crls/crl.pem 397 revokeUsers $SERVER $ca $user $userdir 398 # 2 entries should be added to the base crl for each revocation 399 # since this group of users has re-enrolled (have two e-certs) 400 #curr_revoked=$(openssl crl -noout -text -in $userdir/admin$ca/msp/crls/crl.pem | grep -c 'Serial Number:') 401 curr_revoked="$(fold -w 64 $userdir/admin$ca/msp/crls/crl.pem | openssl crl -noout -text | grep -c 'Serial Number:')" 402 test "$((curr_revoked-prev_revoked))" -eq 2 && 403 echo -e " >>>>>>>>>> revoke/gencrl check for ${stype}CA${ca}...PASSED" || 404 ErrorMsg " >>>>>>>>>> wrong number of certs in CRL for ${stype}CA${ca}...FAILED got ('$curr_revoked') revoked certs on localhost:$SERVER" 405 prev_revoked=$curr_revoked 406 done 407 done 408 done 409 echo "" 410 echo "" 411 done 412 413 # Delete all of the DBs 414 echo -e " >>>>>>>>>> Deleting all databases" 415 $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 416 DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS 417 CleanUp $RC 418 exit $RC