github.com/hyperledger/fabric-ca@v2.0.0-alpha.0.20201120210307-7b4f34729db1+incompatible/scripts/fvt/cluster_test.sh (about)

     1  #!/bin/bash
     2  #
     3  # Copyright IBM Corp. All Rights Reserved.
     4  #
     5  # SPDX-License-Identifier: Apache-2.0
     6  #
     7  
     8  : ${TESTCASE="ca_cluster"}
     9  FABRIC_CA="$GOPATH/src/github.com/hyperledger/fabric-ca"
    10  SCRIPTDIR="$FABRIC_CA/scripts/fvt"
    11  . $SCRIPTDIR/fabric-ca_utils
    12  ROOTDIR=/tmp/cluster
    13  INTDIR=$ROOTDIR/int
    14  ROOTUSERDIR=$ROOTDIR/users
    15  INTUSERDIR=$ROOTDIR/int/users
    16  ENROLLCERT=msp/signcerts/cert.pem
    17  DEFAULT_CA_CONFIG=fabric-ca-config.yaml
    18  RC=0
    19  DBNAME=fabric_ca
    20  INTDBNAME=intfabric_ca
    21  NUMSERVER="$1"  # Number of CA instances behind the proxy
    22  NUMINTERMD="$2"  # Number of intermediate CAs
    23  NUMCAS="$3"      # cacount; as a simplifying assumption,
    24                   #    if NUMSERVER > NUMCAS,
    25                   #    then NUMSERVER % NUMCAS = 0
    26                   #    else NUMCAS % NUMSERVER = 0
    27  ITERATIONS="$4"  # num of commands to run in parallel
    28                   # As a simplifying assumption, ITERATIONS % 4 = 0
    29  NUMJOBS="$5"     # num of concurrent jobs
    30  
    31  # Using the default, the test should complete in ≈ 1 min
    32  : ${NUMSERVER:=2}
    33  : ${NUMINTERMD:=2}
    34  : ${NUMCAS:=2}
    35  : ${ITERATIONS:=8}
    36  : ${NUMJOBS:=1024}   # spawn as many jobs as there are potential requests
    37  NUMUSERS=$((NUMCAS*ITERATIONS)) # NUMUSERS % NUMSERVER should = 0
    38  USERNAME="testuser"
    39  ROOT_CA_ADDR=localhost
    40  INTUSER="intermediateCa16"
    41  INTPSWD="intermediateCa16pw"
    42  SHELL=/bin/bash
    43  
    44  export SHELL PROXY_PORT ROOTDIR ITERATIONS USERNAME PROTO FABRIC_CA_CLIENTEXEC ENROLLCERT
    45  
    46  function enrollAdmin() {
    47     local port="$1"
    48     local ca="$2"
    49     local dir="$3"
    50     : ${port:=$PROXY_PORT}
    51  
    52     mkdir -p $dir/admin$ca
    53     touch $dir/admin$ca/log.txt
    54     FABRIC_CA_CLIENT_HOME=$dir/admin$ca \
    55     $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \
    56       -u ${PROTO}admin:adminpw@localhost:$port \
    57       --csr.hosts admin@fab-client.raleigh.ibm.com \
    58       --csr.hosts admin.fabric.raleigh.ibm.com,127.0.0.2 \
    59       --caname ca$ca >> $dir/admin$ca/log.txt 2>&1
    60  }
    61  
    62  function revokeUsers() {
    63     local port="$1"
    64     local ca="$2"
    65     local user="$3"
    66     local dir="$4"
    67     : ${port:=$PROXY_PORT}
    68  
    69     FABRIC_CA_CLIENT_HOME=$dir/admin$ca \
    70     $FABRIC_CA_CLIENTEXEC revoke --gencrl --debug $TLSOPT \
    71       -u ${PROTO}admin:adminpw@localhost:$port \
    72       --revoke.name ${USERNAME}${ca}-${user} \
    73       --caname ca$ca >> $dir/admin$ca/log.txt 2>&1
    74  }
    75  
    76  function enrollUsers() {
    77     local port="$1"
    78     local ca="$2"
    79     local user="$3"
    80     local dir="$4"
    81     : ${port:=$PROXY_PORT}
    82     mkdir -p $dir/admin$ca
    83     touch $dir/admin$ca/log.txt
    84     FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \
    85     $FABRIC_CA_CLIENTEXEC enroll --debug $TLSOPT \
    86       -u ${PROTO}${USERNAME}${ca}-${user}:${USERNAME}${ca}-${user}@localhost:$port \
    87       --csr.hosts ${USERNAME}${ca}-${user}@fab-client.raleigh.ibm.com \
    88       --csr.hosts ${USERNAME}${ca}-${user}.fabric.raleigh.ibm.com \
    89       --caname ca$ca >> $dir/admin$ca/log.txt 2>&1
    90     test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'= ' '{print $NF}')"
    91  }
    92  
    93  function reenrollUsers() {
    94     local port="$1"
    95     local ca="$2"
    96     local user="$3"
    97     local dir="$4"
    98     : ${port:=$PROXY_PORT}
    99     mkdir -p $dir/admin$ca
   100     touch $dir/admin$ca/log.txt
   101     FABRIC_CA_CLIENT_HOME=$dir/${USERNAME}${ca}-${user} \
   102     $FABRIC_CA_CLIENTEXEC reenroll --debug $TLSOPT \
   103       -u ${PROTO}@localhost:$port \
   104       --caname ca$ca >> $dir/admin$ca/log.txt 2>&1
   105     test "${USERNAME}${ca}-${user}" = "$(openssl x509 -in $dir/${USERNAME}${ca}-${user}/$ENROLLCERT -noout -subject | awk -F'= ' '{print $NF}')"
   106  }
   107  
   108  function register() {
   109     local port="$1"
   110     local ca="$2"
   111     local user="$3"
   112     local dir="$4"
   113     : ${port:=$PROXY_PORT}
   114     FABRIC_CA_CLIENT_HOME=$dir/admin$ca \
   115     $FABRIC_CA_CLIENTEXEC register --debug -u ${PROTO}localhost:$port $TLSOPT \
   116     --id.name ${USERNAME}${ca}-${user} \
   117     --id.secret ${USERNAME}${ca}-${user} \
   118     --id.type client \
   119     --id.maxenrollments $ITERATIONS \
   120     --id.affiliation bank_a \
   121     --id.attrs test=testValue \
   122     --caname ca$ca >> $dir/admin$ca/log.txt 2>&1
   123  }
   124  
   125  function DBvalidateUsers() {
   126     # Query the DB and verify the user state:
   127     #  0 - registered, but not enrolled
   128     #  1 - enrolled
   129     local state="$1"
   130     local dbname="$2"
   131     local StatusField=6
   132     local fsopt=""
   133  
   134     case $DRIVER in
   135        postgres) StatusField=11 ;;
   136     esac
   137  
   138     DBNAME=$dbname $SCRIPTDIR/fabric-ca_setup.sh -L -d $DRIVER \
   139          -n $NUMSERVER -u $NUMCAS 2>/dev/null |
   140     awk -v u="$USERNAME" $fsopt \
   141         -v s="$state" \
   142         -v n="$NUMUSERS" \
   143         -v f="$StatusField" \
   144         -v t=0 '
   145        $1~u && $f==s {t++}
   146        END { if (t!=n) exit 1 }'
   147  }
   148  
   149  function showUsers() {
   150     $SCRIPTDIR/fabric-ca_setup.sh -L -d $DRIVER  \
   151          -n $NUMSERVER -u $NUMCAS 2>/dev/null |
   152     awk -v u="$USERNAME" '$1~u'
   153  }
   154  
   155  export -f enrollAdmin register enrollUsers revokeUsers reenrollUsers
   156  
   157  function checkStatus() {
   158     # Parse the joblog exitstatus (column 7) for all jobs
   159     #  0  - success
   160     #  ^0 - failed
   161     # Success is measured by the number of successful jobs, i.e.,
   162     # there should be one successful job for each request sent:
   163     #   Number of exit '0' entries == NUMUSERS
   164     local log="$1"
   165     local number="$2"
   166     : ${number:="$NUMUSERS"}
   167     awk -v u=$number '
   168           NR!=1 && $7==0 {rc+=1}
   169           END {if (rc!=u) exit 1}' $log
   170     test $? -ne 0 && ErrorMsg "FAILED" || echo "PASSED"
   171  }
   172  
   173  
   174  for DRIVER in mysql postgres; do
   175     echo "Testing $DRIVER >>>>>>>>>>>>>"
   176     # Delete all of the DBs
   177     echo -e "   >>>>>>>>>>  Deleting all databases"
   178     $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS
   179     DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS
   180     rm -rf $ROOTDIR
   181  
   182     # Initialize all of the configs, certs, keys and directories
   183     mkdir -p $ROOTUSERDIR
   184     mkdir -p $INTUSERDIR
   185     echo -e "   >>>>>>>>>>  Initializing Root CAs"
   186     $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -I -n 1 -u $NUMCAS \
   187                                   -n $NUMSERVER -D -d $DRIVER > $ROOTDIR/log.txt 2>&1
   188  
   189     echo -e "   >>>>>>>>>>  Initializing Intermediate CAs"
   190  
   191     # Copy root CA config file to int CA home and change the database name
   192     cp "$ROOTDIR/$DEFAULT_RUN_CONFIG_FILE_NAME" "$INTDIR/$DEFAULT_RUN_CONFIG_FILE_NAME"
   193     sed -i "/datasource:/ s/datasource:\(.*\)fabric_ca\(.*\)/datasource:\1intfabric_ca\2/" "$INTDIR/$DEFAULT_RUN_CONFIG_FILE_NAME"
   194  
   195     ##################################################################
   196     ## Customize enrollment for each CA
   197     ##################################################################
   198     ca=0
   199     rootCafiles=""
   200     intermediateCafiles=""
   201     rootDBconfig=""
   202     intermediateDBconfig=""
   203     # append the customized DB config to each CA's config file
   204     while test $((ca++)) -lt $NUMCAS; do
   205        # Copy CA config files of root CA server to int CA home and change the database name
   206        mkdir -p "$INTDIR/ca/ca$ca" || true
   207        cp "$ROOTDIR/ca/ca$ca/fabric-ca-config.yaml" "$INTDIR/ca/ca$ca/fabric-ca-config.yaml"
   208        sed -i "/datasource:/ s/datasource:\(.*\)fabric_ca_ca$ca\(.*\)/datasource:\1intfabric_ca_ca$ca\2/" "$INTDIR/ca/ca$ca/fabric-ca-config.yaml"
   209  
   210        # build the list of cafiles to be passed to server start
   211        rootCafiles="$rootCafiles,$ROOTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}"
   212        intermediateCafiles="$intermediateCafiles,$INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG}"
   213  
   214        # each intermediate CA needs an enrollment identity and parentserver.url
   215        # each also needs a unique caname, or the sever start will fail
   216        enrollment="
   217  intermediate:
   218    parentserver:
   219      url: ${PROTO}intermediateCa${ca}:intermediateCa${ca}pw@127.0.0.1:$CA_DEFAULT_PORT
   220      caname: ca${ca}
   221    enrollment:
   222      name: intermediateCa${ca}
   223      secret: intermediateCa${ca}pw
   224      hosts:
   225         - localhost
   226  "
   227        # append the intermediate CA config to each CA's config
   228        cat >> $INTDIR/ca/ca$ca/${DEFAULT_CA_CONFIG} <<EOF
   229  $enrollment
   230  EOF
   231     done
   232  
   233     # strip the leading comma from the files list
   234     rootCafiles=${rootCafiles#,}
   235     intermediateCafiles=${intermediateCafiles#,}
   236     # remove the pathlength restriction
   237     sed -i 's/cacount:.*/cacount:/g
   238             s/maxpathlen:.*/maxpathlen:/g
   239             s/pathlength:.*/pathlength:/g' $ROOTDIR/runFabricCaFvt.yaml $INTDIR/runFabricCaFvt.yaml
   240     # Remove all of the CSR.CN data from intermediate CAs --
   241     # otherwise, server startup will fail
   242     find $INTDIR/ca -name $DEFAULT_CA_CONFIG -exec sed -i "s/cn:.*/cn:/g" {} \;
   243  
   244     # Start all Root and intermediate CAs
   245     echo -e "   >>>>>>>>>>  Starting $NUMSERVER Root CA instances with $NUMCAS servers each"
   246     $SCRIPTDIR/fabric-ca_setup.sh -N -X -x $ROOTDIR -S -n $NUMSERVER -D -d $DRIVER \
   247                                   -- "--cafiles" "$rootCafiles" >> $ROOTDIR/log.txt 2>&1 ||
   248                                   ErrorExit "Failed to start root servers"
   249     echo -e "   >>>>>>>>>>  Starting $NUMSERVER Intermediate CA instances with $NUMCAS servers each"
   250  
   251     $SCRIPTDIR/fabric-ca_setup.sh -n $NUMSERVER -S -r $INTERMEDIATE_CA_DEFAULT_PORT -x $INTDIR \
   252                                   -U "https://$INTUSER:$INTPSWD@$ROOT_CA_ADDR:$PROXY_PORT" \
   253                                   --  "--cafiles" "$intermediateCafiles" >> $INTDIR/log.txt 2>&1 ||
   254                                   ErrorExit "Failed to start intermediate servers"
   255  
   256     #########################################################
   257     # The bulk of the work comes here  --
   258     # register and enroll users, in parallel, $NUMJOBS at a time
   259     #########################################################
   260     for SERVER in $PROXY_PORT $INTERMEDIATE_PROXY_PORT ; do
   261        # The intermediate CAs do not share the root CA's DBs;
   262        # each has a unique DB (in a unique directory, if file-based
   263        # If the CA is root, the haproxy backend name is 'server'
   264        # If the CA is intermediate, the haproxy backend name is 'intserver'
   265        if test "$SERVER" = "$INTERMEDIATE_PROXY_PORT"; then
   266           dbname=$INTDBNAME
   267           userdir=$INTUSERDIR
   268           stype=intermediate
   269           backend=intserver
   270        else
   271           dbname=$DBNAME
   272           userdir=$ROOTUSERDIR
   273           stype=root
   274           backend=server
   275        fi
   276  
   277        count=0
   278  
   279        echo -e "      >>>>>>>>>>  Testing $stype CA using DB name $dbname"
   280  
   281        # Enroll the admins -- the total number of enrollment requests
   282        # sent is calulated to be the larger of NUMSERVER | NUMCAS
   283        test $NUMCAS -ge $NUMSERVER && numReq=1 || numReq=$((NUMSERVER/NUMCAS))
   284        printf "         >>>>>>>>>>  Enrolling ${NUMCAS} admins, $numReq times..."
   285        parallel -k --no-notice --jobs $NUMJOBS --joblog $userdir/adminEnroll.log \
   286           enrollAdmin $SERVER {1} $userdir ::: $(seq $NUMCAS) ::: $(seq $numReq)
   287        checkStatus $userdir/adminEnroll.log $((numReq*NUMCAS)) || ErrorExit "Enroll of admins failed"
   288        # Register $NUMUSERS users and validate their status in the DB
   289        test $NUMCAS -lt $NUMSERVER && count=1 || count=$((NUMCAS/NUMSERVER))
   290  
   291        # Register $NUMUSERS users and validate their status in the DB
   292        printf "         >>>>>>>>>>  Registering $NUMUSERS users (${NUMCAS}x${ITERATIONS})..."
   293        parallel --no-notice --jobs $NUMJOBS --joblog $userdir/register.log \
   294           register $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS)
   295        checkStatus $userdir/register.log
   296        DBvalidateUsers 0 $dbname &&
   297            echo -e "         >>>>>>>>>>  Validating user status in DB...PASSED" ||
   298            ErrorMsg "         >>>>>>>>>>  Validating user status in DB...FAILED"
   299        count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) ))
   300  
   301        # Enroll $NUMUSERS users and validate their status in the DB
   302        printf "         >>>>>>>>>>  Enrolling $NUMUSERS users (${NUMCAS}x${ITERATIONS})..."
   303        parallel --no-notice --jobs $NUMJOBS --joblog $userdir/userEnroll.log \
   304           enrollUsers $SERVER {1} {2} $userdir ::: $(seq $NUMCAS) ::: $(seq $ITERATIONS)
   305        checkStatus $userdir/userEnroll.log
   306        DBvalidateUsers 1 $dbname &&
   307            echo -e "         >>>>>>>>>>  Validating user status in DB...PASSED" ||
   308            ErrorMsg "         >>>>>>>>>>  Validating user status in DB...FAILED"
   309        count=$((count+$((ITERATIONS*NUMCAS/NUMSERVER)) ))
   310  
   311        # Do all of the following simultaneously
   312        #  enroll      Enroll an identity
   313        #  getcacert   Get CA certificate chain
   314        #  reenroll    Reenroll an identity
   315        #  register    Register an identity
   316        #  revoke      Revoke an identity
   317        #  gencrl      Generate a CRL
   318  
   319        > /tmp/cmd.lst
   320        for ca in $(seq $NUMCAS); do
   321           # Create the register cmd list of brand new users where
   322           #     the previous register task left off
   323           echo "               >>>>>>>  generating register command list ($((ITERATIONS*NUMCAS)))"
   324           for user in $(seq $((ITERATIONS+1)) $((ITERATIONS+ITERATIONS)) ); do
   325             echo register $SERVER $ca $user $userdir >> /tmp/cmd.lst
   326           done
   327           # Create the enroll cmd list -
   328           #     issue enroll for the first 1/2 of the previously enrolled users
   329           echo "               >>>>>>>  generating enroll command list ($((ITERATIONS/2*NUMCAS)))"
   330           for user in $(seq $((ITERATIONS/2)) ); do
   331              echo enrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst
   332           done
   333           # Create the renroll cmd list -
   334           #     issue renroll for the third 1/4 of the previously enrolled users
   335           echo "               >>>>>>>  generating renroll command list ($((ITERATIONS/4*NUMCAS)))"
   336           for user in  $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do
   337              echo reenrollUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst
   338           done
   339           # Create the revoke cmd list -
   340           #     issue renroll for the last 1/4 of the previously enrolled users
   341           echo "               >>>>>>>  generating revoke command list ($((ITERATIONS/4*NUMCAS)))"
   342           for user in  $(seq $((ITERATIONS/4*3+1)) $ITERATIONS ); do
   343              echo revokeUsers $SERVER $ca $user $userdir >> /tmp/cmd.lst
   344           done
   345           # Create the getcacert cmd list -
   346           echo "               >>>>>>>  generating getcacert command list ($((ITERATIONS*NUMCAS/2)))"
   347           for user in $(seq $((ITERATIONS/2)) ); do
   348              echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC getcacert --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/cacert.txt 2>&1" >> /tmp/cmd.lst
   349           done
   350           # Create the gencrl cmd list -
   351           echo "               >>>>>>>  generating gencrl command list ($((ITERATIONS*NUMCAS/2)))"
   352           for user in $(seq $((ITERATIONS/2+1)) $ITERATIONS); do
   353              echo "FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1" >> /tmp/cmd.lst
   354           done
   355        done
   356  
   357        shuf --output=$userdir/cmd.lst /tmp/cmd.lst
   358        # OK, here goes...
   359        printf "         >>>>>>>>>>  Executing all $((ITERATIONS*3*NUMCAS)) jobs..."
   360        parallel --no-notice --jobs $NUMJOBS --joblog $userdir/cmd.log < $userdir/cmd.lst
   361        checkStatus $userdir/cmd.log $((ITERATIONS*3*NUMCAS))
   362        count=$((count+$((ITERATIONS*3*NUMCAS/NUMSERVER))))
   363  
   364        sleep 1
   365        # Lastly, 1/4 of user certs should be revoked
   366        echo "         >>>>>>>>>>  Checking crl; expect $((ITERATIONS/4)) revoked certificates..."
   367        for ca in $(seq $NUMCAS); do
   368           FABRIC_CA_CLIENT_HOME=$userdir/admin$ca $FABRIC_CA_CLIENTEXEC gencrl --debug -u ${PROTO}localhost:$SERVER $TLSOPT --caname ca$ca > $userdir/admin$ca/crl.txt 2>&1
   369           revoked=$(openssl crl -in  $userdir/admin$ca/msp/crls/crl.pem -text -noout | grep -c 'Serial Number:')
   370           test $revoked -eq $((ITERATIONS/4)) &&
   371              echo -e "            >>>>>>>>>>  crl check for ca$ca ...PASSED" ||
   372              ErrorMsg "            >>>>>>>>>>  crl check for ca$ca...FAILED got ('$revoked') revoked certs on localhost:$SERVER"
   373        done
   374  
   375        # issue revoke for the third 1/4 of the previously enrolled users;
   376        # count the number of entries in the base crl
   377        for ca in $(seq $NUMCAS); do
   378           #prev_revoked=$(openssl crl -noout -text -in $userdir/admin$ca/msp/crls/crl.pem | grep -c 'Serial Number:')
   379           ### @TODO Work-around for FAB-7223: CRL pem file should wrap at 64 characters ###
   380           prev_revoked="$(fold -w 64 $userdir/admin$ca/msp/crls/crl.pem | openssl crl -noout -text | grep -c 'Serial Number:')"
   381           for user in  $(seq $((ITERATIONS/2+1)) $((ITERATIONS/4*3)) ); do
   382              # delete the current crl
   383              rm $userdir/admin$ca/msp/crls/crl.pem
   384              revokeUsers $SERVER $ca $user $userdir
   385              # 2 entries should be added to the base crl for each revocation
   386              # since this group of users has re-enrolled (have two e-certs)
   387              #curr_revoked=$(openssl crl -noout -text -in $userdir/admin$ca/msp/crls/crl.pem | grep -c 'Serial Number:')
   388              curr_revoked="$(fold -w 64 $userdir/admin$ca/msp/crls/crl.pem | openssl crl -noout -text | grep -c 'Serial Number:')"
   389              test "$((curr_revoked-prev_revoked))" -eq 2  &&
   390                 echo -e "            >>>>>>>>>>  revoke/gencrl check for ${stype}CA${ca}...PASSED" ||
   391                 ErrorMsg "            >>>>>>>>>>  wrong number of certs in CRL for ${stype}CA${ca}...FAILED got ('$curr_revoked') revoked certs on localhost:$SERVER"
   392              prev_revoked=$curr_revoked
   393           done
   394        done
   395     done
   396     echo ""
   397     echo ""
   398  done
   399  
   400  # Delete all of the DBs
   401  echo -e "   >>>>>>>>>>  Deleting all databases"
   402  $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS
   403  DBNAME=$INTDBNAME $SCRIPTDIR/fabric-ca_setup.sh -x $ROOTDIR -R -u $NUMCAS
   404  CleanUp $RC
   405  exit $RC