github.com/verrazzano/verrazzano@v1.7.1/ci/scripts/dumpRunLogs.sh (about)

     1  # Copyright (c) 2020, 2022, Oracle and/or its affiliates.
     2  # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  set -u
     5  
     6  POST_DUMP_FAILED_FILE="${POST_DUMP_FAILED_FILE:-${WORKSPACE}/post_dump_failed_file.tmp}"
     7  VZ_LOGS_ROOT="${VZ_LOGS_ROOT:-${WORKSPACE}/logs}"
     8  VERRAZZANO_INSTALL_LOGS_DIR="${VERRAZZANO_INSTALL_LOGS_DIR:-${VZ_LOGS_ROOT}}"
     9  VZ_COMMAND="${VZ_COMMAND:-${GOPATH}/bin/vz}"
    10  TESTS_EXECUTED_FILE="${TESTS_EXECUTED_FILE:-${WORKSPACE}/tests_executed_file.tmp}"
    11  
    12  dumpK8sCluster() {
    13    ANALYSIS_REPORT="analysis.report"
    14    dumpDirectory=$1
    15    ${GO_REPO_PATH}/verrazzano/tools/scripts/k8s-dump-cluster.sh -d ${dumpDirectory} -r ${dumpDirectory}/cluster-snapshot/${ANALYSIS_REPORT}
    16  
    17    # TODO: Handle any error in creating the bug-report or running analyze on that
    18    # Create a bug-report and run analysis tool on the bug-report
    19    # Requires environment variable KUBECONFIG or $HOME/.kube/config
    20    BUG_REPORT_FILE="${dumpDirectory}/bug-report.tar.gz"
    21    if [[ -x ${VZ_COMMAND} ]]; then
    22      $GOPATH/vz bug-report --report-file ${BUG_REPORT_FILE}
    23    else
    24      GO111MODULE=on GOPRIVATE=github.com/verrazzano go run main.go bug-report --report-file ${BUG_REPORT_FILE}
    25    fi
    26  
    27    # Check if the bug-report exists
    28    if [ -f "${BUG_REPORT_FILE}" ]; then
    29      mkdir -p ${dumpDirectory}/bug-report
    30      tar -xvf ${BUG_REPORT_FILE} -C ${dumpDirectory}/bug-report
    31      rm ${BUG_REPORT_FILE} || true
    32  
    33      # Run vz analyze on the extracted directory
    34      if [[ -x ${VZ_COMMAND} ]]; then
    35        ${VZ_COMMAND} analyze --capture-dir ${dumpDirectory}/bug-report --report-format detailed --report-file ${dumpDirectory}/bug-report/${ANALYSIS_REPORT}
    36      else
    37        GO111MODULE=on GOPRIVATE=github.com/verrazzano go run main.go analyze --capture-dir ${dumpDirectory}/bug-report --report-format detailed --report-file ${dumpDirectory}/bug-report/${ANALYSIS_REPORT}
    38      fi
    39    fi
    40  }
    41  
    42  dumpVerrazzanoSystemPods() {
    43    cd ${GO_REPO_PATH}/verrazzano/platform-operator
    44    local sysLogs=${VERRAZZANO_INSTALL_LOGS_DIR}/system
    45    mkdir -p ${sysLogs}
    46    export DIAGNOSTIC_LOG="${sysLogs}/verrazzano-system-pods.log"
    47    ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -m "verrazzano system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
    48    export DIAGNOSTIC_LOG="${sysLogs}/verrazzano-system-certs.log"
    49    ./scripts/install/k8s-dump-objects.sh -o cert -n verrazzano-system -m "verrazzano system certs" || echo "failed" > ${POST_DUMP_FAILED_FILE}
    50    export DIAGNOSTIC_LOG="${sysLogs}/verrazzano-system-osd.log"
    51    ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-osd-*" -m "verrazzano system opensearchdashboards log" -l -c osd || echo "failed" > ${POST_DUMP_FAILED_FILE}
    52    export DIAGNOSTIC_LOG="${sysLogs}/verrazzano-system-os-master.log"
    53    ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "vmi-system-os-master-*" -m "verrazzano system opensearchdashboards log" -l -c es-master || echo "failed" > ${POST_DUMP_FAILED_FILE}
    54  }
    55  
    56  dumpCattleSystemPods() {
    57    cd ${GO_REPO_PATH}/verrazzano/platform-operator
    58    local rancherLogs=${VERRAZZANO_INSTALL_LOGS_DIR}/rancher
    59    mkdir -p ${rancherLogs}
    60    export DIAGNOSTIC_LOG="${rancherLogs}/cattle-system-pods.log"
    61    ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -m "cattle system pods" || echo "failed" > ${POST_DUMP_FAILED_FILE}
    62    export DIAGNOSTIC_LOG="${rancherLogs}/rancher.log"
    63    ./scripts/install/k8s-dump-objects.sh -o pods -n cattle-system -r "rancher-*" -m "Rancher logs" -c rancher -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
    64  }
    65  
    66  dumpNginxIngressControllerLogs() {
    67    cd ${GO_REPO_PATH}/verrazzano/platform-operator
    68    local nginxLogs=${VERRAZZANO_INSTALL_LOGS_DIR}/nginx
    69    mkdir -p ${nginxLogs}
    70    export DIAGNOSTIC_LOG="${nginxLogs}/nginx-ingress-controller.log"
    71    ./scripts/install/k8s-dump-objects.sh -o pods -n ingress-nginx -r "nginx-ingress-controller-*" -m "Nginx Ingress Controller" -c controller -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
    72  }
    73  
    74  dumpVerrazzanoPlatformOperatorLogs() {
    75    ## dump out verrazzano-platform-operator logs
    76    local vpoLogs=${VZ_LOGS_ROOT}/verrazzano-platform-operator
    77    mkdir -p ${vpoLogs}
    78    kubectl -n verrazzano-install logs --selector=app=verrazzano-platform-operator > ${vpoLogs}/verrazzano-platform-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
    79    kubectl -n verrazzano-install describe pod --selector=app=verrazzano-platform-operator > ${vpoLogs}/verrazzano-platform-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
    80    echo "verrazzano-platform-operator logs dumped to verrazzano-platform-operator-pod.log"
    81    echo "verrazzano-platform-operator pod description dumped to verrazzano-platform-operator-pod.out"
    82    echo "------------------------------------------"
    83  }
    84  
    85  dumpVerrazzanoApplicationOperatorLogs() {
    86    ## dump out verrazzano-application-operator logs
    87    local vaoLogs=${VZ_LOGS_ROOT}/verrazzano-application-operator
    88    mkdir -p ${vaoLogs}
    89    kubectl -n verrazzano-system logs --selector=app=verrazzano-application-operator > ${vaoLogs}/verrazzano-application-operator-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
    90    kubectl -n verrazzano-system describe pod --selector=app=verrazzano-application-operator > ${vaoLogs}/verrazzano-application-operator-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
    91    echo "verrazzano-application-operator logs dumped to verrazzano-application-operator-pod.log"
    92    echo "verrazzano-application-operator pod description dumped to verrazzano-application-operator-pod.out"
    93    echo "------------------------------------------"
    94  }
    95  
    96  dumpOamKubernetesRuntimeLogs() {
    97    ## dump out oam-kubernetes-runtime logs
    98    local oamLogs=${VZ_LOGS_ROOT}/oam-kubernetes-runtime
    99    mkdir -p ${oamLogs}
   100    kubectl -n verrazzano-system logs --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${oamLogs}/oam-kubernetes-runtime-pod.log --tail -1 || echo "failed" > ${POST_DUMP_FAILED_FILE}
   101    kubectl -n verrazzano-system describe pod --selector=app.kubernetes.io/instance=oam-kubernetes-runtime > ${oamLogs}/oam-kubernetes-runtime-pod.out || echo "failed" > ${POST_DUMP_FAILED_FILE}
   102    echo "verrazzano-application-operator logs dumped to oam-kubernetes-runtime-pod.log"
   103    echo "verrazzano-application-operator pod description dumped to oam-kubernetes-runtime-pod.out"
   104    echo "------------------------------------------"
   105  }
   106  
   107  dumpVerrazzanoApiLogs() {
   108    cd ${GO_REPO_PATH}/verrazzano/platform-operator
   109    local sysLogs=${VERRAZZANO_INSTALL_LOGS_DIR}/system
   110    mkdir -p ${sysLogs}
   111    export DIAGNOSTIC_LOG="${sysLogs}/verrazzano-authproxy.log"
   112    ./scripts/install/k8s-dump-objects.sh -o pods -n verrazzano-system -r "verrazzano-authproxy-*" -m "verrazzano api" -c verrazzano-authproxy -l || echo "failed" > ${POST_DUMP_FAILED_FILE}
   113  }
   114  
   115  if [ -e ${TESTS_EXECUTED_FILE} ]; then
   116    dumpVerrazzanoSystemPods
   117    dumpCattleSystemPods
   118    dumpNginxIngressControllerLogs
   119    dumpVerrazzanoPlatformOperatorLogs
   120    dumpVerrazzanoApplicationOperatorLogs
   121    dumpOamKubernetesRuntimeLogs
   122    dumpVerrazzanoApiLogs
   123  fi