github.com/k8snetworkplumbingwg/sriov-network-operator@v1.2.1-0.20240408194816-2d2e5a45d453/hack/run-e2e-conformance-virtual-ocp.sh (about) 1 #!/usr/bin/env bash 2 set -xeo pipefail 3 4 OCP_VERSION=${OCP_VERSION:-4.14.0-rc.6} 5 cluster_name=${CLUSTER_NAME:-ocp-virt} 6 domain_name=lab 7 8 api_ip=${API_IP:-192.168.123.253} 9 virtual_router_id=${VIRTUAL_ROUTER_ID:-253} 10 registry="default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}" 11 HOME="/root" 12 13 NUM_OF_WORKERS=${NUM_OF_WORKERS:-3} 14 total_number_of_nodes=$((1 + NUM_OF_WORKERS)) 15 16 if [ "$NUM_OF_WORKERS" -lt 3 ]; then 17 echo "Min number of workers is 3" 18 exit 1 19 fi 20 21 here="$(dirname "$(readlink --canonicalize "${BASH_SOURCE[0]}")")" 22 root="$(readlink --canonicalize "$here/..")" 23 24 source $here/run-e2e-conformance-common 25 26 check_requirements() { 27 for cmd in kcli virsh podman make go jq base64 tar; do 28 if ! command -v "$cmd" &> /dev/null; then 29 echo "$cmd is not available" 30 exit 1 31 fi 32 done 33 return 0 34 } 35 36 echo "## checking requirements" 37 check_requirements 38 echo "## delete existing cluster name $cluster_name" 39 kcli delete cluster $cluster_name -y 40 kcli delete network $cluster_name -y 41 42 function cleanup { 43 kcli delete cluster $cluster_name -y 44 kcli delete network $cluster_name -y 45 } 46 47 if [ -z $SKIP_DELETE ]; then 48 trap cleanup EXIT 49 fi 50 51 kcli create network -c 192.168.123.0/24 ocp 52 kcli create network -c 192.168.${virtual_router_id}.0/24 --nodhcp -i $cluster_name 53 54 cat <<EOF > ./${cluster_name}-plan.yaml 55 tag: $OCP_VERSION 56 ctlplane_memory: 32768 57 worker_memory: 8192 58 pool: default 59 disk_size: 50 60 network: ocp 61 api_ip: $api_ip 62 virtual_router_id: $virtual_router_id 63 domain: $domain_name 64 ctlplanes: 1 65 workers: $NUM_OF_WORKERS 66 machine: q35 67 network_type: OVNKubernetes 68 pull_secret: /root/openshift_pull.json 69 vmrules: 70 - $cluster_name-worker-.*: 71 nets: 72 - name: ocp 73 numa: 0 74 - name: $cluster_name 75 type: igb 76 vfio: true 77 noconf: true 78 numa: 0 79 - name: $cluster_name 80 type: igb 81 vfio: true 82 noconf: true 83 numa: 1 84 numcpus: 6 85 numa: 86 - id: 0 87 vcpus: 0,2,4 88 memory: 4096 89 - id: 1 90 vcpus: 1,3,5 91 memory: 4096 92 93 EOF 94 95 kcli create cluster openshift --paramfile ./${cluster_name}-plan.yaml $cluster_name 96 97 export KUBECONFIG=$HOME/.kcli/clusters/$cluster_name/auth/kubeconfig 98 export PATH=$PWD:$PATH 99 100 # w/a for the registry pull 101 kubectl create clusterrolebinding authenticated-registry-viewer --clusterrole registry-viewer --group system:unauthenticated 102 103 ATTEMPTS=0 104 MAX_ATTEMPTS=72 105 ready=false 106 sleep_time=10 107 108 until $ready || [ $ATTEMPTS -eq $MAX_ATTEMPTS ] 109 do 110 echo "waiting for cluster to be ready" 111 if [ `kubectl get node | grep Ready | wc -l` == $total_number_of_nodes ]; then 112 echo "cluster is ready" 113 ready=true 114 else 115 echo "cluster is not ready yet" 116 sleep $sleep_time 117 fi 118 ATTEMPTS=$((ATTEMPTS+1)) 119 done 120 121 if ! $ready; then 122 echo "Timed out waiting for cluster to be ready" 123 kubectl get nodes 124 exit 1 125 fi 126 127 echo "## label cluster workers as sriov capable" 128 for ((num=0; num<NUM_OF_WORKERS; num++)) 129 do 130 kubectl label node $cluster_name-worker-$num.$domain_name feature.node.kubernetes.io/network-sriov.capable=true --overwrite 131 done 132 133 controller_ip=`kubectl get node -o wide | grep ctlp | awk '{print $6}'` 134 135 if [ `cat /etc/hosts | grep ${api_ip} | grep "default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}" | wc -l` == 0 ]; then 136 echo "adding registry to hosts" 137 sed -i "s/${api_ip}/${api_ip} default-route-openshift-image-registry.apps.${cluster_name}.${domain_name}/g" /etc/hosts 138 fi 139 140 141 cat <<EOF | oc apply -f - 142 apiVersion: v1 143 kind: PersistentVolume 144 metadata: 145 name: registry-pv 146 spec: 147 capacity: 148 storage: 60Gi 149 volumeMode: Filesystem 150 accessModes: 151 - ReadWriteMany 152 persistentVolumeReclaimPolicy: Delete 153 storageClassName: registry-local-storage 154 local: 155 path: /mnt/ 156 nodeAffinity: 157 required: 158 nodeSelectorTerms: 159 - matchExpressions: 160 - key: kubernetes.io/hostname 161 operator: In 162 values: 163 - ${cluster_name}-ctlplane-0.${domain_name} 164 --- 165 apiVersion: v1 166 kind: PersistentVolumeClaim 167 metadata: 168 name: registry-pv-claim 169 namespace: openshift-image-registry 170 spec: 171 accessModes: 172 - ReadWriteMany 173 volumeMode: Filesystem 174 resources: 175 requests: 176 storage: 60Gi 177 storageClassName: registry-local-storage 178 EOF 179 180 kubectl patch configs.imageregistry.operator.openshift.io/cluster --patch '{"spec":{"defaultRoute":true,"storage":{"emptyDir": null,"pvc":{"claim":"registry-pv-claim"}},"topologySpreadConstraints":[],"rolloutStrategy":"Recreate","tolerations":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]}}' --type=merge 181 kubectl patch ingresscontrollers.operator.openshift.io/default -n openshift-ingress-operator --patch '{"spec":{"replicas": 1}}' --type=merge 182 183 export ADMISSION_CONTROLLERS_ENABLED=true 184 export SKIP_VAR_SET="" 185 export NAMESPACE="openshift-sriov-network-operator" 186 export OPERATOR_NAMESPACE=$NAMESPACE 187 export MULTUS_NAMESPACE="openshift-multus" 188 export OPERATOR_EXEC=kubectl 189 export CLUSTER_TYPE=openshift 190 export DEV_MODE=TRUE 191 export CLUSTER_HAS_EMULATED_PF=TRUE 192 193 export SRIOV_NETWORK_OPERATOR_IMAGE="$registry/$NAMESPACE/sriov-network-operator:latest" 194 export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="$registry/$NAMESPACE/sriov-network-config-daemon:latest" 195 export SRIOV_NETWORK_WEBHOOK_IMAGE="$registry/$NAMESPACE/sriov-network-operator-webhook:latest" 196 197 envsubst< deploy/namespace.yaml | kubectl apply -f - 198 199 echo "## build operator image" 200 podman build -t "${SRIOV_NETWORK_OPERATOR_IMAGE}" -f "${root}/Dockerfile" "${root}" 201 202 echo "## build daemon image" 203 podman build -t "${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}" -f "${root}/Dockerfile.sriov-network-config-daemon" "${root}" 204 205 echo "## build webhook image" 206 podman build -t "${SRIOV_NETWORK_WEBHOOK_IMAGE}" -f "${root}/Dockerfile.webhook" "${root}" 207 208 echo "## wait for registry to be available" 209 kubectl wait configs.imageregistry.operator.openshift.io/cluster --for=condition=Available --timeout=120s 210 211 dockercgf=`kubectl -n ${NAMESPACE} get sa builder -oyaml | grep imagePullSecrets -A 1 | grep -o "builder-.*"` 212 auth=`kubectl -n ${NAMESPACE} get secret ${dockercgf} -ojson | jq '.data.".dockercfg"'` 213 auth="${auth:1:-1}" 214 auth=`echo ${auth} | base64 -d` 215 echo ${auth} > registry-login.conf 216 217 internal_registry="image-registry.openshift-image-registry.svc:5000" 218 pass=$( jq .\"$internal_registry\".password registry-login.conf ) 219 podman login -u serviceaccount -p ${pass:1:-1} $registry --tls-verify=false 220 221 MAX_RETRIES=20 222 DELAY_SECONDS=10 223 retry_push() { 224 local command="podman push --tls-verify=false $@" 225 local retries=0 226 227 until [ $retries -ge $MAX_RETRIES ]; do 228 $command && break 229 retries=$((retries+1)) 230 echo "Command failed. Retrying... (Attempt $retries/$MAX_RETRIES)" 231 sleep $DELAY_SECONDS 232 done 233 234 if [ $retries -eq $MAX_RETRIES ]; then 235 echo "Max retries reached. Exiting..." 236 exit 1 237 fi 238 } 239 240 retry_push "${SRIOV_NETWORK_OPERATOR_IMAGE}" 241 podman rmi -fi ${SRIOV_NETWORK_OPERATOR_IMAGE} 242 retry_push "${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE}" 243 podman rmi -fi ${SRIOV_NETWORK_CONFIG_DAEMON_IMAGE} 244 retry_push "${SRIOV_NETWORK_WEBHOOK_IMAGE}" 245 podman rmi -fi ${SRIOV_NETWORK_WEBHOOK_IMAGE} 246 247 podman logout $registry 248 249 echo "## apply CRDs" 250 kubectl apply -k $root/config/crd 251 252 253 cat <<EOF | kubectl apply -f - 254 apiVersion: sriovnetwork.openshift.io/v1 255 kind: SriovOperatorConfig 256 metadata: 257 name: default 258 namespace: ${NAMESPACE} 259 spec: 260 disableDrain: false 261 enableInjector: true 262 enableOperatorWebhook: true 263 logLevel: 2 264 EOF 265 266 export SRIOV_NETWORK_OPERATOR_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator:latest" 267 export SRIOV_NETWORK_CONFIG_DAEMON_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-config-daemon:latest" 268 export SRIOV_NETWORK_WEBHOOK_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-operator-webhook:latest" 269 270 if [[ -v LOCAL_SRIOV_CNI_IMAGE ]]; then 271 podman_tag_and_push ${LOCAL_SRIOV_CNI_IMAGE} "$registry/$NAMESPACE/sriov-cni:latest" 272 export SRIOV_CNI_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-cni:latest" 273 fi 274 275 if [[ -v LOCAL_SRIOV_DEVICE_PLUGIN_IMAGE ]]; then 276 podman_tag_and_push ${LOCAL_SRIOV_DEVICE_PLUGIN_IMAGE} "$registry/$NAMESPACE/sriov-network-device-plugin:latest" 277 export SRIOV_DEVICE_PLUGIN_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/sriov-network-device-plugin:latest" 278 fi 279 280 if [[ -v LOCAL_NETWORK_RESOURCES_INJECTOR_IMAGE ]]; then 281 podman_tag_and_push ${LOCAL_NETWORK_RESOURCES_INJECTOR_IMAGE} "$registry/$NAMESPACE/network-resources-injector:latest" 282 export NETWORK_RESOURCES_INJECTOR_IMAGE="image-registry.openshift-image-registry.svc:5000/$NAMESPACE/network-resources-injector:latest" 283 fi 284 285 echo "## deploying SRIOV Network Operator" 286 hack/deploy-setup.sh $NAMESPACE 287 288 echo "## wait for sriov operator to be ready" 289 hack/deploy-wait.sh 290 291 if [ -z $SKIP_TEST ]; then 292 echo "## run sriov e2e conformance tests" 293 294 if [[ -v TEST_REPORT_PATH ]]; then 295 export JUNIT_OUTPUT="${root}/${TEST_REPORT_PATH}/conformance-test-report" 296 fi 297 298 # Disable exit on error temporarily to gather cluster information 299 set +e 300 SUITE=./test/conformance hack/run-e2e-conformance.sh 301 TEST_EXITE_CODE=$? 302 set -e 303 304 if [[ -v TEST_REPORT_PATH ]]; then 305 kubectl cluster-info dump --namespaces ${NAMESPACE},${MULTUS_NAMESPACE} --output-directory "${root}/${TEST_REPORT_PATH}/cluster-info" 306 fi 307 308 if [[ $TEST_EXITE_CODE -ne 0 ]]; then 309 exit $TEST_EXITE_CODE 310 fi 311 fi