github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/deploy/dev/k8s/deploy_ci.sh (about)

     1  #!/bin/bash
     2  
     3  set -e
     4  
     5  # This script deploys AIS in an existing k8s cluster from within another node in the cluster
     6  # Gitlab Kubernetes runners have access to kubectl but not minikube cmd line
     7  # Each pod deployed will have its own clusterIP service to communicate without relying on host network 
     8  
     9  source ../utils.sh
    10  
    11  check_variable_set() {
    12      local var_name="$1"
    13      local var_value="${!var_name}"
    14      if [[ -z "$var_value" ]]; then
    15          echo "Error: Required variable '$var_name' is not set."
    16          exit 1
    17      fi
    18  }
    19  
    20  # List of required variables
    21  required_vars=("AISNODE_IMAGE" "NUM_TARGET" "NUM_PROXY")
    22  
    23  for var in "${required_vars[@]}"; do
    24      check_variable_set "$var"
    25  done
    26  
    27  is_number ${NUM_TARGET}
    28  is_number ${NUM_PROXY}
    29  if [[ ${NUM_PROXY} -lt 1 ]]; then
    30    exit_error "${NUM_PROXY} is less than 1"
    31  fi
    32  
    33  export TEST_FSPATH_COUNT=${FS_CNT:-3}
    34  PRIMARY_PORT=8080
    35  export AIS_BACKEND_PROVIDERS=$PROVIDERS
    36  INSTANCE=0
    37  
    38  if [[ -n $HTTPS ]]; then
    39    source utils/create_certs.sh
    40    export SCHEME="https://"
    41    export AIS_USE_HTTPS=true
    42    export AIS_SKIP_VERIFY_CRT=true
    43    export AIS_SERVER_CRT="/var/certs/tls.crt"
    44    export AIS_SERVER_KEY="/var/certs/tls.key"
    45    export PROTOCOL="HTTPS"
    46  else
    47    export SCHEME="http://"
    48    export AIS_USE_HTTPS=false
    49    export AIS_SKIP_VERIFY_CRT=false
    50    export AIS_SERVER_CRT=""
    51    export AIS_SERVER_KEY=""
    52    export PROTOCOL="HTTP"
    53  fi
    54  
    55  export AIS_LOG_DIR="/tmp/ais/log"
    56  export AIS_PRIMARY_HOST="ais-proxy-0.default.svc.cluster.local"
    57  export AIS_PRIMARY_URL="${SCHEME}${AIS_PRIMARY_HOST}:${PRIMARY_PORT}"
    58  
    59  if [[ ",$PROVIDERS," == *",gcp,"* ]]; then
    60    echo "Creating GCP credentials secret"
    61    kubectl delete secret gcp-creds || true
    62    kubectl create secret generic gcp-creds --from-file=creds.json=$GOOGLE_APPLICATION_CREDENTIALS
    63  fi
    64  
    65  # Necessary in case another run with different spec is aborted 
    66  echo "Cleaning up any previous deployment"
    67  source utils/cleanup_k8s_ci.sh
    68  
    69  echo "Starting AIS deployment..."
    70  echo "Starting primary proxy deployment..."
    71  for i in $(seq 0 $((NUM_PROXY-1))); do
    72    export POD_NAME="ais-proxy-${i}"
    73    export PORT=$((PRIMARY_PORT+i))
    74    export INSTANCE=${INSTANCE}
    75    # Use a clusterIP service for each pod for communication without host network
    76    export HOSTNAME_LIST="$POD_NAME.default.svc.cluster.local"
    77    envsubst < kube_templates/ci_proxy.yml | kubectl delete -f - || true 
    78    envsubst < kube_templates/ci_proxy.yml | kubectl apply -f -
    79    INSTANCE=$((INSTANCE+1))
    80  done
    81  
    82  echo "Waiting for the primary proxy to be ready..."
    83  kubectl wait --for="condition=ready" --timeout=2m pod ais-proxy-0
    84  
    85  echo "Starting target deployment..."
    86  
    87  INSTANCE=0
    88  for i in $(seq 0 $((NUM_TARGET-1))); do
    89    export POD_NAME="ais-target-${i}"
    90    export PORT=$((9090+i))
    91    export PORT_INTRA_CONTROL=$((9080+i))
    92    export PORT_INTRA_DATA=$((10080+i))
    93    export TARGET_POS_NUM=$i
    94    export INSTANCE=${INSTANCE}
    95    export HOSTNAME_LIST="$POD_NAME.default.svc.cluster.local"
    96    
    97    envsubst < kube_templates/ci_target.yml | kubectl delete -f - || true
    98    envsubst < kube_templates/ci_target.yml | kubectl create -f -
    99    INSTANCE=$((INSTANCE+1))
   100  done
   101  
   102  echo "Waiting for the targets to be ready..."
   103  kubectl wait --for="condition=ready" --timeout=2m pods -l type=ais-target
   104  
   105  echo "List of running pods"
   106  kubectl get pods -o wide