sigs.k8s.io/kueue@v0.6.2/hack/multikueue-e2e-test.sh (about)

     1  #!/usr/bin/env bash
     2  
     3  # Copyright 2023 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  set -o errexit
    18  set -o nounset
    19  set -o pipefail
    20  
    21  SOURCE_DIR="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
    22  ROOT_DIR=$SOURCE_DIR/..
    23  export KUSTOMIZE=$ROOT_DIR/bin/kustomize
    24  export GINKGO=$ROOT_DIR/bin/ginkgo
    25  export KIND=$ROOT_DIR/bin/kind
    26  export YQ=$ROOT_DIR/bin/yq
    27  export E2E_TEST_IMAGE=gcr.io/k8s-staging-perf-tests/sleep:v0.1.0
    28  export MANAGER_KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME}-manager
    29  export WORKER1_KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME}-worker1
    30  export WORKER2_KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME}-worker2
    31  
    32  export JOBSET_MANIFEST=https://github.com/kubernetes-sigs/jobset/releases/download/${JOBSET_VERSION}/manifests.yaml
    33  export JOBSET_IMAGE=registry.k8s.io/jobset/jobset:${JOBSET_VERSION}
    34  export JOBSET_CRDS=${ROOT_DIR}/dep-crds/jobset-operator/
    35  
    36  source ${SOURCE_DIR}/e2e-common.sh
    37  
    38  function cleanup {
    39      if [ $CREATE_KIND_CLUSTER == 'true' ]
    40      then
    41          if [ ! -d "$ARTIFACTS" ]; then
    42              mkdir -p "$ARTIFACTS"
    43          fi
    44  
    45          cluster_cleanup $MANAGER_KIND_CLUSTER_NAME
    46          cluster_cleanup $WORKER1_KIND_CLUSTER_NAME
    47          cluster_cleanup $WORKER2_KIND_CLUSTER_NAME
    48      fi
    49      #do the image restore here for the case when an error happened during deploy
    50      restore_managers_image
    51  }
    52  
    53  
    54  function startup {
    55      if [ $CREATE_KIND_CLUSTER == 'true' ]
    56      then
    57          if [ ! -d "$ARTIFACTS" ]; then
    58              mkdir -p "$ARTIFACTS"
    59          fi
    60  
    61          cluster_create $MANAGER_KIND_CLUSTER_NAME $SOURCE_DIR/multikueue/manager-cluster.kind.yaml
    62  
    63      # NOTE: for local setup, make sure that your firewall allows tcp from manager to the GW ip
    64      # eg. ufw `sudo ufw allow from 172.18.0.0/16 proto tcp to 172.18.0.1`
    65      #
    66      # eg. iptables    `sudo iptables --append INPUT --protocol tcp --src 172.18.0.0/16 --dst 172.18.0.1 --jump ACCEPT
    67      #                  sudo iptables --append OUTPUT --protocol tcp --src 172.18.0.1 --dst 172.18.0./0/16 --jump ACCEPT`
    68  
    69      # have the worker forward the api to the docker gateway address instead of lo
    70      export GW=$(docker inspect ${MANAGER_KIND_CLUSTER_NAME}-control-plane -f '{{.NetworkSettings.Networks.kind.Gateway}}')
    71      $YQ e '.networking.apiServerAddress=env(GW)'  $SOURCE_DIR/multikueue/worker-cluster.kind.yaml > $ARTIFACTS/worker-cluster.yaml
    72  
    73      cluster_create $WORKER1_KIND_CLUSTER_NAME $ARTIFACTS/worker-cluster.yaml
    74      cluster_create $WORKER2_KIND_CLUSTER_NAME $ARTIFACTS/worker-cluster.yaml
    75  
    76      fi
    77  }
    78  
    79  
    80  #$1 - cluster name
    81  function install_jobset {
    82      cluster_kind_load_image ${1} ${JOBSET_IMAGE}
    83      kubectl config use-context kind-${1}
    84      kubectl apply --server-side -f ${JOBSET_MANIFEST}
    85  }
    86  
    87  function kind_load {
    88      if [ $CREATE_KIND_CLUSTER == 'true' ]
    89      then
    90          docker pull $E2E_TEST_IMAGE
    91          cluster_kind_load $MANAGER_KIND_CLUSTER_NAME
    92          cluster_kind_load $WORKER1_KIND_CLUSTER_NAME
    93          cluster_kind_load $WORKER2_KIND_CLUSTER_NAME 
    94  
    95  
    96      # JOBSET SETUP
    97      # MANAGER
    98      # Only install the CRDs and not the controller to be able to
    99      # have JobSets admitted without execution in the manager cluster.
   100      kubectl config use-context kind-${MANAGER_KIND_CLUSTER_NAME}
   101      kubectl apply --server-side -f ${JOBSET_CRDS}/*
   102  
   103      #WORKERS
   104      docker pull registry.k8s.io/jobset/jobset:$JOBSET_VERSION
   105      install_jobset $WORKER1_KIND_CLUSTER_NAME
   106      install_jobset $WORKER2_KIND_CLUSTER_NAME
   107      fi
   108  }
   109  
   110  function kueue_deploy {
   111      (cd config/components/manager && $KUSTOMIZE edit set image controller=$IMAGE_TAG)
   112  
   113      cluster_kueue_deploy $MANAGER_KIND_CLUSTER_NAME
   114      cluster_kueue_deploy $WORKER1_KIND_CLUSTER_NAME
   115      cluster_kueue_deploy $WORKER2_KIND_CLUSTER_NAME
   116  }
   117  
   118  function prepare_secrets {
   119      kubectl config use-context kind-${WORKER1_KIND_CLUSTER_NAME}
   120      source ${SOURCE_DIR}/create-multikueue-kubeconfig.sh ${ARTIFACTS}/worker1.kubeconfig
   121  
   122      kubectl config use-context kind-${WORKER2_KIND_CLUSTER_NAME}
   123      source ${SOURCE_DIR}/create-multikueue-kubeconfig.sh ${ARTIFACTS}/worker2.kubeconfig
   124  
   125      kubectl config use-context kind-${MANAGER_KIND_CLUSTER_NAME}
   126      kubectl create secret generic multikueue1 -n kueue-system --from-file=kubeconfig=${ARTIFACTS}/worker1.kubeconfig
   127      kubectl create secret generic multikueue2 -n kueue-system --from-file=kubeconfig=${ARTIFACTS}/worker2.kubeconfig
   128  }
   129  
   130  trap cleanup EXIT
   131  startup
   132  kind_load
   133  kueue_deploy 
   134  prepare_secrets
   135  
   136  $GINKGO $GINKGO_ARGS --junit-report=junit.xml --output-dir=$ARTIFACTS -v ./test/e2e/multikueue/...