volcano.sh/volcano@v1.9.0/hack/local-up-cluster.sh (about)

     1  #!/bin/bash
     2  
     3  # Copyright 2020 The Volcano Authors.
     4  
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  if [ -z $GOPATH ]; then
    18      echo "Please set GOPATH to start the cluster :)"
    19      exit 1
    20  fi
    21  
    22  K8S_HOME=$GOPATH/src/k8s.io/kubernetes
    23  VC_HOME=$GOPATH/src/volcano.sh/volcano
    24  
    25  CERT_DIR=${VC_HOME}/volcano/certs
    26  
    27  LOCALHOST="127.0.0.1"
    28  API_PORT="6443"
    29  
    30  ROOT_CA=
    31  ROOT_CA_KEY=
    32  
    33  SERVICE_ACCOUNT_KEY=${VC_HOME}/volcano/certs/service-account.key
    34  
    35  function install_tools {
    36      for d in work logs certs config static-pods
    37      do
    38          mkdir -p ${VC_HOME}/volcano/$d
    39      done
    40  
    41      go get -u github.com/cloudflare/cfssl/cmd/...
    42  }
    43  
    44  function build_binaries {
    45      echo "Building Kubernetes ...... "
    46      echo "$(
    47          cd $K8S_HOME
    48          make kubectl kube-controller-manager kube-apiserver kubelet kube-proxy
    49      )"
    50  
    51      echo "Building Volcano ...... "
    52      echo "$(
    53          cd $VC_HOME
    54          make
    55      )"
    56  }
    57  
    58  function create_certkey {
    59      local name=$1
    60      local cn=$2
    61      local org=$3
    62  
    63      local hosts=""
    64      local SEP=""
    65  
    66      shift 3
    67      while [ -n "${1:-}" ]; do
    68          hosts+="${SEP}\"$1\""
    69          SEP=","
    70          shift 1
    71      done
    72  
    73      echo '{"CN":"'${cn}'","hosts":['${hosts}'],"key":{"algo":"rsa","size":2048},"names":[{"O":"'${org}'"}]}' \
    74          | cfssl gencert -ca=${CERT_DIR}/root.pem -ca-key=${CERT_DIR}/root-key.pem -config=${CERT_DIR}/root-ca-config.json - \
    75          | cfssljson -bare ${CERT_DIR}/$name
    76  }
    77  
    78  function generate_cert_files {
    79      openssl genrsa -out "${SERVICE_ACCOUNT_KEY}" 2048 2>/dev/null
    80  
    81      echo '{"signing":{"default":{"expiry":"8760h","usages":["signing","key encipherment","server auth","client auth"]}}}' \
    82          > ${CERT_DIR}/root-ca-config.json
    83  
    84      echo '{"CN":"volcano","key":{"algo":"rsa","size":2048},"names":[{"O":"volcano"}]}' | cfssl gencert -initca - \
    85          | cfssljson -bare ${CERT_DIR}/root
    86  
    87      create_certkey "kube-apiserver" "kubernetes.default" "volcano" "kubernetes.default.svc" "localhost" "127.0.0.1" "10.0.0.1"
    88      create_certkey "admin" "system:admin" "system:masters"
    89      create_certkey "kube-proxy" "system:kube-proxy" "volcano"
    90      create_certkey "kubelet" "system:node:127.0.0.1" "system:nodes"
    91      create_certkey "controller-manager" "system:kube-controller-manager" "volcano"
    92      create_certkey "scheduler" "system:scheduler" "volcano"
    93      create_certkey "webhook-manager" "volcano-webhook-manager" "volcano" "localhost" "127.0.0.1"
    94  
    95      write_kube_config "controller-manager"
    96      write_kube_config "scheduler"
    97      write_kube_config "kubelet"
    98      write_kube_config "admin"
    99  }
   100  
   101  function write_kube_config {
   102      local name=$1
   103  
   104      kubectl config set-cluster local --server=https://${LOCALHOST}:6443 --certificate-authority=${CERT_DIR}/root.pem \
   105              --kubeconfig ${VC_HOME}/volcano/config/${name}.config
   106  
   107      kubectl config set-credentials myself --client-key=${CERT_DIR}/${name}-key.pem \
   108              --client-certificate=${CERT_DIR}/${name}.pem --kubeconfig ${VC_HOME}/volcano/config/${name}.config
   109  
   110      kubectl config set-context local --cluster=local --user=myself --kubeconfig ${VC_HOME}/volcano/config/${name}.config
   111      kubectl config use-context local --kubeconfig ${VC_HOME}/volcano/config/${name}.config
   112  
   113      # kubectl --kubeconfig ./controller-manager.config config view --minify --flatten > ${TOP_DIR}/volcano/config/controller-manager.config
   114  }
   115  
   116  function start_etcd {
   117      nohup ${K8S_HOME}/third_party/etcd/etcd \
   118          --advertise-client-urls="http://${LOCALHOST}:2379" \
   119          --listen-client-urls="http://0.0.0.0:2379" \
   120          --data-dir=${VC_HOME}/volcano/work/etcd \
   121          --debug > ${VC_HOME}/volcano/logs/etcd.log 2>&1 &
   122  }
   123  
   124  function start_apiserver {
   125      nohup ${K8S_HOME}/_output/bin/kube-apiserver \
   126          --logtostderr="false" \
   127          --log-file=${VC_HOME}/volcano/logs/kube-apiserver.log \
   128          --service-account-key-file=${SERVICE_ACCOUNT_KEY} \
   129          --etcd-servers="http://${LOCALHOST}:2379" \
   130          --cert-dir=${CERT_DIR} \
   131          --tls-cert-file=${CERT_DIR}/kube-apiserver.pem \
   132          --tls-private-key-file=${CERT_DIR}/kube-apiserver-key.pem \
   133          --client-ca-file=${CERT_DIR}/root.pem \
   134          --kubelet-client-certificate=${CERT_DIR}/kube-apiserver.pem \
   135          --kubelet-client-key=${CERT_DIR}/kube-apiserver-key.pem \
   136          --insecure-bind-address=0.0.0.0 \
   137          --secure-port=${API_PORT} \
   138          --storage-backend=etcd3 \
   139          --feature-gates=AllAlpha=false \
   140          --service-cluster-ip-range=10.0.0.0/24 &
   141  }
   142  
   143  function start_controller_manager {
   144      nohup ${VC_HOME}/_output/bin/vc-controller-manager \
   145          --v=3 \
   146          --logtostderr=false \
   147          --log-file=${VC_HOME}/volcano/logs/vc-controller-manager.log \
   148          --scheduler-name=default-scheduler \
   149          --kubeconfig=${VC_HOME}/volcano/config/controller-manager.config &
   150  
   151      nohup ${K8S_HOME}/_output/bin/kube-controller-manager \
   152          --v=3 \
   153          --logtostderr="false" \
   154          --log-file=${VC_HOME}/volcano/logs/kube-controller-manager.log \
   155          --service-account-private-key-file=${SERVICE_ACCOUNT_KEY} \
   156          --root-ca-file=${CERT_DIR}/root.pem \
   157          --cluster-signing-cert-file=${CERT_DIR}/root.pem \
   158          --cluster-signing-key-file=${CERT_DIR}/root-key.pem \
   159          --enable-hostpath-provisioner=false \
   160          --pvclaimbinder-sync-period=15s \
   161          --feature-gates=AllAlpha=false \
   162          --kubeconfig ${VC_HOME}/volcano/config/controller-manager.config \
   163          --use-service-account-credentials \
   164          --controllers=* \
   165          --leader-elect=false \
   166          --cert-dir=${CERT_DIR} &
   167  }
   168  
   169  function start_kubelet {
   170      nohup ${K8S_HOME}/_output/bin/kubelet \
   171          --logtostderr="false" \
   172          --log-file=${VC_HOME}/volcano/logs/kubelet.log \
   173          --chaos-chance=0.0 \
   174          --container-runtime=docker \
   175          --hostname-override=${LOCALHOST} \
   176          --address=${LOCALHOST} \
   177          --kubeconfig ${VC_HOME}/volcano/config/kubelet.config \
   178          --feature-gates=AllAlpha=false \
   179          --cpu-cfs-quota=true \
   180          --enable-controller-attach-detach=true \
   181          --cgroups-per-qos=true \
   182          --cgroup-driver=cgroupfs \
   183          --eviction-hard='memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%' \
   184          --eviction-pressure-transition-period=1m \
   185          --pod-manifest-path=${VC_HOME}/volcano/static-pods \
   186          --fail-swap-on=false \
   187          --authorization-mode=Webhook \
   188          --authentication-token-webhook \
   189          --client-ca-file=${CERT_DIR}/root.pem \
   190          --cluster-dns=10.0.0.10 \
   191          --cluster-domain=cluster.local \
   192          --runtime-request-timeout=2m \
   193          --port=10250 &
   194  }
   195  
   196  function start_volcano_scheduler {
   197      nohup ${VC_HOME}/_output/bin/vc-scheduler \
   198          --v=4 \
   199          --logtostderr=false \
   200          --listen-address=":8090" \
   201          --log-file=${VC_HOME}/volcano/logs/vc-scheduler.log \
   202          --scheduler-name=default-scheduler \
   203          --kubeconfig=${VC_HOME}/volcano/config/scheduler.config &
   204  }
   205  
   206  function start_volcano_admission {
   207  	nohup ${VC_HOME}/_output/bin/vc-webhook-manager \
   208  		-v 3 \
   209          --logtostderr=false \
   210          --log-file=${VC_HOME}/volcano/logs/vc-webhook-manager.log \
   211  		--ca-cert-file ${CERT_DIR}/root.pem \
   212          --scheduler-name=default-scheduler \
   213  		--kubeconfig ${VC_HOME}/volcano/config/admin.config \
   214  		--tls-cert-file ${CERT_DIR}/webhook-manager.pem \
   215  		--tls-private-key-file ${CERT_DIR}/webhook-manager-key.pem \
   216  		--webhook-url https://127.0.0.1:443 &
   217  }
   218  
   219  function cleanup_cluster {
   220      killall -9 etcd kube-apiserver kube-controller-manager kubelet vc-controller-manager vc-scheduler vc-webhook-manager
   221      rm -rf ${VC_HOME}/volcano
   222  
   223      # Waiting for TIME_WAIT
   224      sleep 6
   225  }
   226  
   227  function apply_volcano_crds {
   228      kubectl get ns --kubeconfig ${VC_HOME}/volcano/config/admin.config
   229  
   230      for crd in scheduling_v1beta1_podgroup.yaml scheduling_v1beta1_queue.yaml bus_v1alpha1_command.yaml batch_v1alpha1_job.yaml
   231      do
   232          kubectl apply -f ${VC_HOME}/installer/helm/chart/volcano/templates/$crd --kubeconfig ${VC_HOME}/volcano/config/admin.config
   233      done
   234  }
   235  
   236  cleanup_cluster
   237  
   238  install_tools
   239  
   240  # build_binaries
   241  
   242  generate_cert_files
   243  
   244  start_etcd
   245  start_apiserver
   246  apply_volcano_crds
   247  start_controller_manager
   248  start_volcano_admission
   249  start_volcano_scheduler
   250  start_kubelet
   251  
   252