github.phpd.cn/cilium/cilium@v1.6.12/test/provision/k8s_install.sh (about) 1 #!/bin/bash 2 3 set -e 4 HOST=$(hostname) 5 export TOKEN="258062.5d84c017c9b2796c" 6 export CILIUM_CONFIG_DIR="/opt/cilium" 7 export PROVISIONSRC="/tmp/provision/" 8 export SRC_FOLDER="/home/vagrant/go/src/github.com/cilium/cilium" 9 export SYSTEMD_SERVICES="$SRC_FOLDER/contrib/systemd" 10 MOUNT_SYSTEMD="sys-fs-bpf.mount" 11 12 NODE=$1 13 IP=$2 14 K8S_VERSION=$3 15 IPv6=$4 16 CONTAINER_RUNTIME=$5 17 CNI_INTEGRATION=$6 18 # Pinned to the last version of k8s 1.16 branch so we can do 19 # kubectl apply on older k8s test frameworks 20 K8S_KUBECTL_APPLY_FORCE="1.16.15" 21 22 # Kubeadm default parameters 23 export KUBEADM_ADDR='192.168.36.11' 24 export KUBEADM_POD_NETWORK='10.10.0.0' 25 export KUBEADM_POD_CIDR='16' 26 export KUBEADM_SVC_CIDR='10.96.0.0/12' 27 export KUBEADM_CRI_SOCKET="/var/run/dockershim.sock" 28 export KUBEADM_SLAVE_OPTIONS="" 29 export KUBEADM_OPTIONS="" 30 export K8S_FULL_VERSION="" 31 export DNS_DEPLOYMENT="${PROVISIONSRC}/manifest/dns_deployment.yaml" 32 export KUBEDNS_DEPLOYMENT="${PROVISIONSRC}/manifest/kubedns_deployment.yaml" 33 export COREDNS_DEPLOYMENT="${PROVISIONSRC}/manifest/${K8S_VERSION}/coredns_deployment.yaml" 34 if [ ! -f "${COREDNS_DEPLOYMENT}" ]; then 35 export COREDNS_DEPLOYMENT="${PROVISIONSRC}/manifest/coredns_deployment.yaml" 36 fi 37 38 if [ "${CNI_INTEGRATION}" == "flannel" ]; then 39 export KUBEADM_POD_NETWORK="10.244.0.0" 40 fi 41 42 source ${PROVISIONSRC}/helpers.bash 43 44 sudo bash -c "echo MaxSessions 200 >> /etc/ssh/sshd_config" 45 sudo systemctl restart ssh 46 47 retry_function "wget https://get.helm.sh/helm-v2.14.2-linux-amd64.tar.gz" 48 tar xzvf helm-v2.14.2-linux-amd64.tar.gz 49 mv linux-amd64/helm /usr/local/bin/ 50 51 # Install serial ttyS0 server 52 cat <<EOF > /etc/systemd/system/serial-getty@ttyS0.service 53 [Service] 54 ExecStart= 55 ExecStart=/sbin/agetty --autologin root -8 --keep-baud 115200,38400,9600 ttyS0 \$TERM 56 EOF 57 58 systemctl daemon-reload 59 sudo service serial-getty@ttyS0 start 60 61 # TODO: Check if the k8s version is the same 62 if [[ -f "/etc/provision_finished" ]]; then 63 sudo dpkg -l | grep kubelet 64 echo "provision is finished, recompiling" 65 /tmp/provision/compile.sh 66 exit 0 67 fi 68 69 sudo ln -sf $KUBEDNS_DEPLOYMENT $DNS_DEPLOYMENT 70 $PROVISIONSRC/dns.sh 71 72 cat <<EOF > /etc/hosts 73 127.0.0.1 localhost 74 ::1 localhost ip6-localhost ip6-loopback 75 ff02::1 ip6-allnodes 76 ff02::2 ip6-allrouters 77 192.168.36.11 k8s1 78 192.168.36.12 k8s2 79 192.168.36.13 k8s3 80 192.168.36.14 k8s4 81 192.168.36.15 k8s5 82 192.168.36.16 k8s6 83 EOF 84 85 cat <<EOF > /etc/apt/sources.list.d/kubernetes.list 86 deb http://apt.kubernetes.io/ kubernetes-xenial main 87 EOF 88 89 sudo rm /var/lib/apt/lists/lock || true 90 retry_function "wget https://packages.cloud.google.com/apt/doc/apt-key.gpg" 91 apt-key add apt-key.gpg 92 93 KUBEADM_CONFIG_ALPHA1=$(cat <<-EOF 94 apiVersion: kubeadm.k8s.io/v1alpha1 95 kind: MasterConfiguration 96 api: 97 advertiseAddress: "{{ .KUBEADM_ADDR }}" 98 criSocket: "{{ .KUBEADM_CRI_SOCKET }}" 99 kubernetesVersion: "v{{ .K8S_FULL_VERSION }}" 100 token: "{{ .TOKEN }}" 101 networking: 102 podSubnet: "{{ .KUBEADM_POD_NETWORK }}/{{ .KUBEADM_POD_CIDR}}" 103 EOF 104 ) 105 106 KUBEADM_CONFIG="${KUBEADM_CONFIG_ALPHA1}" 107 108 KUBEADM_CONFIG_ALPHA2=$(cat <<-EOF 109 apiVersion: kubeadm.k8s.io/v1alpha2 110 kind: MasterConfiguration 111 api: 112 advertiseAddress: {{ .KUBEADM_ADDR }} 113 bindPort: 6443 114 bootstrapTokens: 115 - groups: 116 - system:bootstrappers:kubeadm:default-node-token 117 token: "{{ .TOKEN }}" 118 kubernetesVersion: "v{{ .K8S_FULL_VERSION }}" 119 networking: 120 dnsDomain: cluster.local 121 podSubnet: "{{ .KUBEADM_POD_NETWORK }}/{{ .KUBEADM_POD_CIDR}}" 122 serviceSubnet: "{{ .KUBEADM_SVC_CIDR }}" 123 nodeRegistration: 124 criSocket: "{{ .KUBEADM_CRI_SOCKET }}" 125 EOF 126 ) 127 128 KUBEADM_CONFIG_ALPHA3=$(cat <<-EOF 129 apiVersion: kubeadm.k8s.io/v1beta1 130 kind: InitConfiguration 131 localAPIEndpoint: 132 advertiseAddress: "{{ .KUBEADM_ADDR }}" 133 bindPort: 6443 134 bootstrapTokens: 135 - groups: 136 - system:bootstrappers:kubeadm:default-node-token 137 token: {{ .TOKEN }} 138 ttl: 24h0m0s 139 usages: 140 - signing 141 - authentication 142 nodeRegistration: 143 criSocket: "{{ .KUBEADM_CRI_SOCKET }}" 144 --- 145 apiVersion: kubeadm.k8s.io/v1beta1 146 kind: ClusterConfiguration 147 kubernetesVersion: "v{{ .K8S_FULL_VERSION }}" 148 networking: 149 dnsDomain: cluster.local 150 podSubnet: "{{ .KUBEADM_POD_NETWORK }}/{{ .KUBEADM_POD_CIDR}}" 151 serviceSubnet: "{{ .KUBEADM_SVC_CIDR }}" 152 EOF 153 ) 154 155 # CRIO bridge disabled. 156 if [[ -f "/etc/cni/net.d/100-crio-bridge.conf" ]]; then 157 echo "Disabling crio CNI bridge" 158 sudo rm -rfv /etc/cni/net.d/100-crio-bridge.conf 159 sudo rm -rfv /etc/cni/net.d/200-loopback.conf || true 160 fi 161 162 # Around the `--ignore-preflight-errors=cri` is used because 163 # /var/run/dockershim.sock is not present (because base image has containerd) 164 # so with that option kubeadm fallback to /var/run/docker.sock 165 # 166 # SystemVerification errors are ignored as net-next VM often triggers them, eg: 167 # [ERROR SystemVerification]: unsupported kernel release: 5.0.0-rc6+ 168 case $K8S_VERSION in 169 "1.8") 170 KUBERNETES_CNI_VERSION="0.5.1" 171 K8S_FULL_VERSION="1.8.14" 172 KUBEADM_OPTIONS="--skip-preflight-checks" 173 KUBEADM_SLAVE_OPTIONS="--skip-preflight-checks" 174 ;; 175 "1.9") 176 KUBERNETES_CNI_VERSION="0.6.0" 177 K8S_FULL_VERSION="1.9.11" 178 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 179 KUBEADM_OPTIONS="--ignore-preflight-errors=cri,SystemVerification" 180 ;; 181 "1.10") 182 KUBERNETES_CNI_VERSION="0.6.0" 183 K8S_FULL_VERSION="1.10.13" 184 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 185 KUBEADM_OPTIONS="--ignore-preflight-errors=cri,SystemVerification" 186 ;; 187 "1.11") 188 KUBERNETES_CNI_VERSION="0.7.5" 189 K8S_FULL_VERSION="1.11.10" 190 KUBEADM_OPTIONS="--ignore-preflight-errors=cri,FileExisting-crictl,SystemVerification" 191 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,FileExisting-crictl,SystemVerification" 192 sudo ln -sf $COREDNS_DEPLOYMENT $DNS_DEPLOYMENT 193 ;; 194 "1.12") 195 KUBERNETES_CNI_VERSION="0.7.5" 196 K8S_FULL_VERSION="1.12.10" 197 KUBEADM_OPTIONS="--ignore-preflight-errors=cri,SystemVerification" 198 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 199 sudo ln -sf $COREDNS_DEPLOYMENT $DNS_DEPLOYMENT 200 KUBEADM_CONFIG="${KUBEADM_CONFIG_ALPHA2}" 201 ;; 202 "1.13") 203 KUBERNETES_CNI_VERSION="0.7.5" 204 K8S_FULL_VERSION="1.13.12" 205 KUBEADM_OPTIONS="--ignore-preflight-errors=cri,SystemVerification" 206 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 207 sudo ln -sf $COREDNS_DEPLOYMENT $DNS_DEPLOYMENT 208 KUBEADM_CONFIG="${KUBEADM_CONFIG_ALPHA3}" 209 ;; 210 "1.14") 211 KUBERNETES_CNI_VERSION="0.7.5" 212 K8S_FULL_VERSION="1.14.10" 213 KUBEADM_OPTIONS="--ignore-preflight-errors=cri" 214 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 215 sudo ln -sf $COREDNS_DEPLOYMENT $DNS_DEPLOYMENT 216 KUBEADM_CONFIG="${KUBEADM_CONFIG_ALPHA3}" 217 ;; 218 "1.15") 219 KUBERNETES_CNI_VERSION="0.7.5" 220 K8S_FULL_VERSION="1.15.12" 221 KUBEADM_OPTIONS="--ignore-preflight-errors=cri" 222 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 223 sudo ln -sf $COREDNS_DEPLOYMENT $DNS_DEPLOYMENT 224 KUBEADM_CONFIG="${KUBEADM_CONFIG_ALPHA3}" 225 ;; 226 "1.16") 227 KUBERNETES_CNI_VERSION="0.8.6" 228 K8S_FULL_VERSION="${K8S_KUBECTL_APPLY_FORCE}" 229 KUBEADM_OPTIONS="--ignore-preflight-errors=cri" 230 KUBEADM_SLAVE_OPTIONS="--discovery-token-unsafe-skip-ca-verification --ignore-preflight-errors=cri,SystemVerification" 231 sudo ln -sf $COREDNS_DEPLOYMENT $DNS_DEPLOYMENT 232 KUBEADM_CONFIG="${KUBEADM_CONFIG_ALPHA3}" 233 ;; 234 esac 235 236 #Install kubernetes 237 case $K8S_VERSION in 238 "1.8"|"1.9"|"1.10"|"1.11"|"1.12"|"1.13"|"1.14"|"1.15"|"1.16") 239 install_k8s_using_packages \ 240 kubernetes-cni=${KUBERNETES_CNI_VERSION}* \ 241 kubelet=${K8S_FULL_VERSION}* \ 242 kubeadm=${K8S_FULL_VERSION}* \ 243 kubectl=${K8S_KUBECTL_APPLY_FORCE}* 244 ;; 245 # "1.16") 246 # install_k8s_using_binary "v${K8S_FULL_VERSION}" "v${KUBERNETES_CNI_VERSION}" 247 # ;; 248 esac 249 250 case $CONTAINER_RUNTIME in 251 "docker") 252 ;; 253 "containerd") 254 KUBEADM_CRI_SOCKET="unix:///run/containerd/containerd.sock" 255 ;; 256 *) 257 echo "Invalid container runtime '${CONTAINER_RUNTIME}'" 258 esac 259 260 if [ "${IPv6}" -eq "1" ]; then 261 KUBEADM_ADDR='[fd04::11]' 262 KUBEADM_POD_NETWORK="fd02::" 263 KUBEADM_POD_CIDR="112" 264 KUBEADM_SVC_CIDR="fd03::/112" 265 fi 266 267 sudo mkdir -p ${CILIUM_CONFIG_DIR} 268 269 sudo cp "$SYSTEMD_SERVICES/$MOUNT_SYSTEMD" /etc/systemd/system/ 270 sudo systemctl enable $MOUNT_SYSTEMD 271 sudo systemctl restart $MOUNT_SYSTEMD 272 sudo rm -rfv /var/lib/kubelet 273 274 #check hostname to know if is kubernetes or runtime test 275 if [[ "${HOST}" == "k8s1" ]]; then 276 277 echo "${KUBEADM_CONFIG}" | envtpl > /tmp/config.yaml 278 279 sudo kubeadm init --config /tmp/config.yaml $KUBEADM_OPTIONS 280 281 mkdir -p /root/.kube 282 sudo cp -i /etc/kubernetes/admin.conf /root/.kube/config 283 sudo chown root:root /root/.kube/config 284 285 sudo -u vagrant mkdir -p /home/vagrant/.kube 286 sudo cp -fi /etc/kubernetes/admin.conf /home/vagrant/.kube/config 287 sudo chown vagrant:vagrant /home/vagrant/.kube/config 288 289 sudo cp -f /etc/kubernetes/admin.conf ${CILIUM_CONFIG_DIR}/kubeconfig 290 kubectl taint nodes --all node-role.kubernetes.io/master- 291 292 sudo systemctl start etcd 293 294 # Install custom DNS deployment 295 kubectl -n kube-system delete -f ${PROVISIONSRC}/manifest/dns_deployment.yaml || true 296 kubectl -n kube-system apply -f ${PROVISIONSRC}/manifest/dns_deployment.yaml 297 298 $PROVISIONSRC/compile.sh 299 else 300 kubeadm join --token=$TOKEN ${KUBEADM_ADDR}:6443 \ 301 ${KUBEADM_SLAVE_OPTIONS} 302 sudo systemctl stop etcd 303 docker pull k8s1:5000/cilium/cilium-dev:latest 304 fi 305 306 # Create world network 307 docker network create --subnet=192.168.9.0/24 outside 308 docker run --net outside --ip 192.168.9.10 --restart=always -d docker.io/cilium/demo-httpd:latest 309 docker run --net outside --ip 192.168.9.11 --restart=always -d docker.io/cilium/demo-httpd:latest 310 311 sudo touch /etc/provision_finished