github.phpd.cn/cilium/cilium@v1.6.12/tests/k8s/Vagrantfile

#!/usr/bin/env bash
# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.require_version ">= 1.8.3"

$build_docker_image = <<SCRIPT
# This may be removed when the box images are based on Ubuntu 17.10+.
sudo bash -c "cat <<EOF > /etc/apt/sources.list
deb http://old-releases.ubuntu.com/ubuntu/ yakkety main restricted
deb http://old-releases.ubuntu.com/ubuntu/ yakkety-updates main restricted
deb http://old-releases.ubuntu.com/ubuntu/ yakkety universe
deb http://old-releases.ubuntu.com/ubuntu/ yakkety-updates universe
deb http://old-releases.ubuntu.com/ubuntu/ yakkety multiverse
deb http://old-releases.ubuntu.com/ubuntu/ yakkety-updates multiverse
deb http://old-releases.ubuntu.com/ubuntu/ yakkety-backports main restricted universe multiverse
EOF
"
curl -s https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo bash -c "cat <<EOF > /etc/apt/sources.list.d/docker.list
deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
EOF
"
sudo apt-get -qq update && sudo apt-get -qq install -y apt-transport-https docker-ce
sudo usermod -aG docker vagrant
docker -v
certs_dir="/home/vagrant/go/src/github.com/cilium/cilium/tests/k8s/cluster/certs"
cd /home/vagrant/go/src/github.com/cilium/cilium/
docker run -d -p 5000:5000 --name registry -v ${certs_dir}:/certs \
        -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/kubernetes.pem \
        -e REGISTRY_HTTP_TLS_KEY=/certs/kubernetes-key.pem \
        registry:2
make docker-image
docker tag cilium/cilium:${DOCKER_IMAGE_TAG} localhost:5000/cilium:${DOCKER_IMAGE_TAG}
docker push localhost:5000/cilium:${DOCKER_IMAGE_TAG}
SCRIPT

$load_docker_image = <<SCRIPT
certs_dir="/home/vagrant/go/src/github.com/cilium/cilium/tests/k8s/cluster/certs"
sudo mkdir -p /etc/docker/certs.d/192.168.36.10:5000
sudo cp ${certs_dir}/ca.pem /etc/docker/certs.d/192.168.36.10:5000/ca.crt
docker pull 192.168.36.10:5000/cilium:${DOCKER_IMAGE_TAG}
docker tag 192.168.36.10:5000/cilium:${DOCKER_IMAGE_TAG} cilium/cilium:${DOCKER_IMAGE_TAG}
SCRIPT

$k8s_install = <<SCRIPT
/home/vagrant/go/src/github.com/cilium/cilium/tests/k8s/cluster/cluster-manager.bash fresh_install
SCRIPT

$cilium_master = <<SCRIPT
ip -6 a a FD01::B/16 dev enp0s8
echo 'FD01::B k8s-1' >> /etc/hosts
echo "FD01::C k8s-2" >> /etc/hosts
SCRIPT

$cilium_slave = <<SCRIPT
ip -6 a a FD01::C/16 dev enp0s8
echo 'FD01::C k8s-1' >> /etc/hosts
echo "FD01::B k8s-2" >> /etc/hosts
SCRIPT

# allow setting up k8s_version remotely when executing the runtime tests via ssh
$install_sshd_env = <<SCRIPT
echo "AcceptEnv k8s_version" >> /etc/ssh/sshd_config
# Load options
sudo service sshd restart
SCRIPT

$job_name = ENV['JOB_BASE_NAME'] || "local"

$build_number = ENV['BUILD_NUMBER'] || "0"
$build_id = "#{$job_name}-#{$build_number}"
$docker_image_tag=ENV['DOCKER_IMAGE_TAG'] || "local_build"

# Only create the build_id_name for Jenkins environment so that
# we can run VMs locally without having any the `build_id` in the name.
if ENV['BUILD_NUMBER'] then
    $build_id_name = "-build-#{$build_id}"
end

# We need this workaround since kube-proxy is not aware of multiple network
# interfaces. If we send a packet to a service IP that packet is sent
# to the default route, because the service IP is unknown by the linux routing
# table, with the source IP of the interface in the default routing table, even
# though the service IP should be routed to a different interface.
# This particular workaround is only needed for cilium, running on a pod on host
# network namespace, to reach out kube-api-server.
$kube_proxy_workaround = <<SCRIPT
sudo iptables -t nat -A POSTROUTING -o enp0s8 ! -s 192.168.36.12 -j MASQUERADE
SCRIPT

Vagrant.configure(2) do |config|
    config.vm.box = "bento/ubuntu-16.10"

    # http://foo-o-rama.com/vagrant--stdin-is-not-a-tty--fix.html
    config.vm.provision "fix-no-tty", type: "shell" do |s|
        s.privileged = false
        s.inline = "sudo sed -i '/tty/!s/mesg n/tty -s \\&\\& mesg n/' /root/.profile"
    end

    # Set up 'builder' as Docker registry, running newer version of Docker to support staged
    # builds and not running k8s.  This VM will be idle after k8s[12] have pulled the
    # newly built cilium image.
    config.vm.define "builder#{$build_id_name}" do |s|
        s.vm.hostname = "builder"

        s.vm.provider "virtualbox" do |vb|
            # Do not inherit DNS server from host, use proxy
            vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
            vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]

            config.vm.synced_folder '../../', '/home/vagrant/go/src/github.com/cilium/cilium'
        end

        s.vm.network "private_network", ip: "192.168.36.10", virtualbox__intnet: "cilium-k8s-multi-test-#{$build_id}"
        s.vm.network "private_network", ip: "192.168.37.10", bridge: "enp0s9"

        s.vm.provision "build", type: "shell", env: {"DOCKER_IMAGE_TAG" => $docker_image_tag}, inline: $build_docker_image
    end

    (1..2).each do |i|
        config.vm.define "k8s#{i}#{$build_id_name}" do |s|
            s.vm.hostname = "k8s-#{i}"
            s.vm.provision "k8s", type: "shell", env: {"k8s_version" => "1.6.6-00"}, inline: $k8s_install
            s.vm.provision "ssh_accept_env", type: "shell", privileged: true, inline: $install_sshd_env

            s.vm.provider "virtualbox" do |vb|
                # Do not inherit DNS server from host, use proxy
                vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
                vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
                vb.memory = 3072

                config.vm.synced_folder '../', '/home/vagrant/go/src/github.com/cilium/cilium/tests'
            end

            s.vm.network "private_network", ip: "192.168.36.1#{i}", virtualbox__intnet: "cilium-k8s-multi-test-#{$build_id}"
            s.vm.network "private_network", ip: "192.168.37.1#{i}", bridge: "enp0s9"

            # Hack to ensure that Kubernetes picks up the node-ip of the private_network
            # instead of the NATed vagrant IP
            s.vm.provision :shell, inline: "sed 's/127\.0\.0\.1.*k8s.*/192\.168\.36\.1#{i} k8s-#{i}/' -i /etc/hosts"

            # Mount BPF filesystem
	    s.vm.provision :shell, inline: "mount bpffs /sys/fs/bpf -t bpf"

            # first node is special and considered the master
            if "#{i}" == "1"
                s.vm.provision "cilium-master-netconfig", type: "shell", run: "always", inline: $cilium_master
            else
                s.vm.provision "kube-proxy-workaround", type: "shell", run: "always", inline: $kube_proxy_workaround
                s.vm.provision "cilium-slave-netconfig", type: "shell", run: "always", inline: $cilium_slave
            end
            s.vm.provision "load-docker-image", type: "shell", env: {"DOCKER_IMAGE_TAG" => $docker_image_tag}, inline: $load_docker_image
        end
    end
end