github.com/cilium/cilium@v1.16.2/test/Vagrantfile

# -*- mode: ruby -*-
# vi: set ft=ruby Vagrant.require_version ">= 2.2.0"

# The source of truth for vagrant box versions.
# Sets SERVER_BOX, SERVER_VERSION, NETNEXT_SERVER_BOXET and NEXT_SERVER_VERSION
# Accepts overrides from env variables
require_relative '../vagrant_box_defaults.rb'
$SERVER_BOX = (ENV['SERVER_BOX'] || $SERVER_BOX)
$SERVER_VERSION= (ENV['SERVER_VERSION'] || $SERVER_VERSION)
$NETNEXT_SERVER_BOX = (ENV['NETNEXT_SERVER_BOX'] || $NETNEXT_SERVER_BOX)
$NETNEXT_SERVER_VERSION= (ENV['NETNEXT_SERVER_VERSION'] || $NETNEXT_SERVER_VERSION)

$BUILD_NUMBER = ENV['BUILD_NUMBER'] || "0"
$JOB_NAME = ENV['JOB_BASE_NAME'] || "LOCAL"
$K8S_VERSION = ENV['K8S_VERSION'] || "1.30"
$K8S_NODES = (ENV['K8S_NODES'] || "2").to_i
$NFS = ENV['NFS']=="0"? false : true
$NFS_OPTS = (ENV['NFS_OPTS'] || "nolock").split(",")
$IPv6=(ENV['IPv6'] || "0")
$CONTAINER_RUNTIME=(ENV['CONTAINER_RUNTIME'] || "docker")
$CNI_INTEGRATION=(ENV['CNI_INTEGRATION'] || "")
$CILIUM_IMAGE = ENV['CILIUM_IMAGE'] || ""
$CILIUM_TAG = ENV['CILIUM_TAG'] || ""
$CILIUM_OPERATOR_IMAGE = ENV['CILIUM_OPERATOR_IMAGE'] || ""
$CILIUM_OPERATOR_TAG = ENV['CILIUM_OPERATOR_TAG'] || ""
$HUBBLE_RELAY_IMAGE = ENV['HUBBLE_RELAY_IMAGE'] || ""
$HUBBLE_RELAY_TAG = ENV['HUBBLE_RELAY_TAG'] || ""
$PRELOAD_VM = ENV['PRELOAD_VM'] || "false"
$PROVISION_EXTERNAL_WORKLOAD = ENV['PROVISION_EXTERNAL_WORKLOAD'] || "false"
$SKIP_K8S_PROVISION = ENV['SKIP_K8S_PROVISION'] || "false"
$NO_CILIUM_ON_NODES = ENV['NO_CILIUM_ON_NODES'] || ENV['NO_CILIUM_ON_NODE'] || ""
$KUBEPROXY = (ENV['KUBEPROXY'] || "1")
$RACE = ENV['RACE'] || ""
$LOCKDEBUG = ENV['LOCKDEBUG'] || ""
$BASE_IMAGE = ENV['BASE_IMAGE'] || ""
$DOCKER_LOGIN = ENV['DOCKER_LOGIN'] || ""
$DOCKER_PASSWORD = ENV['DOCKER_PASSWORD'] || ""
$CILIUM_REGISTRY = ENV['CILIUM_REGISTRY'] || ""

# RAM and CPU settings
$MEMORY = (ENV['VM_MEMORY'] || "4096").to_i
$CPU = (ENV['VM_CPUS'] || "2").to_i

if ENV['NETNEXT'] == "true" || ENV['NETNEXT'] == "1" || ENV['KERNEL'] == "net-next" then
    $SERVER_BOX = $NETNEXT_SERVER_BOX
    $SERVER_VERSION = $NETNEXT_SERVER_VERSION
elsif ENV['KERNEL'] != nil && ENV['KERNEL'] != "" then
    $SERVER_BOX = instance_variable_get("@v"+ ENV['KERNEL'] + "_SERVER_BOX")
    $SERVER_VERSION = instance_variable_get("@v"+ ENV['KERNEL'] + "_SERVER_VERSION")
end

# Workaround issue as described here:
# https://github.com/cilium/cilium/pull/12520
class VagrantPlugins::ProviderVirtualBox::Action::Network
  def dhcp_server_matches_config?(dhcp_server, config)
    true
  end
end

$cleanup = <<SCRIPT
i=1
k8s_nodes="${K8S_NODES:-2}"
while [ "$i" -le "$k8s_nodes" ]; do
    VBoxManage natnetwork add --netname natnet$i --network 192.168.0.0/16 --ipv6 on --enable
    i=$((i+1))
done

res=0
while [ "$res" == "0" ]; do
    VBoxManage natnetwork remove --netname natnet$i
    res=$?
    i=$((i+1))
done

VBoxManage natnetwork list
SCRIPT

$bootstrap = <<SCRIPT
set -o errexit
set -o nounset
set -o pipefail

# Add an exception for the cilium repo for the root user to fix the
# "fatal: unsafe repository ('/home/vagrant/go/src/github.com/cilium/cilium' is owned by someone else)"
# error condition when running `sudo make install`
git config --global --add safe.directory /home/vagrant/go/src/github.com/cilium/cilium

if [ -x /home/vagrant/go/src/github.com/cilium/cilium/.devvmrc ] ; then
   echo "----------------------------------------------------------------"
   echo "Executing .devvmrc"
   /home/vagrant/go/src/github.com/cilium/cilium/.devvmrc || true
fi
echo "----------------------------------------------------------------"
sudo sed -i 's/^mesg n$/tty -s \\&\\& mesg n/g' /root/.profile
echo 'export GOPATH=$(go env GOPATH)' >> /home/vagrant/.bashrc
SCRIPT

Vagrant.configure("2") do |config|
    cilium_dir = '../'
    cilium_path = '/home/vagrant/go/src/github.com/cilium/cilium'
    if ENV["SHARE_PARENT"] == "2" then
        cilium_dir = '../../..'
        cilium_path = '/home/vagrant/go/src/github.com'
    elsif ENV["SHARE_PARENT"] then
        cilium_dir = '../..'
        cilium_path = '/home/vagrant/go/src/github.com/cilium'
    end

    config.trigger.before :up, :provision do |trigger|
        trigger.run = {inline: "bash -c '#{$cleanup}'"}
    end

    config.vm.define "runtime" do |server|
        server.vm.provider "virtualbox" do |vb|
            vb.customize ["modifyvm", :id, "--hwvirtex", "on"]
            vb.cpus = $CPU
            vb.memory= $MEMORY
            vb.linked_clone = true
            vb.default_nic_type = "virtio"
            # Prevent VirtualBox from interfering with host audio stack
            vb.customize ["modifyvm", :id, "--audio", "none"]
            # Enable HPET, the Go scheduler heavily relies on accurate timers.
            vb.customize ["modifyvm", :id, "--hpet", "on"]
        end

        server.vm.box =  "#{$SERVER_BOX}"
        server.vm.box_version = $SERVER_VERSION
        server.vm.boot_timeout = 600
        server.vm.hostname = "runtime"

        server.vm.network "private_network",
            ip: "192.168.56.10",
            virtualbox__intnet: "cilium-k8s#{$BUILD_NUMBER}-#{$JOB_NAME}-#{$K8S_VERSION}"
        server.vm.network "private_network",
            ip: "192.168.57.10",
            virtualbox__intnet: "cilium-k8s-2#{$BUILD_NUMBER}-#{$JOB_NAME}-#{$K8S_VERSION}"

        # @TODO: Clean this one when https://github.com/hashicorp/vagrant/issues/9822 is fixed.
        server.vm.provision "ipv6-config-primary",
            type: "shell",
            run: "always",
            inline: "ip -6 a a fd04::1/96 dev enp0s8 || true"
        server.vm.provision "ipv6-config-secondary",
            type: "shell",
            run: "always",
            inline: "ip -6 a a fd05::1/96 dev enp0s9 || true"

        # This network is only used by NFS
        if $NFS
            # This network is only used by NFS
            server.vm.network "private_network", ip: "192.168.58.10"
            server.vm.synced_folder cilium_dir, cilium_path, type: "nfs", nfs_udp: false, mount_options: $NFS_OPTS
        else
            server.vm.synced_folder cilium_dir, cilium_path
        end

        # Provision section
        server.vm.provision "bootstrap", type: "shell", inline: $bootstrap
        server.vm.provision "file", source: "provision/", destination: "/tmp/"
        server.vm.provision "shell" do |sh|
            sh.path = "./provision/runtime_install.sh"
            sh.env = {
              "RACE" => "#{$RACE}",
              "LOCKDEBUG" => "#{$LOCKDEBUG}",
              "BASE_IMAGE" => "#{$BASE_IMAGE}",
              "PROVISION_EXTERNAL_WORKLOAD" => "#{$PROVISION_EXTERNAL_WORKLOAD}",
              "CILIUM_IMAGE" => "#{$CILIUM_IMAGE}"
            }
        end
    end

    (1..$K8S_NODES).each do |i|
        config.vm.define "k8s#{i}-#{$K8S_VERSION}" do |server|
            server.vm.provider "virtualbox" do |vb|
                vb.customize ["modifyvm", :id, "--hwvirtex", "on"]
                vb.cpus = $CPU
                vb.memory = $MEMORY
                no_cilium_nodes = $NO_CILIUM_ON_NODES.split(',')
                if no_cilium_nodes.include? "k8s#{i}"
                  vb.memory = $MEMORY / 2
                  vb.cpus = 1
                end
                vb.linked_clone = true
                vb.default_nic_type = "virtio"
                # Prevent VirtualBox from interfering with host audio stack
                vb.customize ["modifyvm", :id, "--audio", "none"]
                # Enable HPET, the Go scheduler heavily relies on accurate timers.
                vb.customize ["modifyvm", :id, "--hpet", "on"]

                if ENV['PRELOAD_VM'] == "false" then
                  # Use serial ports if the VM is no longer accessible via SSH
                  vb.customize ["modifyvm", :id, "--uart1", "0x3F8", "4"]
                  vb.customize ["modifyvm", :id, "--uartmode1", "server", "k8s#{i}-#{$K8S_VERSION}-ttyS0.sock"]
                end
            end

            server.vm.box =  "#{$SERVER_BOX}"
            server.vm.box_version = $SERVER_VERSION
            server.vm.hostname = "k8s#{i}"
            server.vm.boot_timeout = 600
            if i == 1 then
                # grafana
                server.vm.network "forwarded_port", guest: 3000, host: 3000,
                  auto_correct: true
                server.vm.network "forwarded_port", guest: 6443, host: 9443,
                  auto_correct: true
            end
            server.vm.network "private_network",
                ip: "192.168.56.1#{i}",
                virtualbox__intnet: "cilium-k8s#{$BUILD_NUMBER}-#{$JOB_NAME}-#{$K8S_VERSION}"
            server.vm.network "private_network",
                ip: "192.168.57.1#{i}",
                virtualbox__intnet: "cilium-k8s-2#{$BUILD_NUMBER}-#{$JOB_NAME}-#{$K8S_VERSION}"

            # @TODO: Clean this one when https://github.com/hashicorp/vagrant/issues/9822 is fixed.
            server.vm.provision "ipv6-config-primary",
                type: "shell",
                run: "always",
                inline: "ip -6 a a fd04::1#{i}/96 dev enp0s8 || true"
            server.vm.provision "ipv6-config-secondary",
                type: "shell",
                run: "always",
                inline: "ip -6 a a fd05::1#{i}/96 dev enp0s9 || true"

            if $NFS
                # This network is only used by NFS
                server.vm.network "private_network", ip: "192.168.58.1#{i}"
                server.vm.synced_folder cilium_dir, cilium_path, type: "nfs", nfs_udp: false, mount_options: $NFS_OPTS
            else
                server.vm.synced_folder cilium_dir, cilium_path
            end

            # Interface for the IPv6 NAT Service. The IP address doesn't matter
            # as it won't be used. We use an IPv4 address as newer versions of
            # VBox reject all IPv6 addresses.
            server.vm.network "private_network",
                ip: "192.168.59.15"
            server.vm.provider "virtualbox" do |vb|
                vb.customize ["modifyvm", :id, "--nic5", "natnetwork"]
                vb.customize ["modifyvm", :id, "--nat-network5", "natnet#{i}"]
            end
            server.vm.provision "ipv6-nat-config",
                type: "shell",
                run: "always",
                inline: "ip -6 r a default via fd17:625c:f037:2::1 dev enp0s16 || true"

            # Provision section
            server.vm.provision "bootstrap", type: "shell", inline: $bootstrap
            server.vm.provision :shell,
                :inline => "sudo sysctl -w net.ipv6.conf.all.forwarding=1"
            server.vm.provision "file", source: "provision/", destination: "/tmp/"
            server.vm.provision "shell" do |sh|
                sh.path = "./provision/k8s_install.sh"
                sh.args = [
                    "k8s#{i}", "192.168.56.1#{i}", "#{$K8S_VERSION}",
                    "#{$IPv6}", "#{$CONTAINER_RUNTIME}", "#{$CNI_INTEGRATION}"]
                sh.env = {"CILIUM_IMAGE" => "#{$CILIUM_IMAGE}",
                          "CILIUM_TAG" => "#{$CILIUM_TAG}",
                          "CILIUM_OPERATOR_IMAGE" => "#{$CILIUM_OPERATOR_IMAGE}",
                          "CILIUM_OPERATOR_TAG" => "#{$CILIUM_OPERATOR_TAG}",
                          "HUBBLE_RELAY_IMAGE" => "#{$HUBBLE_RELAY_IMAGE}",
                          "HUBBLE_RELAY_TAG" => "#{$HUBBLE_RELAY_TAG}",
                          "PRELOAD_VM" => "#{$PRELOAD_VM}",
                          "SKIP_K8S_PROVISION" => "#{$SKIP_K8S_PROVISION}",
                          "KUBEPROXY" => "#{$KUBEPROXY}",
                          "RACE" => "#{$RACE}",
                          "LOCKDEBUG" => "#{$LOCKDEBUG}",
                          "BASE_IMAGE" => "#{$BASE_IMAGE}",
                          "DOCKER_LOGIN" => "#{$DOCKER_LOGIN}",
                          "DOCKER_PASSWORD" => "#{$DOCKER_PASSWORD}",
                          "CILIUM_REGISTRY" => "#{$CILIUM_REGISTRY}"
                }
            end
        end
    end
end