github.com/jlmeeker/kismatic@v1.10.1-0.20180612190640-57f9005a1f1a/integration-tests/disconnected_install_test.go (about) 1 package integration_tests 2 3 import ( 4 "fmt" 5 "os" 6 "time" 7 8 . "github.com/onsi/ginkgo" 9 . "github.com/onsi/gomega" 10 ) 11 12 const ( 13 proxyPort = 3128 14 dockerRegistryPort = 8443 15 ) 16 17 var _ = Describe("disconnected installation", func() { 18 BeforeEach(func() { 19 dir := setupTestWorkingDir() 20 os.Chdir(dir) 21 }) 22 23 Context("with an existing package mirror and image registry", func() { 24 Context("on CentOS", func() { 25 ItOnAWS("should install successfully [slow]", func(aws infrastructureProvisioner) { 26 distro := CentOS7 27 WithInfrastructure(NodeCount{1, 1, 2, 1, 1}, distro, aws, func(nodes provisionedNodes, sshKey string) { 28 // One of the workers is used as the repository and registry 29 repoNode := nodes.worker[1] 30 nodes.worker = nodes.worker[0:1] 31 32 By("Creating a package repository") 33 err := createPackageRepositoryMirror(repoNode, distro, sshKey) 34 FailIfError(err, "Error creating local package repo") 35 36 By("Deploying a docker registry") 37 caFile, err := deployAuthenticatedDockerRegistry(repoNode, dockerRegistryPort, sshKey) 38 FailIfError(err, "Failed to deploy docker registry") 39 40 By("Seeding the local registry") 41 err = seedRegistry(repoNode, caFile, dockerRegistryPort, sshKey) 42 FailIfError(err, "Error seeding local registry") 43 44 By("Disabling internet access") 45 err = disableInternetAccess(nodes.allNodes(), sshKey) 46 FailIfError(err, "Failed to create iptable rule") 47 48 // Configure the repos on the nodes 49 By("Configuring repository on nodes") 50 for _, n := range nodes.allNodes() { 51 err = copyFileToRemote("test-resources/disconnected-installation/configure-rpm-mirrors.sh", "/tmp/configure-rpm-mirrors.sh", n, sshKey, 15*time.Second) 52 FailIfError(err, "Failed to copy script to nodes") 53 } 54 cmds := []string{ 55 "chmod +x /tmp/configure-rpm-mirrors.sh", 56 fmt.Sprintf("sudo /tmp/configure-rpm-mirrors.sh http://%s", repoNode.PrivateIP), 57 } 58 err = runViaSSH(cmds, nodes.allNodes(), sshKey, 5*time.Minute) 59 FailIfError(err, "Failed to run mirror configuration script") 60 61 installOpts := installOptions{ 62 disablePackageInstallation: false, 63 disconnectedInstallation: true, 64 modifyHostsFiles: true, 65 dockerRegistryCAPath: caFile, 66 dockerRegistryServer: fmt.Sprintf("%s:%d", repoNode.PrivateIP, dockerRegistryPort), 67 dockerRegistryUsername: "kismaticuser", 68 dockerRegistryPassword: "kismaticpassword", 69 } 70 71 // installOpts.disableRegistrySeeding = true 72 By("Running kismatic install apply") 73 err = installKismatic(nodes, installOpts, sshKey) 74 Expect(err).ToNot(HaveOccurred()) 75 }) 76 }) 77 }) 78 79 Context("on Ubuntu", func() { 80 ItOnAWS("should install successfully [slow]", func(aws infrastructureProvisioner) { 81 distro := Ubuntu1604LTS 82 WithInfrastructure(NodeCount{1, 1, 2, 1, 1}, distro, aws, func(nodes provisionedNodes, sshKey string) { 83 // One of the workers is used as the repository and registry 84 repoNode := nodes.worker[1] 85 nodes.worker = nodes.worker[0:1] 86 87 By("Creating a package repository") 88 err := createPackageRepositoryMirror(repoNode, distro, sshKey) 89 FailIfError(err, "Error creating local package repo") 90 91 // Deploy a docker registry on the node 92 By("Deploying a docker registry") 93 caFile, err := deployAuthenticatedDockerRegistry(repoNode, dockerRegistryPort, sshKey) 94 FailIfError(err, "Failed to deploy docker registry") 95 96 By("Seeding the local registry") 97 err = seedRegistry(repoNode, caFile, dockerRegistryPort, sshKey) 98 FailIfError(err, "Error seeding local registry") 99 100 By("Disabling internet access") 101 err = disableInternetAccess(nodes.allNodes(), sshKey) 102 FailIfError(err, "Failed to create iptable rule") 103 104 // Configure the repos on the nodes 105 By("Configuring repository on nodes") 106 for _, n := range nodes.allNodes() { 107 err = copyFileToRemote("test-resources/disconnected-installation/configure-deb-mirrors.sh", "/tmp/configure-deb-mirrors.sh", n, sshKey, 15*time.Second) 108 FailIfError(err, "Failed to copy script to nodes") 109 } 110 cmds := []string{ 111 "chmod +x /tmp/configure-deb-mirrors.sh", 112 fmt.Sprintf("sudo /tmp/configure-deb-mirrors.sh http://%s", repoNode.PrivateIP), 113 } 114 err = runViaSSH(cmds, nodes.allNodes(), sshKey, 5*time.Minute) 115 FailIfError(err, "Failed to run mirror configuration script") 116 117 installOpts := installOptions{ 118 disablePackageInstallation: false, 119 disconnectedInstallation: true, 120 modifyHostsFiles: true, 121 dockerRegistryCAPath: caFile, 122 dockerRegistryServer: fmt.Sprintf("%s:%d", repoNode.PrivateIP, dockerRegistryPort), 123 dockerRegistryUsername: "kismaticuser", 124 dockerRegistryPassword: "kismaticpassword", 125 } 126 127 // installOpts.disableRegistrySeeding = true 128 By("Running kismatic install apply") 129 err = installKismatic(nodes, installOpts, sshKey) 130 Expect(err).ToNot(HaveOccurred()) 131 }) 132 }) 133 }) 134 }) 135 136 Context("when there is a proxy between the nodes and the internet", func() { 137 ItOnAWS("should install successfully [slow]", func(aws infrastructureProvisioner) { 138 WithInfrastructure(NodeCount{2, 1, 1, 1, 1}, CentOS7, aws, func(nodes provisionedNodes, sshKey string) { 139 // setup cluster nodes, and a proxy node 140 clusterNodes := provisionedNodes{} 141 clusterNodes.etcd = []NodeDeets{nodes.etcd[1]} 142 clusterNodes.master = nodes.master 143 clusterNodes.worker = nodes.worker 144 clusterNodes.ingress = nodes.ingress 145 clusterNodes.storage = nodes.storage 146 proxyNode := nodes.etcd[0] 147 148 By("Installing the proxy") 149 err := runViaSSH([]string{"sudo yum update -y openssl"}, []NodeDeets{proxyNode}, sshKey, 5*time.Minute) 150 FailIfError(err, "Failed update openssl") 151 err = runViaSSH([]string{"sudo yum install -y squid"}, []NodeDeets{proxyNode}, sshKey, 5*time.Minute) 152 FailIfError(err, "Failed install proxy") 153 err = runViaSSH([]string{"sudo sed -i -e 's/http_access deny all/http_access allow all/g' /etc/squid/squid.conf"}, []NodeDeets{proxyNode}, sshKey, 5*time.Minute) 154 FailIfError(err, "Failed modify squif.conf") 155 err = runViaSSH([]string{"sudo systemctl restart squid"}, []NodeDeets{proxyNode}, sshKey, 5*time.Minute) 156 FailIfError(err, "Failed restart proxy") 157 158 By("Disabling internet access") 159 err = disableInternetAccess(clusterNodes.allNodes(), sshKey) 160 FailIfError(err, "Failed to create iptable rules") 161 162 if err = verifyNoInternetAccess(clusterNodes.allNodes(), sshKey); err == nil { 163 Fail("was able to ping google with outgoing connections blocked") 164 } 165 166 By("Enabling proxy access") 167 err = enableProxyConnection(clusterNodes.allNodes(), sshKey) 168 FailIfError(err, "Failed to create iptable rules") 169 170 By("Verifying connectivity to google.com") 171 err = runViaSSH([]string{fmt.Sprintf("export http_proxy=%s:%d && curl --head www.google.com", proxyNode.PrivateIP, proxyPort)}, clusterNodes.allNodes(), sshKey, 1*time.Minute) 172 FailIfError(err, "Failed to curl google with proxy") 173 174 By("Running kismatic install apply") 175 // don't use the proxy for cluster communication 176 177 installOpts := installOptions{ 178 modifyHostsFiles: true, 179 httpProxy: fmt.Sprintf("%s:%d", proxyNode.PrivateIP, proxyPort), 180 httpsProxy: fmt.Sprintf("%s:%d", proxyNode.PrivateIP, proxyPort), 181 noProxy: fmt.Sprintf("%s,%s", "kubernetes-charts.storage.googleapis.com", "apprenda.github.io"), 182 } 183 err = installKismatic(clusterNodes, installOpts, sshKey) 184 Expect(err).ToNot(HaveOccurred()) 185 }) 186 }) 187 }) 188 }) 189 190 func disableInternetAccess(nodes []NodeDeets, sshKey string) error { 191 By("Blocking all outbound connections") 192 allowSourcePorts := "8888,2379,6666,2380,6660,6443,8443,80,443,10256,10250,10251,10252,10254" // ports needed/checked by inspector 193 allowDestPorts := "8888,2379,6666,2380,6660,6443,8443,80,443,10256,10250,10251,10252,10254" 194 allowedStorgePorts := "8081,111,2049,38465,38466,38467" 195 cmd := []string{ 196 "sudo iptables -A OUTPUT -o lo -j ACCEPT", // allow loopback 197 "sudo iptables -A OUTPUT -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT", // allow SSH 198 fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --match multiport --sports %s -j ACCEPT", allowSourcePorts), // allow inspector 199 fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --match multiport --dports %s -j ACCEPT", allowDestPorts), // allow internal traffic for: inspector, etcd, docker registry 200 fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --match multiport --sports %s -j ACCEPT", allowedStorgePorts), // storage 201 fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --match multiport --dports %s -j ACCEPT", allowedStorgePorts), // storage 202 "sudo iptables -A OUTPUT -s 172.16.0.0/16 -j ACCEPT", 203 "sudo iptables -A OUTPUT -d 172.16.0.0/16 -j ACCEPT", // Allow pod network 204 "sudo iptables -A OUTPUT -s 172.20.0.0/16 -j ACCEPT", 205 "sudo iptables -A OUTPUT -d 172.20.0.0/16 -j ACCEPT", // Allow pod service network 206 "sudo iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT", // ping outside to inside 207 "sudo iptables -A OUTPUT -p icmp --icmp-type echo-reply -j ACCEPT", // ping outside to inside 208 "sudo iptables -A OUTPUT -p icmp --icmp-type echo-request -j ACCEPT", // ping inside to outside 209 "sudo iptables -A INPUT -p icmp --icmp-type echo-reply -j ACCEPT", // ping inside to outside 210 "sudo iptables -P OUTPUT DROP", // drop everything else 211 } 212 return runViaSSH(cmd, nodes, sshKey, 1*time.Minute) 213 } 214 215 func enableProxyConnection(nodes []NodeDeets, sshKey string) error { 216 By("Allowing connection to proxy") 217 // proxy server running on 3128 218 cmd := []string{ 219 fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --match multiport --sports %d -j ACCEPT", proxyPort), 220 fmt.Sprintf("sudo iptables -A OUTPUT -p tcp --match multiport --dports %d -j ACCEPT", proxyPort), 221 } 222 return runViaSSH(cmd, nodes, sshKey, 1*time.Minute) 223 } 224 225 func verifyNoInternetAccess(nodes []NodeDeets, sshKey string) error { 226 By("Verifying that connections are blocked") 227 return runViaSSH([]string{"curl --head www.google.com"}, nodes, sshKey, 1*time.Minute) 228 }