github.com/apprenda/kismatic@v1.12.0/integration-tests/prepare.go (about) 1 package integration_tests 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "os" 7 "path/filepath" 8 "time" 9 10 "github.com/apprenda/kismatic/integration-tests/retry" 11 "github.com/apprenda/kismatic/integration-tests/tls" 12 "github.com/cloudflare/cfssl/csr" 13 . "github.com/onsi/ginkgo" 14 ) 15 16 const ( 17 createKubernetesRepoFileYum = `cat <<EOF > /tmp/kubernetes.repo 18 [kubernetes] 19 name=Kubernetes 20 baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 21 enabled=1 22 gpgcheck=1 23 repo_gpgcheck=1 24 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg 25 https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg 26 EOF 27 ` 28 29 createDockerRepoFileYum = `cat <<EOF > /tmp/docker.repo 30 [docker] 31 name=Docker 32 baseurl=https://download.docker.com/linux/centos/7/x86_64/stable/ 33 enabled=1 34 gpgcheck=1 35 repo_gpgcheck=1 36 gpgkey=https://download.docker.com/linux/centos/gpg 37 EOF 38 ` 39 40 createGlusterRepoFileYum = `cat <<EOF > /tmp/gluster.repo 41 [gluster] 42 name=Gluster 43 baseurl=http://buildlogs.centos.org/centos/7/storage/x86_64/gluster-3.8/ 44 enabled=1 45 gpgcheck=1 46 repo_gpgcheck=0 47 gpgkey=https://download.gluster.org/pub/gluster/glusterfs/3.8/3.8.7/rsa.pub 48 EOF` 49 50 moveKubernetesRepoFileYum = `sudo mv /tmp/kubernetes.repo /etc/yum.repos.d` 51 moveDockerRepoFileYum = `sudo mv /tmp/docker.repo /etc/yum.repos.d` 52 moveGlusterRepoFileYum = `sudo mv /tmp/gluster.repo /etc/yum.repos.d` 53 54 installDockerYum = `sudo yum -y install --setopt=obsoletes=0 docker-ce-17.03.2.ce-1.el7.centos` 55 installKubeletYum = `sudo yum -y install kubelet-1.10.5-0` 56 installNFSUtilsYum = `sudo yum -y install nfs-utils` // required for the kubelet 57 installKubectlYum = `sudo yum -y install kubectl-1.10.5-0` 58 installGlusterfsServerYum = `sudo yum -y install --nogpgcheck glusterfs-server-3.8.15-2.el7` 59 60 updateAptGet = `sudo apt-get update` 61 addDockerRepoKeyApt = `wget -qO - https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -` 62 addDockerRepoApt = `sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable"` 63 installDockerApt = `sudo apt-get -y install docker-ce=17.03.2~ce-0~ubuntu-xenial` 64 65 addKubernetesRepoKeyApt = `wget -qO - https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -` 66 addKubernetesRepoApt = `sudo add-apt-repository "deb https://packages.cloud.google.com/apt/ kubernetes-xenial main"` 67 installKubeletApt = `sudo apt-get -y install kubelet=1.10.5-00` 68 stopKubeletService = `sudo systemctl stop kubelet` 69 installNFSCommonApt = `sudo apt-get -y install nfs-common` 70 installKubectlApt = `sudo apt-get -y install kubectl=1.10.5-00` 71 72 addGlusterRepoApt = `sudo add-apt-repository -y ppa:gluster/glusterfs-3.8` 73 installGlusterfsServerApt = `sudo apt-get -y install glusterfs-server=3.8.15-ubuntu1~xenial1` 74 ) 75 76 type nodePrep struct { 77 CommandsToPrepDockerRepo []string 78 CommandsToInstallDocker []string 79 CommandsToPrepKubernetesRepo []string 80 CommandsToInstallKubelet []string 81 CommandsToInstallKubectl []string 82 CommandsToInstallGlusterfs []string 83 } 84 85 var ubuntu1604Prep = nodePrep{ 86 CommandsToPrepDockerRepo: []string{addDockerRepoKeyApt, addDockerRepoApt, updateAptGet}, 87 CommandsToInstallDocker: []string{installDockerApt}, 88 CommandsToPrepKubernetesRepo: []string{addKubernetesRepoKeyApt, addKubernetesRepoApt, updateAptGet}, 89 CommandsToInstallKubelet: []string{installKubeletApt, installNFSCommonApt, stopKubeletService}, 90 CommandsToInstallKubectl: []string{installKubectlApt}, 91 CommandsToInstallGlusterfs: []string{addGlusterRepoApt, updateAptGet, installGlusterfsServerApt}, 92 } 93 94 var rhel7FamilyPrep = nodePrep{ 95 CommandsToPrepDockerRepo: []string{createDockerRepoFileYum, moveDockerRepoFileYum}, 96 CommandsToInstallDocker: []string{installDockerYum}, 97 CommandsToPrepKubernetesRepo: []string{createKubernetesRepoFileYum, moveKubernetesRepoFileYum}, 98 CommandsToInstallKubelet: []string{installKubeletYum, installNFSUtilsYum}, 99 CommandsToInstallKubectl: []string{installKubectlYum}, 100 CommandsToInstallGlusterfs: []string{createGlusterRepoFileYum, moveGlusterRepoFileYum, installGlusterfsServerYum}, 101 } 102 103 func InstallKismaticPackages(nodes provisionedNodes, distro linuxDistro, sshKey string, disconnected bool) { 104 prep := getPrepForDistro(distro) 105 dockerNodes := append(nodes.etcd, nodes.master...) 106 dockerNodes = append(dockerNodes, nodes.worker...) 107 dockerNodes = append(dockerNodes, nodes.ingress...) 108 dockerNodes = append(dockerNodes, nodes.storage...) 109 By("Configuring docker repository") 110 err := retry.WithBackoff(func() error { 111 return runViaSSH(prep.CommandsToPrepDockerRepo, dockerNodes, sshKey, 5*time.Minute) 112 }, 3) 113 FailIfError(err, "failed to configure package repository over SSH") 114 115 By("Installing Docker") 116 err = retry.WithBackoff(func() error { 117 return runViaSSH(prep.CommandsToInstallDocker, dockerNodes, sshKey, 10*time.Minute) 118 }, 3) 119 FailIfError(err, "failed to install docker") 120 121 kubeNodes := append(nodes.master, nodes.worker...) 122 kubeNodes = append(kubeNodes, nodes.ingress...) 123 kubeNodes = append(kubeNodes, nodes.storage...) 124 125 By("Configuring kubernetes repository") 126 err = retry.WithBackoff(func() error { 127 return runViaSSH(prep.CommandsToPrepKubernetesRepo, kubeNodes, sshKey, 5*time.Minute) 128 }, 3) 129 FailIfError(err, "failed to configure package repository") 130 131 By("Installing Kubelet package") 132 err = retry.WithBackoff(func() error { 133 return runViaSSH(prep.CommandsToInstallKubelet, kubeNodes, sshKey, 15*time.Minute) 134 }, 3) 135 FailIfError(err, "failed to install the kubelet package") 136 137 By("Installing Kubectl") 138 err = retry.WithBackoff(func() error { 139 return runViaSSH(prep.CommandsToInstallKubectl, kubeNodes, sshKey, 10*time.Minute) 140 }, 3) 141 FailIfError(err, "failed to install the kubectl package") 142 143 if len(nodes.storage) > 0 { 144 By("Installing Glusterfs:") 145 err = retry.WithBackoff(func() error { 146 return runViaSSH(prep.CommandsToInstallGlusterfs, nodes.storage, sshKey, 10*time.Minute) 147 }, 3) 148 FailIfError(err, "failed to install glustefs") 149 } 150 } 151 152 func InstallDockerPackage(nodes provisionedNodes, distro linuxDistro, sshKey string) { 153 prep := getPrepForDistro(distro) 154 dockerNodes := append(nodes.etcd, nodes.master...) 155 dockerNodes = append(dockerNodes, nodes.worker...) 156 dockerNodes = append(dockerNodes, nodes.ingress...) 157 dockerNodes = append(dockerNodes, nodes.storage...) 158 By("Configuring docker repository") 159 err := retry.WithBackoff(func() error { 160 return runViaSSH(prep.CommandsToPrepDockerRepo, dockerNodes, sshKey, 5*time.Minute) 161 }, 3) 162 FailIfError(err, "failed to configure package repository over SSH") 163 164 By("Installing Docker") 165 err = retry.WithBackoff(func() error { 166 return runViaSSH(prep.CommandsToInstallDocker, dockerNodes, sshKey, 10*time.Minute) 167 }, 3) 168 FailIfError(err, "failed to install docker") 169 } 170 171 func getPrepForDistro(distro linuxDistro) nodePrep { 172 switch distro { 173 case Ubuntu1604LTS: 174 return ubuntu1604Prep 175 case CentOS7, RedHat7: 176 return rhel7FamilyPrep 177 default: 178 panic(fmt.Sprintf("Unsupported distro %s", distro)) 179 } 180 } 181 182 func deployAuthenticatedDockerRegistry(node NodeDeets, listeningPort int, sshKey string) (string, error) { 183 // Install Docker on the node 184 installDockerCmds := []string{ 185 "sudo curl -sSL https://get.docker.com/ | sh", 186 "sudo systemctl start docker", 187 "mkdir ~/certs", 188 } 189 err := runViaSSH(installDockerCmds, []NodeDeets{node}, sshKey, 10*time.Minute) 190 FailIfError(err, "Failed to install docker over SSH") 191 // Generate CA 192 subject := tls.Subject{ 193 Organization: "someOrg", 194 OrganizationalUnit: "someOrgUnit", 195 } 196 key, caCert, err := tls.NewCACert("test-resources/ca-csr.json", "someCommonName", subject) 197 if err != nil { 198 return "", fmt.Errorf("error generating CA cert for Docker: %v", err) 199 } 200 err = ioutil.WriteFile("docker-ca.pem", caCert, 0644) 201 if err != nil { 202 return "", fmt.Errorf("error writing CA cert to file") 203 } 204 // Generate Certificate 205 ca := &tls.CA{ 206 Key: key, 207 Cert: caCert, 208 ConfigFile: "test-resources/ca-config.json", 209 Profile: "kubernetes", 210 } 211 certHosts := []string{node.Hostname, node.PrivateIP, node.PublicIP} 212 req := csr.CertificateRequest{ 213 CN: node.Hostname, 214 KeyRequest: &csr.BasicKeyRequest{ 215 A: "rsa", 216 S: 2048, 217 }, 218 Hosts: certHosts, 219 Names: []csr.Name{ 220 { 221 C: "US", 222 L: "Troy", 223 O: "Kubernetes", 224 OU: "Cluster", 225 ST: "New York", 226 }, 227 }, 228 } 229 key, cert, err := tls.NewCert(ca, req) 230 if err != nil { 231 return "", fmt.Errorf("error generating certificate for Docker registry: %v", err) 232 } 233 if err = ioutil.WriteFile("docker.pem", cert, 0644); err != nil { 234 return "", fmt.Errorf("error writing certificate to file: %v", err) 235 } 236 if err = ioutil.WriteFile("docker-key.pem", key, 0644); err != nil { 237 return "", fmt.Errorf("error writing private key to file: %v", err) 238 } 239 240 err = copyFileToRemote("docker.pem", "~/certs/docker.pem", node, sshKey, 1*time.Minute) 241 FailIfError(err, "failed to copy docker.pem file") 242 err = copyFileToRemote("docker-key.pem", "~/certs/docker-key.pem", node, sshKey, 1*time.Minute) 243 FailIfError(err, "failed to copy docker-key.pem") 244 245 htpasswdCmd := []string{"mkdir auth", "sudo docker run --entrypoint htpasswd registry -Bbn kismaticuser kismaticpassword > auth/htpasswd"} 246 err = runViaSSH(htpasswdCmd, []NodeDeets{node}, sshKey, 1*time.Minute) 247 FailIfError(err, "Failed to create htpasswd file for Docker registry") 248 249 startDockerRegistryCmd := []string{fmt.Sprintf("sudo docker run -d -p %d:5000 --restart=always ", listeningPort) + 250 " --name registry" + 251 " -v ~/certs:/certs" + 252 " -v `pwd`/auth:/auth" + 253 " -e \"REGISTRY_AUTH=htpasswd\"" + 254 " -e \"REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm\"" + 255 " -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd" + 256 " -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/docker.pem" + 257 " -e REGISTRY_HTTP_TLS_KEY=/certs/docker-key.pem registry"} 258 err = runViaSSH(startDockerRegistryCmd, []NodeDeets{node}, sshKey, 1*time.Minute) 259 FailIfError(err, "Failed to start docker registry over SSH") 260 261 // Need the full path, otherwise ansible looks for it in the wrong place 262 pwd, _ := os.Getwd() 263 return filepath.Join(pwd, "docker-ca.pem"), nil 264 }