github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/kubetest/anywhere.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "errors" 22 "flag" 23 "fmt" 24 "io/ioutil" 25 "log" 26 "os" 27 "os/exec" 28 "regexp" 29 "strings" 30 "text/template" 31 "time" 32 ) 33 34 const defaultKubeadmCNI = "weave" 35 36 var ( 37 // kubernetes-anywhere specific flags. 38 kubernetesAnywherePath = flag.String("kubernetes-anywhere-path", "", 39 "(kubernetes-anywhere only) Path to the kubernetes-anywhere directory. Must be set for kubernetes-anywhere.") 40 kubernetesAnywherePhase2Provider = flag.String("kubernetes-anywhere-phase2-provider", "ignition", 41 "(kubernetes-anywhere only) Provider for phase2 bootstrapping. (Defaults to ignition).") 42 kubernetesAnywhereKubeadmVersion = flag.String("kubernetes-anywhere-kubeadm-version", "stable", 43 "(kubernetes-anywhere only) Version of kubeadm to use, if phase2-provider is kubeadm. May be \"stable\" or a gs:// link to a custom build.") 44 kubernetesAnywhereKubernetesVersion = flag.String("kubernetes-anywhere-kubernetes-version", "", 45 "(kubernetes-anywhere only) Version of Kubernetes to use (e.g. latest, stable, latest-1.6, 1.6.3, etc).") 46 kubernetesAnywhereKubeletVersion = flag.String("kubernetes-anywhere-kubelet-version", "stable", 47 "(kubernetes-anywhere only) Version of Kubelet to use, if phase2-provider is kubeadm. May be \"stable\" or a gs:// link to a custom build.") 48 kubernetesAnywhereKubeletCIVersion = flag.String("kubernetes-anywhere-kubelet-ci-version", "", 49 "(kubernetes-anywhere only) If specified, the ci version for the kubelt to use. Overrides kubernetes-anywhere-kubelet-version.") 50 kubernetesAnywhereCluster = flag.String("kubernetes-anywhere-cluster", "", 51 "(kubernetes-anywhere only) Cluster name. Must be set for kubernetes-anywhere.") 52 kubernetesAnywhereProxyMode = flag.String("kubernetes-anywhere-proxy-mode", "", 53 "(kubernetes-anywhere only) Chose kube-proxy mode.") 54 kubernetesAnywhereUpTimeout = flag.Duration("kubernetes-anywhere-up-timeout", 20*time.Minute, 55 "(kubernetes-anywhere only) Time limit between starting a cluster and making a successful call to the Kubernetes API.") 56 kubernetesAnywhereNumNodes = flag.Int("kubernetes-anywhere-num-nodes", 4, 57 "(kubernetes-anywhere only) Number of nodes to be deployed in the cluster.") 58 kubernetesAnywhereUpgradeMethod = flag.String("kubernetes-anywhere-upgrade-method", "upgrade", 59 "(kubernetes-anywhere only) Indicates whether to do the control plane upgrade with kubeadm method \"init\" or \"upgrade\"") 60 kubernetesAnywhereCNI = flag.String("kubernetes-anywhere-cni", "", 61 "(kubernetes-anywhere only) The name of the CNI plugin used for the cluster's SDN.") 62 kubernetesAnywhereDumpClusterLogs = flag.Bool("kubernetes-anywhere-dump-cluster-logs", true, 63 "(kubernetes-anywhere only) Whether to dump cluster logs.") 64 kubernetesAnywhereOSImage = flag.String("kubernetes-anywhere-os-image", "ubuntu-1604-xenial-v20171212", 65 "(kubernetes-anywhere only) The name of the os_image to use for nodes") 66 kubernetesAnywhereKubeadmFeatureGates = flag.String("kubernetes-anywhere-kubeadm-feature-gates", "", 67 "(kubernetes-anywhere only) A set of key=value pairs that describes feature gates for kubeadm features. If specified, this flag will pass on to kubeadm.") 68 ) 69 70 const kubernetesAnywhereConfigTemplate = ` 71 .phase1.num_nodes={{.NumNodes}} 72 .phase1.cluster_name="{{.Cluster}}" 73 .phase1.ssh_user="" 74 .phase1.cloud_provider="gce" 75 76 .phase1.gce.os_image="{{.OSImage}}" 77 .phase1.gce.instance_type="n1-standard-1" 78 .phase1.gce.project="{{.Project}}" 79 .phase1.gce.region="{{.Region}}" 80 .phase1.gce.zone="{{.Zone}}" 81 .phase1.gce.network="default" 82 83 .phase2.installer_container="docker.io/colemickens/k8s-ignition:latest" 84 .phase2.docker_registry="k8s.gcr.io" 85 .phase2.kubernetes_version="{{.KubernetesVersion}}" 86 .phase2.provider="{{.Phase2Provider}}" 87 .phase2.kubelet_version="{{.KubeletVersion}}" 88 .phase2.kubeadm.version="{{.KubeadmVersion}}" 89 .phase2.kube_context_name="{{.KubeContext}}" 90 .phase2.proxy_mode="{{.KubeproxyMode}}" 91 .phase2.kubeadm.master_upgrade.method="{{.UpgradeMethod}}" 92 .phase2.kubeadm.feature_gates="{{.KubeadmFeatureGates}}" 93 94 .phase3.run_addons=y 95 .phase3.kube_proxy=n 96 .phase3.dashboard=n 97 .phase3.heapster=n 98 .phase3.kube_dns=n 99 .phase3.cni="{{.CNI}}" 100 ` 101 102 const kubernetesAnywhereMultiClusterConfigTemplate = kubernetesAnywhereConfigTemplate + ` 103 .phase2.enable_cloud_provider=y 104 .phase3.gce_storage_class=y 105 ` 106 107 type kubernetesAnywhere struct { 108 path string 109 // These are exported only because their use in the config template requires it. 110 Phase2Provider string 111 KubeadmVersion string 112 KubeletVersion string 113 UpgradeMethod string 114 KubernetesVersion string 115 NumNodes int 116 Project string 117 Cluster string 118 Zone string 119 Region string 120 KubeContext string 121 CNI string 122 KubeproxyMode string 123 OSImage string 124 KubeadmFeatureGates string 125 } 126 127 func initializeKubernetesAnywhere(project, zone string) (*kubernetesAnywhere, error) { 128 if *kubernetesAnywherePath == "" { 129 return nil, fmt.Errorf("--kubernetes-anywhere-path is required") 130 } 131 132 if *kubernetesAnywhereCluster == "" { 133 return nil, fmt.Errorf("--kubernetes-anywhere-cluster is required") 134 } 135 136 if project == "" { 137 return nil, fmt.Errorf("--provider=kubernetes-anywhere requires --gcp-project") 138 } 139 140 if zone == "" { 141 zone = "us-central1-c" 142 } 143 144 kubeletVersion := *kubernetesAnywhereKubeletVersion 145 if *kubernetesAnywhereKubeletCIVersion != "" { 146 // resolvedVersion is EG v1.11.0-alpha.0.1031+d37460147ec956-bazel 147 resolvedVersion, err := resolveCIVersion(*kubernetesAnywhereKubeletCIVersion) 148 if err != nil { 149 return nil, err 150 } 151 kubeletVersion = fmt.Sprintf("gs://kubernetes-release-dev/ci/%v/bin/linux/amd64/", resolvedVersion) 152 } 153 154 // preserve backwards compatibility for e2e tests which never provided cni name 155 if *kubernetesAnywhereCNI == "" && *kubernetesAnywherePhase2Provider == "kubeadm" { 156 *kubernetesAnywhereCNI = defaultKubeadmCNI 157 } 158 159 k := &kubernetesAnywhere{ 160 path: *kubernetesAnywherePath, 161 Phase2Provider: *kubernetesAnywherePhase2Provider, 162 KubeadmVersion: *kubernetesAnywhereKubeadmVersion, 163 KubeletVersion: kubeletVersion, 164 UpgradeMethod: *kubernetesAnywhereUpgradeMethod, 165 KubernetesVersion: *kubernetesAnywhereKubernetesVersion, 166 NumNodes: *kubernetesAnywhereNumNodes, 167 Project: project, 168 Cluster: *kubernetesAnywhereCluster, 169 Zone: zone, 170 Region: regexp.MustCompile(`-[^-]+$`).ReplaceAllString(zone, ""), 171 CNI: *kubernetesAnywhereCNI, 172 KubeproxyMode: *kubernetesAnywhereProxyMode, 173 OSImage: *kubernetesAnywhereOSImage, 174 KubeadmFeatureGates: *kubernetesAnywhereKubeadmFeatureGates, 175 } 176 177 return k, nil 178 } 179 180 func newKubernetesAnywhere(project, zone string) (deployer, error) { 181 k, err := initializeKubernetesAnywhere(project, zone) 182 if err != nil { 183 return nil, err 184 } 185 186 // Set KUBERNETES_CONFORMANCE_TEST so the auth info is picked up 187 // from kubectl instead of bash inference. 188 if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil { 189 return nil, err 190 } 191 192 // Set KUBERNETES_CONFORMANCE_PROVIDER since KUBERNETES_CONFORMANCE_TEST is set 193 // to ensure the right provider is passed onto the test. 194 if err := os.Setenv("KUBERNETES_CONFORMANCE_PROVIDER", "kubernetes-anywhere"); err != nil { 195 return nil, err 196 } 197 198 if err := k.writeConfig(kubernetesAnywhereConfigTemplate); err != nil { 199 return nil, err 200 } 201 return k, nil 202 } 203 204 func resolveCIVersion(version string) (string, error) { 205 file := fmt.Sprintf("gs://kubernetes-release-dev/ci/%v.txt", version) 206 return readGSFile(file) 207 } 208 209 // Implemented as a function var for testing. 210 var readGSFile = readGSFileImpl 211 212 func readGSFileImpl(filepath string) (string, error) { 213 contents, err := control.Output(exec.Command("gsutil", "cat", filepath)) 214 if err != nil { 215 return "", err 216 } 217 return strings.TrimSpace(string(contents)), nil 218 } 219 220 func (k *kubernetesAnywhere) getConfig(configTemplate string) ([]byte, error) { 221 // As needed, plumb through more CLI options to replace these defaults 222 tmpl, err := template.New("kubernetes-anywhere-config").Parse(configTemplate) 223 if err != nil { 224 return nil, fmt.Errorf("Error creating template for KubernetesAnywhere config: %v", err) 225 } 226 227 var buf bytes.Buffer 228 if err = tmpl.Execute(&buf, k); err != nil { 229 return nil, fmt.Errorf("Error executing template for KubernetesAnywhere config: %v", err) 230 } 231 232 return buf.Bytes(), nil 233 } 234 235 func (k *kubernetesAnywhere) writeConfig(configTemplate string) error { 236 config, err := k.getConfig(configTemplate) 237 if err != nil { 238 return fmt.Errorf("Could not generate config: %v", err) 239 } 240 return ioutil.WriteFile(k.path+"/.config", config, 0644) 241 } 242 243 func (k *kubernetesAnywhere) Up() error { 244 cmd := exec.Command("make", "-C", k.path, "WAIT_FOR_KUBECONFIG=y", "deploy") 245 if err := control.FinishRunning(cmd); err != nil { 246 return err 247 } 248 249 if err := k.TestSetup(); err != nil { 250 return err 251 } 252 253 return waitForReadyNodes(k.NumNodes+1, *kubernetesAnywhereUpTimeout, 1) 254 } 255 256 func (k *kubernetesAnywhere) IsUp() error { 257 return isUp(k) 258 } 259 260 func (k *kubernetesAnywhere) DumpClusterLogs(localPath, gcsPath string) error { 261 if !*kubernetesAnywhereDumpClusterLogs { 262 log.Printf("Cluster log dumping disabled for Kubernetes Anywhere.") 263 return nil 264 } 265 return defaultDumpClusterLogs(localPath, gcsPath) 266 } 267 268 func (k *kubernetesAnywhere) TestSetup() error { 269 o, err := control.Output(exec.Command("make", "--silent", "-C", k.path, "kubeconfig-path")) 270 if err != nil { 271 return fmt.Errorf("Could not get kubeconfig-path: %v", err) 272 } 273 kubecfg := strings.TrimSuffix(string(o), "\n") 274 275 if err = os.Setenv("KUBECONFIG", kubecfg); err != nil { 276 return err 277 } 278 return nil 279 } 280 281 func (k *kubernetesAnywhere) Down() error { 282 err := control.FinishRunning(exec.Command("make", "-C", k.path, "kubeconfig-path")) 283 if err != nil { 284 // This is expected if the cluster doesn't exist. 285 return nil 286 } 287 return control.FinishRunning(exec.Command("make", "-C", k.path, "FORCE_DESTROY=y", "destroy")) 288 } 289 290 func (k *kubernetesAnywhere) GetClusterCreated(gcpProject string) (time.Time, error) { 291 return time.Time{}, errors.New("not implemented") 292 } 293 294 const defaultConfigFile = ".config" 295 296 type kubernetesAnywhereMultiCluster struct { 297 *kubernetesAnywhere 298 multiClusters multiClusterDeployment 299 configFile map[string]string 300 kubeContextMap map[string]string 301 } 302 303 // newKubernetesAnywhereMultiCluster returns the deployer based on kubernetes-anywhere 304 // which can be used to deploy multiple clusters simultaneously. 305 func newKubernetesAnywhereMultiCluster(project, zone string, multiClusters multiClusterDeployment) (deployer, error) { 306 if len(multiClusters.clusters) < 1 { 307 return nil, fmt.Errorf("invalid --multi-clusters flag passed") 308 } 309 k, err := initializeKubernetesAnywhere(project, zone) 310 if err != nil { 311 return nil, err 312 } 313 mk := &kubernetesAnywhereMultiCluster{k, multiClusters, make(map[string]string), make(map[string]string)} 314 315 for _, cluster := range mk.multiClusters.clusters { 316 specificZone, specified := mk.multiClusters.zones[cluster] 317 if specified { 318 mk.Zone = specificZone 319 } 320 mk.Cluster = cluster 321 // TODO: revisit the naming of kubecontexts. Currently the federation CI jobs require that the 322 // cluster contexts be prefixed with `federation-` and with particular pattern. 323 mk.KubeContext = "federation-e2e-gce-" + mk.Zone 324 mk.kubeContextMap[cluster] = mk.KubeContext 325 mk.configFile[cluster] = defaultConfigFile + "-" + mk.Cluster 326 if err := mk.writeConfig(kubernetesAnywhereMultiClusterConfigTemplate); err != nil { 327 return nil, err 328 } 329 } 330 return mk, nil 331 } 332 333 // writeConfig writes the kubernetes-anywhere config file to file system after 334 // rendering the template file with configuration in deployer. 335 func (k *kubernetesAnywhereMultiCluster) writeConfig(configTemplate string) error { 336 config, err := k.getConfig(configTemplate) 337 if err != nil { 338 return fmt.Errorf("could not generate config: %v", err) 339 } 340 341 return ioutil.WriteFile(k.path+"/"+k.configFile[k.Cluster], config, 0644) 342 } 343 344 // Up brings up multiple k8s clusters in parallel. 345 func (k *kubernetesAnywhereMultiCluster) Up() error { 346 var cmds []*exec.Cmd 347 for _, cluster := range k.multiClusters.clusters { 348 cmd := exec.Command("make", "-C", k.path, "CONFIG_FILE="+k.configFile[cluster], "deploy") 349 cmds = append(cmds, cmd) 350 } 351 352 if err := control.FinishRunningParallel(cmds...); err != nil { 353 return err 354 } 355 356 return k.TestSetup() 357 } 358 359 // TestSetup sets up test environment by merging kubeconfig of multiple deployments. 360 func (k *kubernetesAnywhereMultiCluster) TestSetup() error { 361 mergedKubeconfigPath := k.path + "/kubeconfig.json" 362 var kubecfg string 363 for _, cluster := range k.multiClusters.clusters { 364 o, err := control.Output(exec.Command("make", "--silent", "-C", k.path, "CONFIG_FILE="+k.configFile[cluster], "kubeconfig-path")) 365 if err != nil { 366 return fmt.Errorf("could not get kubeconfig-path: %v", err) 367 } 368 if len(kubecfg) != 0 { 369 kubecfg += ":" 370 } 371 kubecfg += strings.TrimSuffix(string(o), "\n") 372 } 373 if len(kubecfg) != 0 { 374 kubecfg += ":" + mergedKubeconfigPath 375 } 376 377 if err := os.Setenv("KUBECONFIG", kubecfg); err != nil { 378 return err 379 } 380 381 o, err := control.Output(exec.Command("kubectl", "config", "view", "--flatten=true", "--raw=true")) 382 if err != nil { 383 return fmt.Errorf("could not get kubeconfig-path: %v", err) 384 } 385 386 err = ioutil.WriteFile(mergedKubeconfigPath, o, 0644) 387 if err != nil { 388 return err 389 } 390 391 return os.Setenv("KUBECONFIG", mergedKubeconfigPath) 392 } 393 394 // IsUp checks if all the clusters in the deployer are up. 395 func (k *kubernetesAnywhereMultiCluster) IsUp() error { 396 if err := k.TestSetup(); err != nil { 397 return err 398 } 399 400 for _, cluster := range k.multiClusters.clusters { 401 kubeContext := k.kubeContextMap[cluster] 402 o, err := control.Output(exec.Command("kubectl", "--context="+kubeContext, "get", "nodes", "--no-headers")) 403 if err != nil { 404 log.Printf("kubectl get nodes failed for cluster %s: %s\n%s", cluster, wrapError(err).Error(), string(o)) 405 return err 406 } 407 stdout := strings.TrimSpace(string(o)) 408 log.Printf("Cluster nodes of cluster %s:\n%s", cluster, stdout) 409 410 n := len(strings.Split(stdout, "\n")) 411 if n < k.NumNodes { 412 return fmt.Errorf("cluster %s found, but %d nodes reported", cluster, n) 413 } 414 } 415 return nil 416 } 417 418 // Down brings down multiple k8s clusters in parallel. 419 func (k *kubernetesAnywhereMultiCluster) Down() error { 420 if err := k.TestSetup(); err != nil { 421 // This is expected if the clusters doesn't exist. 422 return nil 423 } 424 425 var cmds []*exec.Cmd 426 for _, cluster := range k.multiClusters.clusters { 427 cmd := exec.Command("make", "-C", k.path, "CONFIG_FILE="+k.configFile[cluster], "FORCE_DESTROY=y", "destroy") 428 cmds = append(cmds, cmd) 429 } 430 return control.FinishRunningParallel(cmds...) 431 }