github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/kubetest/extract.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "fmt" 23 "io/ioutil" 24 "log" 25 "os" 26 "os/exec" 27 "path" 28 "path/filepath" 29 "regexp" 30 "strings" 31 "time" 32 ) 33 34 type extractMode int 35 36 const ( 37 none extractMode = iota 38 local // local 39 gci // gci/FAMILY 40 gciCi // gci/FAMILY/CI_VERSION 41 gke // gke, gke-staging, gke-test 42 ci // ci/latest, ci/latest-1.5 43 rc // release/latest, release/latest-1.5 44 stable // release/stable, release/stable-1.5 45 version // v1.5.0, v1.5.0-beta.2 46 gcs // gs://bucket/prefix/v1.6.0-alpha.0 47 load // Load a --save cluster 48 bazel // A pre/postsubmit bazel build version, prefixed with bazel/ 49 ) 50 51 type extractStrategy struct { 52 mode extractMode 53 option string 54 ciVersion string 55 value string 56 } 57 58 type extractStrategies []extractStrategy 59 60 func (l *extractStrategies) String() string { 61 s := []string{} 62 for _, e := range *l { 63 s = append(s, e.value) 64 } 65 return strings.Join(s, ",") 66 } 67 68 // Converts --extract=release/stable, etc into an extractStrategy{} 69 func (l *extractStrategies) Set(value string) error { 70 var strategies = map[string]extractMode{ 71 `^(local)`: local, 72 `^gke-?(staging|test)?$`: gke, 73 `^gci/([\w-]+)$`: gci, 74 `^gci/([\w-]+)/(.+)$`: gciCi, 75 `^ci/(.+)$`: ci, 76 `^release/(latest.*)$`: rc, 77 `^release/(stable.*)$`: stable, 78 `^(v\d+\.\d+\.\d+[\w.-]*)$`: version, 79 `^(gs://.*)$`: gcs, 80 `^(bazel/.*)$`: bazel, 81 } 82 83 if len(*l) == 2 { 84 return fmt.Errorf("May only define at most 2 --extract strategies: %v %v", *l, value) 85 } 86 for search, mode := range strategies { 87 re := regexp.MustCompile(search) 88 mat := re.FindStringSubmatch(value) 89 if mat == nil { 90 continue 91 } 92 e := extractStrategy{ 93 mode: mode, 94 option: mat[1], 95 value: value, 96 } 97 if len(mat) > 2 { 98 e.ciVersion = mat[2] 99 } 100 *l = append(*l, e) 101 return nil 102 } 103 return fmt.Errorf("Unknown extraction strategy: %v", value) 104 105 } 106 107 // True when this kubetest invocation wants to download and extract a release. 108 func (l *extractStrategies) Enabled() bool { 109 return len(*l) > 0 110 } 111 112 func (e extractStrategy) name() string { 113 return filepath.Base(e.option) 114 } 115 116 func (l extractStrategies) Extract(project, zone string) error { 117 // rm -rf kubernetes* 118 files, err := ioutil.ReadDir(".") 119 if err != nil { 120 return err 121 } 122 for _, file := range files { 123 name := file.Name() 124 if !strings.HasPrefix(name, "kubernetes") { 125 continue 126 } 127 log.Printf("rm %s", name) 128 if err = os.RemoveAll(name); err != nil { 129 return err 130 } 131 } 132 133 for i, e := range l { 134 if i > 0 { 135 // TODO(fejta): new strategy so we support more than 2 --extracts 136 if err := os.Rename("kubernetes", "kubernetes_skew"); err != nil { 137 return err 138 } 139 } 140 if err := e.Extract(project, zone); err != nil { 141 return err 142 } 143 } 144 145 return os.Chdir("kubernetes") 146 } 147 148 // Find get-kube.sh at PWD, in PATH or else download it. 149 func ensureKube() (string, error) { 150 // Does get-kube.sh exist in pwd? 151 i, err := os.Stat("./get-kube.sh") 152 if err == nil && !i.IsDir() && i.Mode()&0111 > 0 { 153 return "./get-kube.sh", nil 154 } 155 156 // How about in the path? 157 p, err := exec.LookPath("get-kube.sh") 158 if err == nil { 159 return p, nil 160 } 161 162 // Download it to a temp file 163 f, err := ioutil.TempFile("", "get-kube") 164 if err != nil { 165 return "", err 166 } 167 defer f.Close() 168 if err := httpRead("https://get.k8s.io", f); err != nil { 169 return "", err 170 } 171 i, err = f.Stat() 172 if err != nil { 173 return "", err 174 } 175 if err := os.Chmod(f.Name(), i.Mode()|0111); err != nil { 176 return "", err 177 } 178 return f.Name(), nil 179 } 180 181 // Download test binaries for kubernetes versions before 1.5. 182 func getTestBinaries(url, version string) error { 183 f, err := os.Create("kubernetes-test.tar.gz") 184 if err != nil { 185 return err 186 } 187 defer f.Close() 188 full := fmt.Sprintf("%v/%v/kubernetes-test.tar.gz", url, version) 189 if err := httpRead(full, f); err != nil { 190 return err 191 } 192 f.Close() 193 o, err := output(exec.Command("md5sum", f.Name())) 194 if err != nil { 195 return err 196 } 197 log.Printf("md5sum: %s", o) 198 if err = finishRunning(exec.Command("tar", "-xzf", f.Name())); err != nil { 199 return err 200 } 201 return nil 202 } 203 204 var ( 205 sleep = time.Sleep 206 ) 207 208 // Calls KUBERNETES_RELASE_URL=url KUBERNETES_RELEASE=version get-kube.sh. 209 // This will download version from the specified url subdir and extract 210 // the tarballs. 211 var getKube = func(url, version string) error { 212 k, err := ensureKube() 213 if err != nil { 214 return err 215 } 216 if err := os.Setenv("KUBERNETES_RELEASE_URL", url); err != nil { 217 return err 218 } 219 220 if err := os.Setenv("KUBERNETES_RELEASE", version); err != nil { 221 return err 222 } 223 if err := os.Setenv("KUBERNETES_SKIP_CONFIRM", "y"); err != nil { 224 return err 225 } 226 if err := os.Setenv("KUBERNETES_SKIP_CREATE_CLUSTER", "y"); err != nil { 227 return err 228 } 229 if err := os.Setenv("KUBERNETES_DOWNLOAD_TESTS", "y"); err != nil { 230 return err 231 } 232 // kube-up in cluster/gke/util.sh depends on this 233 if err := os.Setenv("CLUSTER_API_VERSION", version[1:]); err != nil { 234 return err 235 } 236 log.Printf("U=%s R=%s get-kube.sh", url, version) 237 for i := 0; i < 3; i++ { 238 err = finishRunning(exec.Command(k)) 239 if err == nil { 240 break 241 } 242 err = fmt.Errorf("U=%s R=%s get-kube.sh failed: %v", url, version, err) 243 if i == 2 { 244 return err 245 } 246 log.Println(err) 247 sleep(time.Duration(i) * time.Second) 248 } 249 i, err := os.Stat("./kubernetes/cluster/get-kube-binaries.sh") 250 if err != nil || i.IsDir() { 251 log.Printf("Grabbing test binaries since R=%s < 1.5", version) 252 if err = getTestBinaries(url, version); err != nil { 253 return err 254 } 255 } 256 return nil 257 } 258 259 func setReleaseFromGcs(ci bool, suffix string) error { 260 var prefix string 261 if ci { 262 prefix = "kubernetes-release-dev/ci" 263 } else { 264 prefix = "kubernetes-release/release" 265 } 266 267 url := fmt.Sprintf("https://storage.googleapis.com/%v", prefix) 268 cat := fmt.Sprintf("gs://%v/%v.txt", prefix, suffix) 269 release, err := output(exec.Command("gsutil", "cat", cat)) 270 if err != nil { 271 return err 272 } 273 return getKube(url, strings.TrimSpace(string(release))) 274 } 275 276 func setupGciVars(family string) (string, error) { 277 p := "container-vm-image-staging" 278 b, err := output(exec.Command("gcloud", "compute", "images", "describe-from-family", family, fmt.Sprintf("--project=%v", p), "--format=value(name)")) 279 if err != nil { 280 return "", err 281 } 282 i := strings.TrimSpace(string(b)) 283 g := "gci" 284 m := map[string]string{ 285 "KUBE_GCE_MASTER_PROJECT": p, 286 "KUBE_GCE_MASTER_IMAGE": i, 287 "KUBE_MASTER_OS_DISTRIBUTION": g, 288 289 "KUBE_GCE_NODE_PROJECT": p, 290 "KUBE_GCE_NODE_IMAGE": i, 291 "KUBE_NODE_OS_DISTRIBUTION": g, 292 293 "BUILD_METADATA_GCE_MASTER_IMAGE": i, 294 "BUILD_METADATA_GCE_NODE_IMAGE": i, 295 296 "KUBE_OS_DISTRIBUTION": g, 297 } 298 if family == "gci-canary-test" { 299 var b bytes.Buffer 300 if err := httpRead("https://api.github.com/repos/docker/docker/releases", &b); err != nil { 301 return "", err 302 } 303 var v []map[string]interface{} 304 if err := json.NewDecoder(&b).Decode(&v); err != nil { 305 return "", err 306 } 307 // We want 1.13.0 308 m["KUBE_GCI_DOCKER_VERSION"] = v[0]["name"].(string)[1:] 309 } 310 for k, v := range m { 311 log.Printf("export %s=%s", k, v) 312 if err := os.Setenv(k, v); err != nil { 313 return "", err 314 } 315 } 316 return i, nil 317 } 318 319 func setReleaseFromGci(image string) error { 320 u := fmt.Sprintf("gs://container-vm-image-staging/k8s-version-map/%s", image) 321 b, err := output(exec.Command("gsutil", "cat", u)) 322 if err != nil { 323 return err 324 } 325 r := fmt.Sprintf("v%s", b) 326 return getKube("https://storage.googleapis.com/kubernetes-release/release", strings.TrimSpace(r)) 327 } 328 329 func (e extractStrategy) Extract(project, zone string) error { 330 switch e.mode { 331 case local: 332 url := k8s("kubernetes", "_output", "gcs-stage") 333 files, err := ioutil.ReadDir(url) 334 if err != nil { 335 return err 336 } 337 var release string 338 for _, file := range files { 339 r := file.Name() 340 if strings.HasPrefix(r, "v") { 341 release = r 342 break 343 } 344 } 345 if len(release) == 0 { 346 return fmt.Errorf("No releases found in %v", url) 347 } 348 return getKube(fmt.Sprintf("file://%s", url), release) 349 case gci, gciCi: 350 if i, err := setupGciVars(e.option); err != nil { 351 return err 352 } else if e.ciVersion != "" { 353 return setReleaseFromGcs(true, e.ciVersion) 354 } else { 355 return setReleaseFromGci(i) 356 } 357 case gke: 358 // TODO(fejta): prod v staging v test 359 if project == "" { 360 return fmt.Errorf("--gcp-project unset") 361 } 362 if zone == "" { 363 return fmt.Errorf("--gcp-zone unset") 364 } 365 ci, err := output(exec.Command("gcloud", "container", "get-server-config", fmt.Sprintf("--project=%v", project), fmt.Sprintf("--zone=%v", zone), "--format=value(defaultClusterVersion)")) 366 if err != nil { 367 return err 368 } 369 re := regexp.MustCompile(`(\d+\.\d+)(\..+)?$`) // 1.11.7-beta.0 -> 1.11 370 mat := re.FindStringSubmatch(strings.TrimSpace(string(ci))) 371 if mat == nil { 372 return fmt.Errorf("failed to parse version from %s", ci) 373 } 374 // When JENKINS_USE_SERVER_VERSION=y, we launch the default version as determined 375 // by GKE, but pull the latest version of that branch for tests. e.g. if the default 376 // version is 1.5.3, we would pull test binaries at ci/latest-1.5.txt, but launch 377 // the default (1.5.3). We have to unset CLUSTER_API_VERSION here to allow GKE to 378 // launch the default. 379 // TODO(fejta): clean up this logic. Setting/unsetting the same env var is gross. 380 defer os.Unsetenv("CLUSTER_API_VERSION") 381 return setReleaseFromGcs(true, "latest-"+mat[1]) 382 case ci: 383 return setReleaseFromGcs(true, e.option) 384 case rc, stable: 385 return setReleaseFromGcs(false, e.option) 386 case version: 387 var url string 388 release := e.option 389 if strings.Contains(release, "+") { 390 url = "https://storage.googleapis.com/kubernetes-release-dev/ci" 391 } else { 392 url = "https://storage.googleapis.com/kubernetes-release/release" 393 } 394 return getKube(url, release) 395 case gcs: 396 // strip gs://foo -> /foo 397 withoutGS := e.option[3:] 398 url := "https://storage.googleapis.com" + path.Dir(withoutGS) 399 return getKube(url, path.Base(withoutGS)) 400 case load: 401 return loadState(e.option) 402 case bazel: 403 return getKube("", e.option) 404 } 405 return fmt.Errorf("Unrecognized extraction: %v(%v)", e.mode, e.value) 406 } 407 408 func loadKubeconfig(save string) error { 409 cURL, err := joinURL(save, "kube-config") 410 if err != nil { 411 return fmt.Errorf("bad load url %s: %v", save, err) 412 } 413 if err := os.MkdirAll(home(".kube"), 0775); err != nil { 414 return err 415 } 416 return finishRunning(exec.Command("gsutil", "cp", cURL, home(".kube", "config"))) 417 } 418 419 func loadState(save string) error { 420 log.Printf("Restore state from %s", save) 421 422 uURL, err := joinURL(save, "release-url.txt") 423 if err != nil { 424 return fmt.Errorf("bad load url %s: %v", save, err) 425 } 426 rURL, err := joinURL(save, "release.txt") 427 if err != nil { 428 return fmt.Errorf("bad load url %s: %v", save, err) 429 } 430 431 if err := loadKubeconfig(save); err != nil { 432 return fmt.Errorf("failed loading kubeconfig: %v", err) 433 } 434 435 url, err := output(exec.Command("gsutil", "cat", uURL)) 436 if err != nil { 437 return err 438 } 439 release, err := output(exec.Command("gsutil", "cat", rURL)) 440 if err != nil { 441 return err 442 } 443 return getKube(string(url), string(release)) 444 } 445 446 func saveState(save string) error { 447 url := os.Getenv("KUBERNETES_RELEASE_URL") // TODO(fejta): pass this in to saveState 448 version := os.Getenv("KUBERNETES_RELEASE") 449 log.Printf("Save U=%s R=%s to %s", url, version, save) 450 cURL, err := joinURL(save, "kube-config") 451 if err != nil { 452 return fmt.Errorf("bad save url %s: %v", save, err) 453 } 454 uURL, err := joinURL(save, "release-url.txt") 455 if err != nil { 456 return fmt.Errorf("bad save url %s: %v", save, err) 457 } 458 rURL, err := joinURL(save, "release.txt") 459 if err != nil { 460 return fmt.Errorf("bad save url %s: %v", save, err) 461 } 462 463 if err := finishRunning(exec.Command("gsutil", "cp", home(".kube", "config"), cURL)); err != nil { 464 return fmt.Errorf("failed to save .kube/config to %s: %v", cURL, err) 465 } 466 if cmd, err := inputCommand(url, "gsutil", "cp", "-", uURL); err != nil { 467 return fmt.Errorf("failed to write url %s to %s: %v", url, uURL, err) 468 } else if err = finishRunning(cmd); err != nil { 469 return fmt.Errorf("failed to upload url %s to %s: %v", url, uURL, err) 470 } 471 472 if cmd, err := inputCommand(version, "gsutil", "cp", "-", rURL); err != nil { 473 return fmt.Errorf("failed to write release %s to %s: %v", version, rURL, err) 474 } else if err = finishRunning(cmd); err != nil { 475 return fmt.Errorf("failed to upload release %s to %s: %v", version, rURL, err) 476 } 477 return nil 478 }