github.com/jenkins-x/test-infra@v0.0.7/kubetest/extract_k8s.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "fmt" 23 "io/ioutil" 24 "log" 25 "os" 26 "os/exec" 27 "path" 28 "path/filepath" 29 "regexp" 30 "strings" 31 "time" 32 33 "k8s.io/test-infra/kubetest/util" 34 ) 35 36 type extractMode int 37 38 const ( 39 none extractMode = iota 40 local // local 41 gci // gci/FAMILY 42 gciCi // gci/FAMILY/CI_VERSION 43 gke // gke(deprecated), gke-default, gke-latest 44 ci // ci/latest, ci/latest-1.5 45 rc // release/latest, release/latest-1.5 46 stable // release/stable, release/stable-1.5 47 version // v1.5.0, v1.5.0-beta.2 48 gcs // gs://bucket/prefix/v1.6.0-alpha.0 49 load // Load a --save cluster 50 bazel // A pre/postsubmit bazel build version, prefixed with bazel/ 51 ) 52 53 type extractStrategy struct { 54 mode extractMode 55 option string 56 ciVersion string 57 value string 58 } 59 60 type extractStrategies []extractStrategy 61 62 func (l *extractStrategies) String() string { 63 s := []string{} 64 for _, e := range *l { 65 s = append(s, e.value) 66 } 67 return strings.Join(s, ",") 68 } 69 70 // Converts --extract=release/stable, etc into an extractStrategy{} 71 func (l *extractStrategies) Set(value string) error { 72 var strategies = map[string]extractMode{ 73 `^(local)`: local, 74 `^gke-?(default|latest(-\d+.\d+)?)?$`: gke, 75 `^gci/([\w-]+)$`: gci, 76 `^gci/([\w-]+)/(.+)$`: gciCi, 77 `^ci/(.+)$`: ci, 78 `^release/(latest.*)$`: rc, 79 `^release/(stable.*)$`: stable, 80 `^(v\d+\.\d+\.\d+[\w.\-+]*)$`: version, 81 `^(gs://.*)$`: gcs, 82 `^(bazel/.*)$`: bazel, 83 } 84 85 if len(*l) == 2 { 86 return fmt.Errorf("May only define at most 2 --extract strategies: %v %v", *l, value) 87 } 88 for search, mode := range strategies { 89 re := regexp.MustCompile(search) 90 mat := re.FindStringSubmatch(value) 91 if mat == nil { 92 continue 93 } 94 e := extractStrategy{ 95 mode: mode, 96 option: mat[1], 97 value: value, 98 } 99 if len(mat) > 2 { 100 e.ciVersion = mat[2] 101 } 102 *l = append(*l, e) 103 return nil 104 } 105 return fmt.Errorf("Unknown extraction strategy: %v", value) 106 107 } 108 109 func (l *extractStrategies) Type() string { 110 return "exactStrategies" 111 } 112 113 // True when this kubetest invocation wants to download and extract a release. 114 func (l *extractStrategies) Enabled() bool { 115 return len(*l) > 0 116 } 117 118 func (e extractStrategy) name() string { 119 return filepath.Base(e.option) 120 } 121 122 func (l extractStrategies) Extract(project, zone, region string, extractSrc bool) error { 123 // rm -rf kubernetes* 124 files, err := ioutil.ReadDir(".") 125 if err != nil { 126 return err 127 } 128 for _, file := range files { 129 name := file.Name() 130 if !strings.HasPrefix(name, "kubernetes") { 131 continue 132 } 133 log.Printf("rm %s", name) 134 if err = os.RemoveAll(name); err != nil { 135 return err 136 } 137 } 138 139 for i, e := range l { 140 if i > 0 { 141 // TODO(fejta): new strategy so we support more than 2 --extracts 142 if err := os.Rename("kubernetes", "kubernetes_skew"); err != nil { 143 return err 144 } 145 } 146 if err := e.Extract(project, zone, region, extractSrc); err != nil { 147 return err 148 } 149 } 150 151 return nil 152 } 153 154 // Find get-kube.sh at PWD, in PATH or else download it. 155 func ensureKube() (string, error) { 156 // Does get-kube.sh exist in pwd? 157 i, err := os.Stat("./get-kube.sh") 158 if err == nil && !i.IsDir() && i.Mode()&0111 > 0 { 159 return "./get-kube.sh", nil 160 } 161 162 // How about in the path? 163 p, err := exec.LookPath("get-kube.sh") 164 if err == nil { 165 return p, nil 166 } 167 168 // Download it to a temp file 169 f, err := ioutil.TempFile("", "get-kube") 170 if err != nil { 171 return "", err 172 } 173 defer f.Close() 174 if err := httpRead("https://get.k8s.io", f); err != nil { 175 return "", err 176 } 177 i, err = f.Stat() 178 if err != nil { 179 return "", err 180 } 181 if err := os.Chmod(f.Name(), i.Mode()|0111); err != nil { 182 return "", err 183 } 184 return f.Name(), nil 185 } 186 187 // Download named binaries for kubernetes 188 func getNamedBinaries(url, version, tarball string, retry int) error { 189 f, err := os.Create(tarball) 190 if err != nil { 191 return err 192 } 193 defer f.Close() 194 full := fmt.Sprintf("%s/%s/%s", url, version, tarball) 195 196 for i := 0; i < retry; i++ { 197 log.Printf("downloading %v from %v", tarball, full) 198 if err := httpRead(full, f); err == nil { 199 break 200 } 201 err = fmt.Errorf("url=%s version=%s failed get %v: %v", url, version, tarball, err) 202 if i == retry-1 { 203 return err 204 } 205 log.Println(err) 206 sleep(time.Duration(i) * time.Second) 207 } 208 209 f.Close() 210 o, err := control.Output(exec.Command("md5sum", f.Name())) 211 if err != nil { 212 return err 213 } 214 log.Printf("md5sum: %s", o) 215 216 cwd, err := os.Getwd() 217 if err != nil { 218 return fmt.Errorf("unable to get current directory: %v", err) 219 } 220 log.Printf("Extracting tar file %v into directory %v", f.Name(), cwd) 221 222 if err = control.FinishRunning(exec.Command("tar", "-xzf", f.Name())); err != nil { 223 return err 224 } 225 return nil 226 } 227 228 var ( 229 sleep = time.Sleep 230 ) 231 232 // Calls KUBERNETES_RELEASE_URL=url KUBERNETES_RELEASE=version get-kube.sh. 233 // This will download version from the specified url subdir and extract 234 // the tarballs. 235 var getKube = func(url, version string, getSrc bool) error { 236 // TODO(krzyzacy): migrate rest of the get-kube.sh logic into kubetest, using getNamedBinaries 237 // get/extract the src tarball first since bazel needs a clean tree 238 if getSrc { 239 cwd, err := os.Getwd() 240 if err != nil { 241 return err 242 } 243 if cwd != "kubernetes" { 244 if err = os.Mkdir("kubernetes", 0755); err != nil { 245 return err 246 } 247 if err = os.Chdir("kubernetes"); err != nil { 248 return err 249 } 250 } 251 252 if err := os.Setenv("KUBE_GIT_VERSION", version); err != nil { 253 return err 254 } 255 256 if err := getNamedBinaries(url, version, "kubernetes-src.tar.gz", 3); err != nil { 257 return err 258 } 259 } 260 261 k, err := ensureKube() 262 if err != nil { 263 return err 264 } 265 if err := os.Setenv("KUBERNETES_RELEASE_URL", url); err != nil { 266 return err 267 } 268 269 if err := os.Setenv("KUBERNETES_RELEASE", version); err != nil { 270 return err 271 } 272 if err := os.Setenv("KUBERNETES_SKIP_CONFIRM", "y"); err != nil { 273 return err 274 } 275 if err := os.Setenv("KUBERNETES_SKIP_CREATE_CLUSTER", "y"); err != nil { 276 return err 277 } 278 if err := os.Setenv("KUBERNETES_DOWNLOAD_TESTS", "y"); err != nil { 279 return err 280 } 281 // kube-up in cluster/gke/util.sh depends on this 282 if err := os.Setenv("CLUSTER_API_VERSION", version[1:]); err != nil { 283 return err 284 } 285 log.Printf("U=%s R=%s get-kube.sh", url, version) 286 for i := 0; i < 3; i++ { 287 err = control.FinishRunning(exec.Command(k)) 288 if err == nil { 289 break 290 } 291 err = fmt.Errorf("U=%s R=%s get-kube.sh failed: %v", url, version, err) 292 if i == 2 { 293 return err 294 } 295 log.Println(err) 296 sleep(time.Duration(i) * time.Second) 297 } 298 299 return nil 300 } 301 302 // wrapper for gsutil cat 303 var gsutilCat = func(url string) ([]byte, error) { 304 return control.Output(exec.Command("gsutil", "cat", url)) 305 } 306 307 func setReleaseFromGcs(prefix, suffix string, getSrc bool) error { 308 url := fmt.Sprintf("https://storage.googleapis.com/%v", prefix) 309 release, err := gsutilCat(fmt.Sprintf("gs://%v/%v.txt", prefix, suffix)) 310 if err != nil { 311 return err 312 } 313 return getKube(url, strings.TrimSpace(string(release)), getSrc) 314 } 315 316 func setupGciVars(family string) (string, error) { 317 p := "container-vm-image-staging" 318 b, err := control.Output(exec.Command("gcloud", "compute", "images", "describe-from-family", family, fmt.Sprintf("--project=%v", p), "--format=value(name)")) 319 if err != nil { 320 return "", err 321 } 322 i := strings.TrimSpace(string(b)) 323 g := "gci" 324 m := map[string]string{ 325 "KUBE_GCE_MASTER_PROJECT": p, 326 "KUBE_GCE_MASTER_IMAGE": i, 327 "KUBE_MASTER_OS_DISTRIBUTION": g, 328 329 "KUBE_GCE_NODE_PROJECT": p, 330 "KUBE_GCE_NODE_IMAGE": i, 331 "KUBE_NODE_OS_DISTRIBUTION": g, 332 333 "BUILD_METADATA_GCE_MASTER_IMAGE": i, 334 "BUILD_METADATA_GCE_NODE_IMAGE": i, 335 336 "KUBE_OS_DISTRIBUTION": g, 337 } 338 if family == "gci-canary-test" { 339 var b bytes.Buffer 340 if err := httpRead("https://api.github.com/repos/docker/docker/releases", &b); err != nil { 341 return "", err 342 } 343 var v []map[string]interface{} 344 if err := json.NewDecoder(&b).Decode(&v); err != nil { 345 return "", err 346 } 347 // We want 1.13.0 348 m["KUBE_GCI_DOCKER_VERSION"] = v[0]["name"].(string)[1:] 349 } 350 for k, v := range m { 351 log.Printf("export %s=%s", k, v) 352 if err := os.Setenv(k, v); err != nil { 353 return "", err 354 } 355 } 356 return i, nil 357 } 358 359 func setReleaseFromGci(image string, getSrc bool) error { 360 b, err := gsutilCat(fmt.Sprintf("gs://container-vm-image-staging/k8s-version-map/%s", image)) 361 if err != nil { 362 return err 363 } 364 r := fmt.Sprintf("v%s", b) 365 return getKube("https://storage.googleapis.com/kubernetes-release/release", strings.TrimSpace(r), getSrc) 366 } 367 368 func (e extractStrategy) Extract(project, zone, region string, extractSrc bool) error { 369 switch e.mode { 370 case local: 371 url := util.K8s("kubernetes", "_output", "gcs-stage") 372 files, err := ioutil.ReadDir(url) 373 if err != nil { 374 return err 375 } 376 var release string 377 for _, file := range files { 378 r := file.Name() 379 if strings.HasPrefix(r, "v") { 380 release = r 381 break 382 } 383 } 384 if len(release) == 0 { 385 return fmt.Errorf("No releases found in %v", url) 386 } 387 return getKube(fmt.Sprintf("file://%s", url), release, extractSrc) 388 case gci, gciCi: 389 if i, err := setupGciVars(e.option); err != nil { 390 return err 391 } else if e.ciVersion != "" { 392 return setReleaseFromGcs("kubernetes-release-dev/ci", e.ciVersion, extractSrc) 393 } else { 394 return setReleaseFromGci(i, extractSrc) 395 } 396 case gke: 397 // TODO(fejta): prod v staging v test 398 if project == "" { 399 return fmt.Errorf("--gcp-project unset") 400 } 401 if e.value == "gke" { 402 log.Print("*** --extract=gke is deprecated, migrate to --extract=gke-default ***") 403 } 404 if strings.HasPrefix(e.option, "latest") { 405 // get latest supported master version 406 releasePrefix := "" 407 if strings.HasPrefix(e.option, "latest-") { 408 releasePrefix = strings.TrimPrefix(e.option, "latest-") 409 } 410 version, err := getLatestGKEVersion(project, zone, region, releasePrefix) 411 if err != nil { 412 return fmt.Errorf("failed to get latest gke version: %s", err) 413 } 414 return getKube("https://storage.googleapis.com/kubernetes-release-gke/release", version, extractSrc) 415 } 416 417 // TODO(krzyzacy): clean up gke-default logic 418 if zone == "" { 419 return fmt.Errorf("--gcp-zone unset") 420 } 421 422 // get default cluster version for default extract strategy 423 ci, err := control.Output(exec.Command("gcloud", "container", "get-server-config", fmt.Sprintf("--project=%v", project), fmt.Sprintf("--zone=%v", zone), "--format=value(defaultClusterVersion)")) 424 if err != nil { 425 return err 426 } 427 re := regexp.MustCompile(`(\d+\.\d+)(\..+)?$`) // 1.11.7-beta.0 -> 1.11 428 mat := re.FindStringSubmatch(strings.TrimSpace(string(ci))) 429 if mat == nil { 430 return fmt.Errorf("failed to parse version from %s", ci) 431 } 432 // When JENKINS_USE_SERVER_VERSION=y, we launch the default version as determined 433 // by GKE, but pull the latest version of that branch for tests. e.g. if the default 434 // version is 1.5.3, we would pull test binaries at ci/latest-1.5.txt, but launch 435 // the default (1.5.3). We have to unset CLUSTER_API_VERSION here to allow GKE to 436 // launch the default. 437 // TODO(fejta): clean up this logic. Setting/unsetting the same env var is gross. 438 defer os.Unsetenv("CLUSTER_API_VERSION") 439 return setReleaseFromGcs("kubernetes-release-dev/ci", "latest-"+mat[1], extractSrc) 440 case ci: 441 prefix := "kubernetes-release-dev/ci" 442 if strings.HasPrefix(e.option, "gke-") { 443 prefix = "kubernetes-release-gke/release" 444 } 445 return setReleaseFromGcs(prefix, e.option, extractSrc) 446 case rc, stable: 447 return setReleaseFromGcs("kubernetes-release/release", e.option, extractSrc) 448 case version: 449 var url string 450 release := e.option 451 re := regexp.MustCompile(`(v\d+\.\d+\.\d+-gke.\d+)$`) // v1.8.0-gke.0 452 if re.FindStringSubmatch(release) != nil { 453 url = "https://storage.googleapis.com/kubernetes-release-gke/release" 454 } else if strings.Contains(release, "+") { 455 url = "https://storage.googleapis.com/kubernetes-release-dev/ci" 456 } else { 457 url = "https://storage.googleapis.com/kubernetes-release/release" 458 } 459 return getKube(url, release, extractSrc) 460 case gcs: 461 // strip gs://foo/bar(.txt) -> foo/bar(.txt) 462 withoutGS := e.option[5:] 463 if strings.HasSuffix(e.option, ".txt") { 464 // foo/bar.txt -> bar 465 suffix := strings.TrimSuffix(path.Base(withoutGS), filepath.Ext(withoutGS)) 466 return setReleaseFromGcs(path.Dir(withoutGS), suffix, extractSrc) 467 } 468 url := "https://storage.googleapis.com" + "/" + path.Dir(withoutGS) 469 return getKube(url, path.Base(withoutGS), extractSrc) 470 case load: 471 return loadState(e.option, extractSrc) 472 case bazel: 473 return getKube("", e.option, extractSrc) 474 } 475 return fmt.Errorf("Unrecognized extraction: %v(%v)", e.mode, e.value) 476 } 477 478 func loadKubeconfig(save string) error { 479 cURL, err := util.JoinURL(save, "kube-config") 480 if err != nil { 481 return fmt.Errorf("bad load url %s: %v", save, err) 482 } 483 if err := os.MkdirAll(util.Home(".kube"), 0775); err != nil { 484 return err 485 } 486 return control.FinishRunning(exec.Command("gsutil", "cp", cURL, util.Home(".kube", "config"))) 487 } 488 489 func loadState(save string, getSrc bool) error { 490 log.Printf("Restore state from %s", save) 491 492 uURL, err := util.JoinURL(save, "release-url.txt") 493 if err != nil { 494 return fmt.Errorf("bad load url %s: %v", save, err) 495 } 496 rURL, err := util.JoinURL(save, "release.txt") 497 if err != nil { 498 return fmt.Errorf("bad load url %s: %v", save, err) 499 } 500 501 if err := loadKubeconfig(save); err != nil { 502 return fmt.Errorf("failed loading kubeconfig: %v", err) 503 } 504 505 url, err := gsutilCat(uURL) 506 if err != nil { 507 return err 508 } 509 release, err := gsutilCat(rURL) 510 if err != nil { 511 return err 512 } 513 return getKube(string(url), string(release), getSrc) 514 } 515 516 func saveState(save string) error { 517 url := os.Getenv("KUBERNETES_RELEASE_URL") // TODO(fejta): pass this in to saveState 518 version := os.Getenv("KUBERNETES_RELEASE") 519 log.Printf("Save U=%s R=%s to %s", url, version, save) 520 cURL, err := util.JoinURL(save, "kube-config") 521 if err != nil { 522 return fmt.Errorf("bad save url %s: %v", save, err) 523 } 524 uURL, err := util.JoinURL(save, "release-url.txt") 525 if err != nil { 526 return fmt.Errorf("bad save url %s: %v", save, err) 527 } 528 rURL, err := util.JoinURL(save, "release.txt") 529 if err != nil { 530 return fmt.Errorf("bad save url %s: %v", save, err) 531 } 532 533 if err := control.FinishRunning(exec.Command("gsutil", "cp", util.Home(".kube", "config"), cURL)); err != nil { 534 return fmt.Errorf("failed to save .kube/config to %s: %v", cURL, err) 535 } 536 if cmd, err := control.InputCommand(url, "gsutil", "cp", "-", uURL); err != nil { 537 return fmt.Errorf("failed to write url %s to %s: %v", url, uURL, err) 538 } else if err = control.FinishRunning(cmd); err != nil { 539 return fmt.Errorf("failed to upload url %s to %s: %v", url, uURL, err) 540 } 541 542 if cmd, err := control.InputCommand(version, "gsutil", "cp", "-", rURL); err != nil { 543 return fmt.Errorf("failed to write release %s to %s: %v", version, rURL, err) 544 } else if err = control.FinishRunning(cmd); err != nil { 545 return fmt.Errorf("failed to upload release %s to %s: %v", version, rURL, err) 546 } 547 return nil 548 }