github.com/jenkins-x/test-infra@v0.0.7/kubetest/extract_federation.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "fmt" 21 "io/ioutil" 22 "log" 23 "os" 24 "os/exec" 25 "path" 26 "path/filepath" 27 "regexp" 28 "strings" 29 "time" 30 31 "k8s.io/test-infra/kubetest/util" 32 ) 33 34 type extractFederationStrategy struct { 35 mode extractMode 36 option string 37 ciVersion string 38 value string 39 } 40 41 type extractFederationStrategies []extractFederationStrategy 42 43 func (l *extractFederationStrategies) String() string { 44 s := []string{} 45 for _, e := range *l { 46 s = append(s, e.value) 47 } 48 return strings.Join(s, ",") 49 } 50 51 // Converts --extract-federation=release/stable, etc into an extractFederationStrategy{} 52 func (l *extractFederationStrategies) Set(value string) error { 53 var strategies = map[string]extractMode{ 54 `^(local)`: local, 55 `^ci/(.+)$`: ci, 56 `^release/(latest.*)$`: rc, 57 `^release/(stable.*)$`: stable, 58 `^(v\d+\.\d+\.\d+[\w.\-+]*)$`: version, 59 `^(gs://.*)$`: gcs, 60 } 61 62 if len(*l) == 2 { 63 return fmt.Errorf("may only define at most 2 --extract-federation strategies: %v %v", *l, value) 64 } 65 for search, mode := range strategies { 66 re := regexp.MustCompile(search) 67 mat := re.FindStringSubmatch(value) 68 if mat == nil { 69 continue 70 } 71 e := extractFederationStrategy{ 72 mode: mode, 73 option: mat[1], 74 value: value, 75 } 76 if len(mat) > 2 { 77 e.ciVersion = mat[2] 78 } 79 *l = append(*l, e) 80 return nil 81 } 82 return fmt.Errorf("unknown federation extraction strategy: %v", value) 83 84 } 85 86 func (l *extractFederationStrategies) Type() string { 87 return "extractFederationStrategies" 88 } 89 90 // True when this kubetest invocation wants to download and extract a federation release. 91 func (l *extractFederationStrategies) Enabled() bool { 92 return len(*l) > 0 93 } 94 95 func (e extractFederationStrategy) name() string { 96 return filepath.Base(e.option) 97 } 98 99 func (l extractFederationStrategies) Extract(project, zone string) error { 100 // rm -rf federation* 101 files, err := ioutil.ReadDir(".") 102 if err != nil { 103 return err 104 } 105 for _, file := range files { 106 name := file.Name() 107 if !strings.HasPrefix(name, "federation") { 108 continue 109 } 110 log.Printf("rm %s", name) 111 if err = os.RemoveAll(name); err != nil { 112 return err 113 } 114 } 115 116 for i, e := range l { 117 if i > 0 { 118 if err := os.Rename("federation", "federation_skew"); err != nil { 119 return err 120 } 121 } 122 if err := e.Extract(project, zone); err != nil { 123 return err 124 } 125 } 126 127 return nil 128 } 129 130 // Find get-federation.sh at PWD, in PATH or else download it. 131 func ensureFederation() (string, error) { 132 // Does get-federation.sh exist in pwd? 133 i, err := os.Stat("./get-federation.sh") 134 if err == nil && !i.IsDir() && i.Mode()&0111 > 0 { 135 return "./get-federation.sh", nil 136 } 137 138 // How about in the path? 139 p, err := exec.LookPath("get-federation.sh") 140 if err == nil { 141 return p, nil 142 } 143 144 // Download it to a temp file 145 f, err := ioutil.TempFile("", "get-federation") 146 if err != nil { 147 return "", err 148 } 149 defer f.Close() 150 if err := httpRead("https://raw.githubusercontent.com/kubernetes/federation/master/cluster/get-federation.sh", f); err != nil { 151 return "", err 152 } 153 i, err = f.Stat() 154 if err != nil { 155 return "", err 156 } 157 if err := os.Chmod(f.Name(), i.Mode()|0111); err != nil { 158 return "", err 159 } 160 return f.Name(), nil 161 } 162 163 // Calls FEDERATION_RELEASE_URL=url FEDERATION_RELEASE=version get-federation.sh. 164 // This will download version from the specified url subdir and extract 165 // the tarballs. 166 var getFederation = func(url, version string) error { 167 k, err := ensureFederation() 168 if err != nil { 169 return err 170 } 171 if err := os.Setenv("FEDERATION_RELEASE_URL", url); err != nil { 172 return err 173 } 174 175 if err := os.Setenv("FEDERATION_RELEASE", version); err != nil { 176 return err 177 } 178 if err := os.Setenv("FEDERATION_SKIP_CONFIRM", "y"); err != nil { 179 return err 180 } 181 if err := os.Setenv("FEDERATION_DOWNLOAD_TESTS", "y"); err != nil { 182 return err 183 } 184 log.Printf("url=%s version=%s get-federation.sh", url, version) 185 for i := 0; i < 3; i++ { 186 err = control.FinishRunning(exec.Command(k)) 187 if err == nil { 188 break 189 } 190 err = fmt.Errorf("url=%s version=%s get-federation.sh failed: %v", url, version, err) 191 if i == 2 { 192 return err 193 } 194 log.Println(err) 195 sleep(time.Duration(i) * time.Second) 196 } 197 return nil 198 } 199 200 func setFederationReleaseFromGcs(ci bool, suffix string) error { 201 var prefix string 202 if ci { 203 prefix = "kubernetes-federation-dev/ci" 204 } else { 205 prefix = "kubernetes-federation-release/release" 206 } 207 208 url := fmt.Sprintf("https://storage.googleapis.com/%v", prefix) 209 cat := fmt.Sprintf("gs://%v/%v.txt", prefix, suffix) 210 release, err := control.Output(exec.Command("gsutil", "cat", cat)) 211 if err != nil { 212 return err 213 } 214 return getFederation(url, strings.TrimSpace(string(release))) 215 } 216 217 func (e extractFederationStrategy) Extract(project, zone string) error { 218 switch e.mode { 219 case local: 220 url := util.K8s("federation", "_output", "gcs-stage") 221 files, err := ioutil.ReadDir(url) 222 if err != nil { 223 return err 224 } 225 var release string 226 for _, file := range files { 227 r := file.Name() 228 if strings.HasPrefix(r, "v") { 229 release = r 230 break 231 } 232 } 233 if len(release) == 0 { 234 return fmt.Errorf("no releases found in %v", url) 235 } 236 return getFederation(fmt.Sprintf("file://%s", url), release) 237 case ci: 238 return setFederationReleaseFromGcs(true, e.option) 239 case rc, stable: 240 return setFederationReleaseFromGcs(false, e.option) 241 case version: 242 var url string 243 release := e.option 244 if strings.Contains(release, "+") { 245 url = "https://storage.googleapis.com/kubernetes-federation-dev/ci" 246 } else { 247 url = "https://storage.googleapis.com/kubernetes-federation-release/release" 248 } 249 return getFederation(url, release) 250 case gcs: 251 // strip gs://foo -> /foo 252 withoutGS := strings.TrimPrefix(e.option, "gs://") 253 url := "https://storage.googleapis.com" + path.Dir(withoutGS) 254 return getFederation(url, path.Base(withoutGS)) 255 case bazel: 256 return getFederation("", e.option) 257 } 258 return fmt.Errorf("unrecognized federation extraction: %v(%v)", e.mode, e.value) 259 }