k8s.io/kubernetes@v1.29.3/test/e2e_node/remote/run_remote_suite.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package remote 18 19 import ( 20 "flag" 21 "fmt" 22 "log" 23 "math/rand" 24 "os" 25 "os/exec" 26 "os/signal" 27 "strings" 28 "sync" 29 "time" 30 31 "k8s.io/klog/v2" 32 ) 33 34 var mode = flag.String("mode", "gce", "Mode to operate in. One of gce|ssh. Defaults to gce") 35 var testArgs = flag.String("test_args", "", "Space-separated list of arguments to pass to Ginkgo test runner.") 36 var instanceNamePrefix = flag.String("instance-name-prefix", "", "prefix for instance names") 37 var imageConfigFile = flag.String("image-config-file", "", "yaml file describing images to run") 38 var imageConfigDir = flag.String("image-config-dir", "", "(optional) path to image config files") 39 var images = flag.String("images", "", "images to test") 40 var hosts = flag.String("hosts", "", "hosts to test") 41 var cleanup = flag.Bool("cleanup", true, "If true remove files from remote hosts and delete temporary instances") 42 var deleteInstances = flag.Bool("delete-instances", true, "If true, delete any instances created") 43 var buildOnly = flag.Bool("build-only", false, "If true, build e2e_node_test.tar.gz and exit.") 44 var gubernator = flag.Bool("gubernator", false, "If true, output Gubernator link to view logs") 45 var ginkgoFlags = flag.String("ginkgo-flags", "", "Passed to ginkgo to specify additional flags such as --skip=.") 46 var ( 47 arc Archive 48 ) 49 50 // Archive contains path info in the archive. 51 type Archive struct { 52 sync.Once 53 path string 54 err error 55 } 56 57 func getFlag(name string) string { 58 lookup := flag.Lookup(name) 59 if lookup == nil { 60 return "" 61 } 62 return lookup.Value.String() 63 } 64 65 func RunRemoteTestSuite(testSuite TestSuite) { 66 // Listen for SIGINT and ignore the first one. In case SIGINT is sent to this 67 // process and all its children, we ignore it here, while our children ssh connections 68 // are stopped. This allows us to gather artifacts and print out test state before 69 // being killed. 70 c := make(chan os.Signal, 2) 71 signal.Notify(c, os.Interrupt) 72 go func() { 73 <-c 74 fmt.Printf("Received SIGINT. Will exit on next SIGINT.\n") 75 <-c 76 fmt.Printf("Received another SIGINT. Will exit.\n") 77 os.Exit(1) 78 }() 79 80 rand.Seed(time.Now().UnixNano()) 81 if *buildOnly { 82 // Build the archive and exit 83 CreateTestArchive(testSuite, 84 getFlag("system-spec-name"), 85 getFlag("kubelet-config-file")) 86 return 87 } 88 89 // Append some default ginkgo flags. We use similar defaults here as hack/ginkgo-e2e.sh 90 allGinkgoFlags := fmt.Sprintf("%s --no-color -v", *ginkgoFlags) 91 fmt.Printf("Will use ginkgo flags as: %s", allGinkgoFlags) 92 93 var runner Runner 94 cfg := Config{ 95 InstanceNamePrefix: *instanceNamePrefix, 96 ImageConfigFile: *imageConfigFile, 97 ImageConfigDir: *imageConfigDir, 98 Images: splitCommaList(*images), 99 Hosts: parseHostsList(*hosts), 100 GinkgoFlags: allGinkgoFlags, 101 DeleteInstances: *deleteInstances, 102 Cleanup: *cleanup, 103 TestArgs: *testArgs, 104 ExtraEnvs: getFlag("extra-envs"), 105 RuntimeConfig: getFlag("runtime-config"), 106 SystemSpecName: getFlag("system-spec-name"), 107 } 108 109 var sshRunner Runner 110 111 if *mode == "ssh" { 112 runner = NewSSHRunner(cfg) 113 } else { 114 getRunner, err := GetRunner(*mode) 115 if err != nil { 116 klog.Fatalf("getting runner mode %q : %v", *mode, err) 117 } 118 runner = getRunner(cfg) 119 sshRunner = NewSSHRunner(cfg) 120 } 121 122 if err := runner.Validate(); err != nil { 123 klog.Fatalf("validating remote config, %s", err) 124 } 125 126 // Setup coloring 127 stat, _ := os.Stdout.Stat() 128 useColor := (stat.Mode() & os.ModeCharDevice) != 0 129 blue := "" 130 noColour := "" 131 if useColor { 132 blue = "\033[0;34m" 133 noColour = "\033[0m" 134 } 135 136 results := make(chan *TestResult) 137 138 path, err := arc.getArchive(testSuite) 139 if err != nil { 140 log.Fatalf("unable to create test archive: %s", err) 141 } 142 defer arc.deleteArchive() 143 144 running := runner.StartTests(testSuite, path, results) 145 // You can potentially run SSH based tests while running image based test as well. The GCE provider does this, see 146 // test-e2e-node.sh. 147 if sshRunner != nil && len(cfg.Hosts) > 0 { 148 running += sshRunner.StartTests(testSuite, path, results) 149 } 150 151 // Wait for all tests to complete and emit the results 152 errCount := 0 153 exitOk := true 154 for i := 0; i < running; i++ { 155 tr := <-results 156 host := tr.Host 157 fmt.Println() // Print an empty line 158 fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour) 159 fmt.Printf("%s> START TEST >%s\n", blue, noColour) 160 fmt.Printf("%s>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>%s\n", blue, noColour) 161 fmt.Printf("Start Test Suite on Host %s\n", host) 162 fmt.Printf("%s\n", tr.Output) 163 if tr.Err != nil { 164 errCount++ 165 fmt.Printf("Failure Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n%v\n", host, tr.Err) 166 } else { 167 fmt.Printf("Success Finished Test Suite on Host %s. Refer to artifacts directory for ginkgo log for this host.\n", host) 168 } 169 exitOk = exitOk && tr.ExitOK 170 fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour) 171 fmt.Printf("%s< FINISH TEST <%s\n", blue, noColour) 172 fmt.Printf("%s<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<%s\n", blue, noColour) 173 fmt.Println() // Print an empty line 174 } 175 // Set the exit code if there were failures 176 if !exitOk { 177 fmt.Printf("Failure: %d errors encountered.\n", errCount) 178 callGubernator(*gubernator) 179 arc.deleteArchive() 180 os.Exit(1) 181 } 182 callGubernator(*gubernator) 183 } 184 185 func splitCommaList(s string) []string { 186 if len(s) == 0 { 187 return nil 188 } 189 return strings.Split(s, ",") 190 } 191 192 func callGubernator(gubernator bool) { 193 if gubernator { 194 fmt.Println("Running gubernator.sh") 195 output, err := exec.Command("./test/e2e_node/gubernator.sh", "y").Output() 196 197 if err != nil { 198 fmt.Println("gubernator.sh Failed") 199 fmt.Println(err) 200 return 201 } 202 fmt.Printf("%s", output) 203 } 204 return 205 } 206 207 func (a *Archive) getArchive(suite TestSuite) (string, error) { 208 a.Do(func() { 209 a.path, a.err = CreateTestArchive(suite, 210 getFlag("system-spec-name"), 211 getFlag("kubelet-config-file")) 212 }) 213 return a.path, a.err 214 } 215 216 func (a *Archive) deleteArchive() { 217 path, err := a.getArchive(nil) 218 if err != nil { 219 return 220 } 221 os.Remove(path) 222 } 223 224 // parseHostsList splits a host list of the form a=1.2.3.4,b=5.6.7.8 into the list of hosts [a,b] while registering the 225 // given addresses 226 func parseHostsList(hostList string) []string { 227 if len(hostList) == 0 { 228 return nil 229 } 230 hosts := strings.Split(hostList, ",") 231 var hostsOnly []string 232 for _, host := range hosts { 233 segs := strings.Split(host, "=") 234 if len(segs) == 2 { 235 AddHostnameIP(segs[0], segs[1]) 236 } else if len(segs) > 2 { 237 klog.Fatalf("invalid format of host %q", hostList) 238 } 239 hostsOnly = append(hostsOnly, segs[0]) 240 } 241 return hostsOnly 242 }