github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/kubetest/local.go (about) 1 /* 2 Copyright 2018 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "context" 21 "errors" 22 "flag" 23 "fmt" 24 "io/ioutil" 25 "log" 26 "net" 27 "os" 28 "os/exec" 29 "path/filepath" 30 "sort" 31 "time" 32 33 "github.com/docker/docker/api/types" 34 "github.com/docker/docker/client" 35 ) 36 37 var ( 38 localUpTimeout = flag.Duration("local-up-timeout", 2*time.Minute, "(local only) Time limit between 'local-up-cluster.sh' and a response from the Kubernetes API.") 39 ) 40 41 func removeAllContainers(cli *client.Client) { 42 // list all containers 43 listOptions := types.ContainerListOptions{ 44 Quiet: true, 45 All: true, 46 } 47 containers, err := cli.ContainerList(context.Background(), listOptions) 48 if err != nil { 49 log.Printf("Failed to list containers: %v\n", err) 50 return 51 } 52 53 // reverse sort by Creation time so we delete newest containers first 54 sort.Slice(containers, func(i, j int) bool { 55 return containers[i].Created > containers[j].Created 56 }) 57 58 // stop then remove (which implicitly kills) each container 59 duration := time.Second * 1 60 removeOptions := types.ContainerRemoveOptions{ 61 RemoveVolumes: true, 62 Force: true, 63 } 64 for _, container := range containers { 65 log.Printf("Stopping container: %v %s with ID: %s\n", 66 container.Names, container.Image, container.ID) 67 err = cli.ContainerStop(context.Background(), container.ID, &duration) 68 if err != nil { 69 log.Printf("Error stopping container: %v\n", err) 70 } 71 72 log.Printf("Removing container: %v %s with ID: %s\n", 73 container.Names, container.Image, container.ID) 74 err = cli.ContainerRemove(context.Background(), container.ID, removeOptions) 75 if err != nil { 76 log.Printf("Error removing container: %v\n", err) 77 } 78 } 79 } 80 81 type localCluster struct { 82 tempDir string 83 kubeConfig string 84 } 85 86 var _ deployer = localCluster{} 87 88 func newLocalCluster() *localCluster { 89 tempDir, err := ioutil.TempDir("", "kubetest-local") 90 if err != nil { 91 log.Fatal("unable to create temp directory") 92 } 93 err = os.Chmod(tempDir, 0755) 94 if err != nil { 95 log.Fatal("unable to change temp directory permissions") 96 } 97 return &localCluster{ 98 tempDir: tempDir, 99 } 100 } 101 102 func (n localCluster) getScript(scriptPath string) (string, error) { 103 cwd, err := os.Getwd() 104 if err != nil { 105 return "", err 106 } 107 path := filepath.Join(cwd, scriptPath) 108 if _, err := os.Stat(path); err == nil { 109 return path, nil 110 } 111 return "", fmt.Errorf("unable to find script %v in directory %v", scriptPath, cwd) 112 } 113 114 func (n localCluster) Up() error { 115 script, err := n.getScript("hack/local-up-cluster.sh") 116 if err != nil { 117 return err 118 } 119 120 cmd := exec.Command(script) 121 cmd.Env = os.Environ() 122 123 cmd.Env = append(cmd.Env, "ENABLE_DAEMON=true") 124 cmd.Env = append(cmd.Env, fmt.Sprintf("LOG_DIR=%s", n.tempDir)) 125 126 // Needed for at least one conformance e2e test. Please see issue #59978 127 cmd.Env = append(cmd.Env, "ALLOW_PRIVILEGED=true") 128 129 // when we are running in a DIND scenario, we should use the ip address of 130 // the docker0 network interface, This ensures that when the pods come up 131 // the health checks (for example for kubedns) succeed. If there is no 132 // docker0, just use the defaults in local-up-cluster.sh 133 dockerIP := "" 134 docker0, err := net.InterfaceByName("docker0") 135 if err == nil { 136 addresses, err := docker0.Addrs() 137 if err == nil { 138 for _, address := range addresses { 139 if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { 140 if ipnet.IP.To4() != nil { 141 dockerIP = ipnet.IP.String() 142 break 143 } 144 } 145 } 146 } else { 147 log.Printf("unable to get addresses from docker0 interface : %v", err) 148 } 149 } else { 150 log.Printf("unable to find docker0 interface : %v", err) 151 } 152 if dockerIP != "" { 153 log.Printf("using %v for API_HOST_IP, HOSTNAME_OVERRIDE, KUBELET_HOST", dockerIP) 154 cmd.Env = append(cmd.Env, fmt.Sprintf("API_HOST_IP=%s", dockerIP)) 155 cmd.Env = append(cmd.Env, fmt.Sprintf("HOSTNAME_OVERRIDE=%s", dockerIP)) 156 cmd.Env = append(cmd.Env, fmt.Sprintf("KUBELET_HOST=%s", dockerIP)) 157 } else { 158 log.Println("using local-up-cluster.sh's defaults for API_HOST_IP, HOSTNAME_OVERRIDE, KUBELET_HOST") 159 } 160 161 err = control.FinishRunning(cmd) 162 if err != nil { 163 return err 164 } 165 n.kubeConfig = "/var/run/kubernetes/admin.kubeconfig" 166 _, err = os.Stat(n.kubeConfig) 167 return err 168 } 169 170 func (n localCluster) IsUp() error { 171 if n.kubeConfig != "" { 172 if err := os.Setenv("KUBECONFIG", n.kubeConfig); err != nil { 173 return err 174 } 175 } 176 if err := os.Setenv("KUBERNETES_CONFORMANCE_TEST", "yes"); err != nil { 177 return err 178 } 179 if err := os.Setenv("KUBERNETES_PROVIDER", "local"); err != nil { 180 return err 181 } 182 183 stop := time.Now().Add(*localUpTimeout) 184 for { 185 script, err := n.getScript("cluster/kubectl.sh") 186 if err != nil { 187 return err 188 } 189 nodes, err := kubectlGetNodes(script) 190 if err != nil { 191 return err 192 } 193 readyNodes := countReadyNodes(nodes) 194 if readyNodes > 0 { 195 return nil 196 } 197 if time.Now().After(stop) { 198 break 199 } else { 200 time.Sleep(5 * time.Second) 201 } 202 } 203 return errors.New("local-up-cluster.sh is not ready") 204 } 205 206 func (n localCluster) DumpClusterLogs(localPath, gcsPath string) error { 207 cmd := exec.Command("sudo", "cp", "-r", n.tempDir, localPath) 208 return control.FinishRunning(cmd) 209 } 210 211 func (n localCluster) TestSetup() error { 212 return nil 213 } 214 215 func (n localCluster) Down() error { 216 // create docker client 217 cli, err := client.NewEnvClient() 218 if err != nil { 219 log.Printf("Docker containers cleanup, unable to create Docker client: %v", err) 220 } 221 // make sure all containers are removed 222 removeAllContainers(cli) 223 err = control.FinishRunning(exec.Command("pkill", "hyperkube")) 224 if err != nil { 225 log.Printf("unable to kill hyperkube processes: %v", err) 226 } 227 err = control.FinishRunning(exec.Command("pkill", "etcd")) 228 if err != nil { 229 log.Printf("unable to kill etcd: %v", err) 230 } 231 return nil 232 } 233 234 func (n localCluster) GetClusterCreated(gcpProject string) (time.Time, error) { 235 return time.Time{}, errors.New("GetClusterCreated not implemented in localCluster") 236 }