github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/kubetest/kubernetes.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "encoding/json" 21 "fmt" 22 "log" 23 "os/exec" 24 "time" 25 ) 26 27 // kubectlGetNodes lists nodes by executing kubectl get nodes, parsing the output into a nodeList object 28 func kubectlGetNodes(cmd string) (*nodeList, error) { 29 if cmd == "" { 30 cmd = "kubectl" 31 } 32 o, err := control.Output(exec.Command(cmd, "get", "nodes", "-ojson")) 33 if err != nil { 34 log.Printf("kubectl get nodes failed: %s\n%s", wrapError(err).Error(), string(o)) 35 return nil, err 36 } 37 38 nodes := &nodeList{} 39 if err := json.Unmarshal(o, nodes); err != nil { 40 return nil, fmt.Errorf("error parsing kubectl get nodes output: %v", err) 41 } 42 43 return nodes, nil 44 } 45 46 // isReady checks if the node has a Ready Condition that is True 47 func isReady(node *node) bool { 48 for _, c := range node.Status.Conditions { 49 if c.Type == "Ready" { 50 return c.Status == "True" 51 } 52 } 53 return false 54 } 55 56 // waitForReadyNodes polls the nodes until we see at least desiredCount that are Ready 57 // We can also pass requiredConsecutiveSuccesses to require that we see this N times in a row 58 func waitForReadyNodes(desiredCount int, timeout time.Duration, requiredConsecutiveSuccesses int) error { 59 stop := time.Now().Add(timeout) 60 61 consecutiveSuccesses := 0 62 for { 63 if time.Now().After(stop) { 64 break 65 } 66 67 nodes, err := kubectlGetNodes("") 68 if err != nil { 69 log.Printf("kubectl get nodes failed, sleeping: %v", err) 70 consecutiveSuccesses = 0 71 time.Sleep(30 * time.Second) 72 continue 73 } 74 readyNodes := countReadyNodes(nodes) 75 if readyNodes >= desiredCount { 76 consecutiveSuccesses++ 77 if consecutiveSuccesses >= requiredConsecutiveSuccesses { 78 log.Printf("%d ready nodes found, %d sequential successes - cluster is ready", 79 readyNodes, 80 consecutiveSuccesses) 81 return nil 82 } 83 log.Printf("%d ready nodes found, waiting for %d sequential successes (success count = %d)", 84 readyNodes, 85 requiredConsecutiveSuccesses, 86 consecutiveSuccesses) 87 time.Sleep(2 * time.Second) 88 } else { 89 consecutiveSuccesses = 0 90 log.Printf("%d (ready nodes) < %d (requested instances), sleeping", readyNodes, desiredCount) 91 time.Sleep(30 * time.Second) 92 } 93 } 94 return fmt.Errorf("waiting for ready nodes timed out") 95 } 96 97 // countReadyNodes returns the number of nodes that have isReady == true 98 func countReadyNodes(nodes *nodeList) int { 99 var ready []*node 100 for i := range nodes.Items { 101 node := &nodes.Items[i] 102 if isReady(node) { 103 ready = append(ready, node) 104 } 105 } 106 return len(ready) 107 } 108 109 // nodeList is a simplified version of the v1.NodeList API type 110 type nodeList struct { 111 Items []node `json:"items"` 112 } 113 114 // node is a simplified version of the v1.Node API type 115 type node struct { 116 Metadata metadata `json:"metadata"` 117 Status nodeStatus `json:"status"` 118 } 119 120 // nodeStatus is a simplified version of the v1.NodeStatus API type 121 type nodeStatus struct { 122 Addresses []nodeAddress `json:"addresses"` 123 Conditions []nodeCondition `json:"conditions"` 124 } 125 126 // nodeAddress is a simplified version of the v1.NodeAddress API type 127 type nodeAddress struct { 128 Address string `json:"address"` 129 Type string `json:"type"` 130 } 131 132 // nodeCondition is a simplified version of the v1.NodeCondition API type 133 type nodeCondition struct { 134 Message string `json:"message"` 135 Reason string `json:"reason"` 136 Status string `json:"status"` 137 Type string `json:"type"` 138 } 139 140 // metadata is a simplified version of the kubernetes metadata types 141 type metadata struct { 142 Name string `json:"name"` 143 }