github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/testutil/wait.go (about) 1 package testutil 2 3 import ( 4 "fmt" 5 "os" 6 "time" 7 8 "github.com/hashicorp/nomad/nomad/structs" 9 "github.com/kr/pretty" 10 testing "github.com/mitchellh/go-testing-interface" 11 "github.com/stretchr/testify/assert" 12 "github.com/stretchr/testify/require" 13 ) 14 15 type testFn func() (bool, error) 16 type errorFn func(error) 17 18 func WaitForResult(test testFn, error errorFn) { 19 WaitForResultRetries(500*TestMultiplier(), test, error) 20 } 21 22 func WaitForResultRetries(retries int64, test testFn, error errorFn) { 23 for retries > 0 { 24 time.Sleep(10 * time.Millisecond) 25 retries-- 26 27 success, err := test() 28 if success { 29 return 30 } 31 32 if retries == 0 { 33 error(err) 34 } 35 } 36 } 37 38 // AssertUntil asserts the test function passes throughout the given duration. 39 // Otherwise error is called on failure. 40 func AssertUntil(until time.Duration, test testFn, error errorFn) { 41 deadline := time.Now().Add(until) 42 for time.Now().Before(deadline) { 43 success, err := test() 44 if !success { 45 error(err) 46 return 47 } 48 // Sleep some arbitrary fraction of the deadline 49 time.Sleep(until / 30) 50 } 51 } 52 53 // TestMultiplier returns a multiplier for retries and waits given environment 54 // the tests are being run under. 55 func TestMultiplier() int64 { 56 if IsCI() { 57 return 4 58 } 59 60 return 1 61 } 62 63 // Timeout takes the desired timeout and increases it if running in Travis 64 func Timeout(original time.Duration) time.Duration { 65 return original * time.Duration(TestMultiplier()) 66 } 67 68 func IsCI() bool { 69 _, ok := os.LookupEnv("CI") 70 return ok 71 } 72 73 func IsTravis() bool { 74 _, ok := os.LookupEnv("TRAVIS") 75 return ok 76 } 77 78 func IsAppVeyor() bool { 79 _, ok := os.LookupEnv("APPVEYOR") 80 return ok 81 } 82 83 type rpcFn func(string, interface{}, interface{}) error 84 85 // WaitForLeader blocks until a leader is elected. 86 func WaitForLeader(t testing.T, rpc rpcFn) { 87 t.Helper() 88 WaitForResult(func() (bool, error) { 89 args := &structs.GenericRequest{} 90 var leader string 91 err := rpc("Status.Leader", args, &leader) 92 return leader != "", err 93 }, func(err error) { 94 t.Fatalf("failed to find leader: %v", err) 95 }) 96 } 97 98 // WaitForClient blocks until the client can be found 99 func WaitForClient(t testing.T, rpc rpcFn, nodeID string) { 100 t.Helper() 101 WaitForResult(func() (bool, error) { 102 req := structs.NodeSpecificRequest{ 103 NodeID: nodeID, 104 QueryOptions: structs.QueryOptions{Region: "global"}, 105 } 106 var out structs.SingleNodeResponse 107 108 err := rpc("Node.GetNode", &req, &out) 109 if err != nil { 110 return false, err 111 } 112 if out.Node == nil { 113 return false, fmt.Errorf("node not found") 114 } 115 return out.Node.Status == structs.NodeStatusReady, nil 116 }, func(err error) { 117 t.Fatalf("failed to find node: %v", err) 118 }) 119 } 120 121 // WaitForVotingMembers blocks until autopilot promotes all server peers 122 // to be voting members. 123 // 124 // Useful for tests that change cluster topology (e.g. kill a node) 125 // that should wait until cluster is stable. 126 func WaitForVotingMembers(t testing.T, rpc rpcFn, nPeers int) { 127 WaitForResult(func() (bool, error) { 128 args := &structs.GenericRequest{} 129 args.AllowStale = true 130 args.Region = "global" 131 args.Namespace = structs.DefaultNamespace 132 resp := structs.RaftConfigurationResponse{} 133 err := rpc("Operator.RaftGetConfiguration", args, &resp) 134 if err != nil { 135 return false, fmt.Errorf("failed to query raft: %v", err) 136 } 137 138 if len(resp.Servers) != nPeers { 139 return false, fmt.Errorf("expected %d peers found %d", nPeers, len(resp.Servers)) 140 } 141 142 for _, s := range resp.Servers { 143 if !s.Voter { 144 return false, fmt.Errorf("found nonvoting server: %v", s) 145 } 146 } 147 148 return true, nil 149 }, func(err error) { 150 t.Fatalf("failed to wait until voting members: %v", err) 151 }) 152 } 153 154 // RegisterJobWithToken registers a job and uses the job's Region and Namespace. 155 func RegisterJobWithToken(t testing.T, rpc rpcFn, job *structs.Job, token string) { 156 WaitForResult(func() (bool, error) { 157 args := &structs.JobRegisterRequest{} 158 args.Job = job 159 args.WriteRequest.Region = job.Region 160 args.AuthToken = token 161 args.Namespace = job.Namespace 162 var jobResp structs.JobRegisterResponse 163 err := rpc("Job.Register", args, &jobResp) 164 return err == nil, fmt.Errorf("Job.Register error: %v", err) 165 }, func(err error) { 166 t.Fatalf("error registering job: %v", err) 167 }) 168 169 t.Logf("Job %q registered", job.ID) 170 } 171 172 func RegisterJob(t testing.T, rpc rpcFn, job *structs.Job) { 173 RegisterJobWithToken(t, rpc, job, "") 174 } 175 176 func WaitForRunningWithToken(t testing.T, rpc rpcFn, job *structs.Job, token string) []*structs.AllocListStub { 177 RegisterJobWithToken(t, rpc, job, token) 178 179 var resp structs.JobAllocationsResponse 180 181 WaitForResult(func() (bool, error) { 182 args := &structs.JobSpecificRequest{} 183 args.JobID = job.ID 184 args.QueryOptions.Region = job.Region 185 args.AuthToken = token 186 args.Namespace = job.Namespace 187 err := rpc("Job.Allocations", args, &resp) 188 if err != nil { 189 return false, fmt.Errorf("Job.Allocations error: %v", err) 190 } 191 192 if len(resp.Allocations) == 0 { 193 evals := structs.JobEvaluationsResponse{} 194 require.NoError(t, rpc("Job.Evaluations", args, &evals), "error looking up evals") 195 return false, fmt.Errorf("0 allocations; evals: %s", pretty.Sprint(evals.Evaluations)) 196 } 197 198 for _, alloc := range resp.Allocations { 199 if alloc.ClientStatus == structs.AllocClientStatusPending { 200 return false, fmt.Errorf("alloc not running: id=%v tg=%v status=%v", 201 alloc.ID, alloc.TaskGroup, alloc.ClientStatus) 202 } 203 } 204 205 return true, nil 206 }, func(err error) { 207 require.NoError(t, err) 208 }) 209 210 return resp.Allocations 211 } 212 213 // WaitForRunning runs a job and blocks until all allocs are out of pending. 214 func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocListStub { 215 return WaitForRunningWithToken(t, rpc, job, "") 216 } 217 218 // WaitForFiles blocks until all the files in the slice are present 219 func WaitForFiles(t testing.T, files []string) { 220 assert := assert.New(t) 221 WaitForResult(func() (bool, error) { 222 for _, f := range files { 223 exists := assert.FileExists(f) 224 if !exists { 225 return false, fmt.Errorf("expected file to exist %s", f) 226 } 227 } 228 return true, nil 229 }, func(err error) { 230 t.Fatalf("missing expected files: %v", err) 231 }) 232 }