github.com/adityamillind98/nomad@v0.11.8/testutil/wait.go (about) 1 package testutil 2 3 import ( 4 "fmt" 5 "os" 6 "time" 7 8 "github.com/hashicorp/nomad/nomad/structs" 9 "github.com/kr/pretty" 10 testing "github.com/mitchellh/go-testing-interface" 11 "github.com/stretchr/testify/require" 12 ) 13 14 type testFn func() (bool, error) 15 type errorFn func(error) 16 17 func WaitForResult(test testFn, error errorFn) { 18 WaitForResultRetries(500*TestMultiplier(), test, error) 19 } 20 21 func WaitForResultRetries(retries int64, test testFn, error errorFn) { 22 for retries > 0 { 23 time.Sleep(10 * time.Millisecond) 24 retries-- 25 26 success, err := test() 27 if success { 28 return 29 } 30 31 if retries == 0 { 32 error(err) 33 } 34 } 35 } 36 37 // AssertUntil asserts the test function passes throughout the given duration. 38 // Otherwise error is called on failure. 39 func AssertUntil(until time.Duration, test testFn, error errorFn) { 40 deadline := time.Now().Add(until) 41 for time.Now().Before(deadline) { 42 success, err := test() 43 if !success { 44 error(err) 45 return 46 } 47 // Sleep some arbitrary fraction of the deadline 48 time.Sleep(until / 30) 49 } 50 } 51 52 // TestMultiplier returns a multiplier for retries and waits given environment 53 // the tests are being run under. 54 func TestMultiplier() int64 { 55 if IsCI() { 56 return 4 57 } 58 59 return 1 60 } 61 62 // Timeout takes the desired timeout and increases it if running in Travis 63 func Timeout(original time.Duration) time.Duration { 64 return original * time.Duration(TestMultiplier()) 65 } 66 67 func IsCI() bool { 68 _, ok := os.LookupEnv("CI") 69 return ok 70 } 71 72 func IsTravis() bool { 73 _, ok := os.LookupEnv("TRAVIS") 74 return ok 75 } 76 77 func IsAppVeyor() bool { 78 _, ok := os.LookupEnv("APPVEYOR") 79 return ok 80 } 81 82 type rpcFn func(string, interface{}, interface{}) error 83 84 // WaitForLeader blocks until a leader is elected. 85 func WaitForLeader(t testing.T, rpc rpcFn) { 86 t.Helper() 87 WaitForResult(func() (bool, error) { 88 args := &structs.GenericRequest{} 89 var leader string 90 err := rpc("Status.Leader", args, &leader) 91 return leader != "", err 92 }, func(err error) { 93 t.Fatalf("failed to find leader: %v", err) 94 }) 95 } 96 97 // WaitForVotingMembers blocks until autopilot promotes all server peers 98 // to be voting members. 99 // 100 // Useful for tests that change cluster topology (e.g. kill a node) 101 // that should wait until cluster is stable. 102 func WaitForVotingMembers(t testing.T, rpc rpcFn, nPeers int) { 103 WaitForResult(func() (bool, error) { 104 args := &structs.GenericRequest{} 105 args.AllowStale = true 106 args.Region = "global" 107 args.Namespace = structs.DefaultNamespace 108 resp := structs.RaftConfigurationResponse{} 109 err := rpc("Operator.RaftGetConfiguration", args, &resp) 110 if err != nil { 111 return false, fmt.Errorf("failed to query raft: %v", err) 112 } 113 114 if len(resp.Servers) != nPeers { 115 return false, fmt.Errorf("expected %d peers found %d", nPeers, len(resp.Servers)) 116 } 117 118 for _, s := range resp.Servers { 119 if !s.Voter { 120 return false, fmt.Errorf("found nonvoting server: %v", s) 121 } 122 } 123 124 return true, nil 125 }, func(err error) { 126 t.Fatalf("failed to wait until voting members: %v", err) 127 }) 128 } 129 130 // RegisterJobWithToken registers a job and uses the job's Region and Namespace. 131 func RegisterJobWithToken(t testing.T, rpc rpcFn, job *structs.Job, token string) { 132 WaitForResult(func() (bool, error) { 133 args := &structs.JobRegisterRequest{} 134 args.Job = job 135 args.WriteRequest.Region = job.Region 136 args.AuthToken = token 137 args.Namespace = job.Namespace 138 var jobResp structs.JobRegisterResponse 139 err := rpc("Job.Register", args, &jobResp) 140 return err == nil, fmt.Errorf("Job.Register error: %v", err) 141 }, func(err error) { 142 t.Fatalf("error registering job: %v", err) 143 }) 144 145 t.Logf("Job %q registered", job.ID) 146 } 147 148 func RegisterJob(t testing.T, rpc rpcFn, job *structs.Job) { 149 RegisterJobWithToken(t, rpc, job, "") 150 } 151 152 func WaitForRunningWithToken(t testing.T, rpc rpcFn, job *structs.Job, token string) []*structs.AllocListStub { 153 RegisterJobWithToken(t, rpc, job, token) 154 155 var resp structs.JobAllocationsResponse 156 157 WaitForResult(func() (bool, error) { 158 args := &structs.JobSpecificRequest{} 159 args.JobID = job.ID 160 args.QueryOptions.Region = job.Region 161 args.AuthToken = token 162 args.Namespace = job.Namespace 163 err := rpc("Job.Allocations", args, &resp) 164 if err != nil { 165 return false, fmt.Errorf("Job.Allocations error: %v", err) 166 } 167 168 if len(resp.Allocations) == 0 { 169 evals := structs.JobEvaluationsResponse{} 170 require.NoError(t, rpc("Job.Evaluations", args, &evals), "error looking up evals") 171 return false, fmt.Errorf("0 allocations; evals: %s", pretty.Sprint(evals.Evaluations)) 172 } 173 174 for _, alloc := range resp.Allocations { 175 if alloc.ClientStatus == structs.AllocClientStatusPending { 176 return false, fmt.Errorf("alloc not running: id=%v tg=%v status=%v", 177 alloc.ID, alloc.TaskGroup, alloc.ClientStatus) 178 } 179 } 180 181 return true, nil 182 }, func(err error) { 183 require.NoError(t, err) 184 }) 185 186 return resp.Allocations 187 } 188 189 // WaitForRunning runs a job and blocks until all allocs are out of pending. 190 func WaitForRunning(t testing.T, rpc rpcFn, job *structs.Job) []*structs.AllocListStub { 191 return WaitForRunningWithToken(t, rpc, job, "") 192 }