github.com/ThomasObenaus/nomad@v0.11.1/testutil/server.go (about) 1 package testutil 2 3 // TestServer is a test helper. It uses a fork/exec model to create 4 // a test Nomad server instance in the background and initialize it 5 // with some data and/or services. The test server can then be used 6 // to run a unit test, and offers an easy API to tear itself down 7 // when the test has completed. The only prerequisite is to have a nomad 8 // binary available on the $PATH. 9 // 10 // This package does not use Nomad's official API client. This is 11 // because we use TestServer to test the API client, which would 12 // otherwise cause an import cycle. 13 14 import ( 15 "bytes" 16 "encoding/json" 17 "fmt" 18 "io" 19 "io/ioutil" 20 "net/http" 21 "os" 22 "os/exec" 23 24 cleanhttp "github.com/hashicorp/go-cleanhttp" 25 "github.com/hashicorp/nomad/helper/discover" 26 "github.com/hashicorp/nomad/helper/freeport" 27 testing "github.com/mitchellh/go-testing-interface" 28 ) 29 30 // TestServerConfig is the main server configuration struct. 31 type TestServerConfig struct { 32 NodeName string `json:"name,omitempty"` 33 DataDir string `json:"data_dir,omitempty"` 34 Region string `json:"region,omitempty"` 35 DisableCheckpoint bool `json:"disable_update_check"` 36 LogLevel string `json:"log_level,omitempty"` 37 Consul *Consul `json:"consul,omitempty"` 38 AdvertiseAddrs *Advertise `json:"advertise,omitempty"` 39 Ports *PortsConfig `json:"ports,omitempty"` 40 Server *ServerConfig `json:"server,omitempty"` 41 Client *ClientConfig `json:"client,omitempty"` 42 Vault *VaultConfig `json:"vault,omitempty"` 43 ACL *ACLConfig `json:"acl,omitempty"` 44 DevMode bool `json:"-"` 45 Stdout, Stderr io.Writer `json:"-"` 46 } 47 48 // Consul is used to configure the communication with Consul 49 type Consul struct { 50 Address string `json:"address,omitempty"` 51 Auth string `json:"auth,omitempty"` 52 Token string `json:"token,omitempty"` 53 } 54 55 // Advertise is used to configure the addresses to advertise 56 type Advertise struct { 57 HTTP string `json:"http,omitempty"` 58 RPC string `json:"rpc,omitempty"` 59 Serf string `json:"serf,omitempty"` 60 } 61 62 // PortsConfig is used to configure the network ports we use. 63 type PortsConfig struct { 64 HTTP int `json:"http,omitempty"` 65 RPC int `json:"rpc,omitempty"` 66 Serf int `json:"serf,omitempty"` 67 } 68 69 // ServerConfig is used to configure the nomad server. 70 type ServerConfig struct { 71 Enabled bool `json:"enabled"` 72 BootstrapExpect int `json:"bootstrap_expect"` 73 RaftProtocol int `json:"raft_protocol,omitempty"` 74 } 75 76 // ClientConfig is used to configure the client 77 type ClientConfig struct { 78 Enabled bool `json:"enabled"` 79 } 80 81 // VaultConfig is used to configure Vault 82 type VaultConfig struct { 83 Enabled bool `json:"enabled"` 84 } 85 86 // ACLConfig is used to configure ACLs 87 type ACLConfig struct { 88 Enabled bool `json:"enabled"` 89 } 90 91 // ServerConfigCallback is a function interface which can be 92 // passed to NewTestServerConfig to modify the server config. 93 type ServerConfigCallback func(c *TestServerConfig) 94 95 // defaultServerConfig returns a new TestServerConfig struct 96 // with all of the listen ports incremented by one. 97 func defaultServerConfig() (*TestServerConfig, []int) { 98 ports := freeport.MustTake(3) 99 return &TestServerConfig{ 100 NodeName: fmt.Sprintf("node-%d", ports[0]), 101 DisableCheckpoint: true, 102 LogLevel: "DEBUG", 103 Ports: &PortsConfig{ 104 HTTP: ports[0], 105 RPC: ports[1], 106 Serf: ports[2], 107 }, 108 Server: &ServerConfig{ 109 Enabled: true, 110 BootstrapExpect: 1, 111 }, 112 Client: &ClientConfig{ 113 Enabled: false, 114 }, 115 Vault: &VaultConfig{ 116 Enabled: false, 117 }, 118 ACL: &ACLConfig{ 119 Enabled: false, 120 }, 121 }, ports 122 } 123 124 // TestServer is the main server wrapper struct. 125 type TestServer struct { 126 cmd *exec.Cmd 127 Config *TestServerConfig 128 t testing.T 129 130 // ports (if any) that are reserved through freeport that must be returned 131 // at the end of a test, done when Close() is called. 132 ports []int 133 134 HTTPAddr string 135 SerfAddr string 136 HTTPClient *http.Client 137 } 138 139 // NewTestServer creates a new TestServer, and makes a call to 140 // an optional callback function to modify the configuration. 141 func NewTestServer(t testing.T, cb ServerConfigCallback) *TestServer { 142 path, err := discover.NomadExecutable() 143 if err != nil { 144 t.Skipf("nomad not found, skipping: %v", err) 145 } 146 147 // Do a sanity check that we are actually running nomad 148 vcmd := exec.Command(path, "-version") 149 vcmd.Stdout = nil 150 vcmd.Stderr = nil 151 if err := vcmd.Run(); err != nil { 152 t.Skipf("nomad version failed: %v", err) 153 } 154 155 dataDir, err := ioutil.TempDir("", "nomad") 156 if err != nil { 157 t.Fatalf("err: %s", err) 158 } 159 160 configFile, err := ioutil.TempFile(dataDir, "nomad") 161 if err != nil { 162 defer os.RemoveAll(dataDir) 163 t.Fatalf("err: %s", err) 164 } 165 defer configFile.Close() 166 167 nomadConfig, ports := defaultServerConfig() 168 nomadConfig.DataDir = dataDir 169 170 if cb != nil { 171 cb(nomadConfig) 172 } 173 174 configContent, err := json.Marshal(nomadConfig) 175 if err != nil { 176 t.Fatalf("err: %s", err) 177 } 178 179 if _, err := configFile.Write(configContent); err != nil { 180 t.Fatalf("err: %s", err) 181 } 182 configFile.Close() 183 184 stdout := io.Writer(os.Stdout) 185 if nomadConfig.Stdout != nil { 186 stdout = nomadConfig.Stdout 187 } 188 189 stderr := io.Writer(os.Stderr) 190 if nomadConfig.Stderr != nil { 191 stderr = nomadConfig.Stderr 192 } 193 194 args := []string{"agent", "-config", configFile.Name()} 195 if nomadConfig.DevMode { 196 args = append(args, "-dev") 197 } 198 199 // Start the server 200 cmd := exec.Command(path, args...) 201 cmd.Stdout = stdout 202 cmd.Stderr = stderr 203 if err := cmd.Start(); err != nil { 204 t.Fatalf("err: %s", err) 205 } 206 207 client := cleanhttp.DefaultClient() 208 209 server := &TestServer{ 210 Config: nomadConfig, 211 cmd: cmd, 212 t: t, 213 214 ports: ports, 215 216 HTTPAddr: fmt.Sprintf("127.0.0.1:%d", nomadConfig.Ports.HTTP), 217 SerfAddr: fmt.Sprintf("127.0.0.1:%d", nomadConfig.Ports.Serf), 218 HTTPClient: client, 219 } 220 221 // Wait for the server to be ready 222 if nomadConfig.Server.Enabled && nomadConfig.Server.BootstrapExpect != 0 { 223 server.waitForLeader() 224 } else { 225 server.waitForAPI() 226 } 227 228 // Wait for the client to be ready 229 if nomadConfig.DevMode { 230 server.waitForClient() 231 } 232 return server 233 } 234 235 // Stop stops the test Nomad server, and removes the Nomad data 236 // directory once we are done. 237 func (s *TestServer) Stop() { 238 defer freeport.Return(s.ports) 239 240 defer os.RemoveAll(s.Config.DataDir) 241 242 if err := s.cmd.Process.Kill(); err != nil { 243 s.t.Errorf("err: %s", err) 244 } 245 246 // wait for the process to exit to be sure that the data dir can be 247 // deleted on all platforms. 248 s.cmd.Wait() 249 } 250 251 // waitForAPI waits for only the agent HTTP endpoint to start 252 // responding. This is an indication that the agent has started, 253 // but will likely return before a leader is elected. 254 func (s *TestServer) waitForAPI() { 255 WaitForResult(func() (bool, error) { 256 // Using this endpoint as it is does not have restricted access 257 resp, err := s.HTTPClient.Get(s.url("/v1/metrics")) 258 if err != nil { 259 return false, err 260 } 261 defer resp.Body.Close() 262 if err := s.requireOK(resp); err != nil { 263 return false, err 264 } 265 return true, nil 266 }, func(err error) { 267 defer s.Stop() 268 s.t.Fatalf("err: %s", err) 269 }) 270 } 271 272 // waitForLeader waits for the Nomad server's HTTP API to become 273 // available, and then waits for a known leader and an index of 274 // 1 or more to be observed to confirm leader election is done. 275 func (s *TestServer) waitForLeader() { 276 WaitForResult(func() (bool, error) { 277 // Query the API and check the status code 278 // Using this endpoint as it is does not have restricted access 279 resp, err := s.HTTPClient.Get(s.url("/v1/status/leader")) 280 if err != nil { 281 return false, err 282 } 283 defer resp.Body.Close() 284 if err := s.requireOK(resp); err != nil { 285 return false, err 286 } 287 288 return true, nil 289 }, func(err error) { 290 defer s.Stop() 291 s.t.Fatalf("err: %s", err) 292 }) 293 } 294 295 // waitForClient waits for the Nomad client to be ready. The function returns 296 // immediately if the server is not in dev mode. 297 func (s *TestServer) waitForClient() { 298 if !s.Config.DevMode { 299 return 300 } 301 302 WaitForResult(func() (bool, error) { 303 resp, err := s.HTTPClient.Get(s.url("/v1/nodes")) 304 if err != nil { 305 return false, err 306 } 307 defer resp.Body.Close() 308 if err := s.requireOK(resp); err != nil { 309 return false, err 310 } 311 312 var decoded []struct { 313 ID string 314 Status string 315 } 316 317 dec := json.NewDecoder(resp.Body) 318 if err := dec.Decode(&decoded); err != nil { 319 return false, err 320 } 321 322 if len(decoded) != 1 || decoded[0].Status != "ready" { 323 return false, fmt.Errorf("Node not ready: %v", decoded) 324 } 325 326 return true, nil 327 }, func(err error) { 328 defer s.Stop() 329 s.t.Fatalf("err: %s", err) 330 }) 331 } 332 333 // url is a helper function which takes a relative URL and 334 // makes it into a proper URL against the local Nomad server. 335 func (s *TestServer) url(path string) string { 336 return fmt.Sprintf("http://%s%s", s.HTTPAddr, path) 337 } 338 339 // requireOK checks the HTTP response code and ensures it is acceptable. 340 func (s *TestServer) requireOK(resp *http.Response) error { 341 if resp.StatusCode != 200 { 342 return fmt.Errorf("Bad status code: %d", resp.StatusCode) 343 } 344 return nil 345 } 346 347 // put performs a new HTTP PUT request. 348 func (s *TestServer) put(path string, body io.Reader) *http.Response { 349 req, err := http.NewRequest("PUT", s.url(path), body) 350 if err != nil { 351 s.t.Fatalf("err: %s", err) 352 } 353 resp, err := s.HTTPClient.Do(req) 354 if err != nil { 355 s.t.Fatalf("err: %s", err) 356 } 357 if err := s.requireOK(resp); err != nil { 358 defer resp.Body.Close() 359 s.t.Fatal(err) 360 } 361 return resp 362 } 363 364 // get performs a new HTTP GET request. 365 func (s *TestServer) get(path string) *http.Response { 366 resp, err := s.HTTPClient.Get(s.url(path)) 367 if err != nil { 368 s.t.Fatalf("err: %s", err) 369 } 370 if err := s.requireOK(resp); err != nil { 371 defer resp.Body.Close() 372 s.t.Fatal(err) 373 } 374 return resp 375 } 376 377 // encodePayload returns a new io.Reader wrapping the encoded contents 378 // of the payload, suitable for passing directly to a new request. 379 func (s *TestServer) encodePayload(payload interface{}) io.Reader { 380 var encoded bytes.Buffer 381 enc := json.NewEncoder(&encoded) 382 if err := enc.Encode(payload); err != nil { 383 s.t.Fatalf("err: %s", err) 384 } 385 return &encoded 386 }