github.com/hernad/nomad@v1.6.112/testutil/server.go (about) 1 // Copyright (c) HashiCorp, Inc. 2 // SPDX-License-Identifier: MPL-2.0 3 4 package testutil 5 6 // TestServer is a test helper. It uses a fork/exec model to create 7 // a test Nomad server instance in the background and initialize it 8 // with some data and/or services. The test server can then be used 9 // to run a unit test, and offers an easy API to tear itself down 10 // when the test has completed. The only prerequisite is to have a nomad 11 // binary available on the $PATH. 12 // 13 // This package does not use Nomad's official API client. This is 14 // because we use TestServer to test the API client, which would 15 // otherwise cause an import cycle. 16 17 import ( 18 "bytes" 19 "encoding/json" 20 "fmt" 21 "io" 22 "net/http" 23 "os" 24 "os/exec" 25 "time" 26 27 cleanhttp "github.com/hashicorp/go-cleanhttp" 28 "github.com/hernad/nomad/ci" 29 "github.com/hernad/nomad/helper/discover" 30 testing "github.com/mitchellh/go-testing-interface" 31 ) 32 33 // TestServerConfig is the main server configuration struct. 34 type TestServerConfig struct { 35 NodeName string `json:"name,omitempty"` 36 DataDir string `json:"data_dir,omitempty"` 37 Region string `json:"region,omitempty"` 38 DisableCheckpoint bool `json:"disable_update_check"` 39 LogLevel string `json:"log_level,omitempty"` 40 Consul *Consul `json:"consul,omitempty"` 41 AdvertiseAddrs *Advertise `json:"advertise,omitempty"` 42 Ports *PortsConfig `json:"ports,omitempty"` 43 Server *ServerConfig `json:"server,omitempty"` 44 Client *ClientConfig `json:"client,omitempty"` 45 Vault *VaultConfig `json:"vault,omitempty"` 46 ACL *ACLConfig `json:"acl,omitempty"` 47 DevMode bool `json:"-"` 48 Stdout, Stderr io.Writer `json:"-"` 49 } 50 51 // Consul is used to configure the communication with Consul 52 type Consul struct { 53 Address string `json:"address,omitempty"` 54 Auth string `json:"auth,omitempty"` 55 Token string `json:"token,omitempty"` 56 } 57 58 // Advertise is used to configure the addresses to advertise 59 type Advertise struct { 60 HTTP string `json:"http,omitempty"` 61 RPC string `json:"rpc,omitempty"` 62 Serf string `json:"serf,omitempty"` 63 } 64 65 // PortsConfig is used to configure the network ports we use. 66 type PortsConfig struct { 67 HTTP int `json:"http,omitempty"` 68 RPC int `json:"rpc,omitempty"` 69 Serf int `json:"serf,omitempty"` 70 } 71 72 // ServerConfig is used to configure the nomad server. 73 type ServerConfig struct { 74 Enabled bool `json:"enabled"` 75 BootstrapExpect int `json:"bootstrap_expect"` 76 RaftProtocol int `json:"raft_protocol,omitempty"` 77 } 78 79 // ClientConfig is used to configure the client 80 type ClientConfig struct { 81 Enabled bool `json:"enabled"` 82 } 83 84 // VaultConfig is used to configure Vault 85 type VaultConfig struct { 86 Enabled bool `json:"enabled"` 87 Address string `json:"address"` 88 AllowUnauthenticated bool `json:"allow_unauthenticated"` 89 Token string `json:"token"` 90 } 91 92 // ACLConfig is used to configure ACLs 93 type ACLConfig struct { 94 Enabled bool `json:"enabled"` 95 } 96 97 // ServerConfigCallback is a function interface which can be 98 // passed to NewTestServerConfig to modify the server config. 99 type ServerConfigCallback func(c *TestServerConfig) 100 101 // defaultServerConfig returns a new TestServerConfig struct 102 // with all of the listen ports incremented by one. 103 func defaultServerConfig() *TestServerConfig { 104 ports := ci.PortAllocator.Grab(3) 105 return &TestServerConfig{ 106 NodeName: fmt.Sprintf("node-%d", ports[0]), 107 DisableCheckpoint: true, 108 LogLevel: "DEBUG", 109 Ports: &PortsConfig{ 110 HTTP: ports[0], 111 RPC: ports[1], 112 Serf: ports[2], 113 }, 114 Server: &ServerConfig{ 115 Enabled: true, 116 BootstrapExpect: 1, 117 }, 118 Client: &ClientConfig{ 119 Enabled: false, 120 }, 121 Vault: &VaultConfig{ 122 Enabled: false, 123 AllowUnauthenticated: true, 124 }, 125 ACL: &ACLConfig{ 126 Enabled: false, 127 }, 128 } 129 } 130 131 // TestServer is the main server wrapper struct. 132 type TestServer struct { 133 cmd *exec.Cmd 134 Config *TestServerConfig 135 t testing.T 136 137 HTTPAddr string 138 SerfAddr string 139 HTTPClient *http.Client 140 } 141 142 // NewTestServer creates a new TestServer, and makes a call to 143 // an optional callback function to modify the configuration. 144 func NewTestServer(t testing.T, cb ServerConfigCallback) *TestServer { 145 path, err := discover.NomadExecutable() 146 if err != nil { 147 t.Skipf("nomad not found, skipping: %v", err) 148 } 149 150 // Check that we are actually running nomad 151 vcmd := exec.Command(path, "-version") 152 vcmd.Stdout = nil 153 vcmd.Stderr = nil 154 if err := vcmd.Run(); err != nil { 155 t.Skipf("nomad version failed: %v", err) 156 } 157 158 dataDir, err := os.MkdirTemp("", "nomad") 159 if err != nil { 160 t.Fatalf("err: %s", err) 161 } 162 163 configFile, err := os.CreateTemp(dataDir, "nomad") 164 if err != nil { 165 defer os.RemoveAll(dataDir) 166 t.Fatalf("err: %s", err) 167 } 168 defer configFile.Close() 169 170 nomadConfig := defaultServerConfig() 171 nomadConfig.DataDir = dataDir 172 173 if cb != nil { 174 cb(nomadConfig) 175 } 176 177 configContent, err := json.Marshal(nomadConfig) 178 if err != nil { 179 t.Fatalf("err: %s", err) 180 } 181 182 if _, err := configFile.Write(configContent); err != nil { 183 t.Fatalf("err: %s", err) 184 } 185 configFile.Close() 186 187 stdout := io.Writer(os.Stdout) 188 if nomadConfig.Stdout != nil { 189 stdout = nomadConfig.Stdout 190 } 191 192 stderr := io.Writer(os.Stderr) 193 if nomadConfig.Stderr != nil { 194 stderr = nomadConfig.Stderr 195 } 196 197 args := []string{"agent", "-config", configFile.Name()} 198 if nomadConfig.DevMode { 199 args = append(args, "-dev") 200 } 201 202 // Start the server 203 cmd := exec.Command(path, args...) 204 cmd.Stdout = stdout 205 cmd.Stderr = stderr 206 if err := cmd.Start(); err != nil { 207 t.Fatalf("err: %s", err) 208 } 209 210 client := cleanhttp.DefaultClient() 211 212 server := &TestServer{ 213 Config: nomadConfig, 214 cmd: cmd, 215 t: t, 216 217 HTTPAddr: fmt.Sprintf("127.0.0.1:%d", nomadConfig.Ports.HTTP), 218 SerfAddr: fmt.Sprintf("127.0.0.1:%d", nomadConfig.Ports.Serf), 219 HTTPClient: client, 220 } 221 222 // Wait for the server to be ready 223 if nomadConfig.Server.Enabled && nomadConfig.Server.BootstrapExpect != 0 { 224 server.waitForLeader() 225 } else { 226 server.waitForAPI() 227 } 228 229 // Wait for the client to be ready 230 if nomadConfig.DevMode { 231 server.waitForClient() 232 } 233 return server 234 } 235 236 // Stop stops the test Nomad server, and removes the Nomad data 237 // directory once we are done. 238 func (s *TestServer) Stop() { 239 defer os.RemoveAll(s.Config.DataDir) 240 241 // wait for the process to exit to be sure that the data dir can be 242 // deleted on all platforms. 243 done := make(chan struct{}) 244 go func() { 245 defer close(done) 246 247 s.cmd.Wait() 248 }() 249 250 // kill and wait gracefully 251 if err := s.cmd.Process.Signal(os.Interrupt); err != nil { 252 s.t.Errorf("err: %s", err) 253 } 254 255 select { 256 case <-done: 257 return 258 case <-time.After(5 * time.Second): 259 s.t.Logf("timed out waiting for process to gracefully terminate") 260 } 261 262 if err := s.cmd.Process.Kill(); err != nil { 263 s.t.Errorf("err: %s", err) 264 } 265 select { 266 case <-done: 267 case <-time.After(5 * time.Second): 268 s.t.Logf("timed out waiting for process to be killed") 269 } 270 271 } 272 273 // waitForAPI waits for only the agent HTTP endpoint to start 274 // responding. This is an indication that the agent has started, 275 // but will likely return before a leader is elected. 276 func (s *TestServer) waitForAPI() { 277 WaitForResult(func() (bool, error) { 278 // Using this endpoint as it is does not have restricted access 279 resp, err := s.HTTPClient.Get(s.url("/v1/metrics")) 280 if err != nil { 281 return false, err 282 } 283 defer resp.Body.Close() 284 if err := s.requireOK(resp); err != nil { 285 return false, err 286 } 287 return true, nil 288 }, func(err error) { 289 defer s.Stop() 290 s.t.Fatalf("err: %s", err) 291 }) 292 } 293 294 // waitForLeader waits for the Nomad server's HTTP API to become 295 // available, and then waits for a known leader and an index of 296 // 1 or more to be observed to confirm leader election is done. 297 func (s *TestServer) waitForLeader() { 298 WaitForResult(func() (bool, error) { 299 // Query the API and check the status code 300 // Using this endpoint as it is does not have restricted access 301 resp, err := s.HTTPClient.Get(s.url("/v1/status/leader")) 302 if err != nil { 303 return false, err 304 } 305 defer resp.Body.Close() 306 if err := s.requireOK(resp); err != nil { 307 return false, err 308 } 309 310 return true, nil 311 }, func(err error) { 312 defer s.Stop() 313 s.t.Fatalf("err: %s", err) 314 }) 315 } 316 317 // waitForClient waits for the Nomad client to be ready. The function returns 318 // immediately if the server is not in dev mode. 319 func (s *TestServer) waitForClient() { 320 if !s.Config.DevMode { 321 return 322 } 323 324 WaitForResult(func() (bool, error) { 325 resp, err := s.HTTPClient.Get(s.url("/v1/nodes")) 326 if err != nil { 327 return false, err 328 } 329 defer resp.Body.Close() 330 if err := s.requireOK(resp); err != nil { 331 return false, err 332 } 333 334 var decoded []struct { 335 ID string 336 Status string 337 } 338 339 dec := json.NewDecoder(resp.Body) 340 if err := dec.Decode(&decoded); err != nil { 341 return false, err 342 } 343 344 if len(decoded) != 1 || decoded[0].Status != "ready" { 345 return false, fmt.Errorf("Node not ready: %v", decoded) 346 } 347 348 return true, nil 349 }, func(err error) { 350 defer s.Stop() 351 s.t.Fatalf("err: %s", err) 352 }) 353 } 354 355 // url is a helper function which takes a relative URL and 356 // makes it into a proper URL against the local Nomad server. 357 func (s *TestServer) url(path string) string { 358 return fmt.Sprintf("http://%s%s", s.HTTPAddr, path) 359 } 360 361 // requireOK checks the HTTP response code and ensures it is acceptable. 362 func (s *TestServer) requireOK(resp *http.Response) error { 363 if resp.StatusCode != 200 { 364 return fmt.Errorf("Bad status code: %d", resp.StatusCode) 365 } 366 return nil 367 } 368 369 // put performs a new HTTP PUT request. 370 func (s *TestServer) put(path string, body io.Reader) *http.Response { 371 req, err := http.NewRequest("PUT", s.url(path), body) 372 if err != nil { 373 s.t.Fatalf("err: %s", err) 374 } 375 resp, err := s.HTTPClient.Do(req) 376 if err != nil { 377 s.t.Fatalf("err: %s", err) 378 } 379 if err := s.requireOK(resp); err != nil { 380 defer resp.Body.Close() 381 s.t.Fatal(err) 382 } 383 return resp 384 } 385 386 // get performs a new HTTP GET request. 387 func (s *TestServer) get(path string) *http.Response { 388 resp, err := s.HTTPClient.Get(s.url(path)) 389 if err != nil { 390 s.t.Fatalf("err: %s", err) 391 } 392 if err := s.requireOK(resp); err != nil { 393 defer resp.Body.Close() 394 s.t.Fatal(err) 395 } 396 return resp 397 } 398 399 // encodePayload returns a new io.Reader wrapping the encoded contents 400 // of the payload, suitable for passing directly to a new request. 401 func (s *TestServer) encodePayload(payload interface{}) io.Reader { 402 var encoded bytes.Buffer 403 enc := json.NewEncoder(&encoded) 404 if err := enc.Encode(payload); err != nil { 405 s.t.Fatalf("err: %s", err) 406 } 407 return &encoded 408 }