github.com/hhrutter/nomad@v0.6.0-rc2.0.20170723054333-80c4b03f0705/command/agent/consul/int_test.go (about) 1 package consul_test 2 3 import ( 4 "io/ioutil" 5 "log" 6 "os" 7 "path/filepath" 8 "testing" 9 "time" 10 11 "github.com/boltdb/bolt" 12 consulapi "github.com/hashicorp/consul/api" 13 "github.com/hashicorp/consul/testutil" 14 "github.com/hashicorp/nomad/client" 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/config" 17 "github.com/hashicorp/nomad/client/driver" 18 "github.com/hashicorp/nomad/client/vaultclient" 19 "github.com/hashicorp/nomad/command/agent/consul" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/nomad/structs" 22 ) 23 24 func testLogger() *log.Logger { 25 if testing.Verbose() { 26 return log.New(os.Stderr, "", log.LstdFlags) 27 } 28 return log.New(ioutil.Discard, "", 0) 29 } 30 31 // TestConsul_Integration asserts TaskRunner properly registers and deregisters 32 // services and checks with Consul using an embedded Consul agent. 33 func TestConsul_Integration(t *testing.T) { 34 t.Skip("-short set; skipping") 35 if _, ok := driver.BuiltinDrivers["mock_driver"]; !ok { 36 t.Skip(`test requires mock_driver; run with "-tags nomad_test"`) 37 } 38 if testing.Short() { 39 t.Skip("-short set; skipping") 40 } 41 // Create an embedded Consul server 42 testconsul, err := testutil.NewTestServerConfig(func(c *testutil.TestServerConfig) { 43 // If -v wasn't specified squelch consul logging 44 if !testing.Verbose() { 45 c.Stdout = ioutil.Discard 46 c.Stderr = ioutil.Discard 47 } 48 }) 49 if err != nil { 50 t.Fatalf("error starting test consul server: %v", err) 51 } 52 defer testconsul.Stop() 53 54 conf := config.DefaultConfig() 55 conf.Node = mock.Node() 56 conf.ConsulConfig.Addr = testconsul.HTTPAddr 57 consulConfig, err := conf.ConsulConfig.ApiConfig() 58 if err != nil { 59 t.Fatalf("error generating consul config: %v", err) 60 } 61 62 conf.StateDir, err = ioutil.TempDir("", "nomadtest-consulstate") 63 if err != nil { 64 t.Fatalf("error creating temp dir: %v", err) 65 } 66 defer os.RemoveAll(conf.StateDir) 67 conf.AllocDir, err = ioutil.TempDir("", "nomdtest-consulalloc") 68 if err != nil { 69 t.Fatalf("error creating temp dir: %v", err) 70 } 71 defer os.RemoveAll(conf.AllocDir) 72 73 tmp, err := ioutil.TempFile("", "state-db") 74 if err != nil { 75 t.Fatalf("error creating state db file: %v", err) 76 } 77 db, err := bolt.Open(tmp.Name(), 0600, nil) 78 if err != nil { 79 t.Fatalf("error creating state db: %v", err) 80 } 81 82 alloc := mock.Alloc() 83 task := alloc.Job.TaskGroups[0].Tasks[0] 84 task.Driver = "mock_driver" 85 task.Config = map[string]interface{}{ 86 "run_for": "1h", 87 } 88 // Choose a port that shouldn't be in use 89 task.Resources.Networks[0].ReservedPorts = []structs.Port{{Label: "http", Value: 3}} 90 task.Services = []*structs.Service{ 91 { 92 Name: "httpd", 93 PortLabel: "http", 94 Tags: []string{"nomad", "test", "http"}, 95 Checks: []*structs.ServiceCheck{ 96 { 97 Name: "httpd-http-check", 98 Type: "http", 99 Path: "/", 100 Protocol: "http", 101 PortLabel: "http", 102 Interval: 9000 * time.Hour, 103 Timeout: 1, // fail as fast as possible 104 }, 105 { 106 Name: "httpd-script-check", 107 Type: "script", 108 Command: "/bin/true", 109 Interval: 10 * time.Second, 110 Timeout: 10 * time.Second, 111 }, 112 }, 113 }, 114 { 115 Name: "httpd2", 116 PortLabel: "http", 117 Tags: []string{"test", "http2"}, 118 }, 119 } 120 121 logger := testLogger() 122 logUpdate := func(name, state string, event *structs.TaskEvent) { 123 logger.Printf("[TEST] test.updater: name=%q state=%q event=%v", name, state, event) 124 } 125 allocDir := allocdir.NewAllocDir(logger, filepath.Join(conf.AllocDir, alloc.ID)) 126 if err := allocDir.Build(); err != nil { 127 t.Fatalf("error building alloc dir: %v", err) 128 } 129 taskDir := allocDir.NewTaskDir(task.Name) 130 vclient := vaultclient.NewMockVaultClient() 131 consulClient, err := consulapi.NewClient(consulConfig) 132 if err != nil { 133 t.Fatalf("error creating consul client: %v", err) 134 } 135 serviceClient := consul.NewServiceClient(consulClient.Agent(), true, logger) 136 defer serviceClient.Shutdown() // just-in-case cleanup 137 consulRan := make(chan struct{}) 138 go func() { 139 serviceClient.Run() 140 close(consulRan) 141 }() 142 tr := client.NewTaskRunner(logger, conf, db, logUpdate, taskDir, alloc, task, vclient, serviceClient) 143 tr.MarkReceived() 144 go tr.Run() 145 defer func() { 146 // Make sure we always shutdown task runner when the test exits 147 select { 148 case <-tr.WaitCh(): 149 // Exited cleanly, no need to kill 150 default: 151 tr.Kill("", "", true) // just in case 152 } 153 }() 154 155 // Block waiting for the service to appear 156 catalog := consulClient.Catalog() 157 res, meta, err := catalog.Service("httpd2", "test", nil) 158 for i := 0; len(res) == 0 && i < 10; i++ { 159 //Expected initial request to fail, do a blocking query 160 res, meta, err = catalog.Service("httpd2", "test", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second}) 161 if err != nil { 162 t.Fatalf("error querying for service: %v", err) 163 } 164 } 165 if len(res) != 1 { 166 t.Fatalf("expected 1 service but found %d:\n%#v", len(res), res) 167 } 168 res = res[:] 169 170 // Assert the service with the checks exists 171 for i := 0; len(res) == 0 && i < 10; i++ { 172 res, meta, err = catalog.Service("httpd", "http", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second}) 173 if err != nil { 174 t.Fatalf("error querying for service: %v", err) 175 } 176 } 177 if len(res) != 1 { 178 t.Fatalf("exepcted 1 service but found %d:\n%#v", len(res), res) 179 } 180 181 // Assert the script check passes (mock_driver script checks always 182 // pass) after having time to run once 183 time.Sleep(2 * time.Second) 184 checks, _, err := consulClient.Health().Checks("httpd", nil) 185 if err != nil { 186 t.Fatalf("error querying checks: %v", err) 187 } 188 if expected := 2; len(checks) != expected { 189 t.Fatalf("expected %d checks but found %d:\n%#v", expected, len(checks), checks) 190 } 191 for _, check := range checks { 192 if expected := "httpd"; check.ServiceName != expected { 193 t.Fatalf("expected checks to be for %q but found service name = %q", expected, check.ServiceName) 194 } 195 switch check.Name { 196 case "httpd-http-check": 197 // Port check should fail 198 if expected := consulapi.HealthCritical; check.Status != expected { 199 t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status) 200 } 201 case "httpd-script-check": 202 // mock_driver script checks always succeed 203 if expected := consulapi.HealthPassing; check.Status != expected { 204 t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status) 205 } 206 default: 207 t.Errorf("unexpected check %q with status %q", check.Name, check.Status) 208 } 209 } 210 211 // Assert the service client returns all the checks for the allocation. 212 checksOut, err := serviceClient.Checks(alloc) 213 if err != nil { 214 t.Fatalf("unexpected error retrieving allocation checks: %v", err) 215 } 216 217 if l := len(checksOut); l != 2 { 218 t.Fatalf("got %d checks; want %d", l, 2) 219 } 220 221 logger.Printf("[TEST] consul.test: killing task") 222 223 // Kill the task 224 tr.Kill("", "", false) 225 226 select { 227 case <-tr.WaitCh(): 228 case <-time.After(10 * time.Second): 229 t.Fatalf("timed out waiting for Run() to exit") 230 } 231 232 // Shutdown Consul ServiceClient to ensure all pending operations complete 233 if err := serviceClient.Shutdown(); err != nil { 234 t.Errorf("error shutting down Consul ServiceClient: %v", err) 235 } 236 237 // Ensure Consul is clean 238 services, _, err := catalog.Services(nil) 239 if err != nil { 240 t.Fatalf("error query services: %v", err) 241 } 242 if len(services) != 1 { 243 t.Fatalf("expected only 1 service in Consul but found %d:\n%#v", len(services), services) 244 } 245 if _, ok := services["consul"]; !ok { 246 t.Fatalf(`expected only the "consul" key in Consul but found: %#v`, services) 247 } 248 }