github.com/djenriquez/nomad-1@v0.8.1/command/agent/consul/int_test.go (about) 1 package consul_test 2 3 import ( 4 "io/ioutil" 5 "log" 6 "os" 7 "path/filepath" 8 "testing" 9 "time" 10 11 "github.com/boltdb/bolt" 12 consulapi "github.com/hashicorp/consul/api" 13 "github.com/hashicorp/consul/testutil" 14 "github.com/hashicorp/nomad/client" 15 "github.com/hashicorp/nomad/client/allocdir" 16 "github.com/hashicorp/nomad/client/config" 17 "github.com/hashicorp/nomad/client/driver" 18 "github.com/hashicorp/nomad/client/vaultclient" 19 "github.com/hashicorp/nomad/command/agent/consul" 20 "github.com/hashicorp/nomad/nomad/mock" 21 "github.com/hashicorp/nomad/nomad/structs" 22 "github.com/stretchr/testify/assert" 23 ) 24 25 func testLogger() *log.Logger { 26 if testing.Verbose() { 27 return log.New(os.Stderr, "", log.LstdFlags) 28 } 29 return log.New(ioutil.Discard, "", 0) 30 } 31 32 // TestConsul_Integration asserts TaskRunner properly registers and deregisters 33 // services and checks with Consul using an embedded Consul agent. 34 func TestConsul_Integration(t *testing.T) { 35 if _, ok := driver.BuiltinDrivers["mock_driver"]; !ok { 36 t.Skip(`test requires mock_driver; run with "-tags nomad_test"`) 37 } 38 if testing.Short() { 39 t.Skip("-short set; skipping") 40 } 41 assert := assert.New(t) 42 43 // Create an embedded Consul server 44 testconsul, err := testutil.NewTestServerConfig(func(c *testutil.TestServerConfig) { 45 // If -v wasn't specified squelch consul logging 46 if !testing.Verbose() { 47 c.Stdout = ioutil.Discard 48 c.Stderr = ioutil.Discard 49 } 50 }) 51 if err != nil { 52 t.Fatalf("error starting test consul server: %v", err) 53 } 54 defer testconsul.Stop() 55 56 conf := config.DefaultConfig() 57 conf.Node = mock.Node() 58 conf.ConsulConfig.Addr = testconsul.HTTPAddr 59 consulConfig, err := conf.ConsulConfig.ApiConfig() 60 if err != nil { 61 t.Fatalf("error generating consul config: %v", err) 62 } 63 64 conf.StateDir, err = ioutil.TempDir("", "nomadtest-consulstate") 65 if err != nil { 66 t.Fatalf("error creating temp dir: %v", err) 67 } 68 defer os.RemoveAll(conf.StateDir) 69 conf.AllocDir, err = ioutil.TempDir("", "nomdtest-consulalloc") 70 if err != nil { 71 t.Fatalf("error creating temp dir: %v", err) 72 } 73 defer os.RemoveAll(conf.AllocDir) 74 75 tmp, err := ioutil.TempFile("", "state-db") 76 if err != nil { 77 t.Fatalf("error creating state db file: %v", err) 78 } 79 db, err := bolt.Open(tmp.Name(), 0600, nil) 80 if err != nil { 81 t.Fatalf("error creating state db: %v", err) 82 } 83 84 alloc := mock.Alloc() 85 task := alloc.Job.TaskGroups[0].Tasks[0] 86 task.Driver = "mock_driver" 87 task.Config = map[string]interface{}{ 88 "run_for": "1h", 89 } 90 91 // Choose a port that shouldn't be in use 92 netResource := &structs.NetworkResource{ 93 Device: "eth0", 94 IP: "127.0.0.1", 95 MBits: 50, 96 ReservedPorts: []structs.Port{{Label: "http", Value: 3}}, 97 } 98 alloc.Resources.Networks[0] = netResource 99 alloc.TaskResources["web"].Networks[0] = netResource 100 task.Resources.Networks[0] = netResource 101 task.Services = []*structs.Service{ 102 { 103 Name: "httpd", 104 PortLabel: "http", 105 Tags: []string{"nomad", "test", "http"}, 106 Checks: []*structs.ServiceCheck{ 107 { 108 Name: "httpd-http-check", 109 Type: "http", 110 Path: "/", 111 Protocol: "http", 112 Interval: 9000 * time.Hour, 113 Timeout: 1, // fail as fast as possible 114 }, 115 { 116 Name: "httpd-script-check", 117 Type: "script", 118 Command: "/bin/true", 119 Interval: 10 * time.Second, 120 Timeout: 10 * time.Second, 121 }, 122 }, 123 }, 124 { 125 Name: "httpd2", 126 PortLabel: "http", 127 Tags: []string{ 128 "test", 129 // Use URL-unfriendly tags to test #3620 130 "public-test.ettaviation.com:80/ redirect=302,https://test.ettaviation.com", 131 "public-test.ettaviation.com:443/", 132 }, 133 }, 134 } 135 136 logger := testLogger() 137 logUpdate := func(name, state string, event *structs.TaskEvent, lazySync bool) { 138 logger.Printf("[TEST] test.updater: name=%q state=%q event=%v", name, state, event) 139 } 140 allocDir := allocdir.NewAllocDir(logger, filepath.Join(conf.AllocDir, alloc.ID)) 141 if err := allocDir.Build(); err != nil { 142 t.Fatalf("error building alloc dir: %v", err) 143 } 144 taskDir := allocDir.NewTaskDir(task.Name) 145 vclient := vaultclient.NewMockVaultClient() 146 consulClient, err := consulapi.NewClient(consulConfig) 147 assert.Nil(err) 148 149 serviceClient := consul.NewServiceClient(consulClient.Agent(), logger) 150 defer serviceClient.Shutdown() // just-in-case cleanup 151 consulRan := make(chan struct{}) 152 go func() { 153 serviceClient.Run() 154 close(consulRan) 155 }() 156 tr := client.NewTaskRunner(logger, conf, db, logUpdate, taskDir, alloc, task, vclient, serviceClient) 157 tr.MarkReceived() 158 go tr.Run() 159 defer func() { 160 // Make sure we always shutdown task runner when the test exits 161 select { 162 case <-tr.WaitCh(): 163 // Exited cleanly, no need to kill 164 default: 165 tr.Kill("", "", true) // just in case 166 } 167 }() 168 169 // Block waiting for the service to appear 170 catalog := consulClient.Catalog() 171 res, meta, err := catalog.Service("httpd2", "test", nil) 172 assert.Nil(err) 173 174 for i := 0; len(res) == 0 && i < 10; i++ { 175 //Expected initial request to fail, do a blocking query 176 res, meta, err = catalog.Service("httpd2", "test", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second}) 177 if err != nil { 178 t.Fatalf("error querying for service: %v", err) 179 } 180 } 181 assert.Len(res, 1) 182 183 // Truncate results 184 res = res[:] 185 186 // Assert the service with the checks exists 187 for i := 0; len(res) == 0 && i < 10; i++ { 188 res, meta, err = catalog.Service("httpd", "http", &consulapi.QueryOptions{WaitIndex: meta.LastIndex + 1, WaitTime: 3 * time.Second}) 189 assert.Nil(err) 190 } 191 assert.Len(res, 1) 192 193 // Assert the script check passes (mock_driver script checks always 194 // pass) after having time to run once 195 time.Sleep(2 * time.Second) 196 checks, _, err := consulClient.Health().Checks("httpd", nil) 197 assert.Nil(err) 198 assert.Len(checks, 2) 199 200 for _, check := range checks { 201 if expected := "httpd"; check.ServiceName != expected { 202 t.Fatalf("expected checks to be for %q but found service name = %q", expected, check.ServiceName) 203 } 204 switch check.Name { 205 case "httpd-http-check": 206 // Port check should fail 207 if expected := consulapi.HealthCritical; check.Status != expected { 208 t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status) 209 } 210 case "httpd-script-check": 211 // mock_driver script checks always succeed 212 if expected := consulapi.HealthPassing; check.Status != expected { 213 t.Errorf("expected %q status to be %q but found %q", check.Name, expected, check.Status) 214 } 215 default: 216 t.Errorf("unexpected check %q with status %q", check.Name, check.Status) 217 } 218 } 219 220 // Assert the service client returns all the checks for the allocation. 221 reg, err := serviceClient.AllocRegistrations(alloc.ID) 222 if err != nil { 223 t.Fatalf("unexpected error retrieving allocation checks: %v", err) 224 } 225 if reg == nil { 226 t.Fatalf("Unexpected nil allocation registration") 227 } 228 if snum := reg.NumServices(); snum != 2 { 229 t.Fatalf("Unexpected number of services registered. Got %d; want 2", snum) 230 } 231 if cnum := reg.NumChecks(); cnum != 2 { 232 t.Fatalf("Unexpected number of checks registered. Got %d; want 2", cnum) 233 } 234 235 logger.Printf("[TEST] consul.test: killing task") 236 237 // Kill the task 238 tr.Kill("", "", false) 239 240 select { 241 case <-tr.WaitCh(): 242 case <-time.After(10 * time.Second): 243 t.Fatalf("timed out waiting for Run() to exit") 244 } 245 246 // Shutdown Consul ServiceClient to ensure all pending operations complete 247 if err := serviceClient.Shutdown(); err != nil { 248 t.Errorf("error shutting down Consul ServiceClient: %v", err) 249 } 250 251 // Ensure Consul is clean 252 services, _, err := catalog.Services(nil) 253 assert.Nil(err) 254 assert.Len(services, 1) 255 assert.Contains(services, "consul") 256 }