github.com/diptanu/nomad@v0.5.7-0.20170516172507-d72e86cbe3d9/command/agent/consul/unit_test.go (about)

     1  package consul
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"log"
     8  	"os"
     9  	"reflect"
    10  	"sync"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/hashicorp/consul/api"
    15  	"github.com/hashicorp/nomad/nomad/structs"
    16  )
    17  
    18  const (
    19  	// Ports used in testTask
    20  	xPort = 1234
    21  	yPort = 1235
    22  )
    23  
    24  func testLogger() *log.Logger {
    25  	if testing.Verbose() {
    26  		return log.New(os.Stderr, "", log.LstdFlags)
    27  	}
    28  	return log.New(ioutil.Discard, "", 0)
    29  }
    30  
    31  func testTask() *structs.Task {
    32  	return &structs.Task{
    33  		Name: "taskname",
    34  		Resources: &structs.Resources{
    35  			Networks: []*structs.NetworkResource{
    36  				{
    37  					DynamicPorts: []structs.Port{
    38  						{Label: "x", Value: xPort},
    39  						{Label: "y", Value: yPort},
    40  					},
    41  				},
    42  			},
    43  		},
    44  		Services: []*structs.Service{
    45  			{
    46  				Name:      "taskname-service",
    47  				PortLabel: "x",
    48  				Tags:      []string{"tag1", "tag2"},
    49  			},
    50  		},
    51  	}
    52  }
    53  
    54  // testFakeCtx contains a fake Consul AgentAPI and implements the Exec
    55  // interface to allow testing without running Consul.
    56  type testFakeCtx struct {
    57  	ServiceClient *ServiceClient
    58  	FakeConsul    *fakeConsul
    59  	Task          *structs.Task
    60  
    61  	// Ticked whenever a script is called
    62  	execs chan int
    63  
    64  	// If non-nil will be called by script checks
    65  	ExecFunc func(ctx context.Context, cmd string, args []string) ([]byte, int, error)
    66  }
    67  
    68  // Exec implements the ScriptExecutor interface and will use an alternate
    69  // implementation t.ExecFunc if non-nil.
    70  func (t *testFakeCtx) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
    71  	select {
    72  	case t.execs <- 1:
    73  	default:
    74  	}
    75  	if t.ExecFunc == nil {
    76  		// Default impl is just "ok"
    77  		return []byte("ok"), 0, nil
    78  	}
    79  	return t.ExecFunc(ctx, cmd, args)
    80  }
    81  
    82  var errNoOps = fmt.Errorf("testing error: no pending operations")
    83  
    84  // syncOps simulates one iteration of the ServiceClient.Run loop and returns
    85  // any errors returned by sync() or errNoOps if no pending operations.
    86  func (t *testFakeCtx) syncOnce() error {
    87  	select {
    88  	case ops := <-t.ServiceClient.opCh:
    89  		t.ServiceClient.merge(ops)
    90  		return t.ServiceClient.sync()
    91  	default:
    92  		return errNoOps
    93  	}
    94  }
    95  
    96  // setupFake creates a testFakeCtx with a ServiceClient backed by a fakeConsul.
    97  // A test Task is also provided.
    98  func setupFake() *testFakeCtx {
    99  	fc := newFakeConsul()
   100  	return &testFakeCtx{
   101  		ServiceClient: NewServiceClient(fc, true, testLogger()),
   102  		FakeConsul:    fc,
   103  		Task:          testTask(),
   104  		execs:         make(chan int, 100),
   105  	}
   106  }
   107  
   108  // fakeConsul is a fake in-memory Consul backend for ServiceClient.
   109  type fakeConsul struct {
   110  	// maps of what services and checks have been registered
   111  	services map[string]*api.AgentServiceRegistration
   112  	checks   map[string]*api.AgentCheckRegistration
   113  	mu       sync.Mutex
   114  
   115  	// when UpdateTTL is called the check ID will have its counter inc'd
   116  	checkTTLs map[string]int
   117  
   118  	// What check status to return from Checks()
   119  	checkStatus string
   120  }
   121  
   122  func newFakeConsul() *fakeConsul {
   123  	return &fakeConsul{
   124  		services:    make(map[string]*api.AgentServiceRegistration),
   125  		checks:      make(map[string]*api.AgentCheckRegistration),
   126  		checkTTLs:   make(map[string]int),
   127  		checkStatus: api.HealthPassing,
   128  	}
   129  }
   130  
   131  func (c *fakeConsul) Services() (map[string]*api.AgentService, error) {
   132  	c.mu.Lock()
   133  	defer c.mu.Unlock()
   134  
   135  	r := make(map[string]*api.AgentService, len(c.services))
   136  	for k, v := range c.services {
   137  		r[k] = &api.AgentService{
   138  			ID:                v.ID,
   139  			Service:           v.Name,
   140  			Tags:              make([]string, len(v.Tags)),
   141  			Port:              v.Port,
   142  			Address:           v.Address,
   143  			EnableTagOverride: v.EnableTagOverride,
   144  		}
   145  		copy(r[k].Tags, v.Tags)
   146  	}
   147  	return r, nil
   148  }
   149  
   150  func (c *fakeConsul) Checks() (map[string]*api.AgentCheck, error) {
   151  	c.mu.Lock()
   152  	defer c.mu.Unlock()
   153  
   154  	r := make(map[string]*api.AgentCheck, len(c.checks))
   155  	for k, v := range c.checks {
   156  		r[k] = &api.AgentCheck{
   157  			CheckID:     v.ID,
   158  			Name:        v.Name,
   159  			Status:      c.checkStatus,
   160  			Notes:       v.Notes,
   161  			ServiceID:   v.ServiceID,
   162  			ServiceName: c.services[v.ServiceID].Name,
   163  		}
   164  	}
   165  	return r, nil
   166  }
   167  
   168  func (c *fakeConsul) CheckRegister(check *api.AgentCheckRegistration) error {
   169  	c.mu.Lock()
   170  	defer c.mu.Unlock()
   171  	c.checks[check.ID] = check
   172  	return nil
   173  }
   174  
   175  func (c *fakeConsul) CheckDeregister(checkID string) error {
   176  	c.mu.Lock()
   177  	defer c.mu.Unlock()
   178  	delete(c.checks, checkID)
   179  	delete(c.checkTTLs, checkID)
   180  	return nil
   181  }
   182  
   183  func (c *fakeConsul) ServiceRegister(service *api.AgentServiceRegistration) error {
   184  	c.mu.Lock()
   185  	defer c.mu.Unlock()
   186  	c.services[service.ID] = service
   187  	return nil
   188  }
   189  
   190  func (c *fakeConsul) ServiceDeregister(serviceID string) error {
   191  	c.mu.Lock()
   192  	defer c.mu.Unlock()
   193  	delete(c.services, serviceID)
   194  	return nil
   195  }
   196  
   197  func (c *fakeConsul) UpdateTTL(id string, output string, status string) error {
   198  	c.mu.Lock()
   199  	defer c.mu.Unlock()
   200  	check, ok := c.checks[id]
   201  	if !ok {
   202  		return fmt.Errorf("unknown check id: %q", id)
   203  	}
   204  	// Flip initial status to passing
   205  	check.Status = "passing"
   206  	c.checkTTLs[id]++
   207  	return nil
   208  }
   209  
   210  func TestConsul_ChangeTags(t *testing.T) {
   211  	ctx := setupFake()
   212  
   213  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, nil); err != nil {
   214  		t.Fatalf("unexpected error registering task: %v", err)
   215  	}
   216  
   217  	if err := ctx.syncOnce(); err != nil {
   218  		t.Fatalf("unexpected error syncing task: %v", err)
   219  	}
   220  
   221  	if n := len(ctx.FakeConsul.services); n != 1 {
   222  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   223  	}
   224  
   225  	origKey := ""
   226  	for k, v := range ctx.FakeConsul.services {
   227  		origKey = k
   228  		if v.Name != ctx.Task.Services[0].Name {
   229  			t.Errorf("expected Name=%q != %q", ctx.Task.Services[0].Name, v.Name)
   230  		}
   231  		if !reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   232  			t.Errorf("expected Tags=%v != %v", ctx.Task.Services[0].Tags, v.Tags)
   233  		}
   234  	}
   235  
   236  	origTask := ctx.Task
   237  	ctx.Task = testTask()
   238  	ctx.Task.Services[0].Tags[0] = "newtag"
   239  	if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, nil); err != nil {
   240  		t.Fatalf("unexpected error registering task: %v", err)
   241  	}
   242  	if err := ctx.syncOnce(); err != nil {
   243  		t.Fatalf("unexpected error syncing task: %v", err)
   244  	}
   245  
   246  	if n := len(ctx.FakeConsul.services); n != 1 {
   247  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   248  	}
   249  
   250  	for k, v := range ctx.FakeConsul.services {
   251  		if k == origKey {
   252  			t.Errorf("expected key to change but found %q", k)
   253  		}
   254  		if v.Name != ctx.Task.Services[0].Name {
   255  			t.Errorf("expected Name=%q != %q", ctx.Task.Services[0].Name, v.Name)
   256  		}
   257  		if !reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   258  			t.Errorf("expected Tags=%v != %v", ctx.Task.Services[0].Tags, v.Tags)
   259  		}
   260  	}
   261  }
   262  
   263  // TestConsul_ChangePorts asserts that changing the ports on a service updates
   264  // it in Consul. Since ports are not part of the service ID this is a slightly
   265  // different code path than changing tags.
   266  func TestConsul_ChangePorts(t *testing.T) {
   267  	ctx := setupFake()
   268  	ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
   269  		{
   270  			Name:      "c1",
   271  			Type:      "tcp",
   272  			Interval:  time.Second,
   273  			Timeout:   time.Second,
   274  			PortLabel: "x",
   275  		},
   276  		{
   277  			Name:     "c2",
   278  			Type:     "script",
   279  			Interval: 9000 * time.Hour,
   280  			Timeout:  time.Second,
   281  		},
   282  		{
   283  			Name:      "c3",
   284  			Type:      "http",
   285  			Protocol:  "http",
   286  			Path:      "/",
   287  			Interval:  time.Second,
   288  			Timeout:   time.Second,
   289  			PortLabel: "y",
   290  		},
   291  	}
   292  
   293  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx); err != nil {
   294  		t.Fatalf("unexpected error registering task: %v", err)
   295  	}
   296  
   297  	if err := ctx.syncOnce(); err != nil {
   298  		t.Fatalf("unexpected error syncing task: %v", err)
   299  	}
   300  
   301  	if n := len(ctx.FakeConsul.services); n != 1 {
   302  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   303  	}
   304  
   305  	origServiceKey := ""
   306  	for k, v := range ctx.FakeConsul.services {
   307  		origServiceKey = k
   308  		if v.Name != ctx.Task.Services[0].Name {
   309  			t.Errorf("expected Name=%q != %q", ctx.Task.Services[0].Name, v.Name)
   310  		}
   311  		if !reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   312  			t.Errorf("expected Tags=%v != %v", ctx.Task.Services[0].Tags, v.Tags)
   313  		}
   314  		if v.Port != xPort {
   315  			t.Errorf("expected Port x=%v but found: %v", xPort, v.Port)
   316  		}
   317  	}
   318  
   319  	if n := len(ctx.FakeConsul.checks); n != 3 {
   320  		t.Fatalf("expected 3 checks but found %d:\n%#v", n, ctx.FakeConsul.checks)
   321  	}
   322  
   323  	origTCPKey := ""
   324  	origScriptKey := ""
   325  	origHTTPKey := ""
   326  	for k, v := range ctx.FakeConsul.checks {
   327  		switch v.Name {
   328  		case "c1":
   329  			origTCPKey = k
   330  			if expected := fmt.Sprintf(":%d", xPort); v.TCP != expected {
   331  				t.Errorf("expected Port x=%v but found: %v", expected, v.TCP)
   332  			}
   333  		case "c2":
   334  			origScriptKey = k
   335  			select {
   336  			case <-ctx.execs:
   337  				if n := len(ctx.execs); n > 0 {
   338  					t.Errorf("expected 1 exec but found: %d", n+1)
   339  				}
   340  			case <-time.After(3 * time.Second):
   341  				t.Errorf("script not called in time")
   342  			}
   343  		case "c3":
   344  			origHTTPKey = k
   345  			if expected := fmt.Sprintf("http://:%d/", yPort); v.HTTP != expected {
   346  				t.Errorf("expected Port y=%v but found: %v", expected, v.HTTP)
   347  			}
   348  		default:
   349  			t.Fatalf("unexpected check: %q", v.Name)
   350  		}
   351  	}
   352  
   353  	// Now update the PortLabel on the Service and Check c3
   354  	origTask := ctx.Task
   355  	ctx.Task = testTask()
   356  	ctx.Task.Services[0].PortLabel = "y"
   357  	ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
   358  		{
   359  			Name:      "c1",
   360  			Type:      "tcp",
   361  			Interval:  time.Second,
   362  			Timeout:   time.Second,
   363  			PortLabel: "x",
   364  		},
   365  		{
   366  			Name:     "c2",
   367  			Type:     "script",
   368  			Interval: 9000 * time.Hour,
   369  			Timeout:  time.Second,
   370  		},
   371  		{
   372  			Name:     "c3",
   373  			Type:     "http",
   374  			Protocol: "http",
   375  			Path:     "/",
   376  			Interval: time.Second,
   377  			Timeout:  time.Second,
   378  			// Removed PortLabel
   379  		},
   380  	}
   381  	if err := ctx.ServiceClient.UpdateTask("allocid", origTask, ctx.Task, ctx); err != nil {
   382  		t.Fatalf("unexpected error registering task: %v", err)
   383  	}
   384  	if err := ctx.syncOnce(); err != nil {
   385  		t.Fatalf("unexpected error syncing task: %v", err)
   386  	}
   387  
   388  	if n := len(ctx.FakeConsul.services); n != 1 {
   389  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   390  	}
   391  
   392  	for k, v := range ctx.FakeConsul.services {
   393  		if k != origServiceKey {
   394  			t.Errorf("unexpected key change; was: %q -- but found %q", origServiceKey, k)
   395  		}
   396  		if v.Name != ctx.Task.Services[0].Name {
   397  			t.Errorf("expected Name=%q != %q", ctx.Task.Services[0].Name, v.Name)
   398  		}
   399  		if !reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   400  			t.Errorf("expected Tags=%v != %v", ctx.Task.Services[0].Tags, v.Tags)
   401  		}
   402  		if v.Port != yPort {
   403  			t.Errorf("expected Port y=%v but found: %v", yPort, v.Port)
   404  		}
   405  	}
   406  
   407  	if n := len(ctx.FakeConsul.checks); n != 3 {
   408  		t.Fatalf("expected 3 check but found %d:\n%#v", n, ctx.FakeConsul.checks)
   409  	}
   410  
   411  	for k, v := range ctx.FakeConsul.checks {
   412  		switch v.Name {
   413  		case "c1":
   414  			if k != origTCPKey {
   415  				t.Errorf("unexpected key change for %s from %q to %q", v.Name, origTCPKey, k)
   416  			}
   417  			if expected := fmt.Sprintf(":%d", xPort); v.TCP != expected {
   418  				t.Errorf("expected Port x=%v but found: %v", expected, v.TCP)
   419  			}
   420  		case "c2":
   421  			if k != origScriptKey {
   422  				t.Errorf("unexpected key change for %s from %q to %q", v.Name, origScriptKey, k)
   423  			}
   424  			select {
   425  			case <-ctx.execs:
   426  				if n := len(ctx.execs); n > 0 {
   427  					t.Errorf("expected 1 exec but found: %d", n+1)
   428  				}
   429  			case <-time.After(3 * time.Second):
   430  				t.Errorf("script not called in time")
   431  			}
   432  		case "c3":
   433  			if k == origHTTPKey {
   434  				t.Errorf("expected %s key to change from %q", v.Name, k)
   435  			}
   436  			if expected := fmt.Sprintf("http://:%d/", yPort); v.HTTP != expected {
   437  				t.Errorf("expected Port y=%v but found: %v", expected, v.HTTP)
   438  			}
   439  		default:
   440  			t.Errorf("Unkown check: %q", k)
   441  		}
   442  	}
   443  }
   444  
   445  // TestConsul_RegServices tests basic service registration.
   446  func TestConsul_RegServices(t *testing.T) {
   447  	ctx := setupFake()
   448  
   449  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, nil); err != nil {
   450  		t.Fatalf("unexpected error registering task: %v", err)
   451  	}
   452  
   453  	if err := ctx.syncOnce(); err != nil {
   454  		t.Fatalf("unexpected error syncing task: %v", err)
   455  	}
   456  
   457  	if n := len(ctx.FakeConsul.services); n != 1 {
   458  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   459  	}
   460  	for _, v := range ctx.FakeConsul.services {
   461  		if v.Name != ctx.Task.Services[0].Name {
   462  			t.Errorf("expected Name=%q != %q", ctx.Task.Services[0].Name, v.Name)
   463  		}
   464  		if !reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   465  			t.Errorf("expected Tags=%v != %v", ctx.Task.Services[0].Tags, v.Tags)
   466  		}
   467  		if v.Port != xPort {
   468  			t.Errorf("expected Port=%d != %d", xPort, v.Port)
   469  		}
   470  	}
   471  
   472  	// Make a change which will register a new service
   473  	ctx.Task.Services[0].Name = "taskname-service2"
   474  	ctx.Task.Services[0].Tags[0] = "tag3"
   475  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, nil); err != nil {
   476  		t.Fatalf("unpexpected error registering task: %v", err)
   477  	}
   478  
   479  	// Make sure changes don't take affect until sync() is called (since
   480  	// Run() isn't running)
   481  	if n := len(ctx.FakeConsul.services); n != 1 {
   482  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   483  	}
   484  	for _, v := range ctx.FakeConsul.services {
   485  		if reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   486  			t.Errorf("expected Tags to differ, changes applied before sync()")
   487  		}
   488  	}
   489  
   490  	// Now sync() and re-check for the applied updates
   491  	if err := ctx.syncOnce(); err != nil {
   492  		t.Fatalf("unexpected error syncing task: %v", err)
   493  	}
   494  	if n := len(ctx.FakeConsul.services); n != 2 {
   495  		t.Fatalf("expected 2 services but found %d:\n%#v", n, ctx.FakeConsul.services)
   496  	}
   497  	found := false
   498  	for _, v := range ctx.FakeConsul.services {
   499  		if v.Name == ctx.Task.Services[0].Name {
   500  			if found {
   501  				t.Fatalf("found new service name %q twice", v.Name)
   502  			}
   503  			found = true
   504  			if !reflect.DeepEqual(v.Tags, ctx.Task.Services[0].Tags) {
   505  				t.Errorf("expected Tags=%v != %v", ctx.Task.Services[0].Tags, v.Tags)
   506  			}
   507  		}
   508  	}
   509  	if !found {
   510  		t.Fatalf("did not find new service %q", ctx.Task.Services[0].Name)
   511  	}
   512  
   513  	// Remove the new task
   514  	ctx.ServiceClient.RemoveTask("allocid", ctx.Task)
   515  	if err := ctx.syncOnce(); err != nil {
   516  		t.Fatalf("unexpected error syncing task: %v", err)
   517  	}
   518  	if n := len(ctx.FakeConsul.services); n != 1 {
   519  		t.Fatalf("expected 1 service but found %d:\n%#v", n, ctx.FakeConsul.services)
   520  	}
   521  	for _, v := range ctx.FakeConsul.services {
   522  		if v.Name != "taskname-service" {
   523  			t.Errorf("expected original task to survive not %q", v.Name)
   524  		}
   525  	}
   526  }
   527  
   528  // TestConsul_ShutdownOK tests the ok path for the shutdown logic in
   529  // ServiceClient.
   530  func TestConsul_ShutdownOK(t *testing.T) {
   531  	ctx := setupFake()
   532  
   533  	// Add a script check to make sure its TTL gets updated
   534  	ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
   535  		{
   536  			Name:    "scriptcheck",
   537  			Type:    "script",
   538  			Command: "true",
   539  			// Make check block until shutdown
   540  			Interval:      9000 * time.Hour,
   541  			Timeout:       10 * time.Second,
   542  			InitialStatus: "warning",
   543  		},
   544  	}
   545  
   546  	go ctx.ServiceClient.Run()
   547  
   548  	// Register a task and agent
   549  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx); err != nil {
   550  		t.Fatalf("unexpected error registering task: %v", err)
   551  	}
   552  
   553  	agentServices := []*structs.Service{
   554  		{
   555  			Name:      "http",
   556  			Tags:      []string{"nomad"},
   557  			PortLabel: "localhost:2345",
   558  		},
   559  	}
   560  	if err := ctx.ServiceClient.RegisterAgent("client", agentServices); err != nil {
   561  		t.Fatalf("unexpected error registering agent: %v", err)
   562  	}
   563  
   564  	// Shutdown should block until scripts finish
   565  	if err := ctx.ServiceClient.Shutdown(); err != nil {
   566  		t.Errorf("unexpected error shutting down client: %v", err)
   567  	}
   568  
   569  	// UpdateTTL should have been called once for the script check
   570  	if n := len(ctx.FakeConsul.checkTTLs); n != 1 {
   571  		t.Fatalf("expected 1 checkTTL entry but found: %d", n)
   572  	}
   573  	for _, v := range ctx.FakeConsul.checkTTLs {
   574  		if v != 1 {
   575  			t.Fatalf("expected script check to be updated once but found %d", v)
   576  		}
   577  	}
   578  	for _, v := range ctx.FakeConsul.checks {
   579  		if v.Status != "passing" {
   580  			t.Fatalf("expected check to be passing but found %q", v.Status)
   581  		}
   582  	}
   583  }
   584  
   585  // TestConsul_ShutdownSlow tests the slow but ok path for the shutdown logic in
   586  // ServiceClient.
   587  func TestConsul_ShutdownSlow(t *testing.T) {
   588  	t.Parallel() // run the slow tests in parallel
   589  	ctx := setupFake()
   590  
   591  	// Add a script check to make sure its TTL gets updated
   592  	ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
   593  		{
   594  			Name:    "scriptcheck",
   595  			Type:    "script",
   596  			Command: "true",
   597  			// Make check block until shutdown
   598  			Interval:      9000 * time.Hour,
   599  			Timeout:       5 * time.Second,
   600  			InitialStatus: "warning",
   601  		},
   602  	}
   603  
   604  	// Make Exec slow, but not too slow
   605  	waiter := make(chan struct{})
   606  	ctx.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
   607  		select {
   608  		case <-waiter:
   609  		default:
   610  			close(waiter)
   611  		}
   612  		time.Sleep(time.Second)
   613  		return []byte{}, 0, nil
   614  	}
   615  
   616  	// Make shutdown wait time just a bit longer than ctx.Exec takes
   617  	ctx.ServiceClient.shutdownWait = 3 * time.Second
   618  
   619  	go ctx.ServiceClient.Run()
   620  
   621  	// Register a task and agent
   622  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx); err != nil {
   623  		t.Fatalf("unexpected error registering task: %v", err)
   624  	}
   625  
   626  	// wait for Exec to get called before shutting down
   627  	<-waiter
   628  
   629  	// Shutdown should block until all enqueued operations finish.
   630  	preShutdown := time.Now()
   631  	if err := ctx.ServiceClient.Shutdown(); err != nil {
   632  		t.Errorf("unexpected error shutting down client: %v", err)
   633  	}
   634  
   635  	// Shutdown time should have taken: 1s <= shutdown <= 3s
   636  	shutdownTime := time.Now().Sub(preShutdown)
   637  	if shutdownTime < time.Second || shutdownTime > ctx.ServiceClient.shutdownWait {
   638  		t.Errorf("expected shutdown to take >1s and <%s but took: %s", ctx.ServiceClient.shutdownWait, shutdownTime)
   639  	}
   640  
   641  	// UpdateTTL should have been called once for the script check
   642  	if n := len(ctx.FakeConsul.checkTTLs); n != 1 {
   643  		t.Fatalf("expected 1 checkTTL entry but found: %d", n)
   644  	}
   645  	for _, v := range ctx.FakeConsul.checkTTLs {
   646  		if v != 1 {
   647  			t.Fatalf("expected script check to be updated once but found %d", v)
   648  		}
   649  	}
   650  	for _, v := range ctx.FakeConsul.checks {
   651  		if v.Status != "passing" {
   652  			t.Fatalf("expected check to be passing but found %q", v.Status)
   653  		}
   654  	}
   655  }
   656  
   657  // TestConsul_ShutdownBlocked tests the blocked past deadline path for the
   658  // shutdown logic in ServiceClient.
   659  func TestConsul_ShutdownBlocked(t *testing.T) {
   660  	t.Parallel() // run the slow tests in parallel
   661  	ctx := setupFake()
   662  
   663  	// Add a script check to make sure its TTL gets updated
   664  	ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
   665  		{
   666  			Name:    "scriptcheck",
   667  			Type:    "script",
   668  			Command: "true",
   669  			// Make check block until shutdown
   670  			Interval:      9000 * time.Hour,
   671  			Timeout:       9000 * time.Hour,
   672  			InitialStatus: "warning",
   673  		},
   674  	}
   675  
   676  	block := make(chan struct{})
   677  	defer close(block) // cleanup after test
   678  
   679  	// Make Exec block forever
   680  	waiter := make(chan struct{})
   681  	ctx.ExecFunc = func(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
   682  		close(waiter)
   683  		<-block
   684  		return []byte{}, 0, nil
   685  	}
   686  
   687  	// Use a short shutdown deadline since we're intentionally blocking forever
   688  	ctx.ServiceClient.shutdownWait = time.Second
   689  
   690  	go ctx.ServiceClient.Run()
   691  
   692  	// Register a task and agent
   693  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, ctx); err != nil {
   694  		t.Fatalf("unexpected error registering task: %v", err)
   695  	}
   696  
   697  	// Wait for exec to be called
   698  	<-waiter
   699  
   700  	// Shutdown should block until all enqueued operations finish.
   701  	preShutdown := time.Now()
   702  	err := ctx.ServiceClient.Shutdown()
   703  	if err == nil {
   704  		t.Errorf("expected a timed out error from shutdown")
   705  	}
   706  
   707  	// Shutdown time should have taken shutdownWait; to avoid timing
   708  	// related errors simply test for wait <= shutdown <= wait+3s
   709  	shutdownTime := time.Now().Sub(preShutdown)
   710  	maxWait := ctx.ServiceClient.shutdownWait + (3 * time.Second)
   711  	if shutdownTime < ctx.ServiceClient.shutdownWait || shutdownTime > maxWait {
   712  		t.Errorf("expected shutdown to take >%s and <%s but took: %s", ctx.ServiceClient.shutdownWait, maxWait, shutdownTime)
   713  	}
   714  
   715  	// UpdateTTL should not have been called for the script check
   716  	if n := len(ctx.FakeConsul.checkTTLs); n != 0 {
   717  		t.Fatalf("expected 0 checkTTL entry but found: %d", n)
   718  	}
   719  	for _, v := range ctx.FakeConsul.checks {
   720  		if expected := "warning"; v.Status != expected {
   721  			t.Fatalf("expected check to be %q but found %q", expected, v.Status)
   722  		}
   723  	}
   724  }
   725  
   726  // TestConsul_NoTLSSkipVerifySupport asserts that checks with
   727  // TLSSkipVerify=true are skipped when Consul doesn't support TLSSkipVerify.
   728  func TestConsul_NoTLSSkipVerifySupport(t *testing.T) {
   729  	ctx := setupFake()
   730  	ctx.ServiceClient = NewServiceClient(ctx.FakeConsul, false, testLogger())
   731  	ctx.Task.Services[0].Checks = []*structs.ServiceCheck{
   732  		// This check sets TLSSkipVerify so it should get dropped
   733  		{
   734  			Name:          "tls-check-skip",
   735  			Type:          "http",
   736  			Protocol:      "https",
   737  			Path:          "/",
   738  			TLSSkipVerify: true,
   739  		},
   740  		// This check doesn't set TLSSkipVerify so it should work fine
   741  		{
   742  			Name:          "tls-check-noskip",
   743  			Type:          "http",
   744  			Protocol:      "https",
   745  			Path:          "/",
   746  			TLSSkipVerify: false,
   747  		},
   748  	}
   749  
   750  	if err := ctx.ServiceClient.RegisterTask("allocid", ctx.Task, nil); err != nil {
   751  		t.Fatalf("unexpected error registering task: %v", err)
   752  	}
   753  
   754  	if err := ctx.syncOnce(); err != nil {
   755  		t.Fatalf("unexpected error syncing task: %v", err)
   756  	}
   757  
   758  	if len(ctx.FakeConsul.checks) != 1 {
   759  		t.Errorf("expected 1 check but found %d", len(ctx.FakeConsul.checks))
   760  	}
   761  	for _, v := range ctx.FakeConsul.checks {
   762  		if expected := "tls-check-noskip"; v.Name != expected {
   763  			t.Errorf("only expected %q but found: %q", expected, v.Name)
   764  		}
   765  		if v.TLSSkipVerify {
   766  			t.Errorf("TLSSkipVerify=true when TLSSkipVerify not supported!")
   767  		}
   768  	}
   769  }