github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/command/agent/consul/group_test.go (about)

     1  package consul
     2  
     3  import (
     4  	"io/ioutil"
     5  	"testing"
     6  	"time"
     7  
     8  	consulapi "github.com/hashicorp/consul/api"
     9  	"github.com/hashicorp/consul/sdk/testutil"
    10  	"github.com/hashicorp/nomad/ci"
    11  	"github.com/hashicorp/nomad/client/serviceregistration"
    12  	"github.com/hashicorp/nomad/helper/testlog"
    13  	"github.com/hashicorp/nomad/nomad/mock"
    14  	"github.com/hashicorp/nomad/nomad/structs"
    15  	"github.com/stretchr/testify/require"
    16  )
    17  
    18  func TestConsul_Connect(t *testing.T) {
    19  	ci.Parallel(t)
    20  
    21  	// Create an embedded Consul server
    22  	testconsul, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) {
    23  		c.Peering = nil  // fix for older versions of Consul (<1.13.0) that don't support peering
    24  		// If -v wasn't specified squelch consul logging
    25  		if !testing.Verbose() {
    26  			c.Stdout = ioutil.Discard
    27  			c.Stderr = ioutil.Discard
    28  		}
    29  	})
    30  	if err != nil {
    31  		t.Fatalf("error starting test consul server: %v", err)
    32  	}
    33  	defer testconsul.Stop()
    34  
    35  	consulConfig := consulapi.DefaultConfig()
    36  	consulConfig.Address = testconsul.HTTPAddr
    37  	consulClient, err := consulapi.NewClient(consulConfig)
    38  	require.NoError(t, err)
    39  	namespacesClient := NewNamespacesClient(consulClient.Namespaces(), consulClient.Agent())
    40  	serviceClient := NewServiceClient(consulClient.Agent(), namespacesClient, testlog.HCLogger(t), true)
    41  
    42  	// Lower periodicInterval to ensure periodic syncing doesn't improperly
    43  	// remove Connect services.
    44  	const interval = 50 * time.Millisecond
    45  	serviceClient.periodicInterval = interval
    46  
    47  	// Disable deregistration probation to test syncing
    48  	serviceClient.deregisterProbationExpiry = time.Time{}
    49  
    50  	go serviceClient.Run()
    51  	defer serviceClient.Shutdown()
    52  
    53  	alloc := mock.Alloc()
    54  	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
    55  		{
    56  			Mode: "bridge",
    57  			IP:   "10.0.0.1",
    58  			DynamicPorts: []structs.Port{
    59  				{
    60  					Label: "connect-proxy-testconnect",
    61  					Value: 9999,
    62  					To:    9998,
    63  				},
    64  			},
    65  		},
    66  	}
    67  	tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
    68  	tg.Services = []*structs.Service{
    69  		{
    70  			Name:      "testconnect",
    71  			PortLabel: "9999",
    72  			Meta: map[string]string{
    73  				"alloc_id": "${NOMAD_ALLOC_ID}",
    74  			},
    75  			Connect: &structs.ConsulConnect{
    76  				SidecarService: &structs.ConsulSidecarService{
    77  					Proxy: &structs.ConsulProxy{
    78  						LocalServicePort: 9000,
    79  					},
    80  				},
    81  			},
    82  		},
    83  	}
    84  
    85  	require.NoError(t, serviceClient.RegisterWorkload(BuildAllocServices(mock.Node(), alloc, NoopRestarter())))
    86  
    87  	require.Eventually(t, func() bool {
    88  		services, err := consulClient.Agent().Services()
    89  		require.NoError(t, err)
    90  		return len(services) == 2
    91  	}, 3*time.Second, 100*time.Millisecond)
    92  
    93  	// Test a few times to ensure Nomad doesn't improperly deregister
    94  	// Connect services.
    95  	for i := 10; i > 0; i-- {
    96  		services, err := consulClient.Agent().Services()
    97  		require.NoError(t, err)
    98  		require.Len(t, services, 2)
    99  
   100  		serviceID := serviceregistration.MakeAllocServiceID(alloc.ID, "group-"+alloc.TaskGroup, tg.Services[0])
   101  		connectID := serviceID + "-sidecar-proxy"
   102  
   103  		require.Contains(t, services, serviceID)
   104  		require.True(t, isNomadService(serviceID))
   105  		require.False(t, maybeConnectSidecar(serviceID))
   106  		agentService := services[serviceID]
   107  		require.Equal(t, agentService.Service, "testconnect")
   108  		require.Equal(t, agentService.Address, "10.0.0.1")
   109  		require.Equal(t, agentService.Port, 9999)
   110  		require.Nil(t, agentService.Connect)
   111  		require.Nil(t, agentService.Proxy)
   112  
   113  		require.Contains(t, services, connectID)
   114  		require.True(t, isNomadService(connectID))
   115  		require.True(t, maybeConnectSidecar(connectID))
   116  		connectService := services[connectID]
   117  		require.Equal(t, connectService.Service, "testconnect-sidecar-proxy")
   118  		require.Equal(t, connectService.Address, "10.0.0.1")
   119  		require.Equal(t, connectService.Port, 9999)
   120  		require.Nil(t, connectService.Connect)
   121  		require.Equal(t, connectService.Proxy.DestinationServiceName, "testconnect")
   122  		require.Equal(t, connectService.Proxy.DestinationServiceID, serviceID)
   123  		require.Equal(t, connectService.Proxy.LocalServiceAddress, "127.0.0.1")
   124  		require.Equal(t, connectService.Proxy.LocalServicePort, 9000)
   125  		require.Equal(t, connectService.Proxy.Config, map[string]interface{}{
   126  			"bind_address":     "0.0.0.0",
   127  			"bind_port":        float64(9998),
   128  			"envoy_stats_tags": []interface{}{"nomad.alloc_id=" + alloc.ID, "nomad.group=" + alloc.TaskGroup},
   129  		})
   130  		require.Equal(t, alloc.ID, agentService.Meta["alloc_id"])
   131  
   132  		time.Sleep(interval >> 2)
   133  	}
   134  }