github.com/hernad/nomad@v1.6.112/command/agent/consul/group_test.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package consul
     5  
     6  import (
     7  	"io"
     8  	"testing"
     9  	"time"
    10  
    11  	consulapi "github.com/hashicorp/consul/api"
    12  	"github.com/hashicorp/consul/sdk/testutil"
    13  	"github.com/hernad/nomad/ci"
    14  	"github.com/hernad/nomad/client/serviceregistration"
    15  	"github.com/hernad/nomad/helper/testlog"
    16  	"github.com/hernad/nomad/nomad/mock"
    17  	"github.com/hernad/nomad/nomad/structs"
    18  	"github.com/stretchr/testify/require"
    19  )
    20  
    21  func TestConsul_Connect(t *testing.T) {
    22  	ci.Parallel(t)
    23  
    24  	// Create an embedded Consul server
    25  	testconsul, err := testutil.NewTestServerConfigT(t, func(c *testutil.TestServerConfig) {
    26  		c.Peering = nil // fix for older versions of Consul (<1.13.0) that don't support peering
    27  		// If -v wasn't specified squelch consul logging
    28  		if !testing.Verbose() {
    29  			c.Stdout = io.Discard
    30  			c.Stderr = io.Discard
    31  		}
    32  	})
    33  	if err != nil {
    34  		t.Fatalf("error starting test consul server: %v", err)
    35  	}
    36  	defer testconsul.Stop()
    37  
    38  	consulConfig := consulapi.DefaultConfig()
    39  	consulConfig.Address = testconsul.HTTPAddr
    40  	consulClient, err := consulapi.NewClient(consulConfig)
    41  	require.NoError(t, err)
    42  	namespacesClient := NewNamespacesClient(consulClient.Namespaces(), consulClient.Agent())
    43  	serviceClient := NewServiceClient(consulClient.Agent(), namespacesClient, testlog.HCLogger(t), true)
    44  
    45  	// Lower periodicInterval to ensure periodic syncing doesn't improperly
    46  	// remove Connect services.
    47  	const interval = 50 * time.Millisecond
    48  	serviceClient.periodicInterval = interval
    49  
    50  	// Disable deregistration probation to test syncing
    51  	serviceClient.deregisterProbationExpiry = time.Time{}
    52  
    53  	go serviceClient.Run()
    54  	defer serviceClient.Shutdown()
    55  
    56  	alloc := mock.Alloc()
    57  	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
    58  		{
    59  			Mode: "bridge",
    60  			IP:   "10.0.0.1",
    61  			DynamicPorts: []structs.Port{
    62  				{
    63  					Label: "connect-proxy-testconnect",
    64  					Value: 9999,
    65  					To:    9998,
    66  				},
    67  			},
    68  		},
    69  	}
    70  	tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
    71  	tg.Services = []*structs.Service{
    72  		{
    73  			Name:      "testconnect",
    74  			PortLabel: "9999",
    75  			Meta: map[string]string{
    76  				"alloc_id": "${NOMAD_ALLOC_ID}",
    77  			},
    78  			Connect: &structs.ConsulConnect{
    79  				SidecarService: &structs.ConsulSidecarService{
    80  					Proxy: &structs.ConsulProxy{
    81  						LocalServicePort: 9000,
    82  					},
    83  				},
    84  			},
    85  		},
    86  	}
    87  
    88  	require.NoError(t, serviceClient.RegisterWorkload(BuildAllocServices(mock.Node(), alloc, NoopRestarter())))
    89  
    90  	require.Eventually(t, func() bool {
    91  		services, err := consulClient.Agent().Services()
    92  		require.NoError(t, err)
    93  		return len(services) == 2
    94  	}, 3*time.Second, 100*time.Millisecond)
    95  
    96  	// Test a few times to ensure Nomad doesn't improperly deregister
    97  	// Connect services.
    98  	for i := 10; i > 0; i-- {
    99  		services, err := consulClient.Agent().Services()
   100  		require.NoError(t, err)
   101  		require.Len(t, services, 2)
   102  
   103  		serviceID := serviceregistration.MakeAllocServiceID(alloc.ID, "group-"+alloc.TaskGroup, tg.Services[0])
   104  		connectID := serviceID + "-sidecar-proxy"
   105  
   106  		require.Contains(t, services, serviceID)
   107  		require.True(t, isNomadService(serviceID))
   108  		require.False(t, maybeConnectSidecar(serviceID))
   109  		agentService := services[serviceID]
   110  		require.Equal(t, agentService.Service, "testconnect")
   111  		require.Equal(t, agentService.Address, "10.0.0.1")
   112  		require.Equal(t, agentService.Port, 9999)
   113  		require.Nil(t, agentService.Connect)
   114  		require.Nil(t, agentService.Proxy)
   115  
   116  		require.Contains(t, services, connectID)
   117  		require.True(t, isNomadService(connectID))
   118  		require.True(t, maybeConnectSidecar(connectID))
   119  		connectService := services[connectID]
   120  		require.Equal(t, connectService.Service, "testconnect-sidecar-proxy")
   121  		require.Equal(t, connectService.Address, "10.0.0.1")
   122  		require.Equal(t, connectService.Port, 9999)
   123  		require.Nil(t, connectService.Connect)
   124  		require.Equal(t, connectService.Proxy.DestinationServiceName, "testconnect")
   125  		require.Equal(t, connectService.Proxy.DestinationServiceID, serviceID)
   126  		require.Equal(t, connectService.Proxy.LocalServiceAddress, "127.0.0.1")
   127  		require.Equal(t, connectService.Proxy.LocalServicePort, 9000)
   128  		require.Equal(t, connectService.Proxy.Config, map[string]interface{}{
   129  			"bind_address":     "0.0.0.0",
   130  			"bind_port":        float64(9998),
   131  			"envoy_stats_tags": []interface{}{"nomad.alloc_id=" + alloc.ID, "nomad.group=" + alloc.TaskGroup},
   132  		})
   133  		require.Equal(t, alloc.ID, agentService.Meta["alloc_id"])
   134  
   135  		time.Sleep(interval >> 2)
   136  	}
   137  }