github.com/Ilhicas/nomad@v1.0.4-0.20210304152020-e86851182bc3/client/allocrunner/taskrunner/envoy_bootstrap_hook_test.go (about)

     1  // +build !windows
     2  // todo(shoenig): Once Connect is supported on Windows, we'll need to make this
     3  //  set of tests work there too.
     4  
     5  package taskrunner
     6  
     7  import (
     8  	"context"
     9  	"encoding/json"
    10  	"fmt"
    11  	"io/ioutil"
    12  	"os"
    13  	"path/filepath"
    14  	"testing"
    15  
    16  	consulapi "github.com/hashicorp/consul/api"
    17  	"github.com/hashicorp/nomad/client/allocdir"
    18  	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
    19  	"github.com/hashicorp/nomad/client/taskenv"
    20  	"github.com/hashicorp/nomad/client/testutil"
    21  	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
    22  	"github.com/hashicorp/nomad/helper"
    23  	"github.com/hashicorp/nomad/helper/args"
    24  	"github.com/hashicorp/nomad/helper/testlog"
    25  	"github.com/hashicorp/nomad/helper/uuid"
    26  	"github.com/hashicorp/nomad/nomad/mock"
    27  	"github.com/hashicorp/nomad/nomad/structs"
    28  	"github.com/hashicorp/nomad/nomad/structs/config"
    29  	"github.com/stretchr/testify/require"
    30  	"golang.org/x/sys/unix"
    31  )
    32  
    33  var _ interfaces.TaskPrestartHook = (*envoyBootstrapHook)(nil)
    34  
    35  func writeTmp(t *testing.T, s string, fm os.FileMode) string {
    36  	dir, err := ioutil.TempDir("", "envoy-")
    37  	require.NoError(t, err)
    38  
    39  	fPath := filepath.Join(dir, sidsTokenFile)
    40  	err = ioutil.WriteFile(fPath, []byte(s), fm)
    41  	require.NoError(t, err)
    42  
    43  	return dir
    44  }
    45  
    46  func TestEnvoyBootstrapHook_maybeLoadSIToken(t *testing.T) {
    47  	t.Parallel()
    48  
    49  	// This test fails when running as root because the test case for checking
    50  	// the error condition when the file is unreadable fails (root can read the
    51  	// file even though the permissions are set to 0200).
    52  	if unix.Geteuid() == 0 {
    53  		t.Skip("test only works as non-root")
    54  	}
    55  
    56  	t.Run("file does not exist", func(t *testing.T) {
    57  		h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
    58  		cfg, err := h.maybeLoadSIToken("task1", "/does/not/exist")
    59  		require.NoError(t, err) // absence of token is not an error
    60  		require.Equal(t, "", cfg)
    61  	})
    62  
    63  	t.Run("load token from file", func(t *testing.T) {
    64  		token := uuid.Generate()
    65  		f := writeTmp(t, token, 0440)
    66  		defer cleanupDir(t, f)
    67  
    68  		h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
    69  		cfg, err := h.maybeLoadSIToken("task1", f)
    70  		require.NoError(t, err)
    71  		require.Equal(t, token, cfg)
    72  	})
    73  
    74  	t.Run("file is unreadable", func(t *testing.T) {
    75  		token := uuid.Generate()
    76  		f := writeTmp(t, token, 0200)
    77  		defer cleanupDir(t, f)
    78  
    79  		h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
    80  		cfg, err := h.maybeLoadSIToken("task1", f)
    81  		require.Error(t, err)
    82  		require.False(t, os.IsNotExist(err))
    83  		require.Equal(t, "", cfg)
    84  	})
    85  }
    86  
    87  func TestEnvoyBootstrapHook_decodeTriState(t *testing.T) {
    88  	t.Parallel()
    89  
    90  	require.Equal(t, "", decodeTriState(nil))
    91  	require.Equal(t, "true", decodeTriState(helper.BoolToPtr(true)))
    92  	require.Equal(t, "false", decodeTriState(helper.BoolToPtr(false)))
    93  }
    94  
    95  var (
    96  	consulPlainConfig = consulTransportConfig{
    97  		HTTPAddr: "2.2.2.2",
    98  	}
    99  
   100  	consulTLSConfig = consulTransportConfig{
   101  		HTTPAddr:  "2.2.2.2",            // arg
   102  		Auth:      "user:password",      // env
   103  		SSL:       "true",               // env
   104  		VerifySSL: "true",               // env
   105  		CAFile:    "/etc/tls/ca-file",   // arg
   106  		CertFile:  "/etc/tls/cert-file", // arg
   107  		KeyFile:   "/etc/tls/key-file",  // arg
   108  	}
   109  )
   110  
   111  func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
   112  	t.Parallel()
   113  
   114  	t.Run("excluding SI token", func(t *testing.T) {
   115  		ebArgs := envoyBootstrapArgs{
   116  			sidecarFor:     "s1",
   117  			grpcAddr:       "1.1.1.1",
   118  			consulConfig:   consulPlainConfig,
   119  			envoyAdminBind: "localhost:3333",
   120  		}
   121  		result := ebArgs.args()
   122  		require.Equal(t, []string{"connect", "envoy",
   123  			"-grpc-addr", "1.1.1.1",
   124  			"-http-addr", "2.2.2.2",
   125  			"-admin-bind", "localhost:3333",
   126  			"-bootstrap",
   127  			"-sidecar-for", "s1",
   128  		}, result)
   129  	})
   130  
   131  	t.Run("including SI token", func(t *testing.T) {
   132  		token := uuid.Generate()
   133  		ebArgs := envoyBootstrapArgs{
   134  			sidecarFor:     "s1",
   135  			grpcAddr:       "1.1.1.1",
   136  			consulConfig:   consulPlainConfig,
   137  			envoyAdminBind: "localhost:3333",
   138  			siToken:        token,
   139  		}
   140  		result := ebArgs.args()
   141  		require.Equal(t, []string{"connect", "envoy",
   142  			"-grpc-addr", "1.1.1.1",
   143  			"-http-addr", "2.2.2.2",
   144  			"-admin-bind", "localhost:3333",
   145  			"-bootstrap",
   146  			"-sidecar-for", "s1",
   147  			"-token", token,
   148  		}, result)
   149  	})
   150  
   151  	t.Run("including certificates", func(t *testing.T) {
   152  		ebArgs := envoyBootstrapArgs{
   153  			sidecarFor:     "s1",
   154  			grpcAddr:       "1.1.1.1",
   155  			consulConfig:   consulTLSConfig,
   156  			envoyAdminBind: "localhost:3333",
   157  		}
   158  		result := ebArgs.args()
   159  		require.Equal(t, []string{"connect", "envoy",
   160  			"-grpc-addr", "1.1.1.1",
   161  			"-http-addr", "2.2.2.2",
   162  			"-admin-bind", "localhost:3333",
   163  			"-bootstrap",
   164  			"-sidecar-for", "s1",
   165  			"-ca-file", "/etc/tls/ca-file",
   166  			"-client-cert", "/etc/tls/cert-file",
   167  			"-client-key", "/etc/tls/key-file",
   168  		}, result)
   169  	})
   170  
   171  	t.Run("ingress gateway", func(t *testing.T) {
   172  		ebArgs := envoyBootstrapArgs{
   173  			consulConfig:   consulPlainConfig,
   174  			grpcAddr:       "1.1.1.1",
   175  			envoyAdminBind: "localhost:3333",
   176  			gateway:        "my-ingress-gateway",
   177  			proxyID:        "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
   178  		}
   179  		result := ebArgs.args()
   180  		require.Equal(t, []string{"connect", "envoy",
   181  			"-grpc-addr", "1.1.1.1",
   182  			"-http-addr", "2.2.2.2",
   183  			"-admin-bind", "localhost:3333",
   184  			"-bootstrap",
   185  			"-gateway", "my-ingress-gateway",
   186  			"-proxy-id", "_nomad-task-803cb569-881c-b0d8-9222-360bcc33157e-group-ig-ig-8080",
   187  		}, result)
   188  	})
   189  }
   190  
   191  func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
   192  	t.Parallel()
   193  
   194  	environment := []string{"foo=bar", "baz=1"}
   195  
   196  	t.Run("plain consul config", func(t *testing.T) {
   197  		require.Equal(t, []string{
   198  			"foo=bar", "baz=1",
   199  		}, envoyBootstrapArgs{
   200  			sidecarFor:     "s1",
   201  			grpcAddr:       "1.1.1.1",
   202  			consulConfig:   consulPlainConfig,
   203  			envoyAdminBind: "localhost:3333",
   204  		}.env(environment))
   205  	})
   206  
   207  	t.Run("tls consul config", func(t *testing.T) {
   208  		require.Equal(t, []string{
   209  			"foo=bar", "baz=1",
   210  			"CONSUL_HTTP_AUTH=user:password",
   211  			"CONSUL_HTTP_SSL=true",
   212  			"CONSUL_HTTP_SSL_VERIFY=true",
   213  		}, envoyBootstrapArgs{
   214  			sidecarFor:     "s1",
   215  			grpcAddr:       "1.1.1.1",
   216  			consulConfig:   consulTLSConfig,
   217  			envoyAdminBind: "localhost:3333",
   218  		}.env(environment))
   219  	})
   220  }
   221  
   222  // envoyConfig is used to unmarshal an envoy bootstrap configuration file, so that
   223  // we can inspect the contents in tests.
   224  type envoyConfig struct {
   225  	Admin struct {
   226  		Address struct {
   227  			SocketAddress struct {
   228  				Address string `json:"address"`
   229  				Port    int    `json:"port_value"`
   230  			} `json:"socket_address"`
   231  		} `json:"address"`
   232  	} `json:"admin"`
   233  	Node struct {
   234  		Cluster  string `json:"cluster"`
   235  		ID       string `json:"id"`
   236  		Metadata struct {
   237  			Namespace string `json:"namespace"`
   238  			Version   string `json:"envoy_version"`
   239  		}
   240  	}
   241  	DynamicResources struct {
   242  		ADSConfig struct {
   243  			GRPCServices struct {
   244  				InitialMetadata []struct {
   245  					Key   string `json:"key"`
   246  					Value string `json:"value"`
   247  				} `json:"initial_metadata"`
   248  			} `json:"grpc_services"`
   249  		} `json:"ads_config"`
   250  	} `json:"dynamic_resources"`
   251  }
   252  
   253  // TestEnvoyBootstrapHook_with_SI_token asserts the bootstrap file written for
   254  // Envoy contains a Consul SI token.
   255  func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
   256  	t.Parallel()
   257  	testutil.RequireConsul(t)
   258  
   259  	testConsul := getTestConsul(t)
   260  	defer testConsul.Stop()
   261  
   262  	alloc := mock.ConnectAlloc()
   263  	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
   264  		{
   265  			Mode: "bridge",
   266  			IP:   "10.0.0.1",
   267  			DynamicPorts: []structs.Port{
   268  				{
   269  					Label: "connect-proxy-foo",
   270  					Value: 9999,
   271  					To:    9999,
   272  				},
   273  			},
   274  		},
   275  	}
   276  	tg := alloc.Job.TaskGroups[0]
   277  	tg.Services = []*structs.Service{
   278  		{
   279  			Name:      "foo",
   280  			PortLabel: "9999", // Just need a valid port, nothing will bind to it
   281  			Connect: &structs.ConsulConnect{
   282  				SidecarService: &structs.ConsulSidecarService{},
   283  			},
   284  		},
   285  	}
   286  	sidecarTask := &structs.Task{
   287  		Name: "sidecar",
   288  		Kind: "connect-proxy:foo",
   289  	}
   290  	tg.Tasks = append(tg.Tasks, sidecarTask)
   291  
   292  	logger := testlog.HCLogger(t)
   293  
   294  	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
   295  	defer cleanup()
   296  
   297  	// Register Group Services
   298  	consulConfig := consulapi.DefaultConfig()
   299  	consulConfig.Address = testConsul.HTTPAddr
   300  	consulAPIClient, err := consulapi.NewClient(consulConfig)
   301  	require.NoError(t, err)
   302  
   303  	consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
   304  	go consulClient.Run()
   305  	defer consulClient.Shutdown()
   306  	require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
   307  
   308  	// Run Connect bootstrap Hook
   309  	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
   310  		Addr: consulConfig.Address,
   311  	}, logger))
   312  	req := &interfaces.TaskPrestartRequest{
   313  		Task:    sidecarTask,
   314  		TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
   315  		TaskEnv: taskenv.NewEmptyTaskEnv(),
   316  	}
   317  	require.NoError(t, req.TaskDir.Build(false, nil))
   318  
   319  	// Insert service identity token in the secrets directory
   320  	token := uuid.Generate()
   321  	siTokenFile := filepath.Join(req.TaskDir.SecretsDir, sidsTokenFile)
   322  	err = ioutil.WriteFile(siTokenFile, []byte(token), 0440)
   323  	require.NoError(t, err)
   324  
   325  	resp := &interfaces.TaskPrestartResponse{}
   326  
   327  	// Run the hook
   328  	require.NoError(t, h.Prestart(context.Background(), req, resp))
   329  
   330  	// Assert it is Done
   331  	require.True(t, resp.Done)
   332  
   333  	// Ensure the default path matches
   334  	env := map[string]string{
   335  		taskenv.SecretsDir: req.TaskDir.SecretsDir,
   336  	}
   337  	f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
   338  	require.NoError(t, err)
   339  	defer f.Close()
   340  
   341  	// Assert bootstrap configuration is valid json
   342  	var out envoyConfig
   343  	require.NoError(t, json.NewDecoder(f).Decode(&out))
   344  
   345  	// Assert the SI token got set
   346  	key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
   347  	value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
   348  	require.Equal(t, "x-consul-token", key)
   349  	require.Equal(t, token, value)
   350  }
   351  
   352  // TestTaskRunner_EnvoyBootstrapHook_sidecar_ok asserts the EnvoyBootstrapHook
   353  // creates Envoy's bootstrap.json configuration based on Connect proxy sidecars
   354  // registered for the task.
   355  func TestTaskRunner_EnvoyBootstrapHook_sidecar_ok(t *testing.T) {
   356  	t.Parallel()
   357  	testutil.RequireConsul(t)
   358  
   359  	testConsul := getTestConsul(t)
   360  	defer testConsul.Stop()
   361  
   362  	alloc := mock.ConnectAlloc()
   363  	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
   364  		{
   365  			Mode: "bridge",
   366  			IP:   "10.0.0.1",
   367  			DynamicPorts: []structs.Port{
   368  				{
   369  					Label: "connect-proxy-foo",
   370  					Value: 9999,
   371  					To:    9999,
   372  				},
   373  			},
   374  		},
   375  	}
   376  	tg := alloc.Job.TaskGroups[0]
   377  	tg.Services = []*structs.Service{
   378  		{
   379  			Name:      "foo",
   380  			PortLabel: "9999", // Just need a valid port, nothing will bind to it
   381  			Connect: &structs.ConsulConnect{
   382  				SidecarService: &structs.ConsulSidecarService{},
   383  			},
   384  		},
   385  	}
   386  	sidecarTask := &structs.Task{
   387  		Name: "sidecar",
   388  		Kind: structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
   389  	}
   390  	tg.Tasks = append(tg.Tasks, sidecarTask)
   391  
   392  	logger := testlog.HCLogger(t)
   393  
   394  	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
   395  	defer cleanup()
   396  
   397  	// Register Group Services
   398  	consulConfig := consulapi.DefaultConfig()
   399  	consulConfig.Address = testConsul.HTTPAddr
   400  	consulAPIClient, err := consulapi.NewClient(consulConfig)
   401  	require.NoError(t, err)
   402  
   403  	consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
   404  	go consulClient.Run()
   405  	defer consulClient.Shutdown()
   406  	require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
   407  
   408  	// Run Connect bootstrap Hook
   409  	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
   410  		Addr: consulConfig.Address,
   411  	}, logger))
   412  	req := &interfaces.TaskPrestartRequest{
   413  		Task:    sidecarTask,
   414  		TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
   415  		TaskEnv: taskenv.NewEmptyTaskEnv(),
   416  	}
   417  	require.NoError(t, req.TaskDir.Build(false, nil))
   418  
   419  	resp := &interfaces.TaskPrestartResponse{}
   420  
   421  	// Run the hook
   422  	require.NoError(t, h.Prestart(context.Background(), req, resp))
   423  
   424  	// Assert it is Done
   425  	require.True(t, resp.Done)
   426  
   427  	require.NotNil(t, resp.Env)
   428  	require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
   429  
   430  	// Ensure the default path matches
   431  	env := map[string]string{
   432  		taskenv.SecretsDir: req.TaskDir.SecretsDir,
   433  	}
   434  	f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
   435  	require.NoError(t, err)
   436  	defer f.Close()
   437  
   438  	// Assert bootstrap configuration is valid json
   439  	var out envoyConfig
   440  	require.NoError(t, json.NewDecoder(f).Decode(&out))
   441  
   442  	// Assert no SI token got set
   443  	key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
   444  	value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
   445  	require.Equal(t, "x-consul-token", key)
   446  	require.Equal(t, "", value)
   447  }
   448  
   449  func TestTaskRunner_EnvoyBootstrapHook_gateway_ok(t *testing.T) {
   450  	t.Parallel()
   451  	logger := testlog.HCLogger(t)
   452  
   453  	testConsul := getTestConsul(t)
   454  	defer testConsul.Stop()
   455  
   456  	// Setup an Allocation
   457  	alloc := mock.ConnectIngressGatewayAlloc("bridge")
   458  	allocDir, cleanupDir := allocdir.TestAllocDir(t, logger, "EnvoyBootstrapIngressGateway")
   459  	defer cleanupDir()
   460  
   461  	// Get a Consul client
   462  	consulConfig := consulapi.DefaultConfig()
   463  	consulConfig.Address = testConsul.HTTPAddr
   464  	consulAPIClient, err := consulapi.NewClient(consulConfig)
   465  	require.NoError(t, err)
   466  
   467  	// Register Group Services
   468  	serviceClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
   469  	go serviceClient.Run()
   470  	defer serviceClient.Shutdown()
   471  	require.NoError(t, serviceClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))
   472  
   473  	// Register Configuration Entry
   474  	ceClient := consulAPIClient.ConfigEntries()
   475  	set, _, err := ceClient.Set(&consulapi.IngressGatewayConfigEntry{
   476  		Kind: consulapi.IngressGateway,
   477  		Name: "gateway-service", // matches job
   478  		Listeners: []consulapi.IngressListener{{
   479  			Port:     2000,
   480  			Protocol: "tcp",
   481  			Services: []consulapi.IngressService{{
   482  				Name: "service1",
   483  			}},
   484  		}},
   485  	}, nil)
   486  	require.NoError(t, err)
   487  	require.True(t, set)
   488  
   489  	// Run Connect bootstrap hook
   490  	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
   491  		Addr: consulConfig.Address,
   492  	}, logger))
   493  
   494  	req := &interfaces.TaskPrestartRequest{
   495  		Task:    alloc.Job.TaskGroups[0].Tasks[0],
   496  		TaskDir: allocDir.NewTaskDir(alloc.Job.TaskGroups[0].Tasks[0].Name),
   497  		TaskEnv: taskenv.NewEmptyTaskEnv(),
   498  	}
   499  	require.NoError(t, req.TaskDir.Build(false, nil))
   500  
   501  	var resp interfaces.TaskPrestartResponse
   502  
   503  	// Run the hook
   504  	require.NoError(t, h.Prestart(context.Background(), req, &resp))
   505  
   506  	// Assert the hook is Done
   507  	require.True(t, resp.Done)
   508  	require.NotNil(t, resp.Env)
   509  
   510  	// Read the Envoy Config file
   511  	env := map[string]string{
   512  		taskenv.SecretsDir: req.TaskDir.SecretsDir,
   513  	}
   514  	f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
   515  	require.NoError(t, err)
   516  	defer f.Close()
   517  
   518  	var out envoyConfig
   519  	require.NoError(t, json.NewDecoder(f).Decode(&out))
   520  
   521  	// The only interesting thing on bootstrap is the presence of the cluster,
   522  	// and its associated ID that Nomad sets. Everything is configured at runtime
   523  	// through xDS.
   524  	expID := fmt.Sprintf("_nomad-task-%s-group-web-my-ingress-service-9999", alloc.ID)
   525  	require.Equal(t, expID, out.Node.ID)
   526  	require.Equal(t, "ingress-gateway", out.Node.Cluster)
   527  }
   528  
   529  // TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook
   530  // is a noop for non-Connect proxy sidecar / gateway tasks.
   531  func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) {
   532  	t.Parallel()
   533  	logger := testlog.HCLogger(t)
   534  
   535  	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
   536  	defer cleanup()
   537  
   538  	alloc := mock.Alloc()
   539  	task := alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks[0]
   540  
   541  	// Run Envoy bootstrap Hook. Use invalid Consul address as it should
   542  	// not get hit.
   543  	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
   544  		Addr: "http://127.0.0.2:1",
   545  	}, logger))
   546  	req := &interfaces.TaskPrestartRequest{
   547  		Task:    task,
   548  		TaskDir: allocDir.NewTaskDir(task.Name),
   549  	}
   550  	require.NoError(t, req.TaskDir.Build(false, nil))
   551  
   552  	resp := &interfaces.TaskPrestartResponse{}
   553  
   554  	// Run the hook
   555  	require.NoError(t, h.Prestart(context.Background(), req, resp))
   556  
   557  	// Assert it is Done
   558  	require.True(t, resp.Done)
   559  
   560  	// Assert no file was written
   561  	_, err := os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
   562  	require.Error(t, err)
   563  	require.True(t, os.IsNotExist(err))
   564  }
   565  
   566  // TestTaskRunner_EnvoyBootstrapHook_RecoverableError asserts the Envoy
   567  // bootstrap hook returns a Recoverable error if the bootstrap command runs but
   568  // fails.
   569  func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
   570  	t.Parallel()
   571  	testutil.RequireConsul(t)
   572  
   573  	testConsul := getTestConsul(t)
   574  	defer testConsul.Stop()
   575  
   576  	alloc := mock.ConnectAlloc()
   577  	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
   578  		{
   579  			Mode: "bridge",
   580  			IP:   "10.0.0.1",
   581  			DynamicPorts: []structs.Port{
   582  				{
   583  					Label: "connect-proxy-foo",
   584  					Value: 9999,
   585  					To:    9999,
   586  				},
   587  			},
   588  		},
   589  	}
   590  	tg := alloc.Job.TaskGroups[0]
   591  	tg.Services = []*structs.Service{
   592  		{
   593  			Name:      "foo",
   594  			PortLabel: "9999", // Just need a valid port, nothing will bind to it
   595  			Connect: &structs.ConsulConnect{
   596  				SidecarService: &structs.ConsulSidecarService{},
   597  			},
   598  		},
   599  	}
   600  	sidecarTask := &structs.Task{
   601  		Name: "sidecar",
   602  		Kind: "connect-proxy:foo",
   603  	}
   604  	tg.Tasks = append(tg.Tasks, sidecarTask)
   605  
   606  	logger := testlog.HCLogger(t)
   607  
   608  	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
   609  	defer cleanup()
   610  
   611  	// Unlike the successful test above, do NOT register the group services
   612  	// yet. This should cause a recoverable error similar to if Consul was
   613  	// not running.
   614  
   615  	// Run Connect bootstrap Hook
   616  	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
   617  		Addr: testConsul.HTTPAddr,
   618  	}, logger))
   619  	req := &interfaces.TaskPrestartRequest{
   620  		Task:    sidecarTask,
   621  		TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
   622  		TaskEnv: taskenv.NewEmptyTaskEnv(),
   623  	}
   624  	require.NoError(t, req.TaskDir.Build(false, nil))
   625  
   626  	resp := &interfaces.TaskPrestartResponse{}
   627  
   628  	// Run the hook
   629  	err := h.Prestart(context.Background(), req, resp)
   630  	require.EqualError(t, err, "error creating bootstrap configuration for Connect proxy sidecar: exit status 1")
   631  	require.True(t, structs.IsRecoverable(err))
   632  
   633  	// Assert it is not Done
   634  	require.False(t, resp.Done)
   635  
   636  	// Assert no file was written
   637  	_, err = os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
   638  	require.Error(t, err)
   639  	require.True(t, os.IsNotExist(err))
   640  }
   641  
   642  func TestTaskRunner_EnvoyBootstrapHook_extractNameAndKind(t *testing.T) {
   643  	t.Run("connect sidecar", func(t *testing.T) {
   644  		kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
   645  			structs.NewTaskKind(structs.ConnectProxyPrefix, "foo"),
   646  		)
   647  		require.Nil(t, err)
   648  		require.Equal(t, "connect-proxy", kind)
   649  		require.Equal(t, "foo", name)
   650  	})
   651  
   652  	t.Run("connect gateway", func(t *testing.T) {
   653  		kind, name, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
   654  			structs.NewTaskKind(structs.ConnectIngressPrefix, "foo"),
   655  		)
   656  		require.Nil(t, err)
   657  		require.Equal(t, "connect-ingress", kind)
   658  		require.Equal(t, "foo", name)
   659  	})
   660  
   661  	t.Run("connect native", func(t *testing.T) {
   662  		_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
   663  			structs.NewTaskKind(structs.ConnectNativePrefix, "foo"),
   664  		)
   665  		require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
   666  	})
   667  
   668  	t.Run("normal task", func(t *testing.T) {
   669  		_, _, err := (*envoyBootstrapHook)(nil).extractNameAndKind(
   670  			structs.TaskKind(""),
   671  		)
   672  		require.EqualError(t, err, "envoy must be used as connect sidecar or gateway")
   673  	})
   674  }
   675  
   676  func TestTaskRunner_EnvoyBootstrapHook_grpcAddress(t *testing.T) {
   677  	bridgeH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
   678  		mock.ConnectIngressGatewayAlloc("bridge"),
   679  		new(config.ConsulConfig),
   680  		testlog.HCLogger(t),
   681  	))
   682  
   683  	hostH := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(
   684  		mock.ConnectIngressGatewayAlloc("host"),
   685  		new(config.ConsulConfig),
   686  		testlog.HCLogger(t),
   687  	))
   688  
   689  	t.Run("environment", func(t *testing.T) {
   690  		env := map[string]string{
   691  			grpcConsulVariable: "1.2.3.4:9000",
   692  		}
   693  		require.Equal(t, "1.2.3.4:9000", bridgeH.grpcAddress(env))
   694  		require.Equal(t, "1.2.3.4:9000", hostH.grpcAddress(env))
   695  	})
   696  
   697  	t.Run("defaults", func(t *testing.T) {
   698  		require.Equal(t, "unix://alloc/tmp/consul_grpc.sock", bridgeH.grpcAddress(nil))
   699  		require.Equal(t, "127.0.0.1:8502", hostH.grpcAddress(nil))
   700  	})
   701  }