github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/client/allocrunner/taskrunner/envoybootstrap_hook_test.go

// +build !windows
// todo(shoenig): Once Connect is supported on Windows, we'll need to make this
//  set of tests work there too.

package taskrunner

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	consulapi "github.com/hashicorp/consul/api"
	consultest "github.com/hashicorp/consul/sdk/testutil"
	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/client/taskenv"
	"github.com/hashicorp/nomad/client/testutil"
	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
	"github.com/hashicorp/nomad/helper"
	"github.com/hashicorp/nomad/helper/args"
	"github.com/hashicorp/nomad/helper/testlog"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/nomad/structs/config"
	"github.com/stretchr/testify/require"
	"golang.org/x/sys/unix"
)

var _ interfaces.TaskPrestartHook = (*envoyBootstrapHook)(nil)

func writeTmp(t *testing.T, s string, fm os.FileMode) string {
	dir, err := ioutil.TempDir("", "envoy-")
	require.NoError(t, err)

	fPath := filepath.Join(dir, sidsTokenFile)
	err = ioutil.WriteFile(fPath, []byte(s), fm)
	require.NoError(t, err)

	return dir
}

func TestEnvoyBootstrapHook_maybeLoadSIToken(t *testing.T) {
	t.Parallel()

	// This test fails when running as root because the test case for checking
	// the error condition when the file is unreadable fails (root can read the
	// file even though the permissions are set to 0200).
	if unix.Geteuid() == 0 {
		t.Skip("test only works as non-root")
	}

	t.Run("file does not exist", func(t *testing.T) {
		h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
		cfg, err := h.maybeLoadSIToken("task1", "/does/not/exist")
		require.NoError(t, err) // absence of token is not an error
		require.Equal(t, "", cfg)
	})

	t.Run("load token from file", func(t *testing.T) {
		token := uuid.Generate()
		f := writeTmp(t, token, 0440)
		defer cleanupDir(t, f)

		h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
		cfg, err := h.maybeLoadSIToken("task1", f)
		require.NoError(t, err)
		require.Equal(t, token, cfg)
	})

	t.Run("file is unreadable", func(t *testing.T) {
		token := uuid.Generate()
		f := writeTmp(t, token, 0200)
		defer cleanupDir(t, f)

		h := newEnvoyBootstrapHook(&envoyBootstrapHookConfig{logger: testlog.HCLogger(t)})
		cfg, err := h.maybeLoadSIToken("task1", f)
		require.Error(t, err)
		require.False(t, os.IsNotExist(err))
		require.Equal(t, "", cfg)
	})
}

func TestEnvoyBootstrapHook_decodeTriState(t *testing.T) {
	t.Parallel()

	require.Equal(t, "", decodeTriState(nil))
	require.Equal(t, "true", decodeTriState(helper.BoolToPtr(true)))
	require.Equal(t, "false", decodeTriState(helper.BoolToPtr(false)))
}

var (
	consulPlainConfig = envoyBootstrapConsulConfig{
		HTTPAddr: "2.2.2.2",
	}

	consulTLSConfig = envoyBootstrapConsulConfig{
		HTTPAddr:  "2.2.2.2",            // arg
		Auth:      "user:password",      // env
		SSL:       "true",               // env
		VerifySSL: "true",               // env
		CAFile:    "/etc/tls/ca-file",   // arg
		CertFile:  "/etc/tls/cert-file", // arg
		KeyFile:   "/etc/tls/key-file",  // arg
	}
)

func TestEnvoyBootstrapHook_envoyBootstrapArgs(t *testing.T) {
	t.Parallel()

	t.Run("excluding SI token", func(t *testing.T) {
		ebArgs := envoyBootstrapArgs{
			sidecarFor:     "s1",
			grpcAddr:       "1.1.1.1",
			consulConfig:   consulPlainConfig,
			envoyAdminBind: "localhost:3333",
		}
		result := ebArgs.args()
		require.Equal(t, []string{"connect", "envoy",
			"-grpc-addr", "1.1.1.1",
			"-http-addr", "2.2.2.2",
			"-admin-bind", "localhost:3333",
			"-bootstrap",
			"-sidecar-for", "s1",
		}, result)
	})

	t.Run("including SI token", func(t *testing.T) {
		token := uuid.Generate()
		ebArgs := envoyBootstrapArgs{
			sidecarFor:     "s1",
			grpcAddr:       "1.1.1.1",
			consulConfig:   consulPlainConfig,
			envoyAdminBind: "localhost:3333",
			siToken:        token,
		}
		result := ebArgs.args()
		require.Equal(t, []string{"connect", "envoy",
			"-grpc-addr", "1.1.1.1",
			"-http-addr", "2.2.2.2",
			"-admin-bind", "localhost:3333",
			"-bootstrap",
			"-sidecar-for", "s1",
			"-token", token,
		}, result)
	})

	t.Run("including certificates", func(t *testing.T) {
		ebArgs := envoyBootstrapArgs{
			sidecarFor:     "s1",
			grpcAddr:       "1.1.1.1",
			consulConfig:   consulTLSConfig,
			envoyAdminBind: "localhost:3333",
		}
		result := ebArgs.args()
		require.Equal(t, []string{"connect", "envoy",
			"-grpc-addr", "1.1.1.1",
			"-http-addr", "2.2.2.2",
			"-admin-bind", "localhost:3333",
			"-bootstrap",
			"-sidecar-for", "s1",
			"-ca-file", "/etc/tls/ca-file",
			"-client-cert", "/etc/tls/cert-file",
			"-client-key", "/etc/tls/key-file",
		}, result)
	})
}

func TestEnvoyBootstrapHook_envoyBootstrapEnv(t *testing.T) {
	t.Parallel()

	environment := []string{"foo=bar", "baz=1"}

	t.Run("plain consul config", func(t *testing.T) {
		require.Equal(t, []string{
			"foo=bar", "baz=1",
		}, envoyBootstrapArgs{
			sidecarFor:     "s1",
			grpcAddr:       "1.1.1.1",
			consulConfig:   consulPlainConfig,
			envoyAdminBind: "localhost:3333",
		}.env(environment))
	})

	t.Run("tls consul config", func(t *testing.T) {
		require.Equal(t, []string{
			"foo=bar", "baz=1",
			"CONSUL_HTTP_AUTH=user:password",
			"CONSUL_HTTP_SSL=true",
			"CONSUL_HTTP_SSL_VERIFY=true",
		}, envoyBootstrapArgs{
			sidecarFor:     "s1",
			grpcAddr:       "1.1.1.1",
			consulConfig:   consulTLSConfig,
			envoyAdminBind: "localhost:3333",
		}.env(environment))
	})
}

// dig through envoy config to look for consul token
type envoyConfig struct {
	DynamicResources struct {
		ADSConfig struct {
			GRPCServices struct {
				InitialMetadata []struct {
					Key   string `json:"key"`
					Value string `json:"value"`
				} `json:"initial_metadata"`
			} `json:"grpc_services"`
		} `json:"ads_config"`
	} `json:"dynamic_resources"`
}

// TestEnvoyBootstrapHook_with_SI_token asserts the bootstrap file written for
// Envoy contains a Consul SI token.
func TestEnvoyBootstrapHook_with_SI_token(t *testing.T) {
	t.Parallel()
	testutil.RequireConsul(t)

	testconsul, err := consultest.NewTestServerConfig(func(c *consultest.TestServerConfig) {
		// If -v wasn't specified squelch consul logging
		if !testing.Verbose() {
			c.Stdout = ioutil.Discard
			c.Stderr = ioutil.Discard
		}
	})
	if err != nil {
		t.Fatalf("error starting test consul server: %v", err)
	}
	defer testconsul.Stop()

	alloc := mock.Alloc()
	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
		{
			Mode: "bridge",
			IP:   "10.0.0.1",
			DynamicPorts: []structs.Port{
				{
					Label: "connect-proxy-foo",
					Value: 9999,
					To:    9999,
				},
			},
		},
	}
	tg := alloc.Job.TaskGroups[0]
	tg.Services = []*structs.Service{
		{
			Name:      "foo",
			PortLabel: "9999", // Just need a valid port, nothing will bind to it
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{},
			},
		},
	}
	sidecarTask := &structs.Task{
		Name: "sidecar",
		Kind: "connect-proxy:foo",
	}
	tg.Tasks = append(tg.Tasks, sidecarTask)

	logger := testlog.HCLogger(t)

	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
	defer cleanup()

	// Register Group Services
	consulConfig := consulapi.DefaultConfig()
	consulConfig.Address = testconsul.HTTPAddr
	consulAPIClient, err := consulapi.NewClient(consulConfig)
	require.NoError(t, err)

	consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
	go consulClient.Run()
	defer consulClient.Shutdown()
	require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))

	// Run Connect bootstrap Hook
	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
		Addr: consulConfig.Address,
	}, logger))
	req := &interfaces.TaskPrestartRequest{
		Task:    sidecarTask,
		TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
	}
	require.NoError(t, req.TaskDir.Build(false, nil))

	// Insert service identity token in the secrets directory
	token := uuid.Generate()
	siTokenFile := filepath.Join(req.TaskDir.SecretsDir, sidsTokenFile)
	err = ioutil.WriteFile(siTokenFile, []byte(token), 0440)
	require.NoError(t, err)

	resp := &interfaces.TaskPrestartResponse{}

	// Run the hook
	require.NoError(t, h.Prestart(context.Background(), req, resp))

	// Assert it is Done
	require.True(t, resp.Done)

	// Ensure the default path matches
	env := map[string]string{
		taskenv.SecretsDir: req.TaskDir.SecretsDir,
	}
	f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
	require.NoError(t, err)
	defer f.Close()

	// Assert bootstrap configuration is valid json
	var out envoyConfig
	require.NoError(t, json.NewDecoder(f).Decode(&out))

	// Assert the SI token got set
	key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
	value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
	require.Equal(t, "x-consul-token", key)
	require.Equal(t, token, value)
}

// TestTaskRunner_EnvoyBootstrapHook_Prestart asserts the EnvoyBootstrapHook
// creates Envoy's bootstrap.json configuration based on Connect proxy sidecars
// registered for the task.
func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
	t.Parallel()
	testutil.RequireConsul(t)

	testconsul, err := consultest.NewTestServerConfig(func(c *consultest.TestServerConfig) {
		// If -v wasn't specified squelch consul logging
		if !testing.Verbose() {
			c.Stdout = ioutil.Discard
			c.Stderr = ioutil.Discard
		}
	})
	if err != nil {
		t.Fatalf("error starting test consul server: %v", err)
	}
	defer testconsul.Stop()

	alloc := mock.Alloc()
	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
		{
			Mode: "bridge",
			IP:   "10.0.0.1",
			DynamicPorts: []structs.Port{
				{
					Label: "connect-proxy-foo",
					Value: 9999,
					To:    9999,
				},
			},
		},
	}
	tg := alloc.Job.TaskGroups[0]
	tg.Services = []*structs.Service{
		{
			Name:      "foo",
			PortLabel: "9999", // Just need a valid port, nothing will bind to it
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{},
			},
		},
	}
	sidecarTask := &structs.Task{
		Name: "sidecar",
		Kind: "connect-proxy:foo",
	}
	tg.Tasks = append(tg.Tasks, sidecarTask)

	logger := testlog.HCLogger(t)

	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
	defer cleanup()

	// Register Group Services
	consulConfig := consulapi.DefaultConfig()
	consulConfig.Address = testconsul.HTTPAddr
	consulAPIClient, err := consulapi.NewClient(consulConfig)
	require.NoError(t, err)

	consulClient := agentconsul.NewServiceClient(consulAPIClient.Agent(), logger, true)
	go consulClient.Run()
	defer consulClient.Shutdown()
	require.NoError(t, consulClient.RegisterWorkload(agentconsul.BuildAllocServices(mock.Node(), alloc, agentconsul.NoopRestarter())))

	// Run Connect bootstrap Hook
	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
		Addr: consulConfig.Address,
	}, logger))
	req := &interfaces.TaskPrestartRequest{
		Task:    sidecarTask,
		TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
	}
	require.NoError(t, req.TaskDir.Build(false, nil))

	resp := &interfaces.TaskPrestartResponse{}

	// Run the hook
	require.NoError(t, h.Prestart(context.Background(), req, resp))

	// Assert it is Done
	require.True(t, resp.Done)

	require.NotNil(t, resp.Env)
	require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])

	// Ensure the default path matches
	env := map[string]string{
		taskenv.SecretsDir: req.TaskDir.SecretsDir,
	}
	f, err := os.Open(args.ReplaceEnv(structs.EnvoyBootstrapPath, env))
	require.NoError(t, err)
	defer f.Close()

	// Assert bootstrap configuration is valid json
	var out envoyConfig
	require.NoError(t, json.NewDecoder(f).Decode(&out))

	// Assert no SI token got set
	key := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Key
	value := out.DynamicResources.ADSConfig.GRPCServices.InitialMetadata[0].Value
	require.Equal(t, "x-consul-token", key)
	require.Equal(t, "", value)
}

// TestTaskRunner_EnvoyBootstrapHook_Noop asserts that the Envoy bootstrap hook
// is a noop for non-Connect proxy sidecar tasks.
func TestTaskRunner_EnvoyBootstrapHook_Noop(t *testing.T) {
	t.Parallel()
	logger := testlog.HCLogger(t)

	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
	defer cleanup()

	alloc := mock.Alloc()
	task := alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks[0]

	// Run Envoy bootstrap Hook. Use invalid Consul address as it should
	// not get hit.
	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
		Addr: "http://127.0.0.2:1",
	}, logger))
	req := &interfaces.TaskPrestartRequest{
		Task:    task,
		TaskDir: allocDir.NewTaskDir(task.Name),
	}
	require.NoError(t, req.TaskDir.Build(false, nil))

	resp := &interfaces.TaskPrestartResponse{}

	// Run the hook
	require.NoError(t, h.Prestart(context.Background(), req, resp))

	// Assert it is Done
	require.True(t, resp.Done)

	// Assert no file was written
	_, err := os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
	require.Error(t, err)
	require.True(t, os.IsNotExist(err))
}

// TestTaskRunner_EnvoyBootstrapHook_RecoverableError asserts the Envoy
// bootstrap hook returns a Recoverable error if the bootstrap command runs but
// fails.
func TestTaskRunner_EnvoyBootstrapHook_RecoverableError(t *testing.T) {
	t.Parallel()
	testutil.RequireConsul(t)

	testconsul, err := consultest.NewTestServerConfig(func(c *consultest.TestServerConfig) {
		// If -v wasn't specified squelch consul logging
		if !testing.Verbose() {
			c.Stdout = ioutil.Discard
			c.Stderr = ioutil.Discard
		}
	})
	if err != nil {
		t.Fatalf("error starting test consul server: %v", err)
	}
	defer testconsul.Stop()

	alloc := mock.Alloc()
	alloc.AllocatedResources.Shared.Networks = []*structs.NetworkResource{
		{
			Mode: "bridge",
			IP:   "10.0.0.1",
			DynamicPorts: []structs.Port{
				{
					Label: "connect-proxy-foo",
					Value: 9999,
					To:    9999,
				},
			},
		},
	}
	tg := alloc.Job.TaskGroups[0]
	tg.Services = []*structs.Service{
		{
			Name:      "foo",
			PortLabel: "9999", // Just need a valid port, nothing will bind to it
			Connect: &structs.ConsulConnect{
				SidecarService: &structs.ConsulSidecarService{},
			},
		},
	}
	sidecarTask := &structs.Task{
		Name: "sidecar",
		Kind: "connect-proxy:foo",
	}
	tg.Tasks = append(tg.Tasks, sidecarTask)

	logger := testlog.HCLogger(t)

	allocDir, cleanup := allocdir.TestAllocDir(t, logger, "EnvoyBootstrap")
	defer cleanup()

	// Unlike the successful test above, do NOT register the group services
	// yet. This should cause a recoverable error similar to if Consul was
	// not running.

	// Run Connect bootstrap Hook
	h := newEnvoyBootstrapHook(newEnvoyBootstrapHookConfig(alloc, &config.ConsulConfig{
		Addr: testconsul.HTTPAddr,
	}, logger))
	req := &interfaces.TaskPrestartRequest{
		Task:    sidecarTask,
		TaskDir: allocDir.NewTaskDir(sidecarTask.Name),
	}
	require.NoError(t, req.TaskDir.Build(false, nil))

	resp := &interfaces.TaskPrestartResponse{}

	// Run the hook
	err = h.Prestart(context.Background(), req, resp)
	require.EqualError(t, err, "error creating bootstrap configuration for Connect proxy sidecar: exit status 1")
	require.True(t, structs.IsRecoverable(err))

	// Assert it is not Done
	require.False(t, resp.Done)

	// Assert no file was written
	_, err = os.Open(filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json"))
	require.Error(t, err)
	require.True(t, os.IsNotExist(err))
}