github.com/manicqin/nomad@v0.9.5/client/allocrunner/taskrunner/envoybootstrap_hook.go (about)

     1  package taskrunner
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"fmt"
     7  	"os"
     8  	"os/exec"
     9  	"path/filepath"
    10  	"time"
    11  
    12  	log "github.com/hashicorp/go-hclog"
    13  	"github.com/hashicorp/nomad/client/allocdir"
    14  	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
    15  	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
    16  	"github.com/hashicorp/nomad/helper"
    17  	"github.com/hashicorp/nomad/nomad/structs"
    18  )
    19  
    20  var _ interfaces.TaskPrestartHook = &envoyBootstrapHook{}
    21  
    22  const (
    23  	envoyBaseAdminPort      = 19000
    24  	envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_"
    25  )
    26  
    27  // envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy
    28  // sidecar.
    29  type envoyBootstrapHook struct {
    30  	alloc *structs.Allocation
    31  
    32  	// Bootstrapping Envoy requires talking directly to Consul to generate
    33  	// the bootstrap.json config. Runtime Envoy configuration is done via
    34  	// Consul's gRPC endpoint.
    35  	consulHTTPAddr string
    36  
    37  	logger log.Logger
    38  }
    39  
    40  func newEnvoyBootstrapHook(alloc *structs.Allocation, consulHTTPAddr string, logger log.Logger) *envoyBootstrapHook {
    41  	h := &envoyBootstrapHook{
    42  		alloc:          alloc,
    43  		consulHTTPAddr: consulHTTPAddr,
    44  	}
    45  	h.logger = logger.Named(h.Name())
    46  	return h
    47  }
    48  
    49  func (envoyBootstrapHook) Name() string {
    50  	return "envoy_bootstrap"
    51  }
    52  
    53  func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
    54  	if !req.Task.Kind.IsConnectProxy() {
    55  		// Not a Connect proxy sidecar
    56  		resp.Done = true
    57  		return nil
    58  	}
    59  
    60  	serviceName := req.Task.Kind.Value()
    61  	if serviceName == "" {
    62  		return fmt.Errorf("Connect proxy sidecar does not specify service name")
    63  	}
    64  
    65  	tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
    66  
    67  	var service *structs.Service
    68  	for _, s := range tg.Services {
    69  		if s.Name == serviceName {
    70  			service = s
    71  			break
    72  		}
    73  	}
    74  
    75  	if service == nil {
    76  		return fmt.Errorf("Connect proxy sidecar task exists but no services configured with a sidecar")
    77  	}
    78  
    79  	h.logger.Debug("bootstrapping Connect proxy sidecar", "task", req.Task.Name, "service", serviceName)
    80  
    81  	//TODO Should connect directly to Consul if the sidecar is running on
    82  	//     the host netns.
    83  	grpcAddr := "unix://" + allocdir.AllocGRPCSocket
    84  
    85  	// Envoy runs an administrative API on the loopback interface. If multiple sidecars
    86  	// are running, the bind addresses need to have unique ports.
    87  	// TODO: support running in host netns, using freeport to find available port
    88  	envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name)
    89  	resp.Env = map[string]string{
    90  		helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind,
    91  	}
    92  
    93  	// Envoy bootstrap configuration may contain a Consul token, so write
    94  	// it to the secrets directory like Vault tokens.
    95  	fn := filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json")
    96  
    97  	id := agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tg.Name, service)
    98  	h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "boostrap_file", fn, "sidecar_for_id", id, "grpc_addr", grpcAddr, "admin_bind", envoyAdminBind)
    99  
   100  	// Since Consul services are registered asynchronously with this task
   101  	// hook running, retry a small number of times with backoff.
   102  	for tries := 3; ; tries-- {
   103  		cmd := exec.CommandContext(ctx, "consul", "connect", "envoy",
   104  			"-grpc-addr", grpcAddr,
   105  			"-http-addr", h.consulHTTPAddr,
   106  			"-admin-bind", envoyAdminBind,
   107  			"-bootstrap",
   108  			"-sidecar-for", id,
   109  		)
   110  
   111  		// Redirect output to secrets/envoy_bootstrap.json
   112  		fd, err := os.Create(fn)
   113  		if err != nil {
   114  			return fmt.Errorf("error creating secrets/envoy_bootstrap.json for envoy: %v", err)
   115  		}
   116  		cmd.Stdout = fd
   117  
   118  		buf := bytes.NewBuffer(nil)
   119  		cmd.Stderr = buf
   120  
   121  		// Generate bootstrap
   122  		err = cmd.Run()
   123  
   124  		// Close bootstrap.json
   125  		fd.Close()
   126  
   127  		if err == nil {
   128  			// Happy path! Bootstrap was created, exit.
   129  			break
   130  		}
   131  
   132  		// Check for error from command
   133  		if tries == 0 {
   134  			h.logger.Error("error creating bootstrap configuration for Connect proxy sidecar", "error", err, "stderr", buf.String())
   135  
   136  			// Cleanup the bootstrap file. An errors here is not
   137  			// important as (a) we test to ensure the deletion
   138  			// occurs, and (b) the file will either be rewritten on
   139  			// retry or eventually garbage collected if the task
   140  			// fails.
   141  			os.Remove(fn)
   142  
   143  			// ExitErrors are recoverable since they indicate the
   144  			// command was runnable but exited with a unsuccessful
   145  			// error code.
   146  			_, recoverable := err.(*exec.ExitError)
   147  			return structs.NewRecoverableError(
   148  				fmt.Errorf("error creating bootstrap configuration for Connect proxy sidecar: %v", err),
   149  				recoverable,
   150  			)
   151  		}
   152  
   153  		// Sleep before retrying to give Consul services time to register
   154  		select {
   155  		case <-time.After(2 * time.Second):
   156  		case <-ctx.Done():
   157  			// Killed before bootstrap, exit without setting Done
   158  			return nil
   159  		}
   160  	}
   161  
   162  	// Bootstrap written. Mark as done and move on.
   163  	resp.Done = true
   164  	return nil
   165  }
   166  
   167  func buildEnvoyAdminBind(alloc *structs.Allocation, taskName string) string {
   168  	port := envoyBaseAdminPort
   169  	for idx, task := range alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks {
   170  		if task.Name == taskName {
   171  			port += idx
   172  			break
   173  		}
   174  	}
   175  	return fmt.Sprintf("localhost:%d", port)
   176  }