github.com/manicqin/nomad@v0.9.5/client/allocrunner/taskrunner/envoybootstrap_hook.go (about) 1 package taskrunner 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "os" 8 "os/exec" 9 "path/filepath" 10 "time" 11 12 log "github.com/hashicorp/go-hclog" 13 "github.com/hashicorp/nomad/client/allocdir" 14 "github.com/hashicorp/nomad/client/allocrunner/interfaces" 15 agentconsul "github.com/hashicorp/nomad/command/agent/consul" 16 "github.com/hashicorp/nomad/helper" 17 "github.com/hashicorp/nomad/nomad/structs" 18 ) 19 20 var _ interfaces.TaskPrestartHook = &envoyBootstrapHook{} 21 22 const ( 23 envoyBaseAdminPort = 19000 24 envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_" 25 ) 26 27 // envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy 28 // sidecar. 29 type envoyBootstrapHook struct { 30 alloc *structs.Allocation 31 32 // Bootstrapping Envoy requires talking directly to Consul to generate 33 // the bootstrap.json config. Runtime Envoy configuration is done via 34 // Consul's gRPC endpoint. 35 consulHTTPAddr string 36 37 logger log.Logger 38 } 39 40 func newEnvoyBootstrapHook(alloc *structs.Allocation, consulHTTPAddr string, logger log.Logger) *envoyBootstrapHook { 41 h := &envoyBootstrapHook{ 42 alloc: alloc, 43 consulHTTPAddr: consulHTTPAddr, 44 } 45 h.logger = logger.Named(h.Name()) 46 return h 47 } 48 49 func (envoyBootstrapHook) Name() string { 50 return "envoy_bootstrap" 51 } 52 53 func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error { 54 if !req.Task.Kind.IsConnectProxy() { 55 // Not a Connect proxy sidecar 56 resp.Done = true 57 return nil 58 } 59 60 serviceName := req.Task.Kind.Value() 61 if serviceName == "" { 62 return fmt.Errorf("Connect proxy sidecar does not specify service name") 63 } 64 65 tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup) 66 67 var service *structs.Service 68 for _, s := range tg.Services { 69 if s.Name == serviceName { 70 service = s 71 break 72 } 73 } 74 75 if service == nil { 76 return fmt.Errorf("Connect proxy sidecar task exists but no services configured with a sidecar") 77 } 78 79 h.logger.Debug("bootstrapping Connect proxy sidecar", "task", req.Task.Name, "service", serviceName) 80 81 //TODO Should connect directly to Consul if the sidecar is running on 82 // the host netns. 83 grpcAddr := "unix://" + allocdir.AllocGRPCSocket 84 85 // Envoy runs an administrative API on the loopback interface. If multiple sidecars 86 // are running, the bind addresses need to have unique ports. 87 // TODO: support running in host netns, using freeport to find available port 88 envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name) 89 resp.Env = map[string]string{ 90 helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind, 91 } 92 93 // Envoy bootstrap configuration may contain a Consul token, so write 94 // it to the secrets directory like Vault tokens. 95 fn := filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json") 96 97 id := agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tg.Name, service) 98 h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "boostrap_file", fn, "sidecar_for_id", id, "grpc_addr", grpcAddr, "admin_bind", envoyAdminBind) 99 100 // Since Consul services are registered asynchronously with this task 101 // hook running, retry a small number of times with backoff. 102 for tries := 3; ; tries-- { 103 cmd := exec.CommandContext(ctx, "consul", "connect", "envoy", 104 "-grpc-addr", grpcAddr, 105 "-http-addr", h.consulHTTPAddr, 106 "-admin-bind", envoyAdminBind, 107 "-bootstrap", 108 "-sidecar-for", id, 109 ) 110 111 // Redirect output to secrets/envoy_bootstrap.json 112 fd, err := os.Create(fn) 113 if err != nil { 114 return fmt.Errorf("error creating secrets/envoy_bootstrap.json for envoy: %v", err) 115 } 116 cmd.Stdout = fd 117 118 buf := bytes.NewBuffer(nil) 119 cmd.Stderr = buf 120 121 // Generate bootstrap 122 err = cmd.Run() 123 124 // Close bootstrap.json 125 fd.Close() 126 127 if err == nil { 128 // Happy path! Bootstrap was created, exit. 129 break 130 } 131 132 // Check for error from command 133 if tries == 0 { 134 h.logger.Error("error creating bootstrap configuration for Connect proxy sidecar", "error", err, "stderr", buf.String()) 135 136 // Cleanup the bootstrap file. An errors here is not 137 // important as (a) we test to ensure the deletion 138 // occurs, and (b) the file will either be rewritten on 139 // retry or eventually garbage collected if the task 140 // fails. 141 os.Remove(fn) 142 143 // ExitErrors are recoverable since they indicate the 144 // command was runnable but exited with a unsuccessful 145 // error code. 146 _, recoverable := err.(*exec.ExitError) 147 return structs.NewRecoverableError( 148 fmt.Errorf("error creating bootstrap configuration for Connect proxy sidecar: %v", err), 149 recoverable, 150 ) 151 } 152 153 // Sleep before retrying to give Consul services time to register 154 select { 155 case <-time.After(2 * time.Second): 156 case <-ctx.Done(): 157 // Killed before bootstrap, exit without setting Done 158 return nil 159 } 160 } 161 162 // Bootstrap written. Mark as done and move on. 163 resp.Done = true 164 return nil 165 } 166 167 func buildEnvoyAdminBind(alloc *structs.Allocation, taskName string) string { 168 port := envoyBaseAdminPort 169 for idx, task := range alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks { 170 if task.Name == taskName { 171 port += idx 172 break 173 } 174 } 175 return fmt.Sprintf("localhost:%d", port) 176 }