github.com/kjdelisle/consul@v1.4.5/agent/xds/listeners.go (about) 1 package xds 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 8 envoy "github.com/envoyproxy/go-control-plane/envoy/api/v2" 9 envoyauth "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth" 10 envoycore "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" 11 envoylistener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" 12 extauthz "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/ext_authz/v2" 13 envoytcp "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2" 14 "github.com/envoyproxy/go-control-plane/pkg/util" 15 "github.com/gogo/protobuf/jsonpb" 16 "github.com/gogo/protobuf/proto" 17 "github.com/gogo/protobuf/types" 18 19 "github.com/hashicorp/consul/agent/proxycfg" 20 "github.com/hashicorp/consul/agent/structs" 21 ) 22 23 // listenersFromSnapshot returns the xDS API representation of the "listeners" 24 // in the snapshot. 25 func listenersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot, token string) ([]proto.Message, error) { 26 if cfgSnap == nil { 27 return nil, errors.New("nil config given") 28 } 29 30 // One listener for each upstream plus the public one 31 resources := make([]proto.Message, len(cfgSnap.Proxy.Upstreams)+1) 32 33 // Configure public listener 34 var err error 35 resources[0], err = makePublicListener(cfgSnap, token) 36 if err != nil { 37 return nil, err 38 } 39 for i, u := range cfgSnap.Proxy.Upstreams { 40 resources[i+1], err = makeUpstreamListener(&u) 41 if err != nil { 42 return nil, err 43 } 44 } 45 return resources, nil 46 } 47 48 // makeListener returns a listener with name and bind details set. Filters must 49 // be added before it's useful. 50 // 51 // Note on names: Envoy listeners attempt graceful transitions of connections 52 // when their config changes but that means they can't have their bind address 53 // or port changed in a running instance. Since our users might choose to change 54 // a bind address or port for the public or upstream listeners, we need to 55 // encode those into the unique name for the listener such that if the user 56 // changes them, we actually create a whole new listener on the new address and 57 // port. Envoy should take care of closing the old one once it sees it's no 58 // longer in the config. 59 func makeListener(name, addr string, port int) *envoy.Listener { 60 return &envoy.Listener{ 61 Name: fmt.Sprintf("%s:%s:%d", name, addr, port), 62 Address: makeAddress(addr, port), 63 } 64 } 65 66 // makeListenerFromUserConfig returns the listener config decoded from an 67 // arbitrary proto3 json format string or an error if it's invalid. 68 // 69 // For now we only support embedding in JSON strings because of the hcl parsing 70 // pain (see config.go comment above call to patchSliceOfMaps). Until we 71 // refactor config parser a _lot_ user's opaque config that contains arrays will 72 // be mangled. We could actually fix that up in mapstructure which knows the 73 // type of the target so could resolve the slices to singletons unambiguously 74 // and it would work for us here... but we still have the problem that the 75 // config would render incorrectly in general in our HTTP API responses so we 76 // really need to fix it "properly". 77 // 78 // When we do that we can support just nesting the config directly into the 79 // JSON/hcl naturally but this is a stop-gap that gets us an escape hatch 80 // immediately. It's also probably not a bad thing to support long-term since 81 // any config generated by other systems will likely be in canonical protobuf 82 // from rather than our slight variant in JSON/hcl. 83 func makeListenerFromUserConfig(configJSON string) (*envoy.Listener, error) { 84 // Figure out if there is an @type field. We don't require is since we know 85 // this will be a listener but unmarshalling into types.Any fails if it's not 86 // there and unmarshalling into listener directly fails if it is... 87 var jsonFields map[string]*json.RawMessage 88 if err := json.Unmarshal([]byte(configJSON), &jsonFields); err != nil { 89 return nil, err 90 } 91 92 var l envoy.Listener 93 94 if _, ok := jsonFields["@type"]; ok { 95 // Type field is present so decode it as a types.Any 96 var any types.Any 97 err := jsonpb.UnmarshalString(configJSON, &any) 98 if err != nil { 99 return nil, err 100 } 101 // And then unmarshal the listener again... 102 err = proto.Unmarshal(any.Value, &l) 103 if err != nil { 104 return nil, err 105 } 106 return &l, err 107 } 108 109 // No @type so try decoding as a straight listener. 110 err := jsonpb.UnmarshalString(configJSON, &l) 111 return &l, err 112 } 113 114 // Ensure that the first filter in each filter chain of a public listener is the 115 // authz filter to prevent unauthorized access and that every filter chain uses 116 // our TLS certs. We might allow users to work around this later if there is a 117 // good use case but this is actually a feature for now as it allows them to 118 // specify custom listener params in config but still get our certs delivered 119 // dynamically and intentions enforced without coming up with some complicated 120 // templating/merging solution. 121 func injectConnectFilters(cfgSnap *proxycfg.ConfigSnapshot, token string, listener *envoy.Listener) error { 122 authFilter, err := makeExtAuthFilter(token) 123 if err != nil { 124 return err 125 } 126 for idx := range listener.FilterChains { 127 // Insert our authz filter before any others 128 listener.FilterChains[idx].Filters = 129 append([]envoylistener.Filter{authFilter}, listener.FilterChains[idx].Filters...) 130 131 // Force our TLS for all filter chains on a public listener 132 listener.FilterChains[idx].TlsContext = &envoyauth.DownstreamTlsContext{ 133 CommonTlsContext: makeCommonTLSContext(cfgSnap), 134 RequireClientCertificate: &types.BoolValue{Value: true}, 135 } 136 } 137 return nil 138 } 139 140 func makePublicListener(cfgSnap *proxycfg.ConfigSnapshot, token string) (proto.Message, error) { 141 var l *envoy.Listener 142 var err error 143 144 if listenerJSONRaw, ok := cfgSnap.Proxy.Config["envoy_public_listener_json"]; ok { 145 if listenerJSON, ok := listenerJSONRaw.(string); ok { 146 l, err = makeListenerFromUserConfig(listenerJSON) 147 if err != nil { 148 return l, err 149 } 150 } 151 } 152 153 if l == nil { 154 // No user config, use default listener 155 addr := cfgSnap.Address 156 if addr == "" { 157 addr = "0.0.0.0" 158 } 159 l = makeListener(PublicListenerName, addr, cfgSnap.Port) 160 tcpProxy, err := makeTCPProxyFilter("public_listener", LocalAppClusterName) 161 if err != nil { 162 return l, err 163 } 164 // Setup TCP proxy for now. We inject TLS and authz below for both default 165 // and custom config cases. 166 l.FilterChains = []envoylistener.FilterChain{ 167 { 168 Filters: []envoylistener.Filter{ 169 tcpProxy, 170 }, 171 }, 172 } 173 } 174 175 err = injectConnectFilters(cfgSnap, token, l) 176 return l, err 177 } 178 179 func makeUpstreamListener(u *structs.Upstream) (proto.Message, error) { 180 if listenerJSONRaw, ok := u.Config["envoy_listener_json"]; ok { 181 if listenerJSON, ok := listenerJSONRaw.(string); ok { 182 return makeListenerFromUserConfig(listenerJSON) 183 } 184 } 185 addr := u.LocalBindAddress 186 if addr == "" { 187 addr = "127.0.0.1" 188 } 189 l := makeListener(u.Identifier(), addr, u.LocalBindPort) 190 tcpProxy, err := makeTCPProxyFilter(u.Identifier(), u.Identifier()) 191 if err != nil { 192 return l, err 193 } 194 l.FilterChains = []envoylistener.FilterChain{ 195 { 196 Filters: []envoylistener.Filter{ 197 tcpProxy, 198 }, 199 }, 200 } 201 return l, nil 202 } 203 204 func makeTCPProxyFilter(name, cluster string) (envoylistener.Filter, error) { 205 cfg := &envoytcp.TcpProxy{ 206 StatPrefix: name, 207 Cluster: cluster, 208 } 209 return makeFilter("envoy.tcp_proxy", cfg) 210 } 211 212 func makeExtAuthFilter(token string) (envoylistener.Filter, error) { 213 cfg := &extauthz.ExtAuthz{ 214 StatPrefix: "connect_authz", 215 GrpcService: &envoycore.GrpcService{ 216 // Attach token header so we can authorize the callbacks. Technically 217 // authorize is not really protected data but we locked down the HTTP 218 // implementation to need service:write and since we have the token that 219 // has that it's pretty reasonable to set it up here. 220 InitialMetadata: []*envoycore.HeaderValue{ 221 &envoycore.HeaderValue{ 222 Key: "x-consul-token", 223 Value: token, 224 }, 225 }, 226 TargetSpecifier: &envoycore.GrpcService_EnvoyGrpc_{ 227 EnvoyGrpc: &envoycore.GrpcService_EnvoyGrpc{ 228 ClusterName: LocalAgentClusterName, 229 }, 230 }, 231 }, 232 FailureModeAllow: false, 233 } 234 return makeFilter("envoy.ext_authz", cfg) 235 } 236 237 func makeFilter(name string, cfg proto.Message) (envoylistener.Filter, error) { 238 // Ridiculous dance to make that pbstruct into types.Struct by... encoding it 239 // as JSON and decoding again!! 240 cfgStruct, err := util.MessageToStruct(cfg) 241 if err != nil { 242 return envoylistener.Filter{}, err 243 } 244 245 return envoylistener.Filter{ 246 Name: name, 247 Config: cfgStruct, 248 }, nil 249 } 250 251 func makeCommonTLSContext(cfgSnap *proxycfg.ConfigSnapshot) *envoyauth.CommonTlsContext { 252 // Concatenate all the root PEMs into one. 253 // TODO(banks): verify this actually works with Envoy (docs are not clear). 254 rootPEMS := "" 255 for _, root := range cfgSnap.Roots.Roots { 256 rootPEMS += root.RootCert 257 } 258 259 return &envoyauth.CommonTlsContext{ 260 TlsParams: &envoyauth.TlsParameters{}, 261 TlsCertificates: []*envoyauth.TlsCertificate{ 262 &envoyauth.TlsCertificate{ 263 CertificateChain: &envoycore.DataSource{ 264 Specifier: &envoycore.DataSource_InlineString{ 265 InlineString: cfgSnap.Leaf.CertPEM, 266 }, 267 }, 268 PrivateKey: &envoycore.DataSource{ 269 Specifier: &envoycore.DataSource_InlineString{ 270 InlineString: cfgSnap.Leaf.PrivateKeyPEM, 271 }, 272 }, 273 }, 274 }, 275 ValidationContextType: &envoyauth.CommonTlsContext_ValidationContext{ 276 ValidationContext: &envoyauth.CertificateValidationContext{ 277 // TODO(banks): later for L7 support we may need to configure ALPN here. 278 TrustedCa: &envoycore.DataSource{ 279 Specifier: &envoycore.DataSource_InlineString{ 280 InlineString: rootPEMS, 281 }, 282 }, 283 }, 284 }, 285 } 286 }