istio.io/istio@v0.0.0-20240520182934-d79c90f27776/pilot/pkg/xds/debug.go (about)

     1  // Copyright Istio Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package xds
    16  
    17  import (
    18  	"encoding/json"
    19  	"fmt"
    20  	"html/template"
    21  	"net"
    22  	"net/http"
    23  	"net/http/pprof"
    24  	"net/netip"
    25  	"runtime"
    26  	"sort"
    27  	"strings"
    28  	"sync"
    29  	"time"
    30  
    31  	admin "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
    32  	core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
    33  	wasm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/wasm/v3"
    34  	tls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
    35  	discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
    36  	"google.golang.org/protobuf/proto"
    37  	anypb "google.golang.org/protobuf/types/known/anypb"
    38  
    39  	"istio.io/istio/pilot/pkg/config/kube/crd"
    40  	"istio.io/istio/pilot/pkg/features"
    41  	"istio.io/istio/pilot/pkg/model"
    42  	"istio.io/istio/pilot/pkg/networking/util"
    43  	"istio.io/istio/pilot/pkg/util/protoconv"
    44  	"istio.io/istio/pilot/pkg/xds/endpoints"
    45  	v3 "istio.io/istio/pilot/pkg/xds/v3"
    46  	"istio.io/istio/pkg/config"
    47  	"istio.io/istio/pkg/config/schema/resource"
    48  	"istio.io/istio/pkg/config/xds"
    49  	istiolog "istio.io/istio/pkg/log"
    50  	"istio.io/istio/pkg/security"
    51  	"istio.io/istio/pkg/util/protomarshal"
    52  	"istio.io/istio/pkg/util/sets"
    53  )
    54  
    55  var indexTmpl = template.Must(template.New("index").Parse(`<html>
    56  <head>
    57  <title>Pilot Debug Console</title>
    58  </head>
    59  <style>
    60  #endpoints {
    61    font-family: "Trebuchet MS", Arial, Helvetica, sans-serif;
    62    border-collapse: collapse;
    63  }
    64  
    65  #endpoints td, #endpoints th {
    66    border: 1px solid #ddd;
    67    padding: 8px;
    68  }
    69  
    70  #endpoints tr:nth-child(even){background-color: #f2f2f2;}
    71  
    72  #endpoints tr:hover {background-color: #ddd;}
    73  
    74  #endpoints th {
    75    padding-top: 12px;
    76    padding-bottom: 12px;
    77    text-align: left;
    78    background-color: black;
    79    color: white;
    80  }
    81  </style>
    82  <body>
    83  <br/>
    84  <p style = "font-family:Arial,Helvetica,sans-serif;">
    85  Note: Use <b>pretty</b> in query string (like <b>debug/configz?pretty</b>) to format the output.
    86  </p>
    87  <table id="endpoints">
    88  <tr><th>Endpoint</th><th>Description</th></tr>
    89  {{range .}}
    90  	<tr>
    91  	<td><a href='{{.Href}}'>{{.Name}}</a></td><td>{{.Help}}</td>
    92  	</tr>
    93  {{end}}
    94  </table>
    95  <br/>
    96  </body>
    97  </html>
    98  `))
    99  
   100  // AdsClient defines the data that is displayed on "/adsz" endpoint.
   101  type AdsClient struct {
   102  	ConnectionID string              `json:"connectionId"`
   103  	ConnectedAt  time.Time           `json:"connectedAt"`
   104  	PeerAddress  string              `json:"address"`
   105  	Labels       map[string]string   `json:"labels"`
   106  	Metadata     *model.NodeMetadata `json:"metadata,omitempty"`
   107  	Locality     *core.Locality      `json:"locality,omitempty"`
   108  	Watches      map[string][]string `json:"watches,omitempty"`
   109  }
   110  
   111  // AdsClients is collection of AdsClient connected to this Istiod.
   112  type AdsClients struct {
   113  	Total     int         `json:"totalClients"`
   114  	Connected []AdsClient `json:"clients,omitempty"`
   115  }
   116  
   117  // SyncStatus is the synchronization status between Pilot and a given Envoy
   118  type SyncStatus struct {
   119  	ClusterID            string         `json:"cluster_id,omitempty"`
   120  	ProxyID              string         `json:"proxy,omitempty"`
   121  	ProxyType            model.NodeType `json:"proxy_type,omitempty"`
   122  	ProxyVersion         string         `json:"proxy_version,omitempty"`
   123  	IstioVersion         string         `json:"istio_version,omitempty"`
   124  	ClusterSent          string         `json:"cluster_sent,omitempty"`
   125  	ClusterAcked         string         `json:"cluster_acked,omitempty"`
   126  	ListenerSent         string         `json:"listener_sent,omitempty"`
   127  	ListenerAcked        string         `json:"listener_acked,omitempty"`
   128  	RouteSent            string         `json:"route_sent,omitempty"`
   129  	RouteAcked           string         `json:"route_acked,omitempty"`
   130  	EndpointSent         string         `json:"endpoint_sent,omitempty"`
   131  	EndpointAcked        string         `json:"endpoint_acked,omitempty"`
   132  	ExtensionConfigSent  string         `json:"extensionconfig_sent,omitempty"`
   133  	ExtensionConfigAcked string         `json:"extensionconfig_acked,omitempty"`
   134  }
   135  
   136  // SyncedVersions shows what resourceVersion of a given resource has been acked by Envoy.
   137  type SyncedVersions struct {
   138  	ProxyID         string `json:"proxy,omitempty"`
   139  	ClusterVersion  string `json:"cluster_acked,omitempty"`
   140  	ListenerVersion string `json:"listener_acked,omitempty"`
   141  	RouteVersion    string `json:"route_acked,omitempty"`
   142  	EndpointVersion string `json:"endpoint_acked,omitempty"`
   143  }
   144  
   145  // InitDebug initializes the debug handlers and adds a debug in-memory registry.
   146  func (s *DiscoveryServer) InitDebug(
   147  	mux *http.ServeMux,
   148  	enableProfiling bool,
   149  	fetchWebhook func() map[string]string,
   150  ) *http.ServeMux {
   151  	internalMux := http.NewServeMux()
   152  	s.AddDebugHandlers(mux, internalMux, enableProfiling, fetchWebhook)
   153  	return internalMux
   154  }
   155  
   156  func (s *DiscoveryServer) AddDebugHandlers(mux, internalMux *http.ServeMux, enableProfiling bool, webhook func() map[string]string) {
   157  	// Debug handlers on HTTP ports are added for backward compatibility.
   158  	// They will be exposed on XDS-over-TLS in future releases.
   159  	if !features.EnableDebugOnHTTP {
   160  		return
   161  	}
   162  
   163  	if enableProfiling {
   164  		runtime.SetMutexProfileFraction(features.MutexProfileFraction)
   165  		runtime.SetBlockProfileRate(features.MutexProfileFraction)
   166  		s.addDebugHandler(mux, internalMux, "/debug/pprof/", "Displays pprof index", pprof.Index)
   167  		s.addDebugHandler(mux, internalMux, "/debug/pprof/cmdline", "The command line invocation of the current program", pprof.Cmdline)
   168  		s.addDebugHandler(mux, internalMux, "/debug/pprof/profile", "CPU profile", pprof.Profile)
   169  		s.addDebugHandler(mux, internalMux, "/debug/pprof/symbol", "Symbol looks up the program counters listed in the request", pprof.Symbol)
   170  		s.addDebugHandler(mux, internalMux, "/debug/pprof/trace", "A trace of execution of the current program.", pprof.Trace)
   171  	}
   172  
   173  	mux.HandleFunc("/debug", s.Debug)
   174  
   175  	if features.EnableUnsafeAdminEndpoints {
   176  		s.addDebugHandler(mux, internalMux, "/debug/force_disconnect", "Disconnects a proxy from this Pilot", s.forceDisconnect)
   177  	}
   178  
   179  	s.addDebugHandler(mux, internalMux, "/debug/ecdsz", "Status and debug interface for ECDS", s.ecdsz)
   180  	s.addDebugHandler(mux, internalMux, "/debug/edsz", "Status and debug interface for EDS", s.Edsz)
   181  	s.addDebugHandler(mux, internalMux, "/debug/ndsz", "Status and debug interface for NDS", s.ndsz)
   182  	s.addDebugHandler(mux, internalMux, "/debug/adsz", "Status and debug interface for ADS", s.adsz)
   183  	s.addDebugHandler(mux, internalMux, "/debug/adsz?push=true", "Initiates push of the current state to all connected endpoints", s.adsz)
   184  
   185  	s.addDebugHandler(mux, internalMux, "/debug/syncz", "Synchronization status of all Envoys connected to this Pilot instance", s.Syncz)
   186  	s.addDebugHandler(mux, internalMux, "/debug/config_distribution", "Version status of all Envoys connected to this Pilot instance", s.distributedVersions)
   187  
   188  	s.addDebugHandler(mux, internalMux, "/debug/registryz", "Debug support for registry", s.registryz)
   189  	s.addDebugHandler(mux, internalMux, "/debug/endpointz", "Obsolete, use endpointShardz", s.endpointShardz)
   190  	s.addDebugHandler(mux, internalMux, "/debug/endpointShardz", "Info about the endpoint shards", s.endpointShardz)
   191  	s.addDebugHandler(mux, internalMux, "/debug/cachez", "Info about the internal XDS caches", s.cachez)
   192  	s.addDebugHandler(mux, internalMux, "/debug/cachez?sizes=true", "Info about the size of the internal XDS caches", s.cachez)
   193  	s.addDebugHandler(mux, internalMux, "/debug/cachez?clear=true", "Clear the XDS caches", s.cachez)
   194  	s.addDebugHandler(mux, internalMux, "/debug/configz", "Debug support for config", s.configz)
   195  	s.addDebugHandler(mux, internalMux, "/debug/sidecarz", "Debug sidecar scope for a proxy", s.sidecarz)
   196  	s.addDebugHandler(mux, internalMux, "/debug/resourcesz", "Debug support for watched resources", s.resourcez)
   197  	s.addDebugHandler(mux, internalMux, "/debug/instancesz", "Debug support for service instances", s.instancesz)
   198  
   199  	s.addDebugHandler(mux, internalMux, "/debug/authorizationz", "Internal authorization policies", s.authorizationz)
   200  	s.addDebugHandler(mux, internalMux, "/debug/telemetryz", "Debug Telemetry configuration", s.telemetryz)
   201  	s.addDebugHandler(mux, internalMux, "/debug/config_dump", "ConfigDump in the form of the Envoy admin config dump API for passed in proxyID", s.ConfigDump)
   202  	s.addDebugHandler(mux, internalMux, "/debug/push_status", "Last PushContext Details", s.pushStatusHandler)
   203  	s.addDebugHandler(mux, internalMux, "/debug/pushcontext", "Debug support for current push context", s.pushContextHandler)
   204  	s.addDebugHandler(mux, internalMux, "/debug/connections", "Info about the connected XDS clients", s.connectionsHandler)
   205  
   206  	s.addDebugHandler(mux, internalMux, "/debug/inject", "Active inject template", s.injectTemplateHandler(webhook))
   207  	s.addDebugHandler(mux, internalMux, "/debug/mesh", "Active mesh config", s.meshHandler)
   208  	s.addDebugHandler(mux, internalMux, "/debug/clusterz", "List remote clusters where istiod reads endpoints", s.clusterz)
   209  	s.addDebugHandler(mux, internalMux, "/debug/networkz", "List cross-network gateways", s.networkz)
   210  	s.addDebugHandler(mux, internalMux, "/debug/mcsz", "List information about Kubernetes MCS services", s.mcsz)
   211  
   212  	s.addDebugHandler(mux, internalMux, "/debug/list", "List all supported debug commands in json", s.list)
   213  }
   214  
   215  func (s *DiscoveryServer) addDebugHandler(mux *http.ServeMux, internalMux *http.ServeMux,
   216  	path string, help string, handler func(http.ResponseWriter, *http.Request),
   217  ) {
   218  	s.debugHandlers[path] = help
   219  	// Add handler without auth. This mux is never exposed on an HTTP server and only used internally
   220  	if internalMux != nil {
   221  		internalMux.HandleFunc(path, handler)
   222  	}
   223  	// Add handler with auth; this is expose on an HTTP server
   224  	mux.HandleFunc(path, s.allowAuthenticatedOrLocalhost(http.HandlerFunc(handler)))
   225  }
   226  
   227  func (s *DiscoveryServer) allowAuthenticatedOrLocalhost(next http.Handler) http.HandlerFunc {
   228  	return func(w http.ResponseWriter, req *http.Request) {
   229  		// Request is from localhost, no need to authenticate
   230  		if isRequestFromLocalhost(req) {
   231  			next.ServeHTTP(w, req)
   232  			return
   233  		}
   234  		// Authenticate request with the same method as XDS
   235  		authFailMsgs := make([]string, 0)
   236  		var ids []string
   237  		authRequest := security.AuthContext{Request: req}
   238  		for _, authn := range s.Authenticators {
   239  			u, err := authn.Authenticate(authRequest)
   240  			// If one authenticator passes, return
   241  			if u != nil && u.Identities != nil && err == nil {
   242  				ids = u.Identities
   243  				break
   244  			}
   245  			authFailMsgs = append(authFailMsgs, fmt.Sprintf("Authenticator %s: %v", authn.AuthenticatorType(), err))
   246  		}
   247  		if ids == nil {
   248  			istiolog.Errorf("Failed to authenticate %s %v", req.URL, authFailMsgs)
   249  			// Not including detailed info in the response, XDS doesn't either (returns a generic "authentication failure).
   250  			w.WriteHeader(http.StatusUnauthorized)
   251  			return
   252  		}
   253  		// TODO: Check that the identity contains istio-system namespace, else block or restrict to only info that
   254  		// is visible to the authenticated SA. Will require changes in docs and istioctl too.
   255  		next.ServeHTTP(w, req)
   256  	}
   257  }
   258  
   259  func isRequestFromLocalhost(r *http.Request) bool {
   260  	ip, _, err := net.SplitHostPort(r.RemoteAddr)
   261  	if err != nil {
   262  		return false
   263  	}
   264  
   265  	userIP, _ := netip.ParseAddr(ip)
   266  	return userIP.IsLoopback()
   267  }
   268  
   269  // Syncz dumps the synchronization status of all Envoys connected to this Pilot instance
   270  func (s *DiscoveryServer) Syncz(w http.ResponseWriter, req *http.Request) {
   271  	namespace := req.URL.Query().Get("namespace")
   272  
   273  	syncz := make([]SyncStatus, 0)
   274  	for _, con := range s.SortedClients() {
   275  		node := con.proxy
   276  		if node != nil && (namespace == "" || node.GetNamespace() == namespace) {
   277  			syncz = append(syncz, SyncStatus{
   278  				ProxyID:              node.ID,
   279  				ProxyType:            node.Type,
   280  				ClusterID:            node.GetClusterID().String(),
   281  				IstioVersion:         node.GetIstioVersion(),
   282  				ClusterSent:          con.NonceSent(v3.ClusterType),
   283  				ClusterAcked:         con.NonceAcked(v3.ClusterType),
   284  				ListenerSent:         con.NonceSent(v3.ListenerType),
   285  				ListenerAcked:        con.NonceAcked(v3.ListenerType),
   286  				RouteSent:            con.NonceSent(v3.RouteType),
   287  				RouteAcked:           con.NonceAcked(v3.RouteType),
   288  				EndpointSent:         con.NonceSent(v3.EndpointType),
   289  				EndpointAcked:        con.NonceAcked(v3.EndpointType),
   290  				ExtensionConfigSent:  con.NonceSent(v3.ExtensionConfigurationType),
   291  				ExtensionConfigAcked: con.NonceAcked(v3.ExtensionConfigurationType),
   292  			})
   293  		}
   294  	}
   295  	writeJSON(w, syncz, req)
   296  }
   297  
   298  // registryz providees debug support for registry - adding and listing model items.
   299  // Can be combined with the push debug interface to reproduce changes.
   300  func (s *DiscoveryServer) registryz(w http.ResponseWriter, req *http.Request) {
   301  	all := s.Env.ServiceDiscovery.Services()
   302  	writeJSON(w, all, req)
   303  }
   304  
   305  // Dumps info about the endpoint shards, tracked using the new direct interface.
   306  // Legacy registry provides are synced to the new data structure as well, during
   307  // the full push.
   308  func (s *DiscoveryServer) endpointShardz(w http.ResponseWriter, req *http.Request) {
   309  	writeJSON(w, s.Env.EndpointIndex.Shardz(), req)
   310  }
   311  
   312  func (s *DiscoveryServer) cachez(w http.ResponseWriter, req *http.Request) {
   313  	if err := req.ParseForm(); err != nil {
   314  		w.WriteHeader(http.StatusBadRequest)
   315  		_, _ = w.Write([]byte("Failed to parse request\n"))
   316  		return
   317  	}
   318  	if req.Form.Get("clear") != "" {
   319  		s.Cache.ClearAll()
   320  		_, _ = w.Write([]byte("Cache cleared\n"))
   321  		return
   322  	}
   323  	if req.Form.Get("sizes") != "" {
   324  		snapshot := s.Cache.Snapshot()
   325  		raw := make(map[string]int, len(snapshot))
   326  		totalSize := 0
   327  		for _, resource := range snapshot {
   328  			if resource == nil {
   329  				continue
   330  			}
   331  			resourceType := resource.Resource.TypeUrl
   332  			sz := len(resource.Resource.GetValue())
   333  			raw[resourceType] += sz
   334  			totalSize += sz
   335  		}
   336  		res := make(map[string]string, len(raw))
   337  		for k, v := range raw {
   338  			res[k] = util.ByteCount(v)
   339  		}
   340  		res["total"] = util.ByteCount(totalSize)
   341  		writeJSON(w, res, req)
   342  		return
   343  	}
   344  	snapshot := s.Cache.Snapshot()
   345  	resources := make(map[string][]string, len(snapshot)) // Key is typeUrl and value is resource names.
   346  	for _, resource := range snapshot {
   347  		if resource == nil {
   348  			continue
   349  		}
   350  		resourceType := resource.Resource.TypeUrl
   351  		resources[resourceType] = append(resources[resourceType], resource.Name)
   352  	}
   353  	writeJSON(w, resources, req)
   354  }
   355  
   356  const DistributionTrackingDisabledMessage = "Pilot Version tracking is disabled. It may be enabled by setting the " +
   357  	"PILOT_ENABLE_CONFIG_DISTRIBUTION_TRACKING environment variable to true."
   358  
   359  func (s *DiscoveryServer) distributedVersions(w http.ResponseWriter, req *http.Request) {
   360  	if !features.EnableDistributionTracking {
   361  		w.WriteHeader(http.StatusConflict)
   362  		_, _ = fmt.Fprint(w, DistributionTrackingDisabledMessage)
   363  		return
   364  	}
   365  	if resourceID := req.URL.Query().Get("resource"); resourceID != "" {
   366  		proxyNamespace := req.URL.Query().Get("proxy_namespace")
   367  		knownVersions := make(map[string]string)
   368  		var results []SyncedVersions
   369  		for _, con := range s.SortedClients() {
   370  			// wrap this in independent scope so that panic's don't bypass Unlock...
   371  			con.proxy.RLock()
   372  
   373  			if con.proxy != nil && (proxyNamespace == "" || proxyNamespace == con.proxy.ConfigNamespace) {
   374  				// read nonces from our statusreporter to allow for skipped nonces, etc.
   375  				results = append(results, SyncedVersions{
   376  					ProxyID: con.proxy.ID,
   377  					ClusterVersion: s.getResourceVersion(s.StatusReporter.QueryLastNonce(con.ID(), v3.ClusterType),
   378  						resourceID, knownVersions),
   379  					ListenerVersion: s.getResourceVersion(s.StatusReporter.QueryLastNonce(con.ID(), v3.ListenerType),
   380  						resourceID, knownVersions),
   381  					RouteVersion: s.getResourceVersion(s.StatusReporter.QueryLastNonce(con.ID(), v3.RouteType),
   382  						resourceID, knownVersions),
   383  					EndpointVersion: s.getResourceVersion(s.StatusReporter.QueryLastNonce(con.ID(), v3.EndpointType),
   384  						resourceID, knownVersions),
   385  				})
   386  			}
   387  			con.proxy.RUnlock()
   388  		}
   389  
   390  		writeJSON(w, results, req)
   391  	} else {
   392  		w.WriteHeader(http.StatusUnprocessableEntity)
   393  		_, _ = fmt.Fprintf(w, "querystring parameter 'resource' is required\n")
   394  	}
   395  }
   396  
   397  // VersionLen is the Config Version and is only used as the nonce prefix, but we can reconstruct
   398  // it because is is a b64 encoding of a 64 bit array, which will always be 12 chars in length.
   399  // len = ceil(bitlength/(2^6))+1
   400  const VersionLen = 12
   401  
   402  func (s *DiscoveryServer) getResourceVersion(nonce, key string, cache map[string]string) string {
   403  	if len(nonce) < VersionLen {
   404  		return ""
   405  	}
   406  	configVersion := nonce[:VersionLen]
   407  	result, ok := cache[configVersion]
   408  	if !ok {
   409  		lookupResult, err := s.Env.GetLedger().GetPreviousValue(configVersion, key)
   410  		if err != nil {
   411  			istiolog.Errorf("Unable to retrieve resource %s at version %s: %v", key, configVersion, err)
   412  			lookupResult = ""
   413  		}
   414  		// update the cache even on an error, because errors will not resolve themselves, and we don't want to
   415  		// repeat the same error for many s.adsClients.
   416  		cache[configVersion] = lookupResult
   417  		return lookupResult
   418  	}
   419  	return result
   420  }
   421  
   422  // kubernetesConfig wraps a config.Config with a custom marshaling method that matches a Kubernetes
   423  // object structure.
   424  type kubernetesConfig struct {
   425  	config.Config
   426  }
   427  
   428  func (k kubernetesConfig) MarshalJSON() ([]byte, error) {
   429  	cfg, err := crd.ConvertConfig(k.Config)
   430  	if err != nil {
   431  		return nil, err
   432  	}
   433  	return json.Marshal(cfg)
   434  }
   435  
   436  // Config debugging.
   437  func (s *DiscoveryServer) configz(w http.ResponseWriter, req *http.Request) {
   438  	configs := make([]kubernetesConfig, 0)
   439  	if s.Env == nil || s.Env.ConfigStore == nil {
   440  		return
   441  	}
   442  	s.Env.ConfigStore.Schemas().ForEach(func(schema resource.Schema) bool {
   443  		cfg := s.Env.ConfigStore.List(schema.GroupVersionKind(), "")
   444  		for _, c := range cfg {
   445  			configs = append(configs, kubernetesConfig{c})
   446  		}
   447  		return false
   448  	})
   449  	writeJSON(w, configs, req)
   450  }
   451  
   452  // SidecarScope debugging
   453  func (s *DiscoveryServer) sidecarz(w http.ResponseWriter, req *http.Request) {
   454  	proxyID, con := s.getDebugConnection(req)
   455  	if con == nil {
   456  		s.errorHandler(w, proxyID, con)
   457  		return
   458  	}
   459  	writeJSON(w, con.proxy.SidecarScope, req)
   460  }
   461  
   462  // Resource debugging.
   463  func (s *DiscoveryServer) resourcez(w http.ResponseWriter, req *http.Request) {
   464  	schemas := make([]config.GroupVersionKind, 0)
   465  
   466  	if s.Env != nil && s.Env.ConfigStore != nil {
   467  		s.Env.Schemas().ForEach(func(schema resource.Schema) bool {
   468  			schemas = append(schemas, schema.GroupVersionKind())
   469  			return false
   470  		})
   471  	}
   472  
   473  	writeJSON(w, schemas, req)
   474  }
   475  
   476  // AuthorizationDebug holds debug information for authorization policy.
   477  type AuthorizationDebug struct {
   478  	AuthorizationPolicies *model.AuthorizationPolicies `json:"authorization_policies"`
   479  }
   480  
   481  // authorizationz dumps the internal authorization policies.
   482  func (s *DiscoveryServer) authorizationz(w http.ResponseWriter, req *http.Request) {
   483  	info := AuthorizationDebug{
   484  		AuthorizationPolicies: s.globalPushContext().AuthzPolicies,
   485  	}
   486  	writeJSON(w, info, req)
   487  }
   488  
   489  // AuthorizationDebug holds debug information for authorization policy.
   490  type TelemetryDebug struct {
   491  	Telemetries *model.Telemetries `json:"telemetries"`
   492  }
   493  
   494  func (s *DiscoveryServer) telemetryz(w http.ResponseWriter, req *http.Request) {
   495  	proxyID, con := s.getDebugConnection(req)
   496  	if proxyID != "" && con == nil {
   497  		// We can't guarantee the Pilot we are connected to has a connection to the proxy we requested
   498  		// There isn't a great way around this, but for debugging purposes its suitable to have the caller retry.
   499  		w.WriteHeader(http.StatusNotFound)
   500  		_, _ = w.Write([]byte("Proxy not connected to this Pilot instance. It may be connected to another instance.\n"))
   501  		return
   502  	}
   503  	if con == nil {
   504  		info := TelemetryDebug{
   505  			Telemetries: s.globalPushContext().Telemetry,
   506  		}
   507  		writeJSON(w, info, req)
   508  		return
   509  	}
   510  	writeJSON(w, s.globalPushContext().Telemetry.Debug(con.proxy), req)
   511  }
   512  
   513  // connectionsHandler implements interface for displaying current connections.
   514  // It is mapped to /debug/connections.
   515  func (s *DiscoveryServer) connectionsHandler(w http.ResponseWriter, req *http.Request) {
   516  	adsClients := &AdsClients{}
   517  	connections := s.SortedClients()
   518  	adsClients.Total = len(connections)
   519  
   520  	for _, c := range connections {
   521  		adsClient := AdsClient{
   522  			ConnectionID: c.ID(),
   523  			ConnectedAt:  c.ConnectedAt(),
   524  			PeerAddress:  c.Peer(),
   525  		}
   526  		adsClients.Connected = append(adsClients.Connected, adsClient)
   527  	}
   528  	writeJSON(w, adsClients, req)
   529  }
   530  
   531  // adsz implements a status and debug interface for ADS.
   532  // It is mapped to /debug/adsz
   533  func (s *DiscoveryServer) adsz(w http.ResponseWriter, req *http.Request) {
   534  	if s.handlePushRequest(w, req) {
   535  		return
   536  	}
   537  	proxyID, con := s.getDebugConnection(req)
   538  	if proxyID != "" && con == nil {
   539  		// We can't guarantee the Pilot we are connected to has a connection to the proxy we requested
   540  		// There isn't a great way around this, but for debugging purposes its suitable to have the caller retry.
   541  		w.WriteHeader(http.StatusNotFound)
   542  		_, _ = w.Write([]byte("Proxy not connected to this Pilot instance. It may be connected to another instance.\n"))
   543  		return
   544  	}
   545  	var connections []*Connection
   546  	if con != nil {
   547  		connections = []*Connection{con}
   548  	} else {
   549  		connections = s.SortedClients()
   550  	}
   551  
   552  	adsClients := &AdsClients{}
   553  	adsClients.Total = len(connections)
   554  	for _, c := range connections {
   555  		adsClient := AdsClient{
   556  			ConnectionID: c.ID(),
   557  			ConnectedAt:  c.ConnectedAt(),
   558  			PeerAddress:  c.Peer(),
   559  			Labels:       c.proxy.Labels,
   560  			Metadata:     c.proxy.Metadata,
   561  			Locality:     c.proxy.Locality,
   562  			Watches:      map[string][]string{},
   563  		}
   564  		c.proxy.RLock()
   565  		for k, wr := range c.proxy.WatchedResources {
   566  			r := wr.ResourceNames
   567  			if r == nil {
   568  				r = []string{}
   569  			}
   570  			adsClient.Watches[k] = r
   571  		}
   572  		c.proxy.RUnlock()
   573  		adsClients.Connected = append(adsClients.Connected, adsClient)
   574  	}
   575  	writeJSON(w, adsClients, req)
   576  }
   577  
   578  // ecdsz implements a status and debug interface for ECDS.
   579  // It is mapped to /debug/ecdsz
   580  func (s *DiscoveryServer) ecdsz(w http.ResponseWriter, req *http.Request) {
   581  	if s.handlePushRequest(w, req) {
   582  		return
   583  	}
   584  	proxyID, con := s.getDebugConnection(req)
   585  	if con == nil {
   586  		s.errorHandler(w, proxyID, con)
   587  		return
   588  	}
   589  
   590  	dump := s.getConfigDumpByResourceType(con, nil, []string{v3.ExtensionConfigurationType})
   591  	if len(dump[v3.ExtensionConfigurationType]) == 0 {
   592  		w.WriteHeader(http.StatusNotFound)
   593  		return
   594  	}
   595  	writeJSON(w, dump[v3.ExtensionConfigurationType], req)
   596  }
   597  
   598  // ConfigDump returns information in the form of the Envoy admin API config dump for the specified proxy
   599  // The dump will only contain dynamic listeners/clusters/routes and can be used to compare what an Envoy instance
   600  // should look like according to Pilot vs what it currently does look like.
   601  func (s *DiscoveryServer) ConfigDump(w http.ResponseWriter, req *http.Request) {
   602  	proxyID, con := s.getDebugConnection(req)
   603  	if con == nil {
   604  		s.errorHandler(w, proxyID, con)
   605  		return
   606  	}
   607  	if ts := s.getResourceTypes(req); len(ts) != 0 {
   608  		resources := s.getConfigDumpByResourceType(con, nil, ts)
   609  		configDump := &admin.ConfigDump{}
   610  		for _, resource := range resources {
   611  			for _, rr := range resource {
   612  				configDump.Configs = append(configDump.Configs, rr.Resource)
   613  			}
   614  		}
   615  		writeJSON(w, configDump, req)
   616  		return
   617  	}
   618  
   619  	if con.proxy.IsZTunnel() {
   620  		resources := s.getConfigDumpByResourceType(con, nil, []string{v3.AddressType})
   621  		configDump := &admin.ConfigDump{}
   622  		for _, resource := range resources {
   623  			for _, rr := range resource {
   624  				configDump.Configs = append(configDump.Configs, rr.Resource)
   625  			}
   626  		}
   627  		writeJSON(w, configDump, req)
   628  		return
   629  	}
   630  
   631  	includeEds := req.URL.Query().Get("include_eds") == "true"
   632  	dump, err := s.connectionConfigDump(con, includeEds)
   633  	if err != nil {
   634  		handleHTTPError(w, err)
   635  		return
   636  	}
   637  	writeJSON(w, dump, req)
   638  }
   639  
   640  func (s *DiscoveryServer) getResourceTypes(req *http.Request) []string {
   641  	if shortTypes := req.URL.Query().Get("types"); shortTypes != "" {
   642  		ts := strings.Split(shortTypes, ",")
   643  
   644  		resourceTypes := sets.New[string]()
   645  		for _, t := range ts {
   646  			resourceTypes.Insert(v3.GetResourceType(t))
   647  		}
   648  
   649  		return resourceTypes.UnsortedList()
   650  	}
   651  	return nil
   652  }
   653  
   654  func (s *DiscoveryServer) getConfigDumpByResourceType(conn *Connection, req *model.PushRequest, ts []string) map[string][]*discoveryv3.Resource {
   655  	dumps := make(map[string][]*discoveryv3.Resource)
   656  	if req == nil {
   657  		req = &model.PushRequest{Push: conn.proxy.LastPushContext, Start: time.Now(), Full: true}
   658  	}
   659  
   660  	for _, resourceType := range ts {
   661  		w := conn.proxy.GetWatchedResource(resourceType)
   662  		if w == nil {
   663  			// Not watched, skip
   664  			continue
   665  		}
   666  		gen := s.findGenerator(resourceType, conn)
   667  		if gen == nil {
   668  			// No generator found, skip
   669  			continue
   670  		}
   671  		if resource, _, err := gen.Generate(conn.proxy, w, req); err == nil {
   672  			for _, rr := range resource {
   673  				switch resourceType {
   674  				case v3.SecretType:
   675  					// Secrets must be redacted
   676  					secret := &tls.Secret{}
   677  					if err := rr.Resource.UnmarshalTo(secret); err != nil {
   678  						istiolog.Warnf("failed to unmarshal secret: %v", err)
   679  						continue
   680  					}
   681  					if secret.GetTlsCertificate() != nil {
   682  						secret.GetTlsCertificate().PrivateKey = &core.DataSource{
   683  							Specifier: &core.DataSource_InlineBytes{
   684  								InlineBytes: []byte("[redacted]"),
   685  							},
   686  						}
   687  					}
   688  					rr.Resource = protoconv.MessageToAny(secret)
   689  					dumps[resourceType] = append(dumps[resourceType], rr)
   690  				case v3.ExtensionConfigurationType:
   691  					tce := &core.TypedExtensionConfig{}
   692  					if err := rr.GetResource().UnmarshalTo(tce); err != nil {
   693  						istiolog.Warnf("failed to unmarshal extension: %v", err)
   694  						continue
   695  					}
   696  
   697  					switch tce.TypedConfig.TypeUrl {
   698  					case xds.WasmHTTPFilterType:
   699  						w := &wasm.Wasm{}
   700  						if err := tce.TypedConfig.UnmarshalTo(w); err != nil {
   701  							istiolog.Warnf("failed to unmarshal wasm filter: %v", err)
   702  							continue
   703  						}
   704  						// Redact Wasm secret env variable.
   705  						vmenvs := w.GetConfig().GetVmConfig().EnvironmentVariables
   706  						if vmenvs != nil {
   707  							if _, found := vmenvs.KeyValues[model.WasmSecretEnv]; found {
   708  								vmenvs.KeyValues[model.WasmSecretEnv] = "<Redacted>"
   709  							}
   710  						}
   711  						dumps[resourceType] = append(dumps[resourceType], &discoveryv3.Resource{
   712  							Name:     w.Config.Name,
   713  							Resource: protoconv.MessageToAny(w),
   714  						})
   715  					default:
   716  						dumps[resourceType] = append(dumps[resourceType], rr)
   717  					}
   718  				default:
   719  					dumps[resourceType] = append(dumps[resourceType], rr)
   720  				}
   721  			}
   722  		} else {
   723  			istiolog.Warnf("generate failed for request resource type (%v): %v", resourceType, err)
   724  			continue
   725  		}
   726  	}
   727  
   728  	return dumps
   729  }
   730  
   731  // connectionConfigDump converts the connection internal state into an Envoy Admin API config dump proto
   732  // It is used in debugging to create a consistent object for comparison between Envoy and Pilot outputs
   733  func (s *DiscoveryServer) connectionConfigDump(conn *Connection, includeEds bool) (*admin.ConfigDump, error) {
   734  	req := &model.PushRequest{Push: conn.proxy.LastPushContext, Start: time.Now(), Full: true}
   735  	version := req.Push.PushVersion
   736  
   737  	dump := s.getConfigDumpByResourceType(conn, req, []string{
   738  		v3.ClusterType,
   739  		v3.ListenerType,
   740  		v3.RouteType,
   741  		v3.SecretType,
   742  		v3.EndpointType,
   743  		v3.ExtensionConfigurationType,
   744  	})
   745  
   746  	dynamicActiveClusters := make([]*admin.ClustersConfigDump_DynamicCluster, 0)
   747  	for _, cluster := range dump[v3.ClusterType] {
   748  		dynamicActiveClusters = append(dynamicActiveClusters, &admin.ClustersConfigDump_DynamicCluster{
   749  			Cluster: cluster.Resource,
   750  		})
   751  	}
   752  	clustersAny, err := protoconv.MessageToAnyWithError(&admin.ClustersConfigDump{
   753  		VersionInfo:           version,
   754  		DynamicActiveClusters: dynamicActiveClusters,
   755  	})
   756  	if err != nil {
   757  		return nil, err
   758  	}
   759  
   760  	dynamicActiveListeners := make([]*admin.ListenersConfigDump_DynamicListener, 0)
   761  	for _, listener := range dump[v3.ListenerType] {
   762  		dynamicActiveListeners = append(dynamicActiveListeners, &admin.ListenersConfigDump_DynamicListener{
   763  			Name: listener.Name,
   764  			ActiveState: &admin.ListenersConfigDump_DynamicListenerState{
   765  				Listener:    listener.Resource,
   766  				VersionInfo: version,
   767  			},
   768  		})
   769  	}
   770  	listenersAny, err := protoconv.MessageToAnyWithError(&admin.ListenersConfigDump{
   771  		VersionInfo:      version,
   772  		DynamicListeners: dynamicActiveListeners,
   773  	})
   774  	if err != nil {
   775  		return nil, err
   776  	}
   777  
   778  	dynamicRouteConfig := make([]*admin.RoutesConfigDump_DynamicRouteConfig, 0)
   779  	for _, route := range dump[v3.RouteType] {
   780  		dynamicRouteConfig = append(dynamicRouteConfig, &admin.RoutesConfigDump_DynamicRouteConfig{
   781  			VersionInfo: version,
   782  			RouteConfig: route.Resource,
   783  		})
   784  	}
   785  	routesAny, err := protoconv.MessageToAnyWithError(&admin.RoutesConfigDump{
   786  		DynamicRouteConfigs: dynamicRouteConfig,
   787  	})
   788  	if err != nil {
   789  		return nil, err
   790  	}
   791  
   792  	dynamicSecretsConfig := make([]*admin.SecretsConfigDump_DynamicSecret, 0)
   793  	for _, secret := range dump[v3.SecretType] {
   794  		dynamicSecretsConfig = append(dynamicSecretsConfig, &admin.SecretsConfigDump_DynamicSecret{
   795  			VersionInfo: version,
   796  			Secret:      secret.Resource,
   797  		})
   798  	}
   799  	secretsAny, err := protoconv.MessageToAnyWithError(&admin.SecretsConfigDump{
   800  		DynamicActiveSecrets: dynamicSecretsConfig,
   801  	})
   802  	if err != nil {
   803  		return nil, err
   804  	}
   805  
   806  	extensionsConfig := make([]*admin.EcdsConfigDump_EcdsFilterConfig, 0)
   807  	for _, ext := range dump[v3.ExtensionConfigurationType] {
   808  		extensionsConfig = append(extensionsConfig, &admin.EcdsConfigDump_EcdsFilterConfig{
   809  			VersionInfo: version,
   810  			EcdsFilter:  ext.Resource,
   811  		})
   812  	}
   813  	extensionsAny, err := protoconv.MessageToAnyWithError(&admin.EcdsConfigDump{
   814  		EcdsFilters: extensionsConfig,
   815  	})
   816  	if err != nil {
   817  		return nil, err
   818  	}
   819  
   820  	var endpointsAny *anypb.Any
   821  	// EDS is disabled by default for compatibility with Envoy config_dump interface
   822  	if includeEds {
   823  		endpointConfig := make([]*admin.EndpointsConfigDump_DynamicEndpointConfig, 0)
   824  		for _, endpoint := range dump[v3.EndpointType] {
   825  			endpointConfig = append(endpointConfig, &admin.EndpointsConfigDump_DynamicEndpointConfig{
   826  				VersionInfo:    version,
   827  				EndpointConfig: endpoint.Resource,
   828  			})
   829  		}
   830  		endpointsAny, err = protoconv.MessageToAnyWithError(&admin.EndpointsConfigDump{
   831  			DynamicEndpointConfigs: endpointConfig,
   832  		})
   833  		if err != nil {
   834  			return nil, err
   835  		}
   836  	}
   837  
   838  	bootstrapAny := protoconv.MessageToAny(&admin.BootstrapConfigDump{})
   839  	scopedRoutesAny := protoconv.MessageToAny(&admin.ScopedRoutesConfigDump{})
   840  	// The config dump must have all configs with connections specified in
   841  	// https://www.envoyproxy.io/docs/envoy/latest/api-v2/admin/v2alpha/config_dump.proto
   842  	configs := []*anypb.Any{
   843  		bootstrapAny,
   844  		clustersAny,
   845  	}
   846  	if includeEds {
   847  		configs = append(configs, endpointsAny)
   848  	}
   849  	configs = append(configs,
   850  		listenersAny,
   851  		scopedRoutesAny,
   852  		routesAny,
   853  		secretsAny,
   854  		extensionsAny,
   855  	)
   856  	configDump := &admin.ConfigDump{
   857  		Configs: configs,
   858  	}
   859  	return configDump, nil
   860  }
   861  
   862  // injectTemplateHandler dumps the injection template
   863  // Replaces dumping the template at startup.
   864  func (s *DiscoveryServer) injectTemplateHandler(webhook func() map[string]string) func(http.ResponseWriter, *http.Request) {
   865  	return func(w http.ResponseWriter, req *http.Request) {
   866  		// TODO: we should split the inject template into smaller modules (separate one for dump core, etc),
   867  		// and allow pods to select which patches will be selected. When this happen, this should return
   868  		// all inject templates or take a param to select one.
   869  		if webhook == nil {
   870  			w.WriteHeader(http.StatusNotFound)
   871  			return
   872  		}
   873  
   874  		writeJSON(w, webhook(), req)
   875  	}
   876  }
   877  
   878  // meshHandler dumps the mesh config
   879  func (s *DiscoveryServer) meshHandler(w http.ResponseWriter, req *http.Request) {
   880  	writeJSON(w, s.Env.Mesh(), req)
   881  }
   882  
   883  // pushStatusHandler dumps the last PushContext
   884  func (s *DiscoveryServer) pushStatusHandler(w http.ResponseWriter, req *http.Request) {
   885  	model.LastPushMutex.Lock()
   886  	defer model.LastPushMutex.Unlock()
   887  	if model.LastPushStatus == nil {
   888  		return
   889  	}
   890  	out, err := model.LastPushStatus.StatusJSON()
   891  	if err != nil {
   892  		handleHTTPError(w, err)
   893  		return
   894  	}
   895  	w.Header().Add("Content-Type", "application/json")
   896  
   897  	_, _ = w.Write(out)
   898  }
   899  
   900  // PushContextDebug holds debug information for push context.
   901  type PushContextDebug struct {
   902  	AuthorizationPolicies *model.AuthorizationPolicies
   903  	NetworkGateways       []model.NetworkGateway
   904  	UnresolvedGateways    []model.NetworkGateway
   905  }
   906  
   907  // pushContextHandler dumps the current PushContext
   908  func (s *DiscoveryServer) pushContextHandler(w http.ResponseWriter, req *http.Request) {
   909  	push := PushContextDebug{}
   910  	pc := s.globalPushContext()
   911  	if pc == nil {
   912  		return
   913  	}
   914  	push.AuthorizationPolicies = pc.AuthzPolicies
   915  	if pc.NetworkManager() != nil {
   916  		push.NetworkGateways = pc.NetworkManager().AllGateways()
   917  		push.UnresolvedGateways = pc.NetworkManager().Unresolved.AllGateways()
   918  	}
   919  
   920  	writeJSON(w, push, req)
   921  }
   922  
   923  // Debug lists all the supported debug endpoints.
   924  func (s *DiscoveryServer) Debug(w http.ResponseWriter, req *http.Request) {
   925  	type debugEndpoint struct {
   926  		Name string
   927  		Href string
   928  		Help string
   929  	}
   930  	var deps []debugEndpoint
   931  	for k, v := range s.debugHandlers {
   932  		deps = append(deps, debugEndpoint{
   933  			Name: k,
   934  			Href: k,
   935  			Help: v,
   936  		})
   937  	}
   938  
   939  	sort.Slice(deps, func(i, j int) bool {
   940  		return deps[i].Name < deps[j].Name
   941  	})
   942  
   943  	if err := indexTmpl.Execute(w, deps); err != nil {
   944  		istiolog.Errorf("Error in rendering index template %v", err)
   945  		w.WriteHeader(http.StatusInternalServerError)
   946  	}
   947  }
   948  
   949  // list all the supported debug commands in json.
   950  func (s *DiscoveryServer) list(w http.ResponseWriter, req *http.Request) {
   951  	var cmdNames []string
   952  	for k := range s.debugHandlers {
   953  		key := strings.Replace(k, "/debug/", "", -1)
   954  		// exclude current list command
   955  		if key == "list" {
   956  			continue
   957  		}
   958  		// can not support pprof commands
   959  		if strings.Contains(key, "pprof") {
   960  			continue
   961  		}
   962  		cmdNames = append(cmdNames, key)
   963  	}
   964  	sort.Strings(cmdNames)
   965  	writeJSON(w, cmdNames, req)
   966  }
   967  
   968  // ndsz implements a status and debug interface for NDS.
   969  // It is mapped to /debug/ndsz on the monitor port (15014).
   970  func (s *DiscoveryServer) ndsz(w http.ResponseWriter, req *http.Request) {
   971  	if s.handlePushRequest(w, req) {
   972  		return
   973  	}
   974  	proxyID, con := s.getDebugConnection(req)
   975  	if con == nil {
   976  		s.errorHandler(w, proxyID, con)
   977  		return
   978  	}
   979  	if !con.proxy.Metadata.DNSCapture {
   980  		w.WriteHeader(http.StatusBadRequest)
   981  		_, _ = w.Write([]byte("DNS capture is not enabled in the proxy\n"))
   982  		return
   983  	}
   984  
   985  	if s.Generators[v3.NameTableType] != nil {
   986  		nds, _, _ := s.Generators[v3.NameTableType].Generate(con.proxy, nil, &model.PushRequest{
   987  			Push: con.proxy.LastPushContext,
   988  			Full: true,
   989  		})
   990  		if len(nds) == 0 {
   991  			return
   992  		}
   993  		writeJSON(w, nds[0], req)
   994  	}
   995  }
   996  
   997  // Edsz implements a status and debug interface for EDS.
   998  // It is mapped to /debug/edsz on the monitor port (15014).
   999  func (s *DiscoveryServer) Edsz(w http.ResponseWriter, req *http.Request) {
  1000  	if s.handlePushRequest(w, req) {
  1001  		return
  1002  	}
  1003  
  1004  	proxyID, con := s.getDebugConnection(req)
  1005  	if con == nil {
  1006  		s.errorHandler(w, proxyID, con)
  1007  		return
  1008  	}
  1009  
  1010  	clusters := con.Clusters()
  1011  	eps := make([]jsonMarshalProto, 0, len(clusters))
  1012  	for _, clusterName := range clusters {
  1013  		builder := endpoints.NewEndpointBuilder(clusterName, con.proxy, con.proxy.LastPushContext)
  1014  		eps = append(eps, jsonMarshalProto{builder.BuildClusterLoadAssignment(s.Env.EndpointIndex)})
  1015  	}
  1016  	writeJSON(w, eps, req)
  1017  }
  1018  
  1019  func (s *DiscoveryServer) forceDisconnect(w http.ResponseWriter, req *http.Request) {
  1020  	proxyID, con := s.getDebugConnection(req)
  1021  	if con == nil {
  1022  		s.errorHandler(w, proxyID, con)
  1023  		return
  1024  	}
  1025  	con.Stop()
  1026  	_, _ = w.Write([]byte("OK"))
  1027  }
  1028  
  1029  func cloneProxy(proxy *model.Proxy) *model.Proxy {
  1030  	if proxy == nil {
  1031  		return nil
  1032  	}
  1033  
  1034  	proxy.Lock()
  1035  	defer proxy.Unlock()
  1036  	// nolint: govet
  1037  	copied := *proxy
  1038  	out := &copied
  1039  	out.RWMutex = sync.RWMutex{}
  1040  	// clone WatchedResources which can be mutated when processing request
  1041  	out.WatchedResources = make(map[string]*model.WatchedResource, len(proxy.WatchedResources))
  1042  	for k, v := range proxy.WatchedResources {
  1043  		// nolint: govet
  1044  		v := *v
  1045  		out.WatchedResources[k] = &v
  1046  	}
  1047  	return out
  1048  }
  1049  
  1050  func (s *DiscoveryServer) getProxyConnection(proxyID string) *Connection {
  1051  	for _, con := range s.Clients() {
  1052  		if strings.Contains(con.ID(), proxyID) {
  1053  			out := *con
  1054  			out.proxy = cloneProxy(con.proxy)
  1055  			return &out
  1056  		}
  1057  	}
  1058  
  1059  	return nil
  1060  }
  1061  
  1062  func (s *DiscoveryServer) instancesz(w http.ResponseWriter, req *http.Request) {
  1063  	instances := map[string][]model.ServiceTarget{}
  1064  	for _, con := range s.Clients() {
  1065  		con.proxy.RLock()
  1066  		if con.proxy != nil {
  1067  			instances[con.proxy.ID] = con.proxy.ServiceTargets
  1068  		}
  1069  		con.proxy.RUnlock()
  1070  	}
  1071  	writeJSON(w, instances, req)
  1072  }
  1073  
  1074  func (s *DiscoveryServer) networkz(w http.ResponseWriter, req *http.Request) {
  1075  	if s.Env == nil || s.Env.NetworkManager == nil {
  1076  		return
  1077  	}
  1078  	writeJSON(w, s.Env.NetworkManager.AllGateways(), req)
  1079  }
  1080  
  1081  func (s *DiscoveryServer) mcsz(w http.ResponseWriter, req *http.Request) {
  1082  	svcs := sortMCSServices(s.Env.MCSServices())
  1083  	writeJSON(w, svcs, req)
  1084  }
  1085  
  1086  func sortMCSServices(svcs []model.MCSServiceInfo) []model.MCSServiceInfo {
  1087  	sort.Slice(svcs, func(i, j int) bool {
  1088  		if strings.Compare(svcs[i].Cluster.String(), svcs[j].Cluster.String()) < 0 {
  1089  			return true
  1090  		}
  1091  		if strings.Compare(svcs[i].Namespace, svcs[j].Namespace) < 0 {
  1092  			return true
  1093  		}
  1094  		return strings.Compare(svcs[i].Name, svcs[j].Name) < 0
  1095  	})
  1096  	return svcs
  1097  }
  1098  
  1099  func (s *DiscoveryServer) clusterz(w http.ResponseWriter, req *http.Request) {
  1100  	if s.ListRemoteClusters == nil {
  1101  		w.WriteHeader(http.StatusBadRequest)
  1102  		return
  1103  	}
  1104  	writeJSON(w, s.ListRemoteClusters(), req)
  1105  }
  1106  
  1107  // handlePushRequest handles a ?push=true query param and triggers a push.
  1108  // A boolean response is returned to indicate if the caller should continue
  1109  func (s *DiscoveryServer) handlePushRequest(w http.ResponseWriter, req *http.Request) bool {
  1110  	if err := req.ParseForm(); err != nil {
  1111  		w.WriteHeader(http.StatusBadRequest)
  1112  		_, _ = w.Write([]byte("Failed to parse request\n"))
  1113  		return true
  1114  	}
  1115  	if req.Form.Get("push") != "" {
  1116  		AdsPushAll(s)
  1117  		_, _ = fmt.Fprintf(w, "Pushed to %d servers\n", s.adsClientCount())
  1118  		return true
  1119  	}
  1120  	return false
  1121  }
  1122  
  1123  // getDebugConnection fetches the Connection requested by proxyID
  1124  func (s *DiscoveryServer) getDebugConnection(req *http.Request) (string, *Connection) {
  1125  	if proxyID := req.URL.Query().Get("proxyID"); proxyID != "" {
  1126  		return proxyID, s.getProxyConnection(proxyID)
  1127  	}
  1128  	return "", nil
  1129  }
  1130  
  1131  func (s *DiscoveryServer) errorHandler(w http.ResponseWriter, proxyID string, con *Connection) {
  1132  	if proxyID == "" {
  1133  		w.WriteHeader(http.StatusBadRequest)
  1134  		_, _ = w.Write([]byte("You must provide a proxyID in the query string\n"))
  1135  		return
  1136  	}
  1137  	if con == nil {
  1138  		// We can't guarantee the Pilot we are connected to has a connection to the proxy we requested
  1139  		// There isn't a great way around this, but for debugging purposes its suitable to have the caller retry.
  1140  		w.WriteHeader(http.StatusNotFound)
  1141  		_, _ = w.Write([]byte("Proxy not connected to this Pilot instance. It may be connected to another instance.\n"))
  1142  		return
  1143  	}
  1144  }
  1145  
  1146  // jsonMarshalProto wraps a proto.Message so it can be marshaled with the standard encoding/json library
  1147  type jsonMarshalProto struct {
  1148  	proto.Message
  1149  }
  1150  
  1151  func (p jsonMarshalProto) MarshalJSON() ([]byte, error) {
  1152  	return protomarshal.Marshal(p.Message)
  1153  }
  1154  
  1155  // writeJSON writes a json payload, handling content type, marshaling, and errors
  1156  func writeJSON(w http.ResponseWriter, obj any, req *http.Request) {
  1157  	w.Header().Set("Content-Type", "application/json")
  1158  	var b []byte
  1159  	var err error
  1160  	if req.URL.Query().Has("pretty") {
  1161  		b, err = config.ToPrettyJSON(obj)
  1162  	} else {
  1163  		b, err = config.ToJSON(obj)
  1164  	}
  1165  	if err != nil {
  1166  		w.WriteHeader(http.StatusInternalServerError)
  1167  		_, _ = w.Write([]byte(err.Error()))
  1168  		return
  1169  	}
  1170  	_, err = w.Write(b)
  1171  	if err != nil {
  1172  		w.WriteHeader(http.StatusInternalServerError)
  1173  	}
  1174  }
  1175  
  1176  // handleHTTPError writes an error message to the response
  1177  func handleHTTPError(w http.ResponseWriter, err error) {
  1178  	w.WriteHeader(http.StatusInternalServerError)
  1179  	_, _ = w.Write([]byte(err.Error()))
  1180  }