github.com/psiphon-Labs/psiphon-tunnel-core@v2.0.28+incompatible/psiphon/serverApi.go (about)

     1  /*
     2   * Copyright (c) 2015, Psiphon Inc.
     3   * All rights reserved.
     4   *
     5   * This program is free software: you can redistribute it and/or modify
     6   * it under the terms of the GNU General Public License as published by
     7   * the Free Software Foundation, either version 3 of the License, or
     8   * (at your option) any later version.
     9   *
    10   * This program is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU General Public License for more details.
    14   *
    15   * You should have received a copy of the GNU General Public License
    16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17   *
    18   */
    19  
    20  package psiphon
    21  
    22  import (
    23  	"bytes"
    24  	"context"
    25  	"encoding/base64"
    26  	"encoding/hex"
    27  	"encoding/json"
    28  	"fmt"
    29  	"io"
    30  	"io/ioutil"
    31  	"net"
    32  	"net/http"
    33  	"net/url"
    34  	"strconv"
    35  	"strings"
    36  	"time"
    37  
    38  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
    39  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/buildinfo"
    40  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
    41  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/fragmentor"
    42  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters"
    43  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng"
    44  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
    45  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/tactics"
    46  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/transferstats"
    47  )
    48  
    49  // ServerContext is a utility struct which holds all of the data associated
    50  // with a Psiphon server connection. In addition to the established tunnel, this
    51  // includes data and transport mechanisms for Psiphon API requests. Legacy servers
    52  // offer the Psiphon API through a web service; newer servers offer the Psiphon
    53  // API through SSH requests made directly through the tunnel's SSH client.
    54  type ServerContext struct {
    55  	tunnel                   *Tunnel
    56  	psiphonHttpsClient       *http.Client
    57  	statsRegexps             *transferstats.Regexps
    58  	clientUpgradeVersion     string
    59  	serverHandshakeTimestamp string
    60  	paddingPRNG              *prng.PRNG
    61  }
    62  
    63  // MakeSessionId creates a new session ID. The same session ID is used across
    64  // multi-tunnel controller runs, where each tunnel has its own ServerContext
    65  // instance.
    66  // In server-side stats, we now consider a "session" to be the lifetime of the
    67  // Controller (e.g., the user's commanded start and stop) and we measure this
    68  // duration as well as the duration of each tunnel within the session.
    69  func MakeSessionId() (string, error) {
    70  	randomId, err := common.MakeSecureRandomBytes(protocol.PSIPHON_API_CLIENT_SESSION_ID_LENGTH)
    71  	if err != nil {
    72  		return "", errors.Trace(err)
    73  	}
    74  	return hex.EncodeToString(randomId), nil
    75  }
    76  
    77  // NewServerContext makes the tunneled handshake request to the Psiphon server
    78  // and returns a ServerContext struct for use with subsequent Psiphon server API
    79  // requests (e.g., periodic connected and status requests).
    80  func NewServerContext(tunnel *Tunnel) (*ServerContext, error) {
    81  
    82  	// For legacy servers, set up psiphonHttpsClient for
    83  	// accessing the Psiphon API via the web service.
    84  	var psiphonHttpsClient *http.Client
    85  	if !tunnel.dialParams.ServerEntry.SupportsSSHAPIRequests() ||
    86  		tunnel.config.TargetApiProtocol == protocol.PSIPHON_WEB_API_PROTOCOL {
    87  
    88  		var err error
    89  		psiphonHttpsClient, err = makePsiphonHttpsClient(tunnel)
    90  		if err != nil {
    91  			return nil, errors.Trace(err)
    92  		}
    93  	}
    94  
    95  	serverContext := &ServerContext{
    96  		tunnel:             tunnel,
    97  		psiphonHttpsClient: psiphonHttpsClient,
    98  		paddingPRNG:        prng.NewPRNGWithSeed(tunnel.dialParams.APIRequestPaddingSeed),
    99  	}
   100  
   101  	ignoreRegexps := tunnel.config.GetParameters().Get().Bool(
   102  		parameters.IgnoreHandshakeStatsRegexps)
   103  
   104  	err := serverContext.doHandshakeRequest(ignoreRegexps)
   105  	if err != nil {
   106  		return nil, errors.Trace(err)
   107  	}
   108  
   109  	return serverContext, nil
   110  }
   111  
   112  // doHandshakeRequest performs the "handshake" API request. The handshake
   113  // returns upgrade info, newly discovered server entries -- which are
   114  // stored -- and sponsor info (home pages, stat regexes).
   115  func (serverContext *ServerContext) doHandshakeRequest(
   116  	ignoreStatsRegexps bool) error {
   117  
   118  	params := serverContext.getBaseAPIParameters(baseParametersAll)
   119  
   120  	// The server will return a signed copy of its own server entry when the
   121  	// client specifies this 'missing_server_entry_signature' parameter.
   122  	//
   123  	// The purpose of this mechanism is to rapidly upgrade client local storage
   124  	// from unsigned to signed server entries, and to ensure that the client has
   125  	// a signed server entry for its currently connected server as required for
   126  	// the client-to-client exchange feature.
   127  	//
   128  	// The server entry will be included in handshakeResponse.EncodedServerList,
   129  	// along side discovery servers.
   130  	requestedMissingSignature := false
   131  	if !serverContext.tunnel.dialParams.ServerEntry.HasSignature() {
   132  		requestedMissingSignature = true
   133  		params["missing_server_entry_signature"] =
   134  			serverContext.tunnel.dialParams.ServerEntry.Tag
   135  	}
   136  
   137  	doTactics := !serverContext.tunnel.config.DisableTactics
   138  
   139  	networkID := ""
   140  	if doTactics {
   141  
   142  		// Limitation: it is assumed that the network ID obtained here is the
   143  		// one that is active when the handshake request is received by the
   144  		// server. However, it is remotely possible to switch networks
   145  		// immediately after invoking the GetNetworkID callback and initiating
   146  		// the handshake, if the tunnel protocol is meek.
   147  		//
   148  		// The response handling code below calls GetNetworkID again and ignores
   149  		// any tactics payload if the network ID is not the same. While this
   150  		// doesn't detect all cases of changing networks, it reduces the already
   151  		// narrow window.
   152  
   153  		networkID = serverContext.tunnel.config.GetNetworkID()
   154  
   155  		err := tactics.SetTacticsAPIParameters(
   156  			GetTacticsStorer(serverContext.tunnel.config),
   157  			networkID,
   158  			params)
   159  		if err != nil {
   160  			return errors.Trace(err)
   161  		}
   162  	}
   163  
   164  	// When split tunnel mode is enabled, indicate this to the server. When
   165  	// indicated, the server will perform split tunnel classifications on TCP
   166  	// port forwards and reject, with a distinct response, port forwards which
   167  	// the client should connect to directly, untunneled.
   168  	if serverContext.tunnel.config.SplitTunnelOwnRegion {
   169  		params["split_tunnel"] = "1"
   170  	}
   171  
   172  	// While regular split tunnel mode makes untunneled connections to
   173  	// destinations in the client's own country, selected split tunnel mode
   174  	// allows the client to specify a list of untunneled countries. Either or
   175  	// both modes may be enabled.
   176  	if len(serverContext.tunnel.config.SplitTunnelRegions) > 0 {
   177  		params["split_tunnel_regions"] = serverContext.tunnel.config.SplitTunnelRegions
   178  	}
   179  
   180  	var response []byte
   181  	if serverContext.psiphonHttpsClient == nil {
   182  
   183  		params[protocol.PSIPHON_API_HANDSHAKE_AUTHORIZATIONS] =
   184  			serverContext.tunnel.config.GetAuthorizations()
   185  
   186  		request, err := serverContext.makeSSHAPIRequestPayload(params)
   187  		if err != nil {
   188  			return errors.Trace(err)
   189  		}
   190  
   191  		response, err = serverContext.tunnel.SendAPIRequest(
   192  			protocol.PSIPHON_API_HANDSHAKE_REQUEST_NAME, request)
   193  		if err != nil {
   194  			return errors.Trace(err)
   195  		}
   196  
   197  	} else {
   198  
   199  		// Legacy web service API request
   200  
   201  		responseBody, err := serverContext.doGetRequest(
   202  			makeRequestUrl(serverContext.tunnel, "", "handshake", params))
   203  		if err != nil {
   204  			return errors.Trace(err)
   205  		}
   206  		// Skip legacy format lines and just parse the JSON config line
   207  		configLinePrefix := []byte("Config: ")
   208  		for _, line := range bytes.Split(responseBody, []byte("\n")) {
   209  			if bytes.HasPrefix(line, configLinePrefix) {
   210  				response = line[len(configLinePrefix):]
   211  				break
   212  			}
   213  		}
   214  		if len(response) == 0 {
   215  			return errors.TraceNew("no config line found")
   216  		}
   217  	}
   218  
   219  	// Legacy fields:
   220  	// - 'preemptive_reconnect_lifetime_milliseconds' is unused and ignored
   221  	// - 'ssh_session_id' is ignored; client session ID is used instead
   222  
   223  	var handshakeResponse protocol.HandshakeResponse
   224  
   225  	// Initialize these fields to distinguish between psiphond omitting values in
   226  	// the response and the zero value, which means unlimited rate.
   227  	handshakeResponse.UpstreamBytesPerSecond = -1
   228  	handshakeResponse.DownstreamBytesPerSecond = -1
   229  
   230  	err := json.Unmarshal(response, &handshakeResponse)
   231  	if err != nil {
   232  		return errors.Trace(err)
   233  	}
   234  
   235  	if serverContext.tunnel.config.EmitClientAddress {
   236  		NoticeClientAddress(handshakeResponse.ClientAddress)
   237  	}
   238  
   239  	NoticeClientRegion(handshakeResponse.ClientRegion)
   240  
   241  	// Emit a SplitTunnelRegions notice indicating active split tunnel region.
   242  	// For SplitTunnelOwnRegion, the handshake ClientRegion is the split
   243  	// tunnel region and this region is always listed first.
   244  
   245  	splitTunnelRegions := []string{}
   246  	if serverContext.tunnel.config.SplitTunnelOwnRegion {
   247  		splitTunnelRegions = []string{handshakeResponse.ClientRegion}
   248  	}
   249  	for _, region := range serverContext.tunnel.config.SplitTunnelRegions {
   250  		if !serverContext.tunnel.config.SplitTunnelOwnRegion ||
   251  			region != handshakeResponse.ClientRegion {
   252  
   253  			splitTunnelRegions = append(splitTunnelRegions, region)
   254  		}
   255  	}
   256  	if len(splitTunnelRegions) > 0 {
   257  		NoticeSplitTunnelRegions(splitTunnelRegions)
   258  	}
   259  
   260  	var serverEntries []protocol.ServerEntryFields
   261  
   262  	// Store discovered server entries
   263  	// We use the server's time, as it's available here, for the server entry
   264  	// timestamp since this is more reliable than the client time.
   265  	for _, encodedServerEntry := range handshakeResponse.EncodedServerList {
   266  
   267  		serverEntryFields, err := protocol.DecodeServerEntryFields(
   268  			encodedServerEntry,
   269  			common.TruncateTimestampToHour(handshakeResponse.ServerTimestamp),
   270  			protocol.SERVER_ENTRY_SOURCE_DISCOVERY)
   271  		if err != nil {
   272  			return errors.Trace(err)
   273  		}
   274  
   275  		// Retain the original timestamp and source in the requestedMissingSignature
   276  		// case, as this server entry was not discovered here.
   277  		//
   278  		// Limitation: there is a transient edge case where
   279  		// requestedMissingSignature will be set for a discovery server entry that
   280  		// _is_ also discovered here.
   281  		if requestedMissingSignature &&
   282  			serverEntryFields.GetIPAddress() == serverContext.tunnel.dialParams.ServerEntry.IpAddress {
   283  
   284  			serverEntryFields.SetLocalTimestamp(serverContext.tunnel.dialParams.ServerEntry.LocalTimestamp)
   285  			serverEntryFields.SetLocalSource(serverContext.tunnel.dialParams.ServerEntry.LocalSource)
   286  		}
   287  
   288  		err = protocol.ValidateServerEntryFields(serverEntryFields)
   289  		if err != nil {
   290  			// Skip this entry and continue with the next one
   291  			NoticeWarning("invalid handshake server entry: %s", err)
   292  			continue
   293  		}
   294  
   295  		serverEntries = append(serverEntries, serverEntryFields)
   296  	}
   297  
   298  	err = StoreServerEntries(
   299  		serverContext.tunnel.config,
   300  		serverEntries,
   301  		true)
   302  	if err != nil {
   303  		return errors.Trace(err)
   304  	}
   305  
   306  	NoticeHomepages(handshakeResponse.Homepages)
   307  
   308  	serverContext.clientUpgradeVersion = handshakeResponse.UpgradeClientVersion
   309  	if handshakeResponse.UpgradeClientVersion != "" {
   310  		NoticeClientUpgradeAvailable(handshakeResponse.UpgradeClientVersion)
   311  	} else {
   312  		NoticeClientIsLatestVersion("")
   313  	}
   314  
   315  	if !ignoreStatsRegexps {
   316  
   317  		// The handshake returns page_view_regexes and https_request_regexes.
   318  		// page_view_regexes is obsolete and not used. https_request_regexes, which
   319  		// are actually host/domain name regexes, are used for host/domain name
   320  		// bytes transferred metrics: tunneled traffic TLS SNI server names and HTTP
   321  		// Host header host names are matched against these regexes to select flows
   322  		// for bytes transferred counting.
   323  
   324  		var regexpsNotices []string
   325  		serverContext.statsRegexps, regexpsNotices = transferstats.MakeRegexps(
   326  			handshakeResponse.HttpsRequestRegexes)
   327  
   328  		for _, notice := range regexpsNotices {
   329  			NoticeWarning(notice)
   330  		}
   331  	}
   332  
   333  	diagnosticID := serverContext.tunnel.dialParams.ServerEntry.GetDiagnosticID()
   334  
   335  	serverContext.serverHandshakeTimestamp = handshakeResponse.ServerTimestamp
   336  	NoticeServerTimestamp(diagnosticID, serverContext.serverHandshakeTimestamp)
   337  
   338  	NoticeActiveAuthorizationIDs(diagnosticID, handshakeResponse.ActiveAuthorizationIDs)
   339  
   340  	NoticeTrafficRateLimits(
   341  		diagnosticID,
   342  		handshakeResponse.UpstreamBytesPerSecond,
   343  		handshakeResponse.DownstreamBytesPerSecond)
   344  
   345  	if doTactics && handshakeResponse.TacticsPayload != nil &&
   346  		networkID == serverContext.tunnel.config.GetNetworkID() {
   347  
   348  		var payload *tactics.Payload
   349  		err := json.Unmarshal(handshakeResponse.TacticsPayload, &payload)
   350  		if err != nil {
   351  			return errors.Trace(err)
   352  		}
   353  
   354  		// handshakeResponse.TacticsPayload may be "null", and payload
   355  		// will successfully unmarshal as nil. As a result, the previous
   356  		// handshakeResponse.TacticsPayload != nil test is insufficient.
   357  		if payload != nil {
   358  
   359  			tacticsRecord, err := tactics.HandleTacticsPayload(
   360  				GetTacticsStorer(serverContext.tunnel.config),
   361  				networkID,
   362  				payload)
   363  			if err != nil {
   364  				return errors.Trace(err)
   365  			}
   366  
   367  			if tacticsRecord != nil &&
   368  				prng.FlipWeightedCoin(tacticsRecord.Tactics.Probability) {
   369  
   370  				err := serverContext.tunnel.config.SetParameters(
   371  					tacticsRecord.Tag, true, tacticsRecord.Tactics.Parameters)
   372  				if err != nil {
   373  					NoticeInfo("apply handshake tactics failed: %s", err)
   374  				}
   375  				// The error will be due to invalid tactics values
   376  				// from the server. When SetParameters fails, all
   377  				// previous tactics values are left in place.
   378  			}
   379  		}
   380  	}
   381  
   382  	return nil
   383  }
   384  
   385  // DoConnectedRequest performs the "connected" API request. This request is
   386  // used for statistics, including unique user counting; reporting the full
   387  // tunnel establishment duration including the handshake request; and updated
   388  // fragmentor metrics.
   389  //
   390  // Users are not assigned identifiers. Instead, daily unique users are
   391  // calculated by having clients submit their last connected timestamp
   392  // (truncated to an hour, as a privacy measure). As client clocks are
   393  // unreliable, the server returns new last_connected values for the client to
   394  // store and send next time it connects.
   395  func (serverContext *ServerContext) DoConnectedRequest() error {
   396  
   397  	// Limitation: as currently implemented, the last_connected exchange isn't a
   398  	// distributed, atomic operation. When clients send the connected request,
   399  	// the server may receive the request, count a unique user based on the
   400  	// client's last_connected, and then the tunnel fails before the client
   401  	// receives the response, so the client will not update its last_connected
   402  	// value and submit the same one again, resulting in an inflated unique user
   403  	// count.
   404  	//
   405  	// The SetInFlightConnectedRequest mechanism mitigates one class of connected
   406  	// request interruption, a commanded shutdown in the middle of a connected
   407  	// request, by allowing some time for the request to complete before
   408  	// terminating the tunnel.
   409  	//
   410  	// TODO: consider extending the connected request protocol with additional
   411  	// "acknowledgment" messages so that the server does not commit its unique
   412  	// user count until after the client has acknowledged receipt and durable
   413  	// storage of the new last_connected value.
   414  
   415  	requestDone := make(chan struct{})
   416  	defer close(requestDone)
   417  
   418  	if !serverContext.tunnel.SetInFlightConnectedRequest(requestDone) {
   419  		return errors.TraceNew("tunnel is closing")
   420  	}
   421  	defer serverContext.tunnel.SetInFlightConnectedRequest(nil)
   422  
   423  	params := serverContext.getBaseAPIParameters(
   424  		baseParametersOnlyUpstreamFragmentorDialParameters)
   425  
   426  	lastConnected, err := getLastConnected()
   427  	if err != nil {
   428  		return errors.Trace(err)
   429  	}
   430  
   431  	params["last_connected"] = lastConnected
   432  
   433  	// serverContext.tunnel.establishDuration is nanoseconds; report milliseconds
   434  	params["establishment_duration"] =
   435  		fmt.Sprintf("%d", serverContext.tunnel.establishDuration/time.Millisecond)
   436  
   437  	var response []byte
   438  	if serverContext.psiphonHttpsClient == nil {
   439  
   440  		request, err := serverContext.makeSSHAPIRequestPayload(params)
   441  		if err != nil {
   442  			return errors.Trace(err)
   443  		}
   444  
   445  		response, err = serverContext.tunnel.SendAPIRequest(
   446  			protocol.PSIPHON_API_CONNECTED_REQUEST_NAME, request)
   447  		if err != nil {
   448  			return errors.Trace(err)
   449  		}
   450  
   451  	} else {
   452  
   453  		// Legacy web service API request
   454  
   455  		response, err = serverContext.doGetRequest(
   456  			makeRequestUrl(serverContext.tunnel, "", "connected", params))
   457  		if err != nil {
   458  			return errors.Trace(err)
   459  		}
   460  	}
   461  
   462  	var connectedResponse protocol.ConnectedResponse
   463  	err = json.Unmarshal(response, &connectedResponse)
   464  	if err != nil {
   465  		return errors.Trace(err)
   466  	}
   467  
   468  	err = SetKeyValue(
   469  		datastoreLastConnectedKey, connectedResponse.ConnectedTimestamp)
   470  	if err != nil {
   471  		return errors.Trace(err)
   472  	}
   473  
   474  	return nil
   475  }
   476  
   477  func getLastConnected() (string, error) {
   478  	lastConnected, err := GetKeyValue(datastoreLastConnectedKey)
   479  	if err != nil {
   480  		return "", errors.Trace(err)
   481  	}
   482  	if lastConnected == "" {
   483  		lastConnected = "None"
   484  	}
   485  	return lastConnected, nil
   486  }
   487  
   488  // StatsRegexps gets the Regexps used for the statistics for this tunnel.
   489  func (serverContext *ServerContext) StatsRegexps() *transferstats.Regexps {
   490  	return serverContext.statsRegexps
   491  }
   492  
   493  // DoStatusRequest makes a "status" API request to the server, sending session stats.
   494  func (serverContext *ServerContext) DoStatusRequest(tunnel *Tunnel) error {
   495  
   496  	params := serverContext.getBaseAPIParameters(baseParametersNoDialParameters)
   497  
   498  	// Note: ensure putBackStatusRequestPayload is called, to replace
   499  	// payload for future attempt, in all failure cases.
   500  
   501  	statusPayload, statusPayloadInfo, err := makeStatusRequestPayload(
   502  		serverContext.tunnel.config,
   503  		tunnel.dialParams.ServerEntry.IpAddress)
   504  	if err != nil {
   505  		return errors.Trace(err)
   506  	}
   507  
   508  	// Skip the request when there's no payload to send.
   509  
   510  	if len(statusPayload) == 0 {
   511  		return nil
   512  	}
   513  
   514  	var response []byte
   515  
   516  	if serverContext.psiphonHttpsClient == nil {
   517  
   518  		rawMessage := json.RawMessage(statusPayload)
   519  		params["statusData"] = &rawMessage
   520  
   521  		var request []byte
   522  		request, err = serverContext.makeSSHAPIRequestPayload(params)
   523  
   524  		if err == nil {
   525  			response, err = serverContext.tunnel.SendAPIRequest(
   526  				protocol.PSIPHON_API_STATUS_REQUEST_NAME, request)
   527  		}
   528  
   529  	} else {
   530  
   531  		// Legacy web service API request
   532  		response, err = serverContext.doPostRequest(
   533  			makeRequestUrl(serverContext.tunnel, "", "status", params),
   534  			"application/json",
   535  			bytes.NewReader(statusPayload))
   536  	}
   537  
   538  	if err != nil {
   539  
   540  		// Resend the transfer stats and tunnel stats later
   541  		// Note: potential duplicate reports if the server received and processed
   542  		// the request but the client failed to receive the response.
   543  		putBackStatusRequestPayload(statusPayloadInfo)
   544  
   545  		return errors.Trace(err)
   546  	}
   547  
   548  	confirmStatusRequestPayload(statusPayloadInfo)
   549  
   550  	var statusResponse protocol.StatusResponse
   551  	err = json.Unmarshal(response, &statusResponse)
   552  	if err != nil {
   553  		return errors.Trace(err)
   554  	}
   555  
   556  	for _, serverEntryTag := range statusResponse.InvalidServerEntryTags {
   557  		PruneServerEntry(serverContext.tunnel.config, serverEntryTag)
   558  	}
   559  
   560  	return nil
   561  }
   562  
   563  // statusRequestPayloadInfo is a temporary structure for data used to
   564  // either "clear" or "put back" status request payload data depending
   565  // on whether or not the request succeeded.
   566  type statusRequestPayloadInfo struct {
   567  	serverId        string
   568  	transferStats   *transferstats.AccumulatedStats
   569  	persistentStats map[string][][]byte
   570  }
   571  
   572  func makeStatusRequestPayload(
   573  	config *Config,
   574  	serverId string) ([]byte, *statusRequestPayloadInfo, error) {
   575  
   576  	transferStats := transferstats.TakeOutStatsForServer(serverId)
   577  	hostBytes := transferStats.GetStatsForStatusRequest()
   578  
   579  	persistentStats, err := TakeOutUnreportedPersistentStats(config)
   580  	if err != nil {
   581  		NoticeWarning(
   582  			"TakeOutUnreportedPersistentStats failed: %s", errors.Trace(err))
   583  		persistentStats = nil
   584  		// Proceed with transferStats only
   585  	}
   586  
   587  	if len(hostBytes) == 0 && len(persistentStats) == 0 {
   588  		// There is no payload to send.
   589  		return nil, nil, nil
   590  	}
   591  
   592  	payloadInfo := &statusRequestPayloadInfo{
   593  		serverId, transferStats, persistentStats}
   594  
   595  	payload := make(map[string]interface{})
   596  
   597  	payload["host_bytes"] = hostBytes
   598  
   599  	// We're not recording these fields, but legacy servers require them.
   600  	payload["bytes_transferred"] = 0
   601  	payload["page_views"] = make([]string, 0)
   602  	payload["https_requests"] = make([]string, 0)
   603  
   604  	persistentStatPayloadNames := make(map[string]string)
   605  	persistentStatPayloadNames[datastorePersistentStatTypeRemoteServerList] = "remote_server_list_stats"
   606  	persistentStatPayloadNames[datastorePersistentStatTypeFailedTunnel] = "failed_tunnel_stats"
   607  
   608  	for statType, stats := range persistentStats {
   609  
   610  		// Persistent stats records are already in JSON format
   611  		jsonStats := make([]json.RawMessage, len(stats))
   612  		for i, stat := range stats {
   613  			jsonStats[i] = json.RawMessage(stat)
   614  		}
   615  		payload[persistentStatPayloadNames[statType]] = jsonStats
   616  	}
   617  
   618  	jsonPayload, err := json.Marshal(payload)
   619  	if err != nil {
   620  
   621  		// Send the transfer stats and tunnel stats later
   622  		putBackStatusRequestPayload(payloadInfo)
   623  
   624  		return nil, nil, errors.Trace(err)
   625  	}
   626  
   627  	return jsonPayload, payloadInfo, nil
   628  }
   629  
   630  func putBackStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
   631  	transferstats.PutBackStatsForServer(
   632  		payloadInfo.serverId, payloadInfo.transferStats)
   633  	err := PutBackUnreportedPersistentStats(payloadInfo.persistentStats)
   634  	if err != nil {
   635  		// These persistent stats records won't be resent until after a
   636  		// datastore re-initialization.
   637  		NoticeWarning(
   638  			"PutBackUnreportedPersistentStats failed: %s", errors.Trace(err))
   639  	}
   640  }
   641  
   642  func confirmStatusRequestPayload(payloadInfo *statusRequestPayloadInfo) {
   643  	err := ClearReportedPersistentStats(payloadInfo.persistentStats)
   644  	if err != nil {
   645  		// These persistent stats records may be resent.
   646  		NoticeWarning(
   647  			"ClearReportedPersistentStats failed: %s", errors.Trace(err))
   648  	}
   649  }
   650  
   651  // RecordRemoteServerListStat records a completed common or OSL remote server
   652  // list resource download.
   653  //
   654  // The RSL download event could occur when the client is unable to immediately
   655  // send a status request to a server, so these records are stored in the
   656  // persistent datastore and reported via subsequent status requests sent to
   657  // any Psiphon server.
   658  //
   659  // Note that some common event field values may change between the stat
   660  // recording and reporting, including client geolocation and host_id.
   661  //
   662  // The bytes/duration fields reflect the size and download time for the _last
   663  // chunk only_ in the case of a resumed download. The purpose of these fields
   664  // is to calculate rough data transfer rates. Both bytes and duration are
   665  // included in the log, to allow for filtering out of small transfers which
   666  // may not produce accurate rate numbers.
   667  //
   668  // Multiple "status" requests may be in flight at once (due to multi-tunnel,
   669  // asynchronous final status retry, and aggressive status requests for
   670  // pre-registered tunnels), To avoid duplicate reporting, persistent stats
   671  // records are "taken-out" by a status request and then "put back" in case the
   672  // request fails.
   673  //
   674  // Duplicate reporting may also occur when a server receives and processes a
   675  // status request but the client fails to receive the response.
   676  func RecordRemoteServerListStat(
   677  	config *Config,
   678  	tunneled bool,
   679  	url string,
   680  	etag string,
   681  	bytes int64,
   682  	duration time.Duration,
   683  	authenticated bool) error {
   684  
   685  	if !config.GetParameters().Get().WeightedCoinFlip(
   686  		parameters.RecordRemoteServerListPersistentStatsProbability) {
   687  		return nil
   688  	}
   689  
   690  	params := make(common.APIParameters)
   691  
   692  	params["session_id"] = config.SessionID
   693  	params["propagation_channel_id"] = config.PropagationChannelId
   694  	params["sponsor_id"] = config.GetSponsorID()
   695  	params["client_version"] = config.ClientVersion
   696  	params["client_platform"] = config.ClientPlatform
   697  	params["client_build_rev"] = buildinfo.GetBuildInfo().BuildRev
   698  	if config.DeviceRegion != "" {
   699  		params["device_region"] = config.DeviceRegion
   700  	}
   701  
   702  	params["client_download_timestamp"] = common.TruncateTimestampToHour(common.GetCurrentTimestamp())
   703  	tunneledStr := "0"
   704  	if tunneled {
   705  		tunneledStr = "1"
   706  	}
   707  	params["tunneled"] = tunneledStr
   708  	params["url"] = url
   709  	params["etag"] = etag
   710  	params["bytes"] = fmt.Sprintf("%d", bytes)
   711  
   712  	// duration is nanoseconds; report milliseconds
   713  	params["duration"] = fmt.Sprintf("%d", duration/time.Millisecond)
   714  
   715  	authenticatedStr := "0"
   716  	if authenticated {
   717  		authenticatedStr = "1"
   718  	}
   719  	params["authenticated"] = authenticatedStr
   720  
   721  	remoteServerListStatJson, err := json.Marshal(params)
   722  	if err != nil {
   723  		return errors.Trace(err)
   724  	}
   725  
   726  	return StorePersistentStat(
   727  		config, datastorePersistentStatTypeRemoteServerList, remoteServerListStatJson)
   728  }
   729  
   730  // RecordFailedTunnelStat records metrics for a failed tunnel dial, including
   731  // dial parameters and error condition (tunnelErr). No record is created when
   732  // tunnelErr is nil.
   733  //
   734  // This uses the same reporting facility, with the same caveats, as
   735  // RecordRemoteServerListStat.
   736  func RecordFailedTunnelStat(
   737  	config *Config,
   738  	dialParams *DialParameters,
   739  	livenessTestMetrics *livenessTestMetrics,
   740  	bytesUp int64,
   741  	bytesDown int64,
   742  	tunnelErr error) error {
   743  
   744  	if !config.GetParameters().Get().WeightedCoinFlip(
   745  		parameters.RecordFailedTunnelPersistentStatsProbability) {
   746  		return nil
   747  	}
   748  
   749  	// Callers should not call RecordFailedTunnelStat with a nil tunnelErr, as
   750  	// this is not a useful stat and it results in a nil pointer dereference.
   751  	// This check catches potential bug cases. An example edge case, now
   752  	// fixed, is deferred error handlers, such as the ones in in
   753  	// dialTunnel/tunnel.Activate, which may be invoked in the case of a
   754  	// panic, which can occur before any error value is returned.
   755  	if tunnelErr == nil {
   756  		return errors.TraceNew("no error")
   757  	}
   758  
   759  	lastConnected, err := getLastConnected()
   760  	if err != nil {
   761  		return errors.Trace(err)
   762  	}
   763  
   764  	params := getBaseAPIParameters(baseParametersAll, config, dialParams)
   765  
   766  	delete(params, "server_secret")
   767  	params["server_entry_tag"] = dialParams.ServerEntry.Tag
   768  	params["last_connected"] = lastConnected
   769  	params["client_failed_timestamp"] = common.TruncateTimestampToHour(common.GetCurrentTimestamp())
   770  	if livenessTestMetrics != nil {
   771  		params["liveness_test_upstream_bytes"] = strconv.Itoa(livenessTestMetrics.UpstreamBytes)
   772  		params["liveness_test_sent_upstream_bytes"] = strconv.Itoa(livenessTestMetrics.SentUpstreamBytes)
   773  		params["liveness_test_downstream_bytes"] = strconv.Itoa(livenessTestMetrics.DownstreamBytes)
   774  		params["liveness_test_received_downstream_bytes"] = strconv.Itoa(livenessTestMetrics.ReceivedDownstreamBytes)
   775  	}
   776  	if bytesUp >= 0 {
   777  		params["bytes_up"] = fmt.Sprintf("%d", bytesUp)
   778  	}
   779  	if bytesDown >= 0 {
   780  		params["bytes_down"] = fmt.Sprintf("%d", bytesDown)
   781  	}
   782  
   783  	// Ensure direct server IPs are not exposed in logs. The "net" package, and
   784  	// possibly other 3rd party packages, will include destination addresses in
   785  	// I/O error messages.
   786  	tunnelError := common.RedactIPAddressesString(tunnelErr.Error())
   787  
   788  	params["tunnel_error"] = tunnelError
   789  
   790  	failedTunnelStatJson, err := json.Marshal(params)
   791  	if err != nil {
   792  		return errors.Trace(err)
   793  	}
   794  
   795  	return StorePersistentStat(
   796  		config, datastorePersistentStatTypeFailedTunnel, failedTunnelStatJson)
   797  }
   798  
   799  // doGetRequest makes a tunneled HTTPS request and returns the response body.
   800  func (serverContext *ServerContext) doGetRequest(
   801  	requestUrl string) (responseBody []byte, err error) {
   802  
   803  	request, err := http.NewRequest("GET", requestUrl, nil)
   804  	if err != nil {
   805  		return nil, errors.Trace(err)
   806  	}
   807  
   808  	request.Header.Set("User-Agent", MakePsiphonUserAgent(serverContext.tunnel.config))
   809  
   810  	response, err := serverContext.psiphonHttpsClient.Do(request)
   811  	if err == nil && response.StatusCode != http.StatusOK {
   812  		response.Body.Close()
   813  		err = fmt.Errorf("HTTP GET request failed with response code: %d", response.StatusCode)
   814  	}
   815  	if err != nil {
   816  		// Trim this error since it may include long URLs
   817  		return nil, errors.Trace(TrimError(err))
   818  	}
   819  	defer response.Body.Close()
   820  	body, err := ioutil.ReadAll(response.Body)
   821  	if err != nil {
   822  		return nil, errors.Trace(err)
   823  	}
   824  	return body, nil
   825  }
   826  
   827  // doPostRequest makes a tunneled HTTPS POST request.
   828  func (serverContext *ServerContext) doPostRequest(
   829  	requestUrl string, bodyType string, body io.Reader) (responseBody []byte, err error) {
   830  
   831  	request, err := http.NewRequest("POST", requestUrl, body)
   832  	if err != nil {
   833  		return nil, errors.Trace(err)
   834  	}
   835  
   836  	request.Header.Set("User-Agent", MakePsiphonUserAgent(serverContext.tunnel.config))
   837  	request.Header.Set("Content-Type", bodyType)
   838  
   839  	response, err := serverContext.psiphonHttpsClient.Do(request)
   840  	if err == nil && response.StatusCode != http.StatusOK {
   841  		response.Body.Close()
   842  		err = fmt.Errorf("HTTP POST request failed with response code: %d", response.StatusCode)
   843  	}
   844  	if err != nil {
   845  		// Trim this error since it may include long URLs
   846  		return nil, errors.Trace(TrimError(err))
   847  	}
   848  	defer response.Body.Close()
   849  	responseBody, err = ioutil.ReadAll(response.Body)
   850  	if err != nil {
   851  		return nil, errors.Trace(err)
   852  	}
   853  	return responseBody, nil
   854  }
   855  
   856  // makeSSHAPIRequestPayload makes a JSON payload for an SSH API request.
   857  func (serverContext *ServerContext) makeSSHAPIRequestPayload(
   858  	params common.APIParameters) ([]byte, error) {
   859  	jsonPayload, err := json.Marshal(params)
   860  	if err != nil {
   861  		return nil, errors.Trace(err)
   862  	}
   863  	return jsonPayload, nil
   864  }
   865  
   866  type baseParametersFilter int
   867  
   868  const (
   869  	baseParametersAll baseParametersFilter = iota
   870  	baseParametersOnlyUpstreamFragmentorDialParameters
   871  	baseParametersNoDialParameters
   872  )
   873  
   874  func (serverContext *ServerContext) getBaseAPIParameters(
   875  	filter baseParametersFilter) common.APIParameters {
   876  
   877  	params := getBaseAPIParameters(
   878  		filter,
   879  		serverContext.tunnel.config,
   880  		serverContext.tunnel.dialParams)
   881  
   882  	// Add a random amount of padding to defend against API call traffic size
   883  	// fingerprints. The "pad_response" field instructs the server to pad its
   884  	// response accordingly.
   885  
   886  	p := serverContext.tunnel.config.GetParameters().Get()
   887  	minUpstreamPadding := p.Int(parameters.APIRequestUpstreamPaddingMinBytes)
   888  	maxUpstreamPadding := p.Int(parameters.APIRequestUpstreamPaddingMaxBytes)
   889  	minDownstreamPadding := p.Int(parameters.APIRequestDownstreamPaddingMinBytes)
   890  	maxDownstreamPadding := p.Int(parameters.APIRequestDownstreamPaddingMaxBytes)
   891  
   892  	if maxUpstreamPadding > 0 {
   893  		size := serverContext.paddingPRNG.Range(minUpstreamPadding, maxUpstreamPadding)
   894  		params["padding"] = strings.Repeat(" ", size)
   895  	}
   896  
   897  	if maxDownstreamPadding > 0 {
   898  		size := serverContext.paddingPRNG.Range(minDownstreamPadding, maxDownstreamPadding)
   899  		params["pad_response"] = strconv.Itoa(size)
   900  	}
   901  
   902  	return params
   903  }
   904  
   905  // getBaseAPIParameters returns all the common API parameters that are
   906  // included with each Psiphon API request. These common parameters are used
   907  // for metrics.
   908  func getBaseAPIParameters(
   909  	filter baseParametersFilter,
   910  	config *Config,
   911  	dialParams *DialParameters) common.APIParameters {
   912  
   913  	params := make(common.APIParameters)
   914  
   915  	params["session_id"] = config.SessionID
   916  	params["client_session_id"] = config.SessionID
   917  	params["server_secret"] = dialParams.ServerEntry.WebServerSecret
   918  	params["propagation_channel_id"] = config.PropagationChannelId
   919  	params["sponsor_id"] = config.GetSponsorID()
   920  	params["client_version"] = config.ClientVersion
   921  	params["client_platform"] = config.ClientPlatform
   922  	params["client_features"] = config.clientFeatures
   923  	params["client_build_rev"] = buildinfo.GetBuildInfo().BuildRev
   924  
   925  	// Blank parameters must be omitted.
   926  
   927  	if config.DeviceRegion != "" {
   928  		params["device_region"] = config.DeviceRegion
   929  	}
   930  
   931  	if filter == baseParametersAll {
   932  
   933  		params["relay_protocol"] = dialParams.TunnelProtocol
   934  		params["network_type"] = dialParams.GetNetworkType()
   935  
   936  		if dialParams.BPFProgramName != "" {
   937  			params["client_bpf"] = dialParams.BPFProgramName
   938  		}
   939  
   940  		if dialParams.SelectedSSHClientVersion {
   941  			params["ssh_client_version"] = dialParams.SSHClientVersion
   942  		}
   943  
   944  		if dialParams.UpstreamProxyType != "" {
   945  			params["upstream_proxy_type"] = dialParams.UpstreamProxyType
   946  		}
   947  
   948  		if dialParams.UpstreamProxyCustomHeaderNames != nil {
   949  			params["upstream_proxy_custom_header_names"] = dialParams.UpstreamProxyCustomHeaderNames
   950  		}
   951  
   952  		if dialParams.FrontingProviderID != "" {
   953  			params["fronting_provider_id"] = dialParams.FrontingProviderID
   954  		}
   955  
   956  		if dialParams.MeekDialAddress != "" {
   957  			params["meek_dial_address"] = dialParams.MeekDialAddress
   958  		}
   959  
   960  		if protocol.TunnelProtocolUsesFrontedMeek(dialParams.TunnelProtocol) {
   961  			meekResolvedIPAddress := dialParams.MeekResolvedIPAddress.Load().(string)
   962  			if meekResolvedIPAddress != "" {
   963  				params["meek_resolved_ip_address"] = meekResolvedIPAddress
   964  			}
   965  		}
   966  
   967  		if dialParams.MeekSNIServerName != "" {
   968  			params["meek_sni_server_name"] = dialParams.MeekSNIServerName
   969  		}
   970  
   971  		if dialParams.MeekHostHeader != "" {
   972  			params["meek_host_header"] = dialParams.MeekHostHeader
   973  		}
   974  
   975  		// MeekTransformedHostName is meaningful when meek is used, which is when
   976  		// MeekDialAddress != ""
   977  		if dialParams.MeekDialAddress != "" {
   978  			transformedHostName := "0"
   979  			if dialParams.MeekTransformedHostName {
   980  				transformedHostName = "1"
   981  			}
   982  			params["meek_transformed_host_name"] = transformedHostName
   983  		}
   984  
   985  		if dialParams.SelectedUserAgent {
   986  			params["user_agent"] = dialParams.UserAgent
   987  		}
   988  
   989  		if dialParams.SelectedTLSProfile {
   990  			params["tls_profile"] = dialParams.TLSProfile
   991  			params["tls_version"] = dialParams.GetTLSVersionForMetrics()
   992  		}
   993  
   994  		if dialParams.ServerEntry.Region != "" {
   995  			params["server_entry_region"] = dialParams.ServerEntry.Region
   996  		}
   997  
   998  		if dialParams.ServerEntry.LocalSource != "" {
   999  			params["server_entry_source"] = dialParams.ServerEntry.LocalSource
  1000  		}
  1001  
  1002  		// As with last_connected, this timestamp stat, which may be a precise
  1003  		// handshake request server timestamp, is truncated to hour granularity to
  1004  		// avoid introducing a reconstructable cross-session user trace into server
  1005  		// logs.
  1006  		localServerEntryTimestamp := common.TruncateTimestampToHour(
  1007  			dialParams.ServerEntry.LocalTimestamp)
  1008  		if localServerEntryTimestamp != "" {
  1009  			params["server_entry_timestamp"] = localServerEntryTimestamp
  1010  		}
  1011  
  1012  		params[tactics.APPLIED_TACTICS_TAG_PARAMETER_NAME] =
  1013  			config.GetParameters().Get().Tag()
  1014  
  1015  		if dialParams.DialPortNumber != "" {
  1016  			params["dial_port_number"] = dialParams.DialPortNumber
  1017  		}
  1018  
  1019  		if dialParams.QUICVersion != "" {
  1020  			params["quic_version"] = dialParams.QUICVersion
  1021  		}
  1022  
  1023  		if dialParams.QUICDialSNIAddress != "" {
  1024  			params["quic_dial_sni_address"] = dialParams.QUICDialSNIAddress
  1025  		}
  1026  
  1027  		if dialParams.QUICDisablePathMTUDiscovery {
  1028  			params["quic_disable_client_path_mtu_discovery"] = "1"
  1029  		}
  1030  
  1031  		isReplay := "0"
  1032  		if dialParams.IsReplay {
  1033  			isReplay = "1"
  1034  		}
  1035  		params["is_replay"] = isReplay
  1036  
  1037  		if config.EgressRegion != "" {
  1038  			params["egress_region"] = config.EgressRegion
  1039  		}
  1040  
  1041  		// dialParams.DialDuration is nanoseconds; report milliseconds
  1042  		params["dial_duration"] = fmt.Sprintf("%d", dialParams.DialDuration/time.Millisecond)
  1043  
  1044  		params["candidate_number"] = strconv.Itoa(dialParams.CandidateNumber)
  1045  
  1046  		params["established_tunnels_count"] = strconv.Itoa(dialParams.EstablishedTunnelsCount)
  1047  
  1048  		if dialParams.NetworkLatencyMultiplier != 0.0 {
  1049  			params["network_latency_multiplier"] =
  1050  				fmt.Sprintf("%f", dialParams.NetworkLatencyMultiplier)
  1051  		}
  1052  
  1053  		if dialParams.ConjureTransport != "" {
  1054  			params["conjure_transport"] = dialParams.ConjureTransport
  1055  		}
  1056  
  1057  		if dialParams.ResolveParameters != nil {
  1058  
  1059  			if dialParams.ResolveParameters.PreresolvedIPAddress != "" {
  1060  				params["dns_preresolved"] = dialParams.ResolveParameters.PreresolvedIPAddress
  1061  
  1062  			} else {
  1063  
  1064  				// Log enough information to distinguish several successful or
  1065  				// failed circumvention cases of interest, including preferring
  1066  				// alternate servers and/or using DNS protocol transforms, and
  1067  				// appropriate for both handshake and failed_tunnel logging:
  1068  				//
  1069  				// - The initial attempt made by Resolver.ResolveIP,
  1070  				//   preferring an alternate DNS server and/or using a
  1071  				//   protocol transform succeeds (dns_result = 0, the initial
  1072  				//   attempt, 0, got the first result).
  1073  				//
  1074  				// - A second attempt may be used, still preferring an
  1075  				//   alternate DNS server but no longer using the protocol
  1076  				//   transform, which presumably failed (dns_result = 1, the
  1077  				//   second attempt, 1, got the first result).
  1078  				//
  1079  				// - Subsequent attempts will use the system DNS server and no
  1080  				//   protocol transforms (dns_result > 2).
  1081  				//
  1082  				// Due to the design of Resolver.ResolveIP, the notion
  1083  				// of "success" is approximate; for example a successful
  1084  				// response may arrive after a subsequent attempt succeeds,
  1085  				// simply due to slow network conditions. It's also possible
  1086  				// that, for a given attemp, only one of the two concurrent
  1087  				// requests (A and AAAA) succeeded.
  1088  				//
  1089  				// Note that ResolveParameters.GetFirstAttemptWithAnswer
  1090  				// semantics assume that dialParams.ResolveParameters wasn't
  1091  				// used by or modified by any other dial.
  1092  
  1093  				if dialParams.ResolveParameters.PreferAlternateDNSServer {
  1094  					params["dns_preferred"] = dialParams.ResolveParameters.AlternateDNSServer
  1095  				}
  1096  
  1097  				if dialParams.ResolveParameters.ProtocolTransformName != "" {
  1098  					params["dns_transform"] = dialParams.ResolveParameters.ProtocolTransformName
  1099  				}
  1100  
  1101  				params["dns_attempt"] = strconv.Itoa(
  1102  					dialParams.ResolveParameters.GetFirstAttemptWithAnswer())
  1103  			}
  1104  		}
  1105  
  1106  		if dialParams.DialConnMetrics != nil {
  1107  			metrics := dialParams.DialConnMetrics.GetMetrics()
  1108  			for name, value := range metrics {
  1109  				params[name] = fmt.Sprintf("%v", value)
  1110  			}
  1111  		}
  1112  
  1113  		if dialParams.ObfuscatedSSHConnMetrics != nil {
  1114  			metrics := dialParams.ObfuscatedSSHConnMetrics.GetMetrics()
  1115  			for name, value := range metrics {
  1116  				params[name] = fmt.Sprintf("%v", value)
  1117  			}
  1118  		}
  1119  
  1120  	} else if filter == baseParametersOnlyUpstreamFragmentorDialParameters {
  1121  
  1122  		if dialParams.DialConnMetrics != nil {
  1123  			names := fragmentor.GetUpstreamMetricsNames()
  1124  			metrics := dialParams.DialConnMetrics.GetMetrics()
  1125  			for name, value := range metrics {
  1126  				if common.Contains(names, name) {
  1127  					params[name] = fmt.Sprintf("%v", value)
  1128  				}
  1129  			}
  1130  		}
  1131  	}
  1132  
  1133  	return params
  1134  }
  1135  
  1136  // makeRequestUrl makes a URL for a web service API request.
  1137  func makeRequestUrl(tunnel *Tunnel, port, path string, params common.APIParameters) string {
  1138  	var requestUrl bytes.Buffer
  1139  
  1140  	if port == "" {
  1141  		port = tunnel.dialParams.ServerEntry.WebServerPort
  1142  	}
  1143  
  1144  	requestUrl.WriteString("https://")
  1145  	requestUrl.WriteString(tunnel.dialParams.ServerEntry.IpAddress)
  1146  	requestUrl.WriteString(":")
  1147  	requestUrl.WriteString(port)
  1148  	requestUrl.WriteString("/")
  1149  	requestUrl.WriteString(path)
  1150  
  1151  	if len(params) > 0 {
  1152  
  1153  		queryParams := url.Values{}
  1154  
  1155  		for name, value := range params {
  1156  
  1157  			// Note: this logic skips the tactics.SPEED_TEST_SAMPLES_PARAMETER_NAME
  1158  			// parameter, which has a different type. This parameter is not recognized
  1159  			// by legacy servers.
  1160  
  1161  			switch v := value.(type) {
  1162  			case string:
  1163  				queryParams.Set(name, v)
  1164  			case []string:
  1165  				// String array param encoded as JSON
  1166  				jsonValue, err := json.Marshal(v)
  1167  				if err != nil {
  1168  					break
  1169  				}
  1170  				queryParams.Set(name, string(jsonValue))
  1171  			}
  1172  		}
  1173  
  1174  		requestUrl.WriteString("?")
  1175  		requestUrl.WriteString(queryParams.Encode())
  1176  	}
  1177  
  1178  	return requestUrl.String()
  1179  }
  1180  
  1181  // makePsiphonHttpsClient creates a Psiphon HTTPS client that tunnels web service API
  1182  // requests and which validates the web server using the Psiphon server entry web server
  1183  // certificate.
  1184  func makePsiphonHttpsClient(tunnel *Tunnel) (httpsClient *http.Client, err error) {
  1185  
  1186  	certificate, err := DecodeCertificate(
  1187  		tunnel.dialParams.ServerEntry.WebServerCertificate)
  1188  	if err != nil {
  1189  		return nil, errors.Trace(err)
  1190  	}
  1191  
  1192  	tunneledDialer := func(_ context.Context, _, addr string) (net.Conn, error) {
  1193  		// This case bypasses tunnel.Dial, to avoid its check that the tunnel is
  1194  		// already active (it won't be pre-handshake). This bypass won't handle the
  1195  		// server rejecting the port forward due to split tunnel classification, but
  1196  		// we know that the server won't classify the web API destination as
  1197  		// untunneled.
  1198  		return tunnel.sshClient.Dial("tcp", addr)
  1199  	}
  1200  
  1201  	// Note: as with SSH API requests, there no dial context here. SSH port forward dials
  1202  	// cannot be interrupted directly. Closing the tunnel will interrupt both the dial and
  1203  	// the request. While it's possible to add a timeout here, we leave it with no explicit
  1204  	// timeout which is the same as SSH API requests: if the tunnel has stalled then SSH keep
  1205  	// alives will cause the tunnel to close.
  1206  
  1207  	dialer := NewCustomTLSDialer(
  1208  		&CustomTLSConfig{
  1209  			Parameters:              tunnel.config.GetParameters(),
  1210  			Dial:                    tunneledDialer,
  1211  			VerifyLegacyCertificate: certificate,
  1212  		})
  1213  
  1214  	transport := &http.Transport{
  1215  		DialTLS: func(network, addr string) (net.Conn, error) {
  1216  			return dialer(context.Background(), network, addr)
  1217  		},
  1218  		Dial: func(network, addr string) (net.Conn, error) {
  1219  			return nil, errors.TraceNew("HTTP not supported")
  1220  		},
  1221  	}
  1222  
  1223  	return &http.Client{
  1224  		Transport: transport,
  1225  	}, nil
  1226  }
  1227  
  1228  func HandleServerRequest(
  1229  	tunnelOwner TunnelOwner, tunnel *Tunnel, name string, payload []byte) error {
  1230  
  1231  	switch name {
  1232  	case protocol.PSIPHON_API_OSL_REQUEST_NAME:
  1233  		return HandleOSLRequest(tunnelOwner, tunnel, payload)
  1234  	case protocol.PSIPHON_API_ALERT_REQUEST_NAME:
  1235  		return HandleAlertRequest(tunnelOwner, tunnel, payload)
  1236  	}
  1237  
  1238  	return errors.Tracef("invalid request name: %s", name)
  1239  }
  1240  
  1241  func HandleOSLRequest(
  1242  	tunnelOwner TunnelOwner, tunnel *Tunnel, payload []byte) error {
  1243  
  1244  	var oslRequest protocol.OSLRequest
  1245  	err := json.Unmarshal(payload, &oslRequest)
  1246  	if err != nil {
  1247  		return errors.Trace(err)
  1248  	}
  1249  
  1250  	if oslRequest.ClearLocalSLOKs {
  1251  		DeleteSLOKs()
  1252  	}
  1253  
  1254  	seededNewSLOK := false
  1255  
  1256  	for _, slok := range oslRequest.SeedPayload.SLOKs {
  1257  		duplicate, err := SetSLOK(slok.ID, slok.Key)
  1258  		if err != nil {
  1259  			// TODO: return error to trigger retry?
  1260  			NoticeWarning("SetSLOK failed: %s", errors.Trace(err))
  1261  		} else if !duplicate {
  1262  			seededNewSLOK = true
  1263  		}
  1264  
  1265  		if tunnel.config.EmitSLOKs {
  1266  			NoticeSLOKSeeded(base64.StdEncoding.EncodeToString(slok.ID), duplicate)
  1267  		}
  1268  	}
  1269  
  1270  	if seededNewSLOK {
  1271  		tunnelOwner.SignalSeededNewSLOK()
  1272  	}
  1273  
  1274  	return nil
  1275  }
  1276  
  1277  func HandleAlertRequest(
  1278  	tunnelOwner TunnelOwner, tunnel *Tunnel, payload []byte) error {
  1279  
  1280  	var alertRequest protocol.AlertRequest
  1281  	err := json.Unmarshal(payload, &alertRequest)
  1282  	if err != nil {
  1283  		return errors.Trace(err)
  1284  	}
  1285  
  1286  	if tunnel.config.EmitServerAlerts {
  1287  		NoticeServerAlert(alertRequest)
  1288  	}
  1289  
  1290  	return nil
  1291  }