github.com/psiphon-labs/psiphon-tunnel-core@v2.0.28+incompatible/psiphon/remoteServerList.go (about)

     1  /*
     2   * Copyright (c) 2015, Psiphon Inc.
     3   * All rights reserved.
     4   *
     5   * This program is free software: you can redistribute it and/or modify
     6   * it under the terms of the GNU General Public License as published by
     7   * the Free Software Foundation, either version 3 of the License, or
     8   * (at your option) any later version.
     9   *
    10   * This program is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU General Public License for more details.
    14   *
    15   * You should have received a copy of the GNU General Public License
    16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17   *
    18   */
    19  
    20  package psiphon
    21  
    22  import (
    23  	"context"
    24  	"encoding/hex"
    25  	"fmt"
    26  	"net/url"
    27  	"os"
    28  	"sync/atomic"
    29  	"time"
    30  
    31  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common"
    32  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors"
    33  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/osl"
    34  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters"
    35  	"github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol"
    36  )
    37  
    38  type RemoteServerListFetcher func(
    39  	ctx context.Context, config *Config, attempt int, tunnel *Tunnel, untunneledDialConfig *DialConfig) error
    40  
    41  // FetchCommonRemoteServerList downloads the common remote server list from
    42  // config.RemoteServerListURLs. It validates its digital signature using the
    43  // public key config.RemoteServerListSignaturePublicKey and parses the
    44  // data field into ServerEntry records.
    45  // config.GetRemoteServerListDownloadFilename() is the location to store the
    46  // download. As the download is resumed after failure, this filename must
    47  // be unique and persistent.
    48  func FetchCommonRemoteServerList(
    49  	ctx context.Context,
    50  	config *Config,
    51  	attempt int,
    52  	tunnel *Tunnel,
    53  	untunneledDialConfig *DialConfig) error {
    54  
    55  	NoticeInfo("fetching common remote server list")
    56  
    57  	p := config.GetParameters().Get()
    58  	publicKey := p.String(parameters.RemoteServerListSignaturePublicKey)
    59  	urls := p.TransferURLs(parameters.RemoteServerListURLs)
    60  	downloadTimeout := p.Duration(parameters.FetchRemoteServerListTimeout)
    61  	p.Close()
    62  
    63  	downloadURL := urls.Select(attempt)
    64  	canonicalURL := urls.CanonicalURL()
    65  
    66  	newETag, downloadStatRecorder, err := downloadRemoteServerListFile(
    67  		ctx,
    68  		config,
    69  		tunnel,
    70  		untunneledDialConfig,
    71  		downloadTimeout,
    72  		downloadURL.URL,
    73  		canonicalURL,
    74  		downloadURL.SkipVerify,
    75  		"",
    76  		config.GetRemoteServerListDownloadFilename())
    77  	if err != nil {
    78  		return errors.Tracef("failed to download common remote server list: %s", errors.Trace(err))
    79  	}
    80  
    81  	authenticatedDownload := false
    82  	if downloadStatRecorder != nil {
    83  		defer func() { downloadStatRecorder(authenticatedDownload) }()
    84  	}
    85  
    86  	// When the resource is unchanged, skip.
    87  	if newETag == "" {
    88  		return nil
    89  	}
    90  
    91  	file, err := os.Open(config.GetRemoteServerListDownloadFilename())
    92  	if err != nil {
    93  		return errors.Tracef("failed to open common remote server list: %s", errors.Trace(err))
    94  
    95  	}
    96  	defer file.Close()
    97  
    98  	serverListPayloadReader, err := common.NewAuthenticatedDataPackageReader(
    99  		file, publicKey)
   100  	if err != nil {
   101  		return errors.Tracef("failed to read remote server list: %s", errors.Trace(err))
   102  	}
   103  
   104  	// NewAuthenticatedDataPackageReader authenticates the file before returning.
   105  	authenticatedDownload = true
   106  
   107  	err = StreamingStoreServerEntries(
   108  		ctx,
   109  		config,
   110  		protocol.NewStreamingServerEntryDecoder(
   111  			serverListPayloadReader,
   112  			common.GetCurrentTimestamp(),
   113  			protocol.SERVER_ENTRY_SOURCE_REMOTE),
   114  		true)
   115  	if err != nil {
   116  		return errors.Tracef("failed to store common remote server list: %s", errors.Trace(err))
   117  	}
   118  
   119  	// Now that the server entries are successfully imported, store the response
   120  	// ETag so we won't re-download this same data again.
   121  	err = SetUrlETag(canonicalURL, newETag)
   122  	if err != nil {
   123  		NoticeWarning("failed to set ETag for common remote server list: %s", errors.Trace(err))
   124  		// This fetch is still reported as a success, even if we can't store the etag
   125  	}
   126  
   127  	return nil
   128  }
   129  
   130  // FetchObfuscatedServerLists downloads the obfuscated remote server lists
   131  // from config.ObfuscatedServerListRootURLs.
   132  // It first downloads the OSL registry, and then downloads each seeded OSL
   133  // advertised in the registry. All downloads are resumable, ETags are used
   134  // to skip both an unchanged registry or unchanged OSL files, and when an
   135  // individual download fails, the fetch proceeds if it can.
   136  // Authenticated package digital signatures are validated using the
   137  // public key config.RemoteServerListSignaturePublicKey.
   138  // config.GetObfuscatedServerListDownloadDirectory() is the location to store
   139  // the downloaded files. As  downloads are resumed after failure, this directory
   140  // must be unique and persistent.
   141  func FetchObfuscatedServerLists(
   142  	ctx context.Context,
   143  	config *Config,
   144  	attempt int,
   145  	tunnel *Tunnel,
   146  	untunneledDialConfig *DialConfig) error {
   147  
   148  	NoticeInfo("fetching obfuscated remote server lists")
   149  
   150  	p := config.GetParameters().Get()
   151  	publicKey := p.String(parameters.RemoteServerListSignaturePublicKey)
   152  	urls := p.TransferURLs(parameters.ObfuscatedServerListRootURLs)
   153  	downloadTimeout := p.Duration(parameters.FetchRemoteServerListTimeout)
   154  	p.Close()
   155  
   156  	rootURL := urls.Select(attempt)
   157  	canonicalRootURL := urls.CanonicalURL()
   158  	downloadURL := osl.GetOSLRegistryURL(rootURL.URL)
   159  	canonicalURL := osl.GetOSLRegistryURL(canonicalRootURL)
   160  
   161  	downloadFilename := osl.GetOSLRegistryFilename(config.GetObfuscatedServerListDownloadDirectory())
   162  	cachedFilename := downloadFilename + ".cached"
   163  
   164  	// If the cached registry is not present, we need to download or resume downloading
   165  	// the registry, so clear the ETag to ensure that always happens.
   166  	_, err := os.Stat(cachedFilename)
   167  	if os.IsNotExist(err) {
   168  		SetUrlETag(canonicalURL, "")
   169  	}
   170  
   171  	// failed is set if any operation fails and should trigger a retry. When the OSL registry
   172  	// fails to download, any cached registry is used instead; when any single OSL fails
   173  	// to download, the overall operation proceeds. So this flag records whether to report
   174  	// failure at the end when downloading has proceeded after a failure.
   175  	// TODO: should disk-full conditions not trigger retries?
   176  	var failed bool
   177  
   178  	// updateCache is set when modifed registry content is downloaded. Both the cached
   179  	// file and the persisted ETag will be updated in this case. The update is deferred
   180  	// until after the registry has been authenticated.
   181  	updateCache := false
   182  	registryFilename := cachedFilename
   183  
   184  	newETag, downloadStatRecorder, err := downloadRemoteServerListFile(
   185  		ctx,
   186  		config,
   187  		tunnel,
   188  		untunneledDialConfig,
   189  		downloadTimeout,
   190  		downloadURL,
   191  		canonicalURL,
   192  		rootURL.SkipVerify,
   193  		"",
   194  		downloadFilename)
   195  	if err != nil {
   196  		failed = true
   197  		NoticeWarning("failed to download obfuscated server list registry: %s", errors.Trace(err))
   198  		// Proceed with any existing cached OSL registry.
   199  	}
   200  
   201  	authenticatedDownload := false
   202  	if downloadStatRecorder != nil {
   203  		defer func() { downloadStatRecorder(authenticatedDownload) }()
   204  	}
   205  
   206  	if newETag != "" {
   207  		updateCache = true
   208  		registryFilename = downloadFilename
   209  	}
   210  
   211  	// Prevent excessive notice noise in cases such as a general database
   212  	// failure, as GetSLOK may be called thousands of times per fetch.
   213  	emittedGetSLOKAlert := int32(0)
   214  
   215  	lookupSLOKs := func(slokID []byte) []byte {
   216  		// Lookup SLOKs in local datastore
   217  		key, err := GetSLOK(slokID)
   218  		if err != nil && atomic.CompareAndSwapInt32(&emittedGetSLOKAlert, 0, 1) {
   219  			NoticeWarning("GetSLOK failed: %s", err)
   220  		}
   221  		return key
   222  	}
   223  
   224  	registryFile, err := os.Open(registryFilename)
   225  	if err != nil {
   226  		return errors.Tracef("failed to read obfuscated server list registry: %s", errors.Trace(err))
   227  	}
   228  	defer registryFile.Close()
   229  
   230  	registryStreamer, err := osl.NewRegistryStreamer(
   231  		registryFile,
   232  		publicKey,
   233  		lookupSLOKs)
   234  	if err != nil {
   235  		// TODO: delete file? redownload if corrupt?
   236  		return errors.Tracef("failed to read obfuscated server list registry: %s", errors.Trace(err))
   237  	}
   238  
   239  	authenticatedDownload = true
   240  
   241  	// NewRegistryStreamer authenticates the downloaded registry, so now it would be
   242  	// ok to update the cache. However, we defer that until after processing so we
   243  	// can close the file first before copying it, avoiding related complications on
   244  	// platforms such as Windows.
   245  
   246  	// Note: we proceed to check individual OSLs even if the directory is unchanged,
   247  	// as the set of local SLOKs may have changed.
   248  
   249  	for {
   250  
   251  		oslFileSpec, err := registryStreamer.Next()
   252  		if err != nil {
   253  			failed = true
   254  			NoticeWarning("failed to stream obfuscated server list registry: %s", errors.Trace(err))
   255  			break
   256  		}
   257  
   258  		if oslFileSpec == nil {
   259  			break
   260  		}
   261  
   262  		if !downloadOSLFileSpec(
   263  			ctx,
   264  			config,
   265  			tunnel,
   266  			untunneledDialConfig,
   267  			downloadTimeout,
   268  			rootURL.URL,
   269  			canonicalRootURL,
   270  			rootURL.SkipVerify,
   271  			publicKey,
   272  			lookupSLOKs,
   273  			oslFileSpec) {
   274  
   275  			// downloadOSLFileSpec emits notices with failure information. In the case
   276  			// of a failure, set the retry flag but continue to process other OSL file
   277  			// specs.
   278  			failed = true
   279  		}
   280  
   281  		// Run a garbage collection to reclaim memory from the downloadOSLFileSpec
   282  		// operation before processing the next file.
   283  		DoGarbageCollection()
   284  	}
   285  
   286  	// Now that a new registry is downloaded, validated, and parsed, store
   287  	// the response ETag so we won't re-download this same data again. First
   288  	// close the file to avoid complications on platforms such as Windows.
   289  	if updateCache {
   290  
   291  		registryFile.Close()
   292  
   293  		err := os.Rename(downloadFilename, cachedFilename)
   294  		if err != nil {
   295  			NoticeWarning("failed to set cached obfuscated server list registry: %s", errors.Trace(err))
   296  			// This fetch is still reported as a success, even if we can't update the cache
   297  		}
   298  
   299  		err = SetUrlETag(canonicalURL, newETag)
   300  		if err != nil {
   301  			NoticeWarning("failed to set ETag for obfuscated server list registry: %s", errors.Trace(err))
   302  			// This fetch is still reported as a success, even if we can't store the ETag
   303  		}
   304  	}
   305  
   306  	if failed {
   307  		return errors.TraceNew("one or more operations failed")
   308  	}
   309  
   310  	return nil
   311  }
   312  
   313  // downloadOSLFileSpec downloads, authenticates, and imports the OSL specified
   314  // by oslFileSpec. The return value indicates whether the operation succeeded.
   315  // Failure information is emitted in notices.
   316  func downloadOSLFileSpec(
   317  	ctx context.Context,
   318  	config *Config,
   319  	tunnel *Tunnel,
   320  	untunneledDialConfig *DialConfig,
   321  	downloadTimeout time.Duration,
   322  	rootURL string,
   323  	canonicalRootURL string,
   324  	skipVerify bool,
   325  	publicKey string,
   326  	lookupSLOKs func(slokID []byte) []byte,
   327  	oslFileSpec *osl.OSLFileSpec) bool {
   328  
   329  	downloadFilename := osl.GetOSLFilename(
   330  		config.GetObfuscatedServerListDownloadDirectory(), oslFileSpec.ID)
   331  
   332  	downloadURL := osl.GetOSLFileURL(rootURL, oslFileSpec.ID)
   333  	canonicalURL := osl.GetOSLFileURL(canonicalRootURL, oslFileSpec.ID)
   334  
   335  	hexID := hex.EncodeToString(oslFileSpec.ID)
   336  
   337  	// Note: the MD5 checksum step assumes the remote server list host's ETag uses MD5
   338  	// with a hex encoding. If this is not the case, the sourceETag should be left blank.
   339  	sourceETag := fmt.Sprintf("\"%s\"", hex.EncodeToString(oslFileSpec.MD5Sum))
   340  
   341  	newETag, downloadStatRecorder, err := downloadRemoteServerListFile(
   342  		ctx,
   343  		config,
   344  		tunnel,
   345  		untunneledDialConfig,
   346  		downloadTimeout,
   347  		downloadURL,
   348  		canonicalURL,
   349  		skipVerify,
   350  		sourceETag,
   351  		downloadFilename)
   352  	if err != nil {
   353  		NoticeWarning("failed to download obfuscated server list file (%s): %s", hexID, errors.Trace(err))
   354  		return false
   355  	}
   356  
   357  	authenticatedDownload := false
   358  	if downloadStatRecorder != nil {
   359  		defer func() { downloadStatRecorder(authenticatedDownload) }()
   360  	}
   361  
   362  	// When the resource is unchanged, skip.
   363  	if newETag == "" {
   364  		return true
   365  	}
   366  
   367  	file, err := os.Open(downloadFilename)
   368  	if err != nil {
   369  		NoticeWarning("failed to open obfuscated server list file (%s): %s", hexID, errors.Trace(err))
   370  		return false
   371  	}
   372  	defer file.Close()
   373  
   374  	serverListPayloadReader, err := osl.NewOSLReader(
   375  		file,
   376  		oslFileSpec,
   377  		lookupSLOKs,
   378  		publicKey)
   379  	if err != nil {
   380  		NoticeWarning("failed to read obfuscated server list file (%s): %s", hexID, errors.Trace(err))
   381  		return false
   382  	}
   383  
   384  	// NewOSLReader authenticates the file before returning.
   385  	authenticatedDownload = true
   386  
   387  	err = StreamingStoreServerEntries(
   388  		ctx,
   389  		config,
   390  		protocol.NewStreamingServerEntryDecoder(
   391  			serverListPayloadReader,
   392  			common.GetCurrentTimestamp(),
   393  			protocol.SERVER_ENTRY_SOURCE_OBFUSCATED),
   394  		true)
   395  	if err != nil {
   396  		NoticeWarning("failed to store obfuscated server list file (%s): %s", hexID, errors.Trace(err))
   397  		return false
   398  	}
   399  
   400  	// Now that the server entries are successfully imported, store the response
   401  	// ETag so we won't re-download this same data again.
   402  	err = SetUrlETag(canonicalURL, newETag)
   403  	if err != nil {
   404  		NoticeWarning("failed to set ETag for obfuscated server list file (%s): %s", hexID, errors.Trace(err))
   405  		// This fetch is still reported as a success, even if we can't store the ETag
   406  		return true
   407  	}
   408  
   409  	return true
   410  }
   411  
   412  // downloadRemoteServerListFile downloads the source URL to the destination
   413  // file, performing a resumable download. When the download completes and the
   414  // file content has changed, the new resource ETag is returned. Otherwise,
   415  // blank is returned. The caller is responsible for calling SetUrlETag once
   416  // the file content has been validated.
   417  //
   418  // The downloadStatReporter return value is a function that will invoke
   419  // RecordRemoteServerListStat to record a remote server list download event.
   420  // The caller must call this function if the return value is not nil,
   421  // providing a boolean argument indicating whether the download was
   422  // successfully authenticated.
   423  func downloadRemoteServerListFile(
   424  	ctx context.Context,
   425  	config *Config,
   426  	tunnel *Tunnel,
   427  	untunneledDialConfig *DialConfig,
   428  	downloadTimeout time.Duration,
   429  	sourceURL string,
   430  	canonicalURL string,
   431  	skipVerify bool,
   432  	sourceETag string,
   433  	destinationFilename string) (string, func(bool), error) {
   434  
   435  	// All download URLs with the same canonicalURL
   436  	// must have the same entity and ETag.
   437  	lastETag, err := GetUrlETag(canonicalURL)
   438  	if err != nil {
   439  		return "", nil, errors.Trace(err)
   440  	}
   441  
   442  	// sourceETag, when specified, is prior knowledge of the
   443  	// remote ETag that can be used to skip the request entirely.
   444  	// This will be set in the case of OSL files, from the MD5Sum
   445  	// values stored in the registry.
   446  	if lastETag != "" && sourceETag == lastETag {
   447  		// TODO: notice?
   448  		return "", nil, nil
   449  	}
   450  
   451  	var cancelFunc context.CancelFunc
   452  	ctx, cancelFunc = context.WithTimeout(ctx, downloadTimeout)
   453  	defer cancelFunc()
   454  
   455  	// MakeDownloadHttpClient will select either a tunneled
   456  	// or untunneled configuration.
   457  
   458  	httpClient, tunneled, err := MakeDownloadHTTPClient(
   459  		ctx,
   460  		config,
   461  		tunnel,
   462  		untunneledDialConfig,
   463  		skipVerify)
   464  	if err != nil {
   465  		return "", nil, errors.Trace(err)
   466  	}
   467  
   468  	startTime := time.Now()
   469  
   470  	bytes, responseETag, err := ResumeDownload(
   471  		ctx,
   472  		httpClient,
   473  		sourceURL,
   474  		MakePsiphonUserAgent(config),
   475  		destinationFilename,
   476  		lastETag)
   477  
   478  	duration := time.Since(startTime)
   479  
   480  	NoticeRemoteServerListResourceDownloadedBytes(sourceURL, bytes, duration)
   481  
   482  	if err != nil {
   483  		return "", nil, errors.Trace(err)
   484  	}
   485  
   486  	if responseETag == lastETag {
   487  		return "", nil, nil
   488  	}
   489  
   490  	NoticeRemoteServerListResourceDownloaded(sourceURL)
   491  
   492  	downloadStatRecorder := func(authenticated bool) {
   493  
   494  		// Invoke DNS cache extension (if enabled in the resolver) now that
   495  		// the download succeeded and the payload is authenticated. Only
   496  		// extend when authenticated, as this demonstrates that any domain
   497  		// name resolved to an endpoint that served a valid Psiphon remote
   498  		// server list.
   499  		//
   500  		// TODO: when !skipVerify, invoke DNS cache extension earlier, in
   501  		// ResumeDownload, after making the request but before downloading
   502  		// the response body?
   503  		resolver := config.GetResolver()
   504  		url, err := url.Parse(sourceURL)
   505  		if authenticated && resolver != nil && err == nil {
   506  			resolver.VerifyCacheExtension(url.Hostname())
   507  		}
   508  
   509  		_ = RecordRemoteServerListStat(
   510  			config, tunneled, sourceURL, responseETag, bytes, duration, authenticated)
   511  	}
   512  
   513  	return responseETag, downloadStatRecorder, nil
   514  }