github.com/psiphon-labs/psiphon-tunnel-core@v2.0.28+incompatible/psiphon/server/meek.go (about) 1 /* 2 * Copyright (c) 2016, Psiphon Inc. 3 * All rights reserved. 4 * 5 * This program is free software: you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation, either version 3 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 package server 21 22 import ( 23 "bytes" 24 "context" 25 "crypto/rand" 26 "crypto/tls" 27 "encoding/base64" 28 "encoding/hex" 29 "encoding/json" 30 std_errors "errors" 31 "hash/crc64" 32 "io" 33 "net" 34 "net/http" 35 "runtime" 36 "strconv" 37 "strings" 38 "sync" 39 "sync/atomic" 40 "time" 41 42 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common" 43 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" 44 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/monotime" 45 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/obfuscator" 46 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/parameters" 47 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng" 48 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/protocol" 49 "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/values" 50 tris "github.com/Psiphon-Labs/tls-tris" 51 lrucache "github.com/cognusion/go-cache-lru" 52 "github.com/juju/ratelimit" 53 "golang.org/x/crypto/nacl/box" 54 ) 55 56 // MeekServer is based on meek-server.go from Tor and Psiphon: 57 // 58 // https://gitweb.torproject.org/pluggable-transports/meek.git/blob/HEAD:/meek-client/meek-client.go 59 // CC0 1.0 Universal 60 // 61 // https://bitbucket.org/psiphon/psiphon-circumvention-system/src/default/go/meek-client/meek-client.go 62 63 const ( 64 65 // Protocol version 1 clients can handle arbitrary length response bodies. Older clients 66 // report no version number and expect at most 64K response bodies. 67 MEEK_PROTOCOL_VERSION_1 = 1 68 69 // Protocol version 2 clients initiate a session by sending an encrypted and obfuscated meek 70 // cookie with their initial HTTP request. Connection information is contained within the 71 // encrypted cookie payload. The server inspects the cookie and establishes a new session and 72 // returns a new random session ID back to client via Set-Cookie header. The client uses this 73 // session ID on all subsequent requests for the remainder of the session. 74 MEEK_PROTOCOL_VERSION_2 = 2 75 76 // Protocol version 3 clients include resiliency enhancements and will add a Range header 77 // when retrying a request for a partially downloaded response payload. 78 MEEK_PROTOCOL_VERSION_3 = 3 79 80 MEEK_MAX_REQUEST_PAYLOAD_LENGTH = 65536 81 MEEK_MIN_SESSION_ID_LENGTH = 8 82 MEEK_MAX_SESSION_ID_LENGTH = 20 83 MEEK_DEFAULT_TURN_AROUND_TIMEOUT = 10 * time.Millisecond 84 MEEK_DEFAULT_EXTENDED_TURN_AROUND_TIMEOUT = 100 * time.Millisecond 85 MEEK_DEFAULT_SKIP_EXTENDED_TURN_AROUND_THRESHOLD = 8192 86 MEEK_DEFAULT_MAX_SESSION_STALENESS = 45 * time.Second 87 MEEK_DEFAULT_HTTP_CLIENT_IO_TIMEOUT = 45 * time.Second 88 MEEK_DEFAULT_RESPONSE_BUFFER_LENGTH = 65536 89 MEEK_DEFAULT_POOL_BUFFER_LENGTH = 65536 90 MEEK_DEFAULT_POOL_BUFFER_COUNT = 2048 91 ) 92 93 // MeekServer implements the meek protocol, which tunnels TCP traffic (in the case of Psiphon, 94 // Obfuscated SSH traffic) over HTTP. Meek may be fronted (through a CDN) or direct and may be 95 // HTTP or HTTPS. 96 // 97 // Upstream traffic arrives in HTTP request bodies and downstream traffic is sent in response 98 // bodies. The sequence of traffic for a given flow is associated using a session ID that's 99 // set as a HTTP cookie for the client to submit with each request. 100 // 101 // MeekServer hooks into TunnelServer via the net.Conn interface by transforming the 102 // HTTP payload traffic for a given session into net.Conn conforming Read()s and Write()s via 103 // the meekConn struct. 104 type MeekServer struct { 105 support *SupportServices 106 listener net.Listener 107 listenerTunnelProtocol string 108 listenerPort int 109 passthroughAddress string 110 turnAroundTimeout time.Duration 111 extendedTurnAroundTimeout time.Duration 112 skipExtendedTurnAroundThreshold int 113 maxSessionStaleness time.Duration 114 httpClientIOTimeout time.Duration 115 tlsConfig *tris.Config 116 obfuscatorSeedHistory *obfuscator.SeedHistory 117 clientHandler func(clientTunnelProtocol string, clientConn net.Conn) 118 openConns *common.Conns 119 stopBroadcast <-chan struct{} 120 sessionsLock sync.RWMutex 121 sessions map[string]*meekSession 122 checksumTable *crc64.Table 123 bufferPool *CachedResponseBufferPool 124 rateLimitLock sync.Mutex 125 rateLimitHistory *lrucache.Cache 126 rateLimitCount int 127 rateLimitSignalGC chan struct{} 128 } 129 130 // NewMeekServer initializes a new meek server. 131 func NewMeekServer( 132 support *SupportServices, 133 listener net.Listener, 134 listenerTunnelProtocol string, 135 listenerPort int, 136 useTLS, isFronted, useObfuscatedSessionTickets bool, 137 clientHandler func(clientTunnelProtocol string, clientConn net.Conn), 138 stopBroadcast <-chan struct{}) (*MeekServer, error) { 139 140 passthroughAddress := support.Config.TunnelProtocolPassthroughAddresses[listenerTunnelProtocol] 141 142 turnAroundTimeout := MEEK_DEFAULT_TURN_AROUND_TIMEOUT 143 if support.Config.MeekTurnAroundTimeoutMilliseconds != nil { 144 turnAroundTimeout = time.Duration( 145 *support.Config.MeekTurnAroundTimeoutMilliseconds) * time.Millisecond 146 } 147 148 extendedTurnAroundTimeout := MEEK_DEFAULT_EXTENDED_TURN_AROUND_TIMEOUT 149 if support.Config.MeekExtendedTurnAroundTimeoutMilliseconds != nil { 150 extendedTurnAroundTimeout = time.Duration( 151 *support.Config.MeekExtendedTurnAroundTimeoutMilliseconds) * time.Millisecond 152 } 153 154 skipExtendedTurnAroundThreshold := MEEK_DEFAULT_SKIP_EXTENDED_TURN_AROUND_THRESHOLD 155 if support.Config.MeekSkipExtendedTurnAroundThresholdBytes != nil { 156 skipExtendedTurnAroundThreshold = *support.Config.MeekSkipExtendedTurnAroundThresholdBytes 157 } 158 159 maxSessionStaleness := MEEK_DEFAULT_MAX_SESSION_STALENESS 160 if support.Config.MeekMaxSessionStalenessMilliseconds != nil { 161 maxSessionStaleness = time.Duration( 162 *support.Config.MeekMaxSessionStalenessMilliseconds) * time.Millisecond 163 } 164 165 httpClientIOTimeout := MEEK_DEFAULT_HTTP_CLIENT_IO_TIMEOUT 166 if support.Config.MeekHTTPClientIOTimeoutMilliseconds != nil { 167 httpClientIOTimeout = time.Duration( 168 *support.Config.MeekHTTPClientIOTimeoutMilliseconds) * time.Millisecond 169 } 170 171 checksumTable := crc64.MakeTable(crc64.ECMA) 172 173 bufferLength := MEEK_DEFAULT_POOL_BUFFER_LENGTH 174 if support.Config.MeekCachedResponsePoolBufferSize != 0 { 175 bufferLength = support.Config.MeekCachedResponsePoolBufferSize 176 } 177 178 bufferCount := MEEK_DEFAULT_POOL_BUFFER_COUNT 179 if support.Config.MeekCachedResponsePoolBufferCount != 0 { 180 bufferCount = support.Config.MeekCachedResponsePoolBufferCount 181 } 182 183 _, thresholdSeconds, _, _, _, _, _, _, reapFrequencySeconds, maxEntries := 184 support.TrafficRulesSet.GetMeekRateLimiterConfig() 185 186 rateLimitHistory := lrucache.NewWithLRU( 187 time.Duration(thresholdSeconds)*time.Second, 188 time.Duration(reapFrequencySeconds)*time.Second, 189 maxEntries) 190 191 bufferPool := NewCachedResponseBufferPool(bufferLength, bufferCount) 192 193 meekServer := &MeekServer{ 194 support: support, 195 listener: listener, 196 listenerTunnelProtocol: listenerTunnelProtocol, 197 listenerPort: listenerPort, 198 passthroughAddress: passthroughAddress, 199 turnAroundTimeout: turnAroundTimeout, 200 extendedTurnAroundTimeout: extendedTurnAroundTimeout, 201 skipExtendedTurnAroundThreshold: skipExtendedTurnAroundThreshold, 202 maxSessionStaleness: maxSessionStaleness, 203 httpClientIOTimeout: httpClientIOTimeout, 204 obfuscatorSeedHistory: obfuscator.NewSeedHistory(nil), 205 clientHandler: clientHandler, 206 openConns: common.NewConns(), 207 stopBroadcast: stopBroadcast, 208 sessions: make(map[string]*meekSession), 209 checksumTable: checksumTable, 210 bufferPool: bufferPool, 211 rateLimitHistory: rateLimitHistory, 212 rateLimitSignalGC: make(chan struct{}, 1), 213 } 214 215 if useTLS { 216 tlsConfig, err := meekServer.makeMeekTLSConfig( 217 isFronted, useObfuscatedSessionTickets) 218 if err != nil { 219 return nil, errors.Trace(err) 220 } 221 meekServer.tlsConfig = tlsConfig 222 } 223 224 return meekServer, nil 225 } 226 227 type meekContextKey struct { 228 key string 229 } 230 231 var meekNetConnContextKey = &meekContextKey{"net.Conn"} 232 233 // Run runs the meek server; this function blocks while serving HTTP or 234 // HTTPS connections on the specified listener. This function also runs 235 // a goroutine which cleans up expired meek client sessions. 236 // 237 // To stop the meek server, both Close() the listener and set the stopBroadcast 238 // signal specified in NewMeekServer. 239 func (server *MeekServer) Run() error { 240 241 waitGroup := new(sync.WaitGroup) 242 243 waitGroup.Add(1) 244 go func() { 245 defer waitGroup.Done() 246 ticker := time.NewTicker(server.maxSessionStaleness / 2) 247 defer ticker.Stop() 248 for { 249 select { 250 case <-ticker.C: 251 server.deleteExpiredSessions() 252 case <-server.stopBroadcast: 253 return 254 } 255 } 256 }() 257 258 waitGroup.Add(1) 259 go func() { 260 defer waitGroup.Done() 261 server.rateLimitWorker() 262 }() 263 264 // Serve HTTP or HTTPS 265 // 266 // - WriteTimeout may include time awaiting request, as per: 267 // https://blog.cloudflare.com/the-complete-guide-to-golang-net-http-timeouts 268 // - Legacy meek-server wrapped each client HTTP connection with an explicit idle 269 // timeout net.Conn and didn't use http.Server timeouts. We could do the same 270 // here (use ActivityMonitoredConn) but the stock http.Server timeouts should 271 // now be sufficient. 272 273 httpServer := &http.Server{ 274 ReadTimeout: server.httpClientIOTimeout, 275 WriteTimeout: server.httpClientIOTimeout, 276 Handler: server, 277 ConnState: server.httpConnStateCallback, 278 ConnContext: func(ctx context.Context, conn net.Conn) context.Context { 279 return context.WithValue(ctx, meekNetConnContextKey, conn) 280 }, 281 282 // Disable auto HTTP/2 (https://golang.org/doc/go1.6) 283 TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), 284 } 285 286 // Note: Serve() will be interrupted by listener.Close() call 287 var err error 288 if server.tlsConfig != nil { 289 httpsServer := HTTPSServer{Server: httpServer} 290 err = httpsServer.ServeTLS(server.listener, server.tlsConfig) 291 } else { 292 err = httpServer.Serve(server.listener) 293 } 294 295 // Can't check for the exact error that Close() will cause in Accept(), 296 // (see: https://code.google.com/p/go/issues/detail?id=4373). So using an 297 // explicit stop signal to stop gracefully. 298 select { 299 case <-server.stopBroadcast: 300 err = nil 301 default: 302 } 303 304 // deleteExpiredSessions calls deleteSession which may block waiting 305 // for active request handlers to complete; timely shutdown requires 306 // stopping the listener and closing all existing connections before 307 // awaiting the reaperWaitGroup. 308 309 server.listener.Close() 310 server.openConns.CloseAll() 311 312 waitGroup.Wait() 313 314 return err 315 } 316 317 // ServeHTTP handles meek client HTTP requests, where the request body 318 // contains upstream traffic and the response will contain downstream 319 // traffic. 320 func (server *MeekServer) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { 321 322 // Note: no longer requiring that the request method is POST 323 324 // Check for the expected meek/session ID cookie. 325 // Also check for prohibited HTTP headers. 326 327 var meekCookie *http.Cookie 328 for _, c := range request.Cookies() { 329 meekCookie = c 330 break 331 } 332 if meekCookie == nil || len(meekCookie.Value) == 0 { 333 log.WithTrace().Warning("missing meek cookie") 334 common.TerminateHTTPConnection(responseWriter, request) 335 return 336 } 337 338 if len(server.support.Config.MeekProhibitedHeaders) > 0 { 339 for _, header := range server.support.Config.MeekProhibitedHeaders { 340 value := request.Header.Get(header) 341 if header != "" { 342 log.WithTraceFields(LogFields{ 343 "header": header, 344 "value": value, 345 }).Warning("prohibited meek header") 346 common.TerminateHTTPConnection(responseWriter, request) 347 return 348 } 349 } 350 } 351 352 // A valid meek cookie indicates which class of request this is: 353 // 354 // 1. A new meek session. Create a new session ID and proceed with 355 // relaying tunnel traffic. 356 // 357 // 2. An existing meek session. Resume relaying tunnel traffic. 358 // 359 // 3. A request to an endpoint. This meek connection is not for relaying 360 // tunnel traffic. Instead, the request is handed off to a custom handler. 361 362 sessionID, 363 session, 364 underlyingConn, 365 endPoint, 366 endPointGeoIPData, 367 err := server.getSessionOrEndpoint(request, meekCookie) 368 369 if err != nil { 370 // Debug since session cookie errors commonly occur during 371 // normal operation. 372 log.WithTraceFields(LogFields{"error": err}).Debug("session lookup failed") 373 common.TerminateHTTPConnection(responseWriter, request) 374 return 375 } 376 377 if endPoint != "" { 378 379 // Endpoint mode. Currently, this means it's handled by the tactics 380 // request handler. 381 382 handled := server.support.TacticsServer.HandleEndPoint( 383 endPoint, common.GeoIPData(*endPointGeoIPData), responseWriter, request) 384 if !handled { 385 log.WithTraceFields(LogFields{"endPoint": endPoint}).Info("unhandled endpoint") 386 common.TerminateHTTPConnection(responseWriter, request) 387 } 388 return 389 } 390 391 // Tunnel relay mode. 392 393 // Ensure that there's only one concurrent request handler per client 394 // session. Depending on the nature of a network disruption, it can 395 // happen that a client detects a failure and retries while the server 396 // is still streaming response in the handler for the _previous_ client 397 // request. 398 // 399 // Even if the session.cachedResponse were safe for concurrent 400 // use (it is not), concurrent handling could lead to loss of session 401 // since upstream data read by the first request may not reach the 402 // cached response before the second request reads the cached data. 403 // 404 // The existing handler will stream response data, holding the lock, 405 // for no more than MEEK_EXTENDED_TURN_AROUND_TIMEOUT. 406 // 407 // TODO: interrupt an existing handler? The existing handler will be 408 // sending data to the cached response, but if that buffer fills, the 409 // session will be lost. 410 411 requestNumber := atomic.AddInt64(&session.requestCount, 1) 412 413 // Wait for the existing request to complete. 414 session.lock.Lock() 415 defer session.lock.Unlock() 416 417 // Count this metric once the lock is acquired, to avoid concurrent and 418 // potentially incorrect session.underlyingConn updates. 419 // 420 // It should never be the case that a new underlyingConn has the same 421 // value as the previous session.underlyingConn, as each is a net.Conn 422 // interface which includes a pointer, and the previous value cannot 423 // be garbage collected until session.underlyingConn is updated. 424 if session.underlyingConn != underlyingConn { 425 atomic.AddInt64(&session.metricUnderlyingConnCount, 1) 426 session.underlyingConn = underlyingConn 427 } 428 429 // If a newer request has arrived while waiting, discard this one. 430 // Do not delay processing the newest request. 431 // 432 // If the session expired and was deleted while this request was waiting, 433 // discard this request. The session is no longer valid, and the final call 434 // to session.cachedResponse.Reset may have already occured, so any further 435 // session.cachedResponse access may deplete resources (fail to refill the pool). 436 if atomic.LoadInt64(&session.requestCount) > requestNumber || session.deleted { 437 common.TerminateHTTPConnection(responseWriter, request) 438 return 439 } 440 441 // pumpReads causes a TunnelServer/SSH goroutine blocking on a Read to 442 // read the request body as upstream traffic. 443 // TODO: run pumpReads and pumpWrites concurrently? 444 445 // pumpReads checksums the request payload and skips relaying it when 446 // it matches the immediately previous request payload. This allows 447 // clients to resend request payloads, when retrying due to connection 448 // interruption, without knowing whether the server has received or 449 // relayed the data. 450 451 requestSize, err := session.clientConn.pumpReads(request.Body) 452 if err != nil { 453 if err != io.EOF { 454 // Debug since errors such as "i/o timeout" occur during normal operation; 455 // also, golang network error messages may contain client IP. 456 log.WithTraceFields(LogFields{"error": err}).Debug("read request failed") 457 } 458 common.TerminateHTTPConnection(responseWriter, request) 459 460 // Note: keep session open to allow client to retry 461 462 return 463 } 464 465 // The extended turn around mechanism optimizes for downstream flows by 466 // sending more data in the response as long as it's available. As a 467 // heuristic, when the request size meets a threshold, optimize instead 468 // of upstream flows by skipping the extended turn around. 469 skipExtendedTurnAround := requestSize >= int64(server.skipExtendedTurnAroundThreshold) 470 471 // Set cookie before writing the response. 472 473 if session.meekProtocolVersion >= MEEK_PROTOCOL_VERSION_2 && !session.sessionIDSent { 474 // Replace the meek cookie with the session ID. 475 // SetCookie for the the session ID cookie is only set once, to reduce overhead. This 476 // session ID value replaces the original meek cookie value. 477 http.SetCookie(responseWriter, &http.Cookie{Name: meekCookie.Name, Value: sessionID}) 478 session.sessionIDSent = true 479 } 480 481 // When streaming data into the response body, a copy is 482 // retained in the cachedResponse buffer. This allows the 483 // client to retry and request that the response be resent 484 // when the HTTP connection is interrupted. 485 // 486 // If a Range header is present, the client is retrying, 487 // possibly after having received a partial response. In 488 // this case, use any cached response to attempt to resend 489 // the response, starting from the resend position the client 490 // indicates. 491 // 492 // When the resend position is not available -- because the 493 // cachedResponse buffer could not hold it -- the client session 494 // is closed, as there's no way to resume streaming the payload 495 // uninterrupted. 496 // 497 // The client may retry before a cached response is prepared, 498 // so a cached response is not always used when a Range header 499 // is present. 500 // 501 // TODO: invalid Range header is ignored; should it be otherwise? 502 503 position, isRetry := checkRangeHeader(request) 504 if isRetry { 505 atomic.AddInt64(&session.metricClientRetries, 1) 506 } 507 508 hasCompleteCachedResponse := session.cachedResponse.HasPosition(0) 509 510 // The client is not expected to send position > 0 when there is 511 // no cached response; let that case fall through to the next 512 // HasPosition check which will fail and close the session. 513 514 var responseSize int 515 var responseError error 516 517 if isRetry && (hasCompleteCachedResponse || position > 0) { 518 519 if !session.cachedResponse.HasPosition(position) { 520 greaterThanSwapInt64(&session.metricCachedResponseMissPosition, int64(position)) 521 common.TerminateHTTPConnection(responseWriter, request) 522 session.delete(true) 523 return 524 } 525 526 responseWriter.WriteHeader(http.StatusPartialContent) 527 528 // TODO: 529 // - enforce a max extended buffer count per client, for 530 // fairness? Throttling may make this unnecessary. 531 // - cachedResponse can now start releasing extended buffers, 532 // as response bytes before "position" will never be requested 533 // again? 534 535 responseSize, responseError = session.cachedResponse.CopyFromPosition(position, responseWriter) 536 greaterThanSwapInt64(&session.metricPeakCachedResponseHitSize, int64(responseSize)) 537 538 // The client may again fail to receive the payload and may again 539 // retry, so not yet releasing cachedResponse buffers. 540 541 } else { 542 543 // _Now_ we release buffers holding data from the previous 544 // response. And then immediately stream the new response into 545 // newly acquired buffers. 546 session.cachedResponse.Reset() 547 548 // Note: this code depends on an implementation detail of 549 // io.MultiWriter: a Write() to the MultiWriter writes first 550 // to the cache, and then to the response writer. So if the 551 // write to the response writer fails, the payload is cached. 552 multiWriter := io.MultiWriter(session.cachedResponse, responseWriter) 553 554 // The client expects 206, not 200, whenever it sets a Range header, 555 // which it may do even when no cached response is prepared. 556 if isRetry { 557 responseWriter.WriteHeader(http.StatusPartialContent) 558 } 559 560 // pumpWrites causes a TunnelServer/SSH goroutine blocking on a Write to 561 // write its downstream traffic through to the response body. 562 563 responseSize, responseError = session.clientConn.pumpWrites(multiWriter, skipExtendedTurnAround) 564 greaterThanSwapInt64(&session.metricPeakResponseSize, int64(responseSize)) 565 greaterThanSwapInt64(&session.metricPeakCachedResponseSize, int64(session.cachedResponse.Available())) 566 } 567 568 // responseError is the result of writing the body either from CopyFromPosition or pumpWrites 569 if responseError != nil { 570 if responseError != io.EOF { 571 // Debug since errors such as "i/o timeout" occur during normal operation; 572 // also, golang network error messages may contain client IP. 573 log.WithTraceFields(LogFields{"error": responseError}).Debug("write response failed") 574 } 575 common.TerminateHTTPConnection(responseWriter, request) 576 577 // Note: keep session open to allow client to retry 578 579 return 580 } 581 } 582 583 func checkRangeHeader(request *http.Request) (int, bool) { 584 rangeHeader := request.Header.Get("Range") 585 if rangeHeader == "" { 586 return 0, false 587 } 588 589 prefix := "bytes=" 590 suffix := "-" 591 592 if !strings.HasPrefix(rangeHeader, prefix) || 593 !strings.HasSuffix(rangeHeader, suffix) { 594 595 return 0, false 596 } 597 598 rangeHeader = strings.TrimPrefix(rangeHeader, prefix) 599 rangeHeader = strings.TrimSuffix(rangeHeader, suffix) 600 position, err := strconv.Atoi(rangeHeader) 601 602 if err != nil { 603 return 0, false 604 } 605 606 return position, true 607 } 608 609 // getSessionOrEndpoint checks if the cookie corresponds to an existing tunnel 610 // relay session ID. If no session is found, the cookie must be an obfuscated 611 // meek cookie. A new session is created when the meek cookie indicates relay 612 // mode; or the endpoint is returned when the meek cookie indicates endpoint 613 // mode. 614 func (server *MeekServer) getSessionOrEndpoint( 615 request *http.Request, meekCookie *http.Cookie) (string, *meekSession, net.Conn, string, *GeoIPData, error) { 616 617 underlyingConn := request.Context().Value(meekNetConnContextKey).(net.Conn) 618 619 // Check for an existing session. 620 621 server.sessionsLock.RLock() 622 existingSessionID := meekCookie.Value 623 session, ok := server.sessions[existingSessionID] 624 server.sessionsLock.RUnlock() 625 if ok { 626 // TODO: can multiple http client connections using same session cookie 627 // cause race conditions on session struct? 628 session.touch() 629 return existingSessionID, session, underlyingConn, "", nil, nil 630 } 631 632 // Determine the client remote address, which is used for geolocation 633 // stats, rate limiting, anti-probing, discovery, and tactics selection 634 // logic. 635 // 636 // When an intermediate proxy or CDN is in use, we may be 637 // able to determine the original client address by inspecting HTTP 638 // headers such as X-Forwarded-For. 639 // 640 // We trust only headers provided by CDNs. Fronted Psiphon server hosts 641 // should be configured to accept tunnel connections only from CDN edges. 642 // When the CDN passes along a chain of IPs, as in X-Forwarded-For, we 643 // trust only the right-most IP, which is provided by the CDN. 644 645 clientIP, _, err := net.SplitHostPort(request.RemoteAddr) 646 if err != nil { 647 return "", nil, nil, "", nil, errors.Trace(err) 648 } 649 if net.ParseIP(clientIP) == nil { 650 return "", nil, nil, "", nil, errors.TraceNew("invalid IP address") 651 } 652 653 if protocol.TunnelProtocolUsesFrontedMeek(server.listenerTunnelProtocol) && 654 len(server.support.Config.MeekProxyForwardedForHeaders) > 0 { 655 656 // When there are multiple header names in MeekProxyForwardedForHeaders, 657 // the first valid match is preferred. MeekProxyForwardedForHeaders should be 658 // configured to use header names that are always provided by the CDN(s) and 659 // not header names that may be passed through from clients. 660 for _, header := range server.support.Config.MeekProxyForwardedForHeaders { 661 662 // In the case where there are multiple headers, 663 // request.Header.Get returns the first header, but we want the 664 // last header; so use request.Header.Values and select the last 665 // value. As per RFC 2616 section 4.2, a proxy must not change 666 // the order of field values, which implies that it should append 667 // values to the last header. 668 values := request.Header.Values(header) 669 if len(values) > 0 { 670 value := values[len(values)-1] 671 672 // Some headers, such as X-Forwarded-For, are a comma-separated 673 // list of IPs (each proxy in a chain). Select the last IP. 674 IPs := strings.Split(value, ",") 675 IP := IPs[len(IPs)-1] 676 677 // Remove optional whitespace surrounding the commas. 678 IP = strings.TrimSpace(IP) 679 680 if net.ParseIP(IP) != nil { 681 clientIP = IP 682 break 683 } 684 } 685 } 686 } 687 688 geoIPData := server.support.GeoIPService.Lookup(clientIP) 689 690 // The session is new (or expired). Treat the cookie value as a new meek 691 // cookie, extract the payload, and create a new session. 692 693 payloadJSON, err := server.getMeekCookiePayload(clientIP, meekCookie.Value) 694 if err != nil { 695 return "", nil, nil, "", nil, errors.Trace(err) 696 } 697 698 // Note: this meek server ignores legacy values PsiphonClientSessionId 699 // and PsiphonServerAddress. 700 var clientSessionData protocol.MeekCookieData 701 702 err = json.Unmarshal(payloadJSON, &clientSessionData) 703 if err != nil { 704 return "", nil, nil, "", nil, errors.Trace(err) 705 } 706 707 tunnelProtocol := server.listenerTunnelProtocol 708 709 if clientSessionData.ClientTunnelProtocol != "" { 710 711 if !protocol.IsValidClientTunnelProtocol( 712 clientSessionData.ClientTunnelProtocol, 713 server.listenerTunnelProtocol, 714 server.support.Config.GetRunningProtocols()) { 715 716 return "", nil, nil, "", nil, errors.Tracef( 717 "invalid client tunnel protocol: %s", clientSessionData.ClientTunnelProtocol) 718 } 719 720 tunnelProtocol = clientSessionData.ClientTunnelProtocol 721 } 722 723 // Any rate limit is enforced after the meek cookie is validated, so a prober 724 // without the obfuscation secret will be unable to fingerprint the server 725 // based on response time combined with the rate limit configuration. The 726 // rate limit is primarily intended to limit memory resource consumption and 727 // not the overhead incurred by cookie validation. 728 729 if server.rateLimit(clientIP, geoIPData, tunnelProtocol) { 730 return "", nil, nil, "", nil, errors.TraceNew("rate limit exceeded") 731 } 732 733 // Handle endpoints before enforcing CheckEstablishTunnels. 734 // Currently, endpoints are tactics requests, and we allow these to be 735 // handled by servers which would otherwise reject new tunnels. 736 737 if clientSessionData.EndPoint != "" { 738 return "", nil, nil, clientSessionData.EndPoint, &geoIPData, nil 739 } 740 741 // Don't create new sessions when not establishing. A subsequent SSH handshake 742 // will not succeed, so creating a meek session just wastes resources. 743 744 if server.support.TunnelServer != nil && 745 !server.support.TunnelServer.CheckEstablishTunnels() { 746 return "", nil, nil, "", nil, errors.TraceNew("not establishing tunnels") 747 } 748 749 // Disconnect immediately if the tactics for the client restricts usage of 750 // the fronting provider ID. The probability may be used to influence 751 // usage of a given fronting provider; but when only that provider works 752 // for a given client, and the probability is less than 1.0, the client 753 // can retry until it gets a successful coin flip. 754 // 755 // Clients will also skip candidates with restricted fronting provider IDs. 756 // The client-side probability, RestrictFrontingProviderIDsClientProbability, 757 // is applied independently of the server-side coin flip here. 758 // 759 // At this stage, GeoIP tactics filters are active, but handshake API 760 // parameters are not. 761 // 762 // See the comment in server.LoadConfig regarding fronting provider ID 763 // limitations. 764 765 if protocol.TunnelProtocolUsesFrontedMeek(server.listenerTunnelProtocol) && 766 server.support.ServerTacticsParametersCache != nil { 767 768 p, err := server.support.ServerTacticsParametersCache.Get(geoIPData) 769 if err != nil { 770 return "", nil, nil, "", nil, errors.Trace(err) 771 } 772 773 if !p.IsNil() && 774 common.Contains( 775 p.Strings(parameters.RestrictFrontingProviderIDs), 776 server.support.Config.GetFrontingProviderID()) { 777 if p.WeightedCoinFlip( 778 parameters.RestrictFrontingProviderIDsServerProbability) { 779 return "", nil, nil, "", nil, errors.TraceNew("restricted fronting provider") 780 } 781 } 782 } 783 784 // Create a new session 785 786 bufferLength := MEEK_DEFAULT_RESPONSE_BUFFER_LENGTH 787 if server.support.Config.MeekCachedResponseBufferSize != 0 { 788 bufferLength = server.support.Config.MeekCachedResponseBufferSize 789 } 790 cachedResponse := NewCachedResponse(bufferLength, server.bufferPool) 791 792 session = &meekSession{ 793 meekProtocolVersion: clientSessionData.MeekProtocolVersion, 794 sessionIDSent: false, 795 cachedResponse: cachedResponse, 796 cookieName: meekCookie.Name, 797 contentType: request.Header.Get("Content-Type"), 798 } 799 800 session.touch() 801 802 // Create a new meek conn that will relay the payload 803 // between meek request/responses and the tunnel server client 804 // handler. The client IP is also used to initialize the 805 // meek conn with a useful value to return when the tunnel 806 // server calls conn.RemoteAddr() to get the client's IP address. 807 808 // Assumes clientIP is a valid IP address; the port value is a stub 809 // and is expected to be ignored. 810 clientConn := newMeekConn( 811 server, 812 session, 813 underlyingConn, 814 &net.TCPAddr{ 815 IP: net.ParseIP(clientIP), 816 Port: 0, 817 }, 818 clientSessionData.MeekProtocolVersion) 819 820 session.clientConn = clientConn 821 822 // Note: MEEK_PROTOCOL_VERSION_1 doesn't support changing the 823 // meek cookie to a session ID; v1 clients always send the 824 // original meek cookie value with each request. The issue with 825 // v1 is that clients which wake after a device sleep will attempt 826 // to resume a meek session and the server can't differentiate 827 // between resuming a session and creating a new session. This 828 // causes the v1 client connection to hang/timeout. 829 sessionID := meekCookie.Value 830 if clientSessionData.MeekProtocolVersion >= MEEK_PROTOCOL_VERSION_2 { 831 sessionID, err = makeMeekSessionID() 832 if err != nil { 833 return "", nil, nil, "", nil, errors.Trace(err) 834 } 835 } 836 837 server.sessionsLock.Lock() 838 server.sessions[sessionID] = session 839 server.sessionsLock.Unlock() 840 841 // Note: from the tunnel server's perspective, this client connection 842 // will close when session.delete calls Close() on the meekConn. 843 server.clientHandler(clientSessionData.ClientTunnelProtocol, session.clientConn) 844 845 return sessionID, session, underlyingConn, "", nil, nil 846 } 847 848 func (server *MeekServer) rateLimit( 849 clientIP string, geoIPData GeoIPData, tunnelProtocol string) bool { 850 851 historySize, 852 thresholdSeconds, 853 tunnelProtocols, 854 regions, 855 ISPs, 856 ASNs, 857 cities, 858 GCTriggerCount, _, _ := 859 server.support.TrafficRulesSet.GetMeekRateLimiterConfig() 860 861 if historySize == 0 { 862 return false 863 } 864 865 if len(tunnelProtocols) > 0 { 866 if !common.Contains(tunnelProtocols, tunnelProtocol) { 867 return false 868 } 869 } 870 871 if len(regions) > 0 || len(ISPs) > 0 || len(ASNs) > 0 || len(cities) > 0 { 872 873 if len(regions) > 0 { 874 if !common.Contains(regions, geoIPData.Country) { 875 return false 876 } 877 } 878 879 if len(ISPs) > 0 { 880 if !common.Contains(ISPs, geoIPData.ISP) { 881 return false 882 } 883 } 884 885 if len(ASNs) > 0 { 886 if !common.Contains(ASNs, geoIPData.ASN) { 887 return false 888 } 889 } 890 891 if len(cities) > 0 { 892 if !common.Contains(cities, geoIPData.City) { 893 return false 894 } 895 } 896 } 897 898 // With IPv6, individual users or sites are users commonly allocated a /64 899 // or /56, so rate limit by /56. 900 rateLimitIP := clientIP 901 IP := net.ParseIP(clientIP) 902 if IP != nil && IP.To4() == nil { 903 rateLimitIP = IP.Mask(net.CIDRMask(56, 128)).String() 904 } 905 906 // go-cache-lru is safe for concurrent access, but lacks an atomic 907 // compare-and-set type operations to check if an entry exists before 908 // adding a new one. This mutex ensures the Get and Add are atomic 909 // (as well as synchronizing access to rateLimitCount). 910 server.rateLimitLock.Lock() 911 912 var rateLimiter *ratelimit.Bucket 913 entry, ok := server.rateLimitHistory.Get(rateLimitIP) 914 if ok { 915 rateLimiter = entry.(*ratelimit.Bucket) 916 } else { 917 rateLimiter = ratelimit.NewBucketWithQuantum( 918 time.Duration(thresholdSeconds)*time.Second, 919 int64(historySize), 920 int64(historySize)) 921 server.rateLimitHistory.Set( 922 rateLimitIP, 923 rateLimiter, 924 time.Duration(thresholdSeconds)*time.Second) 925 } 926 927 limit := rateLimiter.TakeAvailable(1) < 1 928 929 triggerGC := false 930 if limit { 931 server.rateLimitCount += 1 932 if server.rateLimitCount >= GCTriggerCount { 933 triggerGC = true 934 server.rateLimitCount = 0 935 } 936 } 937 938 server.rateLimitLock.Unlock() 939 940 if triggerGC { 941 select { 942 case server.rateLimitSignalGC <- struct{}{}: 943 default: 944 } 945 } 946 947 return limit 948 } 949 950 func (server *MeekServer) rateLimitWorker() { 951 for { 952 select { 953 case <-server.rateLimitSignalGC: 954 runtime.GC() 955 case <-server.stopBroadcast: 956 return 957 } 958 } 959 } 960 961 func (server *MeekServer) deleteSession(sessionID string) { 962 963 // Don't obtain the server.sessionsLock write lock until modifying 964 // server.sessions, as the session.delete can block for up to 965 // MEEK_HTTP_CLIENT_IO_TIMEOUT. Allow new sessions to be added 966 // concurrently. 967 // 968 // Since a lock isn't held for the duration, concurrent calls to 969 // deleteSession with the same sessionID could happen; this is 970 // not expected since only the reaper goroutine calls deleteExpiredSessions 971 // (and in any case concurrent execution of the ok block is not an issue). 972 server.sessionsLock.RLock() 973 session, ok := server.sessions[sessionID] 974 server.sessionsLock.RUnlock() 975 976 if ok { 977 session.delete(false) 978 979 server.sessionsLock.Lock() 980 delete(server.sessions, sessionID) 981 server.sessionsLock.Unlock() 982 } 983 } 984 985 func (server *MeekServer) deleteExpiredSessions() { 986 987 // A deleteSession call may block for up to MEEK_HTTP_CLIENT_IO_TIMEOUT, 988 // so grab a snapshot list of expired sessions and do not hold a lock for 989 // the duration of deleteExpiredSessions. This allows new sessions to be 990 // added concurrently. 991 // 992 // New sessions added after the snapshot is taken will be checked for 993 // expiry on subsequent periodic calls to deleteExpiredSessions. 994 // 995 // To avoid long delays in releasing resources, individual deletes are 996 // performed concurrently. 997 998 server.sessionsLock.Lock() 999 expiredSessionIDs := make([]string, 0) 1000 for sessionID, session := range server.sessions { 1001 if session.expired() { 1002 expiredSessionIDs = append(expiredSessionIDs, sessionID) 1003 } 1004 } 1005 server.sessionsLock.Unlock() 1006 1007 start := time.Now() 1008 1009 deleteWaitGroup := new(sync.WaitGroup) 1010 for _, sessionID := range expiredSessionIDs { 1011 deleteWaitGroup.Add(1) 1012 go func(sessionID string) { 1013 defer deleteWaitGroup.Done() 1014 server.deleteSession(sessionID) 1015 }(sessionID) 1016 } 1017 deleteWaitGroup.Wait() 1018 1019 log.WithTraceFields( 1020 LogFields{"elapsed time": time.Since(start)}).Debug("deleted expired sessions") 1021 } 1022 1023 // httpConnStateCallback tracks open persistent HTTP/HTTPS connections to the 1024 // meek server. 1025 func (server *MeekServer) httpConnStateCallback(conn net.Conn, connState http.ConnState) { 1026 switch connState { 1027 case http.StateNew: 1028 server.openConns.Add(conn) 1029 case http.StateHijacked, http.StateClosed: 1030 server.openConns.Remove(conn) 1031 } 1032 } 1033 1034 // getMeekCookiePayload extracts the payload from a meek cookie. The cookie 1035 // payload is base64 encoded, obfuscated, and NaCl encrypted. 1036 func (server *MeekServer) getMeekCookiePayload( 1037 clientIP string, cookieValue string) ([]byte, error) { 1038 1039 decodedValue, err := base64.StdEncoding.DecodeString(cookieValue) 1040 if err != nil { 1041 return nil, errors.Trace(err) 1042 } 1043 1044 // The data consists of an obfuscated seed message prepended 1045 // to the obfuscated, encrypted payload. The server obfuscator 1046 // will read the seed message, leaving the remaining encrypted 1047 // data in the reader. 1048 1049 reader := bytes.NewReader(decodedValue[:]) 1050 1051 obfuscator, err := obfuscator.NewServerObfuscator( 1052 &obfuscator.ObfuscatorConfig{ 1053 Keyword: server.support.Config.MeekObfuscatedKey, 1054 SeedHistory: server.obfuscatorSeedHistory, 1055 IrregularLogger: func(clientIP string, err error, logFields common.LogFields) { 1056 logIrregularTunnel( 1057 server.support, 1058 server.listenerTunnelProtocol, 1059 server.listenerPort, 1060 clientIP, 1061 errors.Trace(err), 1062 LogFields(logFields)) 1063 }, 1064 }, 1065 clientIP, 1066 reader) 1067 if err != nil { 1068 return nil, errors.Trace(err) 1069 } 1070 1071 offset, err := reader.Seek(0, 1) 1072 if err != nil { 1073 return nil, errors.Trace(err) 1074 } 1075 encryptedPayload := decodedValue[offset:] 1076 1077 obfuscator.ObfuscateClientToServer(encryptedPayload) 1078 1079 var nonce [24]byte 1080 var privateKey, ephemeralPublicKey [32]byte 1081 1082 decodedPrivateKey, err := base64.StdEncoding.DecodeString( 1083 server.support.Config.MeekCookieEncryptionPrivateKey) 1084 if err != nil { 1085 return nil, errors.Trace(err) 1086 } 1087 copy(privateKey[:], decodedPrivateKey) 1088 1089 if len(encryptedPayload) < 32 { 1090 return nil, errors.TraceNew("unexpected encrypted payload size") 1091 } 1092 copy(ephemeralPublicKey[0:32], encryptedPayload[0:32]) 1093 1094 payload, ok := box.Open(nil, encryptedPayload[32:], &nonce, &ephemeralPublicKey, &privateKey) 1095 if !ok { 1096 return nil, errors.TraceNew("open box failed") 1097 } 1098 1099 return payload, nil 1100 } 1101 1102 // makeMeekTLSConfig creates a TLS config for a meek HTTPS listener. 1103 // Currently, this config is optimized for fronted meek where the nature 1104 // of the connection is non-circumvention; it's optimized for performance 1105 // assuming the peer is an uncensored CDN. 1106 func (server *MeekServer) makeMeekTLSConfig( 1107 isFronted bool, useObfuscatedSessionTickets bool) (*tris.Config, error) { 1108 1109 certificate, privateKey, err := common.GenerateWebServerCertificate(values.GetHostName()) 1110 if err != nil { 1111 return nil, errors.Trace(err) 1112 } 1113 1114 tlsCertificate, err := tris.X509KeyPair( 1115 []byte(certificate), []byte(privateKey)) 1116 if err != nil { 1117 return nil, errors.Trace(err) 1118 } 1119 1120 // Vary the minimum version to frustrate scanning/fingerprinting of unfronted servers. 1121 // Limitation: like the certificate, this value changes on restart. 1122 minVersionCandidates := []uint16{tris.VersionTLS10, tris.VersionTLS11, tris.VersionTLS12} 1123 minVersion := minVersionCandidates[prng.Intn(len(minVersionCandidates))] 1124 1125 config := &tris.Config{ 1126 Certificates: []tris.Certificate{tlsCertificate}, 1127 NextProtos: []string{"http/1.1"}, 1128 MinVersion: minVersion, 1129 UseExtendedMasterSecret: true, 1130 } 1131 1132 if isFronted { 1133 // This is a reordering of the supported CipherSuites in golang 1.6[*]. Non-ephemeral key 1134 // CipherSuites greatly reduce server load, and we try to select these since the meek 1135 // protocol is providing obfuscation, not privacy/integrity (this is provided by the 1136 // tunneled SSH), so we don't benefit from the perfect forward secrecy property provided 1137 // by ephemeral key CipherSuites. 1138 // https://github.com/golang/go/blob/1cb3044c9fcd88e1557eca1bf35845a4108bc1db/src/crypto/tls/cipher_suites.go#L75 1139 // 1140 // This optimization is applied only when there's a CDN in front of the meek server; in 1141 // unfronted cases we prefer a more natural TLS handshake. 1142 // 1143 // [*] the list has since been updated, removing CipherSuites using RC4 and 3DES. 1144 config.CipherSuites = []uint16{ 1145 tris.TLS_RSA_WITH_AES_128_GCM_SHA256, 1146 tris.TLS_RSA_WITH_AES_256_GCM_SHA384, 1147 tris.TLS_RSA_WITH_AES_128_CBC_SHA, 1148 tris.TLS_RSA_WITH_AES_256_CBC_SHA, 1149 tris.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 1150 tris.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 1151 tris.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 1152 tris.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, 1153 tris.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, 1154 tris.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, 1155 tris.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, 1156 tris.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, 1157 } 1158 config.PreferServerCipherSuites = true 1159 } 1160 1161 if useObfuscatedSessionTickets { 1162 1163 // See obfuscated session ticket overview 1164 // in NewObfuscatedClientSessionCache. 1165 1166 config.UseObfuscatedSessionTickets = true 1167 1168 var obfuscatedSessionTicketKey [32]byte 1169 key, err := hex.DecodeString(server.support.Config.MeekObfuscatedKey) 1170 if err == nil && len(key) != 32 { 1171 err = std_errors.New("invalid obfuscated session key length") 1172 } 1173 if err != nil { 1174 return nil, errors.Trace(err) 1175 } 1176 copy(obfuscatedSessionTicketKey[:], key) 1177 1178 var standardSessionTicketKey [32]byte 1179 _, err = rand.Read(standardSessionTicketKey[:]) 1180 if err != nil { 1181 return nil, errors.Trace(err) 1182 } 1183 1184 // Note: SessionTicketKey needs to be set, or else, it appears, 1185 // tris.Config.serverInit() will clobber the value set by 1186 // SetSessionTicketKeys. 1187 config.SessionTicketKey = obfuscatedSessionTicketKey 1188 config.SetSessionTicketKeys([][32]byte{ 1189 standardSessionTicketKey, 1190 obfuscatedSessionTicketKey}) 1191 } 1192 1193 // When configured, initialize passthrough mode, an anti-probing defense. 1194 // Clients must prove knowledge of the obfuscated key via a message sent in 1195 // the TLS ClientHello random field. 1196 // 1197 // When clients fail to provide a valid message, the client connection is 1198 // relayed to the designated passthrough address, typically another web site. 1199 // The entire flow is relayed, including the original ClientHello, so the 1200 // client will perform a TLS handshake with the passthrough target. 1201 // 1202 // Irregular events are logged for invalid client activity. 1203 1204 if server.passthroughAddress != "" { 1205 1206 config.PassthroughAddress = server.passthroughAddress 1207 1208 config.PassthroughVerifyMessage = func( 1209 message []byte) bool { 1210 1211 return obfuscator.VerifyTLSPassthroughMessage( 1212 !server.support.Config.LegacyPassthrough, 1213 server.support.Config.MeekObfuscatedKey, 1214 message) 1215 } 1216 1217 config.PassthroughLogInvalidMessage = func( 1218 clientIP string) { 1219 1220 logIrregularTunnel( 1221 server.support, 1222 server.listenerTunnelProtocol, 1223 server.listenerPort, 1224 clientIP, 1225 errors.TraceNew("invalid passthrough message"), 1226 nil) 1227 } 1228 1229 config.PassthroughHistoryAddNew = func( 1230 clientIP string, 1231 clientRandom []byte) bool { 1232 1233 // Use a custom, shorter TTL based on the validity period of the 1234 // passthrough message. 1235 TTL := obfuscator.TLS_PASSTHROUGH_TIME_PERIOD 1236 if server.support.Config.LegacyPassthrough { 1237 TTL = obfuscator.HISTORY_SEED_TTL 1238 } 1239 1240 // strictMode is true as, unlike with meek cookies, legitimate meek clients 1241 // never retry TLS connections using a previous random value. 1242 1243 ok, logFields := server.obfuscatorSeedHistory.AddNewWithTTL( 1244 true, 1245 clientIP, 1246 "client-random", 1247 clientRandom, 1248 TTL) 1249 1250 if logFields != nil { 1251 logIrregularTunnel( 1252 server.support, 1253 server.listenerTunnelProtocol, 1254 server.listenerPort, 1255 clientIP, 1256 errors.TraceNew("duplicate passthrough message"), 1257 LogFields(*logFields)) 1258 } 1259 1260 return ok 1261 } 1262 } 1263 1264 return config, nil 1265 } 1266 1267 type meekSession struct { 1268 // Note: 64-bit ints used with atomic operations are placed 1269 // at the start of struct to ensure 64-bit alignment. 1270 // (https://golang.org/pkg/sync/atomic/#pkg-note-BUG) 1271 lastActivity int64 1272 requestCount int64 1273 metricClientRetries int64 1274 metricPeakResponseSize int64 1275 metricPeakCachedResponseSize int64 1276 metricPeakCachedResponseHitSize int64 1277 metricCachedResponseMissPosition int64 1278 metricUnderlyingConnCount int64 1279 lock sync.Mutex 1280 deleted bool 1281 underlyingConn net.Conn 1282 clientConn *meekConn 1283 meekProtocolVersion int 1284 sessionIDSent bool 1285 cachedResponse *CachedResponse 1286 cookieName string 1287 contentType string 1288 } 1289 1290 func (session *meekSession) touch() { 1291 atomic.StoreInt64(&session.lastActivity, int64(monotime.Now())) 1292 } 1293 1294 func (session *meekSession) expired() bool { 1295 if session.clientConn == nil { 1296 // Not fully initialized. meekSession.clientConn will be set before adding 1297 // the session to MeekServer.sessions. 1298 return false 1299 } 1300 lastActivity := monotime.Time(atomic.LoadInt64(&session.lastActivity)) 1301 return monotime.Since(lastActivity) > 1302 session.clientConn.meekServer.maxSessionStaleness 1303 } 1304 1305 // delete releases all resources allocated by a session. 1306 func (session *meekSession) delete(haveLock bool) { 1307 1308 // TODO: close the persistent HTTP client connection, if one exists? 1309 1310 // This final call session.cachedResponse.Reset releases shared resources. 1311 // 1312 // This call requires exclusive access. session.lock is be obtained before 1313 // calling session.cachedResponse.Reset. Once the lock is obtained, no 1314 // request for this session is being processed concurrently, and pending 1315 // requests will block at session.lock. 1316 // 1317 // This logic assumes that no further session.cachedResponse access occurs, 1318 // or else resources may deplete (buffers won't be returned to the pool). 1319 // These requirements are achieved by obtaining the lock, setting 1320 // session.deleted, and any subsequent request handlers checking 1321 // session.deleted immediately after obtaining the lock. 1322 // 1323 // session.lock.Lock may block for up to MEEK_HTTP_CLIENT_IO_TIMEOUT, 1324 // the timeout for any active request handler processing a session 1325 // request. 1326 // 1327 // When the lock must be acquired, clientConn.Close is called first, to 1328 // interrupt any existing request handler blocking on pumpReads or pumpWrites. 1329 1330 session.clientConn.Close() 1331 1332 if !haveLock { 1333 session.lock.Lock() 1334 } 1335 1336 // Release all extended buffers back to the pool. 1337 // session.cachedResponse.Reset is not safe for concurrent calls. 1338 session.cachedResponse.Reset() 1339 1340 session.deleted = true 1341 1342 if !haveLock { 1343 session.lock.Unlock() 1344 } 1345 } 1346 1347 // GetMetrics implements the common.MetricsSource interface. 1348 func (session *meekSession) GetMetrics() common.LogFields { 1349 logFields := make(common.LogFields) 1350 logFields["meek_client_retries"] = atomic.LoadInt64(&session.metricClientRetries) 1351 logFields["meek_peak_response_size"] = atomic.LoadInt64(&session.metricPeakResponseSize) 1352 logFields["meek_peak_cached_response_size"] = atomic.LoadInt64(&session.metricPeakCachedResponseSize) 1353 logFields["meek_peak_cached_response_hit_size"] = atomic.LoadInt64(&session.metricPeakCachedResponseHitSize) 1354 logFields["meek_cached_response_miss_position"] = atomic.LoadInt64(&session.metricCachedResponseMissPosition) 1355 logFields["meek_underlying_connection_count"] = atomic.LoadInt64(&session.metricUnderlyingConnCount) 1356 logFields["meek_cookie_name"] = session.cookieName 1357 logFields["meek_content_type"] = session.contentType 1358 return logFields 1359 } 1360 1361 // makeMeekSessionID creates a new session ID. The variable size is intended to 1362 // frustrate traffic analysis of both plaintext and TLS meek traffic. 1363 func makeMeekSessionID() (string, error) { 1364 1365 size := MEEK_MIN_SESSION_ID_LENGTH + 1366 prng.Intn(MEEK_MAX_SESSION_ID_LENGTH-MEEK_MIN_SESSION_ID_LENGTH) 1367 1368 sessionID, err := common.MakeSecureRandomBytes(size) 1369 if err != nil { 1370 return "", errors.Trace(err) 1371 } 1372 1373 // Omit padding to maximize variable size space. To the client, the session 1374 // ID is an opaque string cookie value. 1375 1376 return base64.RawStdEncoding.EncodeToString(sessionID), nil 1377 } 1378 1379 // meekConn implements the net.Conn interface and is to be used as a client 1380 // connection by the tunnel server (being passed to sshServer.handleClient). 1381 // meekConn bridges net/http request/response payload readers and writers 1382 // and goroutines calling Read()s and Write()s. 1383 type meekConn struct { 1384 meekServer *MeekServer 1385 meekSession *meekSession 1386 firstUnderlyingConn net.Conn 1387 remoteAddr net.Addr 1388 protocolVersion int 1389 closeBroadcast chan struct{} 1390 closed int32 1391 lastReadChecksum *uint64 1392 readLock sync.Mutex 1393 emptyReadBuffer chan *bytes.Buffer 1394 partialReadBuffer chan *bytes.Buffer 1395 fullReadBuffer chan *bytes.Buffer 1396 writeLock sync.Mutex 1397 nextWriteBuffer chan []byte 1398 writeResult chan error 1399 } 1400 1401 func newMeekConn( 1402 meekServer *MeekServer, 1403 meekSession *meekSession, 1404 underlyingConn net.Conn, 1405 remoteAddr net.Addr, 1406 protocolVersion int) *meekConn { 1407 1408 // In order to inspect its properties, meekConn will hold a reference to 1409 // firstUnderlyingConn, the _first_ underlying TCP conn, for the full 1410 // lifetime of meekConn, which may exceed the lifetime of firstUnderlyingConn 1411 // and include subsequent underlying TCP conns. In this case, it is expected 1412 // that firstUnderlyingConn will be closed by "net/http", so no OS resources 1413 // (e.g., a socket) are retained longer than necessary. 1414 1415 conn := &meekConn{ 1416 meekServer: meekServer, 1417 meekSession: meekSession, 1418 firstUnderlyingConn: underlyingConn, 1419 remoteAddr: remoteAddr, 1420 protocolVersion: protocolVersion, 1421 closeBroadcast: make(chan struct{}), 1422 closed: 0, 1423 emptyReadBuffer: make(chan *bytes.Buffer, 1), 1424 partialReadBuffer: make(chan *bytes.Buffer, 1), 1425 fullReadBuffer: make(chan *bytes.Buffer, 1), 1426 nextWriteBuffer: make(chan []byte, 1), 1427 writeResult: make(chan error, 1), 1428 } 1429 // Read() calls and pumpReads() are synchronized by exchanging control 1430 // of a single readBuffer. This is the same scheme used in and described 1431 // in psiphon.MeekConn. 1432 conn.emptyReadBuffer <- new(bytes.Buffer) 1433 return conn 1434 } 1435 1436 // GetMetrics implements the common.MetricsSource interface. The metrics are 1437 // maintained in the meek session type; but logTunnel, which calls 1438 // MetricsSource.GetMetrics, has a pointer only to this conn, so it calls 1439 // through to the session. 1440 func (conn *meekConn) GetMetrics() common.LogFields { 1441 1442 logFields := conn.meekSession.GetMetrics() 1443 1444 if conn.meekServer.passthroughAddress != "" { 1445 logFields["passthrough_address"] = conn.meekServer.passthroughAddress 1446 } 1447 1448 // Include metrics, such as fragmentor metrics, from the _first_ underlying 1449 // TCP conn. Properties of subsequent underlying TCP conns are not reflected 1450 // in these metrics; we assume that the first TCP conn, which most likely 1451 // transits the various protocol handshakes, is most significant. 1452 underlyingMetrics, ok := conn.firstUnderlyingConn.(common.MetricsSource) 1453 if ok { 1454 logFields.Add(underlyingMetrics.GetMetrics()) 1455 } 1456 1457 return logFields 1458 } 1459 1460 // GetUnderlyingTCPAddrs implements the common.UnderlyingTCPAddrSource 1461 // interface, returning the TCP addresses for the _first_ underlying TCP 1462 // connection in the meek tunnel. 1463 func (conn *meekConn) GetUnderlyingTCPAddrs() (*net.TCPAddr, *net.TCPAddr, bool) { 1464 localAddr, ok := conn.firstUnderlyingConn.LocalAddr().(*net.TCPAddr) 1465 if !ok { 1466 return nil, nil, false 1467 } 1468 remoteAddr, ok := conn.firstUnderlyingConn.RemoteAddr().(*net.TCPAddr) 1469 if !ok { 1470 return nil, nil, false 1471 } 1472 return localAddr, remoteAddr, true 1473 } 1474 1475 // SetReplay implements the common.FragmentorReplayAccessor interface, applying 1476 // the inputs to the _first_ underlying TCP connection in the meek tunnel. If 1477 // the underlying connection is closed, the SetSeed call will have no effect. 1478 func (conn *meekConn) SetReplay(PRNG *prng.PRNG) { 1479 fragmentor, ok := conn.firstUnderlyingConn.(common.FragmentorReplayAccessor) 1480 if ok { 1481 fragmentor.SetReplay(PRNG) 1482 } 1483 } 1484 1485 // GetReplay implements the FragmentorReplayAccessor interface, getting the 1486 // outputs from the _first_ underlying TCP connection in the meek tunnel. 1487 // 1488 // We assume that the first TCP conn is most significant: the initial TCP 1489 // connection most likely fragments protocol handshakes; and, in the case the 1490 // packet manipulation, any selected packet manipulation spec would have been 1491 // successful. 1492 func (conn *meekConn) GetReplay() (*prng.Seed, bool) { 1493 fragmentor, ok := conn.firstUnderlyingConn.(common.FragmentorReplayAccessor) 1494 if ok { 1495 return fragmentor.GetReplay() 1496 } 1497 return nil, false 1498 } 1499 1500 // pumpReads causes goroutines blocking on meekConn.Read() to read 1501 // from the specified reader. This function blocks until the reader 1502 // is fully consumed or the meekConn is closed. A read buffer allows 1503 // up to MEEK_MAX_REQUEST_PAYLOAD_LENGTH bytes to be read and buffered 1504 // without a Read() immediately consuming the bytes, but there's still 1505 // a possibility of a stall if no Read() calls are made after this 1506 // read buffer is full. 1507 // Returns the number of request bytes read. 1508 // Note: assumes only one concurrent call to pumpReads 1509 func (conn *meekConn) pumpReads(reader io.Reader) (int64, error) { 1510 1511 // Use either an empty or partial buffer. By using a partial 1512 // buffer, pumpReads will not block if the Read() caller has 1513 // not fully drained the read buffer. 1514 1515 var readBuffer *bytes.Buffer 1516 select { 1517 case readBuffer = <-conn.emptyReadBuffer: 1518 case readBuffer = <-conn.partialReadBuffer: 1519 case <-conn.closeBroadcast: 1520 return 0, io.EOF 1521 } 1522 1523 newDataOffset := readBuffer.Len() 1524 1525 // Since we need to read the full request payload in order to 1526 // take its checksum before relaying it, the read buffer can 1527 // grow to up to 2 x MEEK_MAX_REQUEST_PAYLOAD_LENGTH + 1. 1528 1529 // +1 allows for an explicit check for request payloads that 1530 // exceed the maximum permitted length. 1531 limitReader := io.LimitReader(reader, MEEK_MAX_REQUEST_PAYLOAD_LENGTH+1) 1532 n, err := readBuffer.ReadFrom(limitReader) 1533 1534 if err == nil && n == MEEK_MAX_REQUEST_PAYLOAD_LENGTH+1 { 1535 err = std_errors.New("invalid request payload length") 1536 } 1537 1538 // If the request read fails, don't relay the new data. This allows 1539 // the client to retry and resend its request payload without 1540 // interrupting/duplicating the payload flow. 1541 if err != nil { 1542 readBuffer.Truncate(newDataOffset) 1543 conn.replaceReadBuffer(readBuffer) 1544 return 0, errors.Trace(err) 1545 } 1546 1547 // Check if request payload checksum matches immediately 1548 // previous payload. On match, assume this is a client retry 1549 // sending payload that was already relayed and skip this 1550 // payload. Payload is OSSH ciphertext and almost surely 1551 // will not repeat. In the highly unlikely case that it does, 1552 // the underlying SSH connection will fail and the client 1553 // must reconnect. 1554 1555 checksum := crc64.Checksum( 1556 readBuffer.Bytes()[newDataOffset:], conn.meekServer.checksumTable) 1557 1558 if conn.lastReadChecksum == nil { 1559 conn.lastReadChecksum = new(uint64) 1560 } else if *conn.lastReadChecksum == checksum { 1561 readBuffer.Truncate(newDataOffset) 1562 } 1563 1564 *conn.lastReadChecksum = checksum 1565 1566 conn.replaceReadBuffer(readBuffer) 1567 1568 return n, nil 1569 } 1570 1571 var errMeekConnectionHasClosed = std_errors.New("meek connection has closed") 1572 1573 // Read reads from the meekConn into buffer. Read blocks until 1574 // some data is read or the meekConn closes. Under the hood, it 1575 // waits for pumpReads to submit a reader to read from. 1576 // Note: lock is to conform with net.Conn concurrency semantics 1577 func (conn *meekConn) Read(buffer []byte) (int, error) { 1578 conn.readLock.Lock() 1579 defer conn.readLock.Unlock() 1580 1581 var readBuffer *bytes.Buffer 1582 select { 1583 case readBuffer = <-conn.partialReadBuffer: 1584 case readBuffer = <-conn.fullReadBuffer: 1585 case <-conn.closeBroadcast: 1586 return 0, errors.Trace(errMeekConnectionHasClosed) 1587 } 1588 1589 n, err := readBuffer.Read(buffer) 1590 1591 conn.replaceReadBuffer(readBuffer) 1592 1593 return n, err 1594 } 1595 1596 func (conn *meekConn) replaceReadBuffer(readBuffer *bytes.Buffer) { 1597 length := readBuffer.Len() 1598 if length >= MEEK_MAX_REQUEST_PAYLOAD_LENGTH { 1599 conn.fullReadBuffer <- readBuffer 1600 } else if length == 0 { 1601 conn.emptyReadBuffer <- readBuffer 1602 } else { 1603 conn.partialReadBuffer <- readBuffer 1604 } 1605 } 1606 1607 // pumpWrites causes goroutines blocking on meekConn.Write() to write 1608 // to the specified writer. This function blocks until the meek response 1609 // body limits (size for protocol v1, turn around time for protocol v2+) 1610 // are met, or the meekConn is closed. 1611 // 1612 // Note: channel scheme assumes only one concurrent call to pumpWrites 1613 func (conn *meekConn) pumpWrites( 1614 writer io.Writer, skipExtendedTurnAround bool) (int, error) { 1615 1616 startTime := time.Now() 1617 timeout := time.NewTimer(conn.meekServer.turnAroundTimeout) 1618 defer timeout.Stop() 1619 1620 n := 0 1621 for { 1622 select { 1623 case buffer := <-conn.nextWriteBuffer: 1624 written, err := writer.Write(buffer) 1625 n += written 1626 // Assumes that writeResult won't block. 1627 // Note: always send the err to writeResult, 1628 // as the Write() caller is blocking on this. 1629 conn.writeResult <- err 1630 1631 if err != nil { 1632 return n, err 1633 } 1634 1635 if conn.protocolVersion < MEEK_PROTOCOL_VERSION_1 { 1636 // Pre-protocol version 1 clients expect at most 1637 // MEEK_MAX_REQUEST_PAYLOAD_LENGTH response bodies 1638 return n, nil 1639 } 1640 1641 if skipExtendedTurnAround { 1642 // When fast turn around is indicated, skip the extended turn 1643 // around timeout. This optimizes for upstream flows. 1644 return n, nil 1645 } 1646 1647 totalElapsedTime := time.Since(startTime) / time.Millisecond 1648 if totalElapsedTime >= conn.meekServer.extendedTurnAroundTimeout { 1649 return n, nil 1650 } 1651 timeout.Reset(conn.meekServer.turnAroundTimeout) 1652 1653 case <-timeout.C: 1654 return n, nil 1655 1656 case <-conn.closeBroadcast: 1657 return n, errors.Trace(errMeekConnectionHasClosed) 1658 } 1659 } 1660 } 1661 1662 // Write writes the buffer to the meekConn. It blocks until the 1663 // entire buffer is written to or the meekConn closes. Under the 1664 // hood, it waits for sufficient pumpWrites calls to consume the 1665 // write buffer. 1666 // Note: lock is to conform with net.Conn concurrency semantics 1667 func (conn *meekConn) Write(buffer []byte) (int, error) { 1668 conn.writeLock.Lock() 1669 defer conn.writeLock.Unlock() 1670 1671 // TODO: may be more efficient to send whole buffer 1672 // and have pumpWrites stash partial buffer when can't 1673 // send it all. 1674 1675 n := 0 1676 for n < len(buffer) { 1677 end := n + MEEK_MAX_REQUEST_PAYLOAD_LENGTH 1678 if end > len(buffer) { 1679 end = len(buffer) 1680 } 1681 1682 // Only write MEEK_MAX_REQUEST_PAYLOAD_LENGTH at a time, 1683 // to ensure compatibility with v1 protocol. 1684 chunk := buffer[n:end] 1685 1686 select { 1687 case conn.nextWriteBuffer <- chunk: 1688 case <-conn.closeBroadcast: 1689 return n, errors.Trace(errMeekConnectionHasClosed) 1690 } 1691 1692 // Wait for the buffer to be processed. 1693 select { 1694 case <-conn.writeResult: 1695 // The err from conn.writeResult comes from the 1696 // io.MultiWriter used in pumpWrites, which writes 1697 // to both the cached response and the HTTP response. 1698 // 1699 // Don't stop on error here, since only writing 1700 // to the HTTP response will fail, and the client 1701 // may retry and use the cached response. 1702 // 1703 // It's possible that the cached response buffer 1704 // is too small for the client to successfully 1705 // retry, but that cannot be determined. In this 1706 // case, the meek connection will eventually fail. 1707 // 1708 // err is already logged in ServeHTTP. 1709 case <-conn.closeBroadcast: 1710 return n, errors.Trace(errMeekConnectionHasClosed) 1711 } 1712 n += len(chunk) 1713 } 1714 return n, nil 1715 } 1716 1717 // Close closes the meekConn. This will interrupt any blocked 1718 // Read, Write, pumpReads, and pumpWrites. 1719 func (conn *meekConn) Close() error { 1720 if atomic.CompareAndSwapInt32(&conn.closed, 0, 1) { 1721 close(conn.closeBroadcast) 1722 1723 // In general, we reply on "net/http" to close underlying TCP conns. In this 1724 // case, we can directly close the first once, if it's still open. 1725 conn.firstUnderlyingConn.Close() 1726 } 1727 return nil 1728 } 1729 1730 // Stub implementation of net.Conn.LocalAddr 1731 func (conn *meekConn) LocalAddr() net.Addr { 1732 return nil 1733 } 1734 1735 // RemoteAddr returns the remoteAddr specified in newMeekConn. This 1736 // acts as a proxy for the actual remote address, which is either a 1737 // direct HTTP/HTTPS connection remote address, or in the case of 1738 // downstream proxy of CDN fronts, some other value determined via 1739 // HTTP headers. 1740 func (conn *meekConn) RemoteAddr() net.Addr { 1741 return conn.remoteAddr 1742 } 1743 1744 // SetDeadline is not a true implementation of net.Conn.SetDeadline. It 1745 // merely checks that the requested timeout exceeds the MEEK_MAX_SESSION_STALENESS 1746 // period. When it does, and the session is idle, the meekConn Read/Write will 1747 // be interrupted and return an error (not a timeout error) before the deadline. 1748 // In other words, this conn will approximate the desired functionality of 1749 // timing out on idle on or before the requested deadline. 1750 func (conn *meekConn) SetDeadline(t time.Time) error { 1751 // Overhead: nanoseconds (https://blog.cloudflare.com/its-go-time-on-linux/) 1752 if time.Now().Add(conn.meekServer.maxSessionStaleness).Before(t) { 1753 return nil 1754 } 1755 return errors.TraceNew("not supported") 1756 } 1757 1758 // Stub implementation of net.Conn.SetReadDeadline 1759 func (conn *meekConn) SetReadDeadline(t time.Time) error { 1760 return errors.TraceNew("not supported") 1761 } 1762 1763 // Stub implementation of net.Conn.SetWriteDeadline 1764 func (conn *meekConn) SetWriteDeadline(t time.Time) error { 1765 return errors.TraceNew("not supported") 1766 }