github.com/elfadel/cilium@v1.6.12/pkg/proxy/socket.go (about) 1 // Copyright 2016-2017 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package proxy 16 17 import ( 18 "fmt" 19 "net" 20 "os" 21 "sync" 22 "syscall" 23 "time" 24 25 "golang.org/x/sys/unix" 26 27 "github.com/cilium/cilium/pkg/flowdebug" 28 "github.com/cilium/cilium/pkg/lock" 29 "github.com/cilium/cilium/pkg/logging/logfields" 30 "github.com/cilium/cilium/pkg/maps/ctmap" 31 "github.com/cilium/cilium/pkg/u8proto" 32 33 "github.com/sirupsen/logrus" 34 ) 35 36 const ( 37 fieldConn = "conn" 38 fieldSize = "size" 39 fieldConnPair = "connPair" 40 ) 41 42 type proxySocket struct { 43 // listener is the TCP listener. 44 listener net.Listener 45 46 // locker protects closing the closing channel and accessing pairs. 47 locker lock.Mutex 48 49 closing chan struct{} 50 51 // pairs is the set of active connection pairs. 52 pairs []*connectionPair 53 } 54 55 func listenSocket(address string, mark int, transparent bool) (*proxySocket, error) { 56 socket := &proxySocket{ 57 closing: make(chan struct{}), 58 } 59 60 addr, err := net.ResolveTCPAddr("tcp", address) 61 if err != nil { 62 return nil, err 63 } 64 65 family := syscall.AF_INET 66 if addr.IP.To4() == nil { 67 family = syscall.AF_INET6 68 } 69 70 fd, err := syscall.Socket(family, syscall.SOCK_STREAM, 0) 71 if err != nil { 72 return nil, err 73 } 74 75 if err = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { 76 return nil, fmt.Errorf("unable to set SO_REUSEADDR socket option: %s", err) 77 } 78 79 if transparent { 80 if family == syscall.AF_INET { 81 err = syscall.SetsockoptInt(fd, unix.SOL_IP, unix.IP_TRANSPARENT, 1) 82 } else { 83 err = syscall.SetsockoptInt(fd, unix.SOL_IPV6, unix.IPV6_TRANSPARENT, 1) 84 } 85 if err != nil { 86 return nil, fmt.Errorf("unable to set SO_TRANSPARENT socket option: %s", err) 87 } 88 } 89 90 if mark != 0 { 91 setFdMark(fd, mark) 92 } 93 94 sockAddr, err := ipToSockaddr(family, addr.IP, addr.Port, addr.Zone) 95 if err != nil { 96 syscall.Close(fd) 97 return nil, err 98 } 99 100 if err := syscall.Bind(fd, sockAddr); err != nil { 101 syscall.Close(fd) 102 return nil, err 103 } 104 105 if err := syscall.Listen(fd, 128); err != nil { 106 syscall.Close(fd) 107 return nil, err 108 } 109 110 f := os.NewFile(uintptr(fd), addr.String()) 111 defer f.Close() 112 113 socket.listener, err = net.FileListener(f) 114 if err != nil { 115 return nil, err 116 } 117 118 return socket, nil 119 } 120 121 func setLinger(c net.Conn, linger time.Duration) error { 122 if tcp, ok := c.(*net.TCPConn); ok { 123 if err := tcp.SetLinger(int(linger.Seconds())); err != nil { 124 return fmt.Errorf("unable to set SO_LINGER socket option: %s", err) 125 } 126 } 127 128 return nil 129 } 130 131 // Accept calls Accept() on the listen socket of the proxy. 132 // If not nil, afterClose is called after the connection pair has been closed. 133 // If cascadeClose is true, the returned connectionPair will immediately be 134 // closed when this listen socket is closed. 135 func (s *proxySocket) Accept(cascadeClose bool) (*connectionPair, error) { 136 c, err := s.listener.Accept() 137 if err != nil { 138 return nil, err 139 } 140 141 // Set the SO_LINGER socket option so that a request connection is 142 // guaranteed to be closed within the proxyConnectionCloseTimeout. 143 // If the linger timeout expires, the connection is closed with a RST, 144 // which is useful to signal to the client that the termination is 145 // abnormal. 146 if err = setLinger(c, proxyConnectionCloseTimeout); err != nil { 147 c.Close() 148 return nil, err 149 } 150 151 // Enable keepalive on all accepted connections to force data on the 152 // TCP connection in regular intervals to ensure that the datapath 153 // never expires state associated with this connection. 154 if err = setKeepAlive(c); err != nil { 155 c.Close() 156 return nil, err 157 } 158 159 var afterClose func(*connectionPair) 160 if cascadeClose { 161 afterClose = s.connectionPairClosed 162 } 163 pair := newConnectionPair(afterClose) 164 165 s.locker.Lock() 166 if cascadeClose { 167 s.pairs = append(s.pairs, pair) 168 } 169 pair.Rx.SetConnection(c) 170 s.locker.Unlock() 171 172 return pair, nil 173 } 174 175 func (s *proxySocket) connectionPairClosed(pair *connectionPair) { 176 scopedLog := log.WithField(fieldConnPair, pair) 177 scopedLog.Debug("Connection pair closed, removing from proxy socket cascading delete list") 178 179 s.locker.Lock() 180 defer s.locker.Unlock() 181 for i, p := range s.pairs { 182 if p == pair { 183 scopedLog.Debug("Connection pair removed from proxy socket cascading delete list") 184 // Delete the pair from the list. 185 numPairs := len(s.pairs) 186 s.pairs[i] = s.pairs[numPairs-1] 187 s.pairs = s.pairs[:numPairs-1] 188 return 189 } 190 } 191 } 192 193 // Close closes the proxy socket and stops accepting new connections. 194 func (s *proxySocket) Close() { 195 s.locker.Lock() 196 197 select { 198 case <-s.closing: 199 s.locker.Unlock() 200 return 201 default: 202 } 203 204 close(s.closing) 205 s.listener.Close() 206 207 pairs := s.pairs 208 s.pairs = nil 209 210 s.locker.Unlock() 211 212 // Immediately close all connection pairs for which cascading close was 213 // requested in Accept. 214 for _, pair := range pairs { 215 pair.Rx.Close() 216 } 217 } 218 219 type socketQueue chan []byte 220 221 type proxyConnection struct { 222 // isRequestDirection indicates the direction of the TCP connection: 223 // ingress/request (true) or egress/response (false). 224 isRequestDirection bool 225 226 // conn is the underlying TCP connection. 227 conn net.Conn 228 229 // queue is the queue of messages to send. 230 queue socketQueue 231 232 // closeLocker is used to ensure the close channel may only be closed once. 233 closeLocker lock.Mutex 234 235 // close is the channel that is closed to indicate that the connection 236 // must be closed. closeLocker must be held when attempting to close this 237 // channel to prevent closing it multiple times. 238 close chan struct{} 239 240 // afterClose is a function that is called after the connection's queue is 241 // closed. 242 afterClose func() 243 } 244 245 func newProxyConnection(rx bool, afterClose func()) *proxyConnection { 246 return &proxyConnection{ 247 isRequestDirection: rx, 248 queue: make(socketQueue, socketQueueSize), 249 close: make(chan struct{}), 250 afterClose: afterClose, 251 } 252 } 253 254 // SetConnection associates an established connection to the proxy connection. 255 // It starts a goroutine to write Enqueue()ed messages into the connection. 256 func (c *proxyConnection) SetConnection(conn net.Conn) { 257 if c.conn != nil { 258 log.WithField(fieldConn, c).Panic("Established connection is already associated") 259 } 260 261 c.conn = conn 262 go c.writeQueuedMessages() 263 } 264 265 func fmtAddress(a net.Addr) string { 266 if a == nil { 267 return "nil" 268 } 269 return a.String() 270 } 271 272 // Closed returns true if the connection is closed 273 func (c *proxyConnection) Closed() bool { 274 return c == nil || c.conn == nil 275 } 276 277 func (c *proxyConnection) String() string { 278 if c.isRequestDirection { 279 if c.Closed() { 280 return "rx:closed" 281 } 282 283 return fmt.Sprintf("rx:%s->%s", 284 fmtAddress(c.conn.RemoteAddr()), 285 fmtAddress(c.conn.LocalAddr())) 286 } 287 288 if c.Closed() { 289 return "tx:closed" 290 } 291 292 return fmt.Sprintf("tx:%s->%s", 293 fmtAddress(c.conn.LocalAddr()), 294 fmtAddress(c.conn.RemoteAddr())) 295 } 296 297 func (c *proxyConnection) writeQueuedMessages() { 298 scopedLog := log.WithField(fieldConn, c) 299 defer c.Close() 300 301 for { 302 select { 303 case <-c.close: 304 scopedLog.Debug("Connection closed, message queue exiting") 305 return 306 307 case msg, more := <-c.queue: 308 if !more { 309 // This should never happen since the queue channel is never closed. 310 return 311 } 312 313 // Write the entire message into the socket. 314 _, err := c.conn.Write(msg) 315 316 // Ignore any write errors in case the socket has been closed by this proxy. 317 select { 318 case <-c.close: 319 scopedLog.Debug("Connection closed, message queue exiting") 320 return 321 default: 322 } 323 324 if err != nil { 325 scopedLog.WithError(err).Warn("Error while writing to socket, closing socket") 326 return 327 } 328 } 329 } 330 } 331 332 func (c *proxyConnection) direction() string { 333 if c.isRequestDirection { 334 return "request" 335 } 336 return "response" 337 } 338 339 // Enqueue queues a message to be written into the connection. 340 func (c *proxyConnection) Enqueue(msg []byte) { 341 scopedLog := log.WithFields(logrus.Fields{ 342 fieldConn: c, 343 fieldSize: len(msg), 344 }) 345 346 flowdebug.Log(scopedLog, fmt.Sprintf("Enqueueing %s message", c.direction())) 347 348 select { 349 case <-c.close: 350 flowdebug.Log(scopedLog, fmt.Sprintf("%s connection is closed; dropping message", c.direction())) 351 case c.queue <- msg: 352 flowdebug.Log(scopedLog, fmt.Sprintf("Enqueued %s message", c.direction())) 353 } 354 } 355 356 // Close closes this connection. 357 // The connection on the other side of the proxy is closed after it is queued 358 // for closing or after proxyConnectionCloseTimeout. 359 func (c *proxyConnection) Close() { 360 scopedLog := log.WithField(fieldConn, c) 361 362 c.closeLocker.Lock() 363 select { 364 case <-c.close: 365 // Already closed. Nothing to do. 366 c.closeLocker.Unlock() 367 return 368 default: 369 } 370 // Cause writeQueuedMessages to terminate. 371 close(c.close) 372 c.closeLocker.Unlock() 373 374 // Actually close the TCP connection. This will unblock any eventually 375 // blocking c.conn.Write call in writeQueuedMessages. 376 if !c.Closed() { 377 scopedLog.Debug("Closing socket") 378 c.conn.Close() 379 } 380 381 // Call connectionPair.close() concurrently so that this call doesn't block 382 // while waiting for the other connection in the pair to close. 383 go c.afterClose() 384 } 385 386 type connectionPair struct { 387 Rx, Tx *proxyConnection 388 afterCloseOnce sync.Once 389 afterClose func() 390 } 391 392 func newConnectionPair(afterClose func(*connectionPair)) *connectionPair { 393 pair := &connectionPair{} 394 pair.Rx = newProxyConnection(true, pair.close) 395 pair.Tx = newProxyConnection(false, pair.close) 396 if afterClose != nil { 397 pair.afterClose = func() { afterClose(pair) } 398 } 399 return pair 400 } 401 402 func (p *connectionPair) String() string { 403 return p.Rx.String() + "<->" + p.Tx.String() 404 } 405 406 func (p *connectionPair) close() { 407 scopedLog := log.WithField(fieldConnPair, p) 408 409 // Wait for both Rx and Tx to be closed or for timeout. 410 timeout := time.NewTimer(proxyConnectionCloseTimeout) 411 var bothClosed bool 412 select { 413 case <-p.Rx.close: 414 scopedLog.Debug("Rx is already closed, waiting for Tx to close") 415 // Rx is closed. Wait for Tx to close or timeout. 416 select { 417 case <-p.Tx.close: 418 bothClosed = true 419 case <-timeout.C: 420 scopedLog.Debug("Timeout while waiting for Tx to close; closing Tx") 421 p.Tx.Close() 422 } 423 case <-p.Tx.close: 424 // Tx is closed. Wait for Rx to close or timeout. 425 scopedLog.Debug("Tx is already closed, waiting for Rx to close") 426 select { 427 case <-p.Rx.close: 428 bothClosed = true 429 case <-timeout.C: 430 scopedLog.Debug("Timeout while waiting for Rx to close; closing Rx") 431 p.Rx.Close() 432 } 433 default: 434 // This case should never be selected, since connectionPair.close() is 435 // called only from proxyConnection.Close() after the close channel is 436 // closed, so at least one of the cases above can always be selected. 437 } 438 timeout.Stop() 439 440 if bothClosed { 441 scopedLog.Debug("Both Rx and Tx are closed") 442 if p.afterClose != nil { 443 p.afterCloseOnce.Do(p.afterClose) 444 } 445 } 446 } 447 448 func lookupSrcID(mapname, remoteAddr, localAddr string, ingress bool) (uint32, error) { 449 val, err := ctmap.Lookup(mapname, remoteAddr, localAddr, u8proto.TCP, ingress) 450 if err != nil { 451 flowdebug.Log(log.WithField(logfields.Object, logfields.Repr(val)), "Did not find proxy entry!") 452 return 0, err 453 } 454 455 flowdebug.Log(log.WithField(logfields.Object, logfields.Repr(val)), "Found proxy entry") 456 457 return val.SourceSecurityID, nil 458 } 459 460 func setFdMark(fd, mark int) { 461 scopedLog := log.WithFields(logrus.Fields{ 462 fieldFd: fd, 463 fieldMarker: mark, 464 }) 465 flowdebug.Log(scopedLog, "Setting packet marker of socket") 466 467 err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_MARK, mark) 468 if err != nil { 469 470 scopedLog.WithError(err).Warn("Unable to set SO_MARK") 471 } 472 } 473 474 func setSocketMark(c net.Conn, mark int) { 475 if tc, ok := c.(*net.TCPConn); ok { 476 if f, err := tc.File(); err == nil { 477 defer f.Close() 478 setFdMark(int(f.Fd()), mark) 479 } 480 } 481 }