github.com/devwanda/aphelion-staking@v0.33.9/p2p/conn/connection.go (about) 1 package conn 2 3 import ( 4 "bufio" 5 "runtime/debug" 6 7 "fmt" 8 "io" 9 "math" 10 "net" 11 "reflect" 12 "sync" 13 "sync/atomic" 14 "time" 15 16 "github.com/pkg/errors" 17 18 amino "github.com/evdatsion/go-amino" 19 20 flow "github.com/devwanda/aphelion-staking/libs/flowrate" 21 "github.com/devwanda/aphelion-staking/libs/log" 22 tmmath "github.com/devwanda/aphelion-staking/libs/math" 23 "github.com/devwanda/aphelion-staking/libs/service" 24 "github.com/devwanda/aphelion-staking/libs/timer" 25 ) 26 27 const ( 28 defaultMaxPacketMsgPayloadSize = 1024 29 30 numBatchPacketMsgs = 10 31 minReadBufferSize = 1024 32 minWriteBufferSize = 65536 33 updateStats = 2 * time.Second 34 35 // some of these defaults are written in the user config 36 // flushThrottle, sendRate, recvRate 37 // TODO: remove values present in config 38 defaultFlushThrottle = 100 * time.Millisecond 39 40 defaultSendQueueCapacity = 1 41 defaultRecvBufferCapacity = 4096 42 defaultRecvMessageCapacity = 22020096 // 21MB 43 defaultSendRate = int64(512000) // 500KB/s 44 defaultRecvRate = int64(512000) // 500KB/s 45 defaultSendTimeout = 10 * time.Second 46 defaultPingInterval = 60 * time.Second 47 defaultPongTimeout = 45 * time.Second 48 ) 49 50 type receiveCbFunc func(chID byte, msgBytes []byte) 51 type errorCbFunc func(interface{}) 52 53 /* 54 Each peer has one `MConnection` (multiplex connection) instance. 55 56 __multiplex__ *noun* a system or signal involving simultaneous transmission of 57 several messages along a single channel of communication. 58 59 Each `MConnection` handles message transmission on multiple abstract communication 60 `Channel`s. Each channel has a globally unique byte id. 61 The byte id and the relative priorities of each `Channel` are configured upon 62 initialization of the connection. 63 64 There are two methods for sending messages: 65 func (m MConnection) Send(chID byte, msgBytes []byte) bool {} 66 func (m MConnection) TrySend(chID byte, msgBytes []byte}) bool {} 67 68 `Send(chID, msgBytes)` is a blocking call that waits until `msg` is 69 successfully queued for the channel with the given id byte `chID`, or until the 70 request times out. The message `msg` is serialized using Go-Amino. 71 72 `TrySend(chID, msgBytes)` is a nonblocking call that returns false if the 73 channel's queue is full. 74 75 Inbound message bytes are handled with an onReceive callback function. 76 */ 77 type MConnection struct { 78 service.BaseService 79 80 conn net.Conn 81 bufConnReader *bufio.Reader 82 bufConnWriter *bufio.Writer 83 sendMonitor *flow.Monitor 84 recvMonitor *flow.Monitor 85 send chan struct{} 86 pong chan struct{} 87 channels []*Channel 88 channelsIdx map[byte]*Channel 89 onReceive receiveCbFunc 90 onError errorCbFunc 91 errored uint32 92 config MConnConfig 93 94 // Closing quitSendRoutine will cause the sendRoutine to eventually quit. 95 // doneSendRoutine is closed when the sendRoutine actually quits. 96 quitSendRoutine chan struct{} 97 doneSendRoutine chan struct{} 98 99 // Closing quitRecvRouting will cause the recvRouting to eventually quit. 100 quitRecvRoutine chan struct{} 101 102 // used to ensure FlushStop and OnStop 103 // are safe to call concurrently. 104 stopMtx sync.Mutex 105 106 flushTimer *timer.ThrottleTimer // flush writes as necessary but throttled. 107 pingTimer *time.Ticker // send pings periodically 108 109 // close conn if pong is not received in pongTimeout 110 pongTimer *time.Timer 111 pongTimeoutCh chan bool // true - timeout, false - peer sent pong 112 113 chStatsTimer *time.Ticker // update channel stats periodically 114 115 created time.Time // time of creation 116 117 _maxPacketMsgSize int 118 } 119 120 // MConnConfig is a MConnection configuration. 121 type MConnConfig struct { 122 SendRate int64 `mapstructure:"send_rate"` 123 RecvRate int64 `mapstructure:"recv_rate"` 124 125 // Maximum payload size 126 MaxPacketMsgPayloadSize int `mapstructure:"max_packet_msg_payload_size"` 127 128 // Interval to flush writes (throttled) 129 FlushThrottle time.Duration `mapstructure:"flush_throttle"` 130 131 // Interval to send pings 132 PingInterval time.Duration `mapstructure:"ping_interval"` 133 134 // Maximum wait time for pongs 135 PongTimeout time.Duration `mapstructure:"pong_timeout"` 136 } 137 138 // DefaultMConnConfig returns the default config. 139 func DefaultMConnConfig() MConnConfig { 140 return MConnConfig{ 141 SendRate: defaultSendRate, 142 RecvRate: defaultRecvRate, 143 MaxPacketMsgPayloadSize: defaultMaxPacketMsgPayloadSize, 144 FlushThrottle: defaultFlushThrottle, 145 PingInterval: defaultPingInterval, 146 PongTimeout: defaultPongTimeout, 147 } 148 } 149 150 // NewMConnection wraps net.Conn and creates multiplex connection 151 func NewMConnection( 152 conn net.Conn, 153 chDescs []*ChannelDescriptor, 154 onReceive receiveCbFunc, 155 onError errorCbFunc, 156 ) *MConnection { 157 return NewMConnectionWithConfig( 158 conn, 159 chDescs, 160 onReceive, 161 onError, 162 DefaultMConnConfig()) 163 } 164 165 // NewMConnectionWithConfig wraps net.Conn and creates multiplex connection with a config 166 func NewMConnectionWithConfig( 167 conn net.Conn, 168 chDescs []*ChannelDescriptor, 169 onReceive receiveCbFunc, 170 onError errorCbFunc, 171 config MConnConfig, 172 ) *MConnection { 173 if config.PongTimeout >= config.PingInterval { 174 panic("pongTimeout must be less than pingInterval (otherwise, next ping will reset pong timer)") 175 } 176 177 mconn := &MConnection{ 178 conn: conn, 179 bufConnReader: bufio.NewReaderSize(conn, minReadBufferSize), 180 bufConnWriter: bufio.NewWriterSize(conn, minWriteBufferSize), 181 sendMonitor: flow.New(0, 0), 182 recvMonitor: flow.New(0, 0), 183 send: make(chan struct{}, 1), 184 pong: make(chan struct{}, 1), 185 onReceive: onReceive, 186 onError: onError, 187 config: config, 188 created: time.Now(), 189 } 190 191 // Create channels 192 var channelsIdx = map[byte]*Channel{} 193 var channels = []*Channel{} 194 195 for _, desc := range chDescs { 196 channel := newChannel(mconn, *desc) 197 channelsIdx[channel.desc.ID] = channel 198 channels = append(channels, channel) 199 } 200 mconn.channels = channels 201 mconn.channelsIdx = channelsIdx 202 203 mconn.BaseService = *service.NewBaseService(nil, "MConnection", mconn) 204 205 // maxPacketMsgSize() is a bit heavy, so call just once 206 mconn._maxPacketMsgSize = mconn.maxPacketMsgSize() 207 208 return mconn 209 } 210 211 func (c *MConnection) SetLogger(l log.Logger) { 212 c.BaseService.SetLogger(l) 213 for _, ch := range c.channels { 214 ch.SetLogger(l) 215 } 216 } 217 218 // OnStart implements BaseService 219 func (c *MConnection) OnStart() error { 220 if err := c.BaseService.OnStart(); err != nil { 221 return err 222 } 223 c.flushTimer = timer.NewThrottleTimer("flush", c.config.FlushThrottle) 224 c.pingTimer = time.NewTicker(c.config.PingInterval) 225 c.pongTimeoutCh = make(chan bool, 1) 226 c.chStatsTimer = time.NewTicker(updateStats) 227 c.quitSendRoutine = make(chan struct{}) 228 c.doneSendRoutine = make(chan struct{}) 229 c.quitRecvRoutine = make(chan struct{}) 230 go c.sendRoutine() 231 go c.recvRoutine() 232 return nil 233 } 234 235 // stopServices stops the BaseService and timers and closes the quitSendRoutine. 236 // if the quitSendRoutine was already closed, it returns true, otherwise it returns false. 237 // It uses the stopMtx to ensure only one of FlushStop and OnStop can do this at a time. 238 func (c *MConnection) stopServices() (alreadyStopped bool) { 239 c.stopMtx.Lock() 240 defer c.stopMtx.Unlock() 241 242 select { 243 case <-c.quitSendRoutine: 244 // already quit 245 return true 246 default: 247 } 248 249 select { 250 case <-c.quitRecvRoutine: 251 // already quit 252 return true 253 default: 254 } 255 256 c.BaseService.OnStop() 257 c.flushTimer.Stop() 258 c.pingTimer.Stop() 259 c.chStatsTimer.Stop() 260 261 // inform the recvRouting that we are shutting down 262 close(c.quitRecvRoutine) 263 close(c.quitSendRoutine) 264 return false 265 } 266 267 // FlushStop replicates the logic of OnStop. 268 // It additionally ensures that all successful 269 // .Send() calls will get flushed before closing 270 // the connection. 271 func (c *MConnection) FlushStop() { 272 if c.stopServices() { 273 return 274 } 275 276 // this block is unique to FlushStop 277 { 278 // wait until the sendRoutine exits 279 // so we dont race on calling sendSomePacketMsgs 280 <-c.doneSendRoutine 281 282 // Send and flush all pending msgs. 283 // Since sendRoutine has exited, we can call this 284 // safely 285 eof := c.sendSomePacketMsgs() 286 for !eof { 287 eof = c.sendSomePacketMsgs() 288 } 289 c.flush() 290 291 // Now we can close the connection 292 } 293 294 c.conn.Close() // nolint: errcheck 295 296 // We can't close pong safely here because 297 // recvRoutine may write to it after we've stopped. 298 // Though it doesn't need to get closed at all, 299 // we close it @ recvRoutine. 300 301 // c.Stop() 302 } 303 304 // OnStop implements BaseService 305 func (c *MConnection) OnStop() { 306 if c.stopServices() { 307 return 308 } 309 310 c.conn.Close() // nolint: errcheck 311 312 // We can't close pong safely here because 313 // recvRoutine may write to it after we've stopped. 314 // Though it doesn't need to get closed at all, 315 // we close it @ recvRoutine. 316 } 317 318 func (c *MConnection) String() string { 319 return fmt.Sprintf("MConn{%v}", c.conn.RemoteAddr()) 320 } 321 322 func (c *MConnection) flush() { 323 c.Logger.Debug("Flush", "conn", c) 324 err := c.bufConnWriter.Flush() 325 if err != nil { 326 c.Logger.Error("MConnection flush failed", "err", err) 327 } 328 } 329 330 // Catch panics, usually caused by remote disconnects. 331 func (c *MConnection) _recover() { 332 if r := recover(); r != nil { 333 c.Logger.Error("MConnection panicked", "err", r, "stack", string(debug.Stack())) 334 c.stopForError(errors.Errorf("recovered from panic: %v", r)) 335 } 336 } 337 338 func (c *MConnection) stopForError(r interface{}) { 339 c.Stop() 340 if atomic.CompareAndSwapUint32(&c.errored, 0, 1) { 341 if c.onError != nil { 342 c.onError(r) 343 } 344 } 345 } 346 347 // Queues a message to be sent to channel. 348 func (c *MConnection) Send(chID byte, msgBytes []byte) bool { 349 if !c.IsRunning() { 350 return false 351 } 352 353 c.Logger.Debug("Send", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) 354 355 // Send message to channel. 356 channel, ok := c.channelsIdx[chID] 357 if !ok { 358 c.Logger.Error(fmt.Sprintf("Cannot send bytes, unknown channel %X", chID)) 359 return false 360 } 361 362 success := channel.sendBytes(msgBytes) 363 if success { 364 // Wake up sendRoutine if necessary 365 select { 366 case c.send <- struct{}{}: 367 default: 368 } 369 } else { 370 c.Logger.Debug("Send failed", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) 371 } 372 return success 373 } 374 375 // Queues a message to be sent to channel. 376 // Nonblocking, returns true if successful. 377 func (c *MConnection) TrySend(chID byte, msgBytes []byte) bool { 378 if !c.IsRunning() { 379 return false 380 } 381 382 c.Logger.Debug("TrySend", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) 383 384 // Send message to channel. 385 channel, ok := c.channelsIdx[chID] 386 if !ok { 387 c.Logger.Error(fmt.Sprintf("Cannot send bytes, unknown channel %X", chID)) 388 return false 389 } 390 391 ok = channel.trySendBytes(msgBytes) 392 if ok { 393 // Wake up sendRoutine if necessary 394 select { 395 case c.send <- struct{}{}: 396 default: 397 } 398 } 399 400 return ok 401 } 402 403 // CanSend returns true if you can send more data onto the chID, false 404 // otherwise. Use only as a heuristic. 405 func (c *MConnection) CanSend(chID byte) bool { 406 if !c.IsRunning() { 407 return false 408 } 409 410 channel, ok := c.channelsIdx[chID] 411 if !ok { 412 c.Logger.Error(fmt.Sprintf("Unknown channel %X", chID)) 413 return false 414 } 415 return channel.canSend() 416 } 417 418 // sendRoutine polls for packets to send from channels. 419 func (c *MConnection) sendRoutine() { 420 defer c._recover() 421 422 FOR_LOOP: 423 for { 424 var _n int64 425 var err error 426 SELECTION: 427 select { 428 case <-c.flushTimer.Ch: 429 // NOTE: flushTimer.Set() must be called every time 430 // something is written to .bufConnWriter. 431 c.flush() 432 case <-c.chStatsTimer.C: 433 for _, channel := range c.channels { 434 channel.updateStats() 435 } 436 case <-c.pingTimer.C: 437 c.Logger.Debug("Send Ping") 438 _n, err = cdc.MarshalBinaryLengthPrefixedWriter(c.bufConnWriter, PacketPing{}) 439 if err != nil { 440 break SELECTION 441 } 442 c.sendMonitor.Update(int(_n)) 443 c.Logger.Debug("Starting pong timer", "dur", c.config.PongTimeout) 444 c.pongTimer = time.AfterFunc(c.config.PongTimeout, func() { 445 select { 446 case c.pongTimeoutCh <- true: 447 default: 448 } 449 }) 450 c.flush() 451 case timeout := <-c.pongTimeoutCh: 452 if timeout { 453 c.Logger.Debug("Pong timeout") 454 err = errors.New("pong timeout") 455 } else { 456 c.stopPongTimer() 457 } 458 case <-c.pong: 459 c.Logger.Debug("Send Pong") 460 _n, err = cdc.MarshalBinaryLengthPrefixedWriter(c.bufConnWriter, PacketPong{}) 461 if err != nil { 462 break SELECTION 463 } 464 c.sendMonitor.Update(int(_n)) 465 c.flush() 466 case <-c.quitSendRoutine: 467 break FOR_LOOP 468 case <-c.send: 469 // Send some PacketMsgs 470 eof := c.sendSomePacketMsgs() 471 if !eof { 472 // Keep sendRoutine awake. 473 select { 474 case c.send <- struct{}{}: 475 default: 476 } 477 } 478 } 479 480 if !c.IsRunning() { 481 break FOR_LOOP 482 } 483 if err != nil { 484 c.Logger.Error("Connection failed @ sendRoutine", "conn", c, "err", err) 485 c.stopForError(err) 486 break FOR_LOOP 487 } 488 } 489 490 // Cleanup 491 c.stopPongTimer() 492 close(c.doneSendRoutine) 493 } 494 495 // Returns true if messages from channels were exhausted. 496 // Blocks in accordance to .sendMonitor throttling. 497 func (c *MConnection) sendSomePacketMsgs() bool { 498 // Block until .sendMonitor says we can write. 499 // Once we're ready we send more than we asked for, 500 // but amortized it should even out. 501 c.sendMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.SendRate), true) 502 503 // Now send some PacketMsgs. 504 for i := 0; i < numBatchPacketMsgs; i++ { 505 if c.sendPacketMsg() { 506 return true 507 } 508 } 509 return false 510 } 511 512 // Returns true if messages from channels were exhausted. 513 func (c *MConnection) sendPacketMsg() bool { 514 // Choose a channel to create a PacketMsg from. 515 // The chosen channel will be the one whose recentlySent/priority is the least. 516 var leastRatio float32 = math.MaxFloat32 517 var leastChannel *Channel 518 for _, channel := range c.channels { 519 // If nothing to send, skip this channel 520 if !channel.isSendPending() { 521 continue 522 } 523 // Get ratio, and keep track of lowest ratio. 524 ratio := float32(channel.recentlySent) / float32(channel.desc.Priority) 525 if ratio < leastRatio { 526 leastRatio = ratio 527 leastChannel = channel 528 } 529 } 530 531 // Nothing to send? 532 if leastChannel == nil { 533 return true 534 } 535 // c.Logger.Info("Found a msgPacket to send") 536 537 // Make & send a PacketMsg from this channel 538 _n, err := leastChannel.writePacketMsgTo(c.bufConnWriter) 539 if err != nil { 540 c.Logger.Error("Failed to write PacketMsg", "err", err) 541 c.stopForError(err) 542 return true 543 } 544 c.sendMonitor.Update(int(_n)) 545 c.flushTimer.Set() 546 return false 547 } 548 549 // recvRoutine reads PacketMsgs and reconstructs the message using the channels' "recving" buffer. 550 // After a whole message has been assembled, it's pushed to onReceive(). 551 // Blocks depending on how the connection is throttled. 552 // Otherwise, it never blocks. 553 func (c *MConnection) recvRoutine() { 554 defer c._recover() 555 556 FOR_LOOP: 557 for { 558 // Block until .recvMonitor says we can read. 559 c.recvMonitor.Limit(c._maxPacketMsgSize, atomic.LoadInt64(&c.config.RecvRate), true) 560 561 // Peek into bufConnReader for debugging 562 /* 563 if numBytes := c.bufConnReader.Buffered(); numBytes > 0 { 564 bz, err := c.bufConnReader.Peek(tmmath.MinInt(numBytes, 100)) 565 if err == nil { 566 // return 567 } else { 568 c.Logger.Debug("Error peeking connection buffer", "err", err) 569 // return nil 570 } 571 c.Logger.Info("Peek connection buffer", "numBytes", numBytes, "bz", bz) 572 } 573 */ 574 575 // Read packet type 576 var packet Packet 577 var _n int64 578 var err error 579 _n, err = cdc.UnmarshalBinaryLengthPrefixedReader(c.bufConnReader, &packet, int64(c._maxPacketMsgSize)) 580 c.recvMonitor.Update(int(_n)) 581 582 if err != nil { 583 // stopServices was invoked and we are shutting down 584 // receiving is excpected to fail since we will close the connection 585 select { 586 case <-c.quitRecvRoutine: 587 break FOR_LOOP 588 default: 589 } 590 591 if c.IsRunning() { 592 if err == io.EOF { 593 c.Logger.Info("Connection is closed @ recvRoutine (likely by the other side)", "conn", c) 594 } else { 595 c.Logger.Error("Connection failed @ recvRoutine (reading byte)", "conn", c, "err", err) 596 } 597 c.stopForError(err) 598 } 599 break FOR_LOOP 600 } 601 602 // Read more depending on packet type. 603 switch pkt := packet.(type) { 604 case PacketPing: 605 // TODO: prevent abuse, as they cause flush()'s. 606 // https://github.com/devwanda/aphelion-staking/issues/1190 607 c.Logger.Debug("Receive Ping") 608 select { 609 case c.pong <- struct{}{}: 610 default: 611 // never block 612 } 613 case PacketPong: 614 c.Logger.Debug("Receive Pong") 615 select { 616 case c.pongTimeoutCh <- false: 617 default: 618 // never block 619 } 620 case PacketMsg: 621 channel, ok := c.channelsIdx[pkt.ChannelID] 622 if !ok || channel == nil { 623 err := fmt.Errorf("unknown channel %X", pkt.ChannelID) 624 c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) 625 c.stopForError(err) 626 break FOR_LOOP 627 } 628 629 msgBytes, err := channel.recvPacketMsg(pkt) 630 if err != nil { 631 if c.IsRunning() { 632 c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) 633 c.stopForError(err) 634 } 635 break FOR_LOOP 636 } 637 if msgBytes != nil { 638 c.Logger.Debug("Received bytes", "chID", pkt.ChannelID, "msgBytes", fmt.Sprintf("%X", msgBytes)) 639 // NOTE: This means the reactor.Receive runs in the same thread as the p2p recv routine 640 c.onReceive(pkt.ChannelID, msgBytes) 641 } 642 default: 643 err := fmt.Errorf("unknown message type %v", reflect.TypeOf(packet)) 644 c.Logger.Error("Connection failed @ recvRoutine", "conn", c, "err", err) 645 c.stopForError(err) 646 break FOR_LOOP 647 } 648 } 649 650 // Cleanup 651 close(c.pong) 652 for range c.pong { 653 // Drain 654 } 655 } 656 657 // not goroutine-safe 658 func (c *MConnection) stopPongTimer() { 659 if c.pongTimer != nil { 660 _ = c.pongTimer.Stop() 661 c.pongTimer = nil 662 } 663 } 664 665 // maxPacketMsgSize returns a maximum size of PacketMsg, including the overhead 666 // of amino encoding. 667 func (c *MConnection) maxPacketMsgSize() int { 668 return len(cdc.MustMarshalBinaryLengthPrefixed(PacketMsg{ 669 ChannelID: 0x01, 670 EOF: 1, 671 Bytes: make([]byte, c.config.MaxPacketMsgPayloadSize), 672 })) + 10 // leave room for changes in amino 673 } 674 675 type ConnectionStatus struct { 676 Duration time.Duration 677 SendMonitor flow.Status 678 RecvMonitor flow.Status 679 Channels []ChannelStatus 680 } 681 682 type ChannelStatus struct { 683 ID byte 684 SendQueueCapacity int 685 SendQueueSize int 686 Priority int 687 RecentlySent int64 688 } 689 690 func (c *MConnection) Status() ConnectionStatus { 691 var status ConnectionStatus 692 status.Duration = time.Since(c.created) 693 status.SendMonitor = c.sendMonitor.Status() 694 status.RecvMonitor = c.recvMonitor.Status() 695 status.Channels = make([]ChannelStatus, len(c.channels)) 696 for i, channel := range c.channels { 697 status.Channels[i] = ChannelStatus{ 698 ID: channel.desc.ID, 699 SendQueueCapacity: cap(channel.sendQueue), 700 SendQueueSize: int(atomic.LoadInt32(&channel.sendQueueSize)), 701 Priority: channel.desc.Priority, 702 RecentlySent: atomic.LoadInt64(&channel.recentlySent), 703 } 704 } 705 return status 706 } 707 708 //----------------------------------------------------------------------------- 709 710 type ChannelDescriptor struct { 711 ID byte 712 Priority int 713 SendQueueCapacity int 714 RecvBufferCapacity int 715 RecvMessageCapacity int 716 } 717 718 func (chDesc ChannelDescriptor) FillDefaults() (filled ChannelDescriptor) { 719 if chDesc.SendQueueCapacity == 0 { 720 chDesc.SendQueueCapacity = defaultSendQueueCapacity 721 } 722 if chDesc.RecvBufferCapacity == 0 { 723 chDesc.RecvBufferCapacity = defaultRecvBufferCapacity 724 } 725 if chDesc.RecvMessageCapacity == 0 { 726 chDesc.RecvMessageCapacity = defaultRecvMessageCapacity 727 } 728 filled = chDesc 729 return 730 } 731 732 // TODO: lowercase. 733 // NOTE: not goroutine-safe. 734 type Channel struct { 735 conn *MConnection 736 desc ChannelDescriptor 737 sendQueue chan []byte 738 sendQueueSize int32 // atomic. 739 recving []byte 740 sending []byte 741 recentlySent int64 // exponential moving average 742 743 maxPacketMsgPayloadSize int 744 745 Logger log.Logger 746 } 747 748 func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel { 749 desc = desc.FillDefaults() 750 if desc.Priority <= 0 { 751 panic("Channel default priority must be a positive integer") 752 } 753 return &Channel{ 754 conn: conn, 755 desc: desc, 756 sendQueue: make(chan []byte, desc.SendQueueCapacity), 757 recving: make([]byte, 0, desc.RecvBufferCapacity), 758 maxPacketMsgPayloadSize: conn.config.MaxPacketMsgPayloadSize, 759 } 760 } 761 762 func (ch *Channel) SetLogger(l log.Logger) { 763 ch.Logger = l 764 } 765 766 // Queues message to send to this channel. 767 // Goroutine-safe 768 // Times out (and returns false) after defaultSendTimeout 769 func (ch *Channel) sendBytes(bytes []byte) bool { 770 select { 771 case ch.sendQueue <- bytes: 772 atomic.AddInt32(&ch.sendQueueSize, 1) 773 return true 774 case <-time.After(defaultSendTimeout): 775 return false 776 } 777 } 778 779 // Queues message to send to this channel. 780 // Nonblocking, returns true if successful. 781 // Goroutine-safe 782 func (ch *Channel) trySendBytes(bytes []byte) bool { 783 select { 784 case ch.sendQueue <- bytes: 785 atomic.AddInt32(&ch.sendQueueSize, 1) 786 return true 787 default: 788 return false 789 } 790 } 791 792 // Goroutine-safe 793 func (ch *Channel) loadSendQueueSize() (size int) { 794 return int(atomic.LoadInt32(&ch.sendQueueSize)) 795 } 796 797 // Goroutine-safe 798 // Use only as a heuristic. 799 func (ch *Channel) canSend() bool { 800 return ch.loadSendQueueSize() < defaultSendQueueCapacity 801 } 802 803 // Returns true if any PacketMsgs are pending to be sent. 804 // Call before calling nextPacketMsg() 805 // Goroutine-safe 806 func (ch *Channel) isSendPending() bool { 807 if len(ch.sending) == 0 { 808 if len(ch.sendQueue) == 0 { 809 return false 810 } 811 ch.sending = <-ch.sendQueue 812 } 813 return true 814 } 815 816 // Creates a new PacketMsg to send. 817 // Not goroutine-safe 818 func (ch *Channel) nextPacketMsg() PacketMsg { 819 packet := PacketMsg{} 820 packet.ChannelID = ch.desc.ID 821 maxSize := ch.maxPacketMsgPayloadSize 822 packet.Bytes = ch.sending[:tmmath.MinInt(maxSize, len(ch.sending))] 823 if len(ch.sending) <= maxSize { 824 packet.EOF = byte(0x01) 825 ch.sending = nil 826 atomic.AddInt32(&ch.sendQueueSize, -1) // decrement sendQueueSize 827 } else { 828 packet.EOF = byte(0x00) 829 ch.sending = ch.sending[tmmath.MinInt(maxSize, len(ch.sending)):] 830 } 831 return packet 832 } 833 834 // Writes next PacketMsg to w and updates c.recentlySent. 835 // Not goroutine-safe 836 func (ch *Channel) writePacketMsgTo(w io.Writer) (n int64, err error) { 837 var packet = ch.nextPacketMsg() 838 n, err = cdc.MarshalBinaryLengthPrefixedWriter(w, packet) 839 atomic.AddInt64(&ch.recentlySent, n) 840 return 841 } 842 843 // Handles incoming PacketMsgs. It returns a message bytes if message is 844 // complete. NOTE message bytes may change on next call to recvPacketMsg. 845 // Not goroutine-safe 846 func (ch *Channel) recvPacketMsg(packet PacketMsg) ([]byte, error) { 847 ch.Logger.Debug("Read PacketMsg", "conn", ch.conn, "packet", packet) 848 var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Bytes) 849 if recvCap < recvReceived { 850 return nil, fmt.Errorf("received message exceeds available capacity: %v < %v", recvCap, recvReceived) 851 } 852 ch.recving = append(ch.recving, packet.Bytes...) 853 if packet.EOF == byte(0x01) { 854 msgBytes := ch.recving 855 856 // clear the slice without re-allocating. 857 // http://stackoverflow.com/questions/16971741/how-do-you-clear-a-slice-in-go 858 // suggests this could be a memory leak, but we might as well keep the memory for the channel until it closes, 859 // at which point the recving slice stops being used and should be garbage collected 860 ch.recving = ch.recving[:0] // make([]byte, 0, ch.desc.RecvBufferCapacity) 861 return msgBytes, nil 862 } 863 return nil, nil 864 } 865 866 // Call this periodically to update stats for throttling purposes. 867 // Not goroutine-safe 868 func (ch *Channel) updateStats() { 869 // Exponential decay of stats. 870 // TODO: optimize. 871 atomic.StoreInt64(&ch.recentlySent, int64(float64(atomic.LoadInt64(&ch.recentlySent))*0.8)) 872 } 873 874 //---------------------------------------- 875 // Packet 876 877 type Packet interface { 878 AssertIsPacket() 879 } 880 881 func RegisterPacket(cdc *amino.Codec) { 882 cdc.RegisterInterface((*Packet)(nil), nil) 883 cdc.RegisterConcrete(PacketPing{}, "tendermint/p2p/PacketPing", nil) 884 cdc.RegisterConcrete(PacketPong{}, "tendermint/p2p/PacketPong", nil) 885 cdc.RegisterConcrete(PacketMsg{}, "tendermint/p2p/PacketMsg", nil) 886 } 887 888 func (PacketPing) AssertIsPacket() {} 889 func (PacketPong) AssertIsPacket() {} 890 func (PacketMsg) AssertIsPacket() {} 891 892 type PacketPing struct { 893 } 894 895 type PacketPong struct { 896 } 897 898 type PacketMsg struct { 899 ChannelID byte 900 EOF byte // 1 means message ends here. 901 Bytes []byte 902 } 903 904 func (mp PacketMsg) String() string { 905 return fmt.Sprintf("PacketMsg{%X:%X T:%X}", mp.ChannelID, mp.Bytes, mp.EOF) 906 }