github.com/slackhq/nebula@v1.9.0/handshake_ix.go (about)

     1  package nebula
     2  
     3  import (
     4  	"time"
     5  
     6  	"github.com/flynn/noise"
     7  	"github.com/sirupsen/logrus"
     8  	"github.com/slackhq/nebula/header"
     9  	"github.com/slackhq/nebula/iputil"
    10  	"github.com/slackhq/nebula/udp"
    11  )
    12  
    13  // NOISE IX Handshakes
    14  
    15  // This function constructs a handshake packet, but does not actually send it
    16  // Sending is done by the handshake manager
    17  func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
    18  	err := f.handshakeManager.allocateIndex(hh)
    19  	if err != nil {
    20  		f.l.WithError(err).WithField("vpnIp", hh.hostinfo.vpnIp).
    21  			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to generate index")
    22  		return false
    23  	}
    24  
    25  	certState := f.pki.GetCertState()
    26  	ci := NewConnectionState(f.l, f.cipher, certState, true, noise.HandshakeIX, []byte{}, 0)
    27  	hh.hostinfo.ConnectionState = ci
    28  
    29  	hsProto := &NebulaHandshakeDetails{
    30  		InitiatorIndex: hh.hostinfo.localIndexId,
    31  		Time:           uint64(time.Now().UnixNano()),
    32  		Cert:           certState.RawCertificateNoKey,
    33  	}
    34  
    35  	hsBytes := []byte{}
    36  
    37  	hs := &NebulaHandshake{
    38  		Details: hsProto,
    39  	}
    40  	hsBytes, err = hs.Marshal()
    41  
    42  	if err != nil {
    43  		f.l.WithError(err).WithField("vpnIp", hh.hostinfo.vpnIp).
    44  			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
    45  		return false
    46  	}
    47  
    48  	h := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, 0, 1)
    49  	ci.messageCounter.Add(1)
    50  
    51  	msg, _, _, err := ci.H.WriteMessage(h, hsBytes)
    52  	if err != nil {
    53  		f.l.WithError(err).WithField("vpnIp", hh.hostinfo.vpnIp).
    54  			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
    55  		return false
    56  	}
    57  
    58  	// We are sending handshake packet 1, so we don't expect to receive
    59  	// handshake packet 1 from the responder
    60  	ci.window.Update(f.l, 1)
    61  
    62  	hh.hostinfo.HandshakePacket[0] = msg
    63  	hh.ready = true
    64  	return true
    65  }
    66  
    67  func ixHandshakeStage1(f *Interface, addr *udp.Addr, via *ViaSender, packet []byte, h *header.H) {
    68  	certState := f.pki.GetCertState()
    69  	ci := NewConnectionState(f.l, f.cipher, certState, false, noise.HandshakeIX, []byte{}, 0)
    70  	// Mark packet 1 as seen so it doesn't show up as missed
    71  	ci.window.Update(f.l, 1)
    72  
    73  	msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:])
    74  	if err != nil {
    75  		f.l.WithError(err).WithField("udpAddr", addr).
    76  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.ReadMessage")
    77  		return
    78  	}
    79  
    80  	hs := &NebulaHandshake{}
    81  	err = hs.Unmarshal(msg)
    82  	/*
    83  		l.Debugln("GOT INDEX: ", hs.Details.InitiatorIndex)
    84  	*/
    85  	if err != nil || hs.Details == nil {
    86  		f.l.WithError(err).WithField("udpAddr", addr).
    87  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
    88  		return
    89  	}
    90  
    91  	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool())
    92  	if err != nil {
    93  		e := f.l.WithError(err).WithField("udpAddr", addr).
    94  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"})
    95  
    96  		if f.l.Level > logrus.DebugLevel {
    97  			e = e.WithField("cert", remoteCert)
    98  		}
    99  
   100  		e.Info("Invalid certificate from host")
   101  		return
   102  	}
   103  	vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP)
   104  	certName := remoteCert.Details.Name
   105  	fingerprint, _ := remoteCert.Sha256Sum()
   106  	issuer := remoteCert.Details.Issuer
   107  
   108  	if vpnIp == f.myVpnIp {
   109  		f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   110  			WithField("certName", certName).
   111  			WithField("fingerprint", fingerprint).
   112  			WithField("issuer", issuer).
   113  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
   114  		return
   115  	}
   116  
   117  	if addr != nil {
   118  		if !f.lightHouse.GetRemoteAllowList().Allow(vpnIp, addr.IP) {
   119  			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
   120  			return
   121  		}
   122  	}
   123  
   124  	myIndex, err := generateIndex(f.l)
   125  	if err != nil {
   126  		f.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   127  			WithField("certName", certName).
   128  			WithField("fingerprint", fingerprint).
   129  			WithField("issuer", issuer).
   130  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index")
   131  		return
   132  	}
   133  
   134  	hostinfo := &HostInfo{
   135  		ConnectionState:   ci,
   136  		localIndexId:      myIndex,
   137  		remoteIndexId:     hs.Details.InitiatorIndex,
   138  		vpnIp:             vpnIp,
   139  		HandshakePacket:   make(map[uint8][]byte, 0),
   140  		lastHandshakeTime: hs.Details.Time,
   141  		relayState: RelayState{
   142  			relays:        map[iputil.VpnIp]struct{}{},
   143  			relayForByIp:  map[iputil.VpnIp]*Relay{},
   144  			relayForByIdx: map[uint32]*Relay{},
   145  		},
   146  	}
   147  
   148  	f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   149  		WithField("certName", certName).
   150  		WithField("fingerprint", fingerprint).
   151  		WithField("issuer", issuer).
   152  		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   153  		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
   154  		Info("Handshake message received")
   155  
   156  	hs.Details.ResponderIndex = myIndex
   157  	hs.Details.Cert = certState.RawCertificateNoKey
   158  	// Update the time in case their clock is way off from ours
   159  	hs.Details.Time = uint64(time.Now().UnixNano())
   160  
   161  	hsBytes, err := hs.Marshal()
   162  	if err != nil {
   163  		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   164  			WithField("certName", certName).
   165  			WithField("fingerprint", fingerprint).
   166  			WithField("issuer", issuer).
   167  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
   168  		return
   169  	}
   170  
   171  	nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2)
   172  	msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes)
   173  	if err != nil {
   174  		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   175  			WithField("certName", certName).
   176  			WithField("fingerprint", fingerprint).
   177  			WithField("issuer", issuer).
   178  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
   179  		return
   180  	} else if dKey == nil || eKey == nil {
   181  		f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   182  			WithField("certName", certName).
   183  			WithField("fingerprint", fingerprint).
   184  			WithField("issuer", issuer).
   185  			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Noise did not arrive at a key")
   186  		return
   187  	}
   188  
   189  	hostinfo.HandshakePacket[0] = make([]byte, len(packet[header.Len:]))
   190  	copy(hostinfo.HandshakePacket[0], packet[header.Len:])
   191  
   192  	// Regardless of whether you are the sender or receiver, you should arrive here
   193  	// and complete standing up the connection.
   194  	hostinfo.HandshakePacket[2] = make([]byte, len(msg))
   195  	copy(hostinfo.HandshakePacket[2], msg)
   196  
   197  	// We are sending handshake packet 2, so we don't expect to receive
   198  	// handshake packet 2 from the initiator.
   199  	ci.window.Update(f.l, 2)
   200  
   201  	ci.peerCert = remoteCert
   202  	ci.dKey = NewNebulaCipherState(dKey)
   203  	ci.eKey = NewNebulaCipherState(eKey)
   204  
   205  	hostinfo.remotes = f.lightHouse.QueryCache(vpnIp)
   206  	hostinfo.SetRemote(addr)
   207  	hostinfo.CreateRemoteCIDR(remoteCert)
   208  
   209  	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f)
   210  	if err != nil {
   211  		switch err {
   212  		case ErrAlreadySeen:
   213  			// Update remote if preferred
   214  			if existing.SetRemoteIfPreferred(f.hostMap, addr) {
   215  				// Send a test packet to ensure the other side has also switched to
   216  				// the preferred remote
   217  				f.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
   218  			}
   219  
   220  			msg = existing.HandshakePacket[2]
   221  			f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
   222  			if addr != nil {
   223  				err := f.outside.WriteTo(msg, addr)
   224  				if err != nil {
   225  					f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr).
   226  						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
   227  						WithError(err).Error("Failed to send handshake message")
   228  				} else {
   229  					f.l.WithField("vpnIp", existing.vpnIp).WithField("udpAddr", addr).
   230  						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
   231  						Info("Handshake message sent")
   232  				}
   233  				return
   234  			} else {
   235  				if via == nil {
   236  					f.l.Error("Handshake send failed: both addr and via are nil.")
   237  					return
   238  				}
   239  				hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp)
   240  				f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
   241  				f.l.WithField("vpnIp", existing.vpnIp).WithField("relay", via.relayHI.vpnIp).
   242  					WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
   243  					Info("Handshake message sent")
   244  				return
   245  			}
   246  		case ErrExistingHostInfo:
   247  			// This means there was an existing tunnel and this handshake was older than the one we are currently based on
   248  			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   249  				WithField("certName", certName).
   250  				WithField("oldHandshakeTime", existing.lastHandshakeTime).
   251  				WithField("newHandshakeTime", hostinfo.lastHandshakeTime).
   252  				WithField("fingerprint", fingerprint).
   253  				WithField("issuer", issuer).
   254  				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   255  				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
   256  				Info("Handshake too old")
   257  
   258  			// Send a test packet to trigger an authenticated tunnel test, this should suss out any lingering tunnel issues
   259  			f.SendMessageToVpnIp(header.Test, header.TestRequest, vpnIp, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
   260  			return
   261  		case ErrLocalIndexCollision:
   262  			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
   263  			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   264  				WithField("certName", certName).
   265  				WithField("fingerprint", fingerprint).
   266  				WithField("issuer", issuer).
   267  				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   268  				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
   269  				WithField("localIndex", hostinfo.localIndexId).WithField("collision", existing.vpnIp).
   270  				Error("Failed to add HostInfo due to localIndex collision")
   271  			return
   272  		default:
   273  			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
   274  			// And we forget to update it here
   275  			f.l.WithError(err).WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   276  				WithField("certName", certName).
   277  				WithField("fingerprint", fingerprint).
   278  				WithField("issuer", issuer).
   279  				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   280  				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
   281  				Error("Failed to add HostInfo to HostMap")
   282  			return
   283  		}
   284  	}
   285  
   286  	// Do the send
   287  	f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
   288  	if addr != nil {
   289  		err = f.outside.WriteTo(msg, addr)
   290  		if err != nil {
   291  			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   292  				WithField("certName", certName).
   293  				WithField("fingerprint", fingerprint).
   294  				WithField("issuer", issuer).
   295  				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   296  				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
   297  				WithError(err).Error("Failed to send handshake")
   298  		} else {
   299  			f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   300  				WithField("certName", certName).
   301  				WithField("fingerprint", fingerprint).
   302  				WithField("issuer", issuer).
   303  				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   304  				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
   305  				Info("Handshake message sent")
   306  		}
   307  	} else {
   308  		if via == nil {
   309  			f.l.Error("Handshake send failed: both addr and via are nil.")
   310  			return
   311  		}
   312  		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp)
   313  		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
   314  		f.l.WithField("vpnIp", vpnIp).WithField("relay", via.relayHI.vpnIp).
   315  			WithField("certName", certName).
   316  			WithField("fingerprint", fingerprint).
   317  			WithField("issuer", issuer).
   318  			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   319  			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
   320  			Info("Handshake message sent")
   321  	}
   322  
   323  	f.connectionManager.AddTrafficWatch(hostinfo.localIndexId)
   324  	hostinfo.ConnectionState.messageCounter.Store(2)
   325  	hostinfo.remotes.ResetBlockedRemotes()
   326  
   327  	return
   328  }
   329  
   330  func ixHandshakeStage2(f *Interface, addr *udp.Addr, via *ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
   331  	if hh == nil {
   332  		// Nothing here to tear down, got a bogus stage 2 packet
   333  		return true
   334  	}
   335  
   336  	hh.Lock()
   337  	defer hh.Unlock()
   338  
   339  	hostinfo := hh.hostinfo
   340  	if addr != nil {
   341  		if !f.lightHouse.GetRemoteAllowList().Allow(hostinfo.vpnIp, addr.IP) {
   342  			f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
   343  			return false
   344  		}
   345  	}
   346  
   347  	ci := hostinfo.ConnectionState
   348  	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:])
   349  	if err != nil {
   350  		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   351  			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
   352  			Error("Failed to call noise.ReadMessage")
   353  
   354  		// We don't want to tear down the connection on a bad ReadMessage because it could be an attacker trying
   355  		// to DOS us. Every other error condition after should to allow a possible good handshake to complete in the
   356  		// near future
   357  		return false
   358  	} else if dKey == nil || eKey == nil {
   359  		f.l.WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   360  			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
   361  			Error("Noise did not arrive at a key")
   362  
   363  		// This should be impossible in IX but just in case, if we get here then there is no chance to recover
   364  		// the handshake state machine. Tear it down
   365  		return true
   366  	}
   367  
   368  	hs := &NebulaHandshake{}
   369  	err = hs.Unmarshal(msg)
   370  	if err != nil || hs.Details == nil {
   371  		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   372  			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
   373  
   374  		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
   375  		return true
   376  	}
   377  
   378  	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool())
   379  	if err != nil {
   380  		e := f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
   381  			WithField("handshake", m{"stage": 2, "style": "ix_psk0"})
   382  
   383  		if f.l.Level > logrus.DebugLevel {
   384  			e = e.WithField("cert", remoteCert)
   385  		}
   386  
   387  		e.Error("Invalid certificate from host")
   388  
   389  		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
   390  		return true
   391  	}
   392  
   393  	vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP)
   394  	certName := remoteCert.Details.Name
   395  	fingerprint, _ := remoteCert.Sha256Sum()
   396  	issuer := remoteCert.Details.Issuer
   397  
   398  	// Ensure the right host responded
   399  	if vpnIp != hostinfo.vpnIp {
   400  		f.l.WithField("intendedVpnIp", hostinfo.vpnIp).WithField("haveVpnIp", vpnIp).
   401  			WithField("udpAddr", addr).WithField("certName", certName).
   402  			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
   403  			Info("Incorrect host responded to handshake")
   404  
   405  		// Release our old handshake from pending, it should not continue
   406  		f.handshakeManager.DeleteHostInfo(hostinfo)
   407  
   408  		// Create a new hostinfo/handshake for the intended vpn ip
   409  		f.handshakeManager.StartHandshake(hostinfo.vpnIp, func(newHH *HandshakeHostInfo) {
   410  			//TODO: this doesnt know if its being added or is being used for caching a packet
   411  			// Block the current used address
   412  			newHH.hostinfo.remotes = hostinfo.remotes
   413  			newHH.hostinfo.remotes.BlockRemote(addr)
   414  
   415  			// Get the correct remote list for the host we did handshake with
   416  			hostinfo.remotes = f.lightHouse.QueryCache(vpnIp)
   417  
   418  			f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).WithField("vpnIp", vpnIp).
   419  				WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.GetPreferredRanges())).
   420  				Info("Blocked addresses for handshakes")
   421  
   422  			// Swap the packet store to benefit the original intended recipient
   423  			newHH.packetStore = hh.packetStore
   424  			hh.packetStore = []*cachedPacket{}
   425  
   426  			// Finally, put the correct vpn ip in the host info, tell them to close the tunnel, and return true to tear down
   427  			hostinfo.vpnIp = vpnIp
   428  			f.sendCloseTunnel(hostinfo)
   429  		})
   430  
   431  		return true
   432  	}
   433  
   434  	// Mark packet 2 as seen so it doesn't show up as missed
   435  	ci.window.Update(f.l, 2)
   436  
   437  	duration := time.Since(hh.startTime).Nanoseconds()
   438  	f.l.WithField("vpnIp", vpnIp).WithField("udpAddr", addr).
   439  		WithField("certName", certName).
   440  		WithField("fingerprint", fingerprint).
   441  		WithField("issuer", issuer).
   442  		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
   443  		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
   444  		WithField("durationNs", duration).
   445  		WithField("sentCachedPackets", len(hh.packetStore)).
   446  		Info("Handshake message received")
   447  
   448  	hostinfo.remoteIndexId = hs.Details.ResponderIndex
   449  	hostinfo.lastHandshakeTime = hs.Details.Time
   450  
   451  	// Store their cert and our symmetric keys
   452  	ci.peerCert = remoteCert
   453  	ci.dKey = NewNebulaCipherState(dKey)
   454  	ci.eKey = NewNebulaCipherState(eKey)
   455  
   456  	// Make sure the current udpAddr being used is set for responding
   457  	if addr != nil {
   458  		hostinfo.SetRemote(addr)
   459  	} else {
   460  		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnIp)
   461  	}
   462  
   463  	// Build up the radix for the firewall if we have subnets in the cert
   464  	hostinfo.CreateRemoteCIDR(remoteCert)
   465  
   466  	// Complete our handshake and update metrics, this will replace any existing tunnels for this vpnIp
   467  	f.handshakeManager.Complete(hostinfo, f)
   468  	f.connectionManager.AddTrafficWatch(hostinfo.localIndexId)
   469  
   470  	hostinfo.ConnectionState.messageCounter.Store(2)
   471  
   472  	if f.l.Level >= logrus.DebugLevel {
   473  		hostinfo.logger(f.l).Debugf("Sending %d stored packets", len(hh.packetStore))
   474  	}
   475  
   476  	if len(hh.packetStore) > 0 {
   477  		nb := make([]byte, 12, 12)
   478  		out := make([]byte, mtu)
   479  		for _, cp := range hh.packetStore {
   480  			cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out)
   481  		}
   482  		f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
   483  	}
   484  
   485  	hostinfo.remotes.ResetBlockedRemotes()
   486  	f.metricHandshakes.Update(duration)
   487  
   488  	return false
   489  }