github.com/zhuohuang-hust/src-cbuild@v0.0.0-20230105071821-c7aab3e7c840/mergeCode/libnetwork/networkdb/cluster.go (about)

     1  package networkdb
     2  
     3  import (
     4  	"bytes"
     5  	"crypto/rand"
     6  	"encoding/hex"
     7  	"fmt"
     8  	"math/big"
     9  	rnd "math/rand"
    10  	"net"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/Sirupsen/logrus"
    15  	"github.com/hashicorp/memberlist"
    16  )
    17  
    18  const (
    19  	reapInterval  = 60 * time.Second
    20  	reapPeriod    = 5 * time.Second
    21  	retryInterval = 1 * time.Second
    22  )
    23  
    24  type logWriter struct{}
    25  
    26  func (l *logWriter) Write(p []byte) (int, error) {
    27  	str := string(p)
    28  
    29  	switch {
    30  	case strings.Contains(str, "[WARN]"):
    31  		logrus.Warn(str)
    32  	case strings.Contains(str, "[DEBUG]"):
    33  		logrus.Debug(str)
    34  	case strings.Contains(str, "[INFO]"):
    35  		logrus.Info(str)
    36  	case strings.Contains(str, "[ERR]"):
    37  		logrus.Warn(str)
    38  	}
    39  
    40  	return len(p), nil
    41  }
    42  
    43  // SetKey adds a new key to the key ring
    44  func (nDB *NetworkDB) SetKey(key []byte) {
    45  	logrus.Debugf("Adding key %s", hex.EncodeToString(key)[0:5])
    46  	for _, dbKey := range nDB.config.Keys {
    47  		if bytes.Equal(key, dbKey) {
    48  			return
    49  		}
    50  	}
    51  	nDB.config.Keys = append(nDB.config.Keys, key)
    52  	if nDB.keyring != nil {
    53  		nDB.keyring.AddKey(key)
    54  	}
    55  }
    56  
    57  // SetPrimaryKey sets the given key as the primary key. This should have
    58  // been added apriori through SetKey
    59  func (nDB *NetworkDB) SetPrimaryKey(key []byte) {
    60  	logrus.Debugf("Primary Key %s", hex.EncodeToString(key)[0:5])
    61  	for _, dbKey := range nDB.config.Keys {
    62  		if bytes.Equal(key, dbKey) {
    63  			if nDB.keyring != nil {
    64  				nDB.keyring.UseKey(dbKey)
    65  			}
    66  			break
    67  		}
    68  	}
    69  }
    70  
    71  // RemoveKey removes a key from the key ring. The key being removed
    72  // can't be the primary key
    73  func (nDB *NetworkDB) RemoveKey(key []byte) {
    74  	logrus.Debugf("Remove Key %s", hex.EncodeToString(key)[0:5])
    75  	for i, dbKey := range nDB.config.Keys {
    76  		if bytes.Equal(key, dbKey) {
    77  			nDB.config.Keys = append(nDB.config.Keys[:i], nDB.config.Keys[i+1:]...)
    78  			if nDB.keyring != nil {
    79  				nDB.keyring.RemoveKey(dbKey)
    80  			}
    81  			break
    82  		}
    83  	}
    84  }
    85  
    86  func (nDB *NetworkDB) clusterInit() error {
    87  	config := memberlist.DefaultLANConfig()
    88  	config.Name = nDB.config.NodeName
    89  	config.BindAddr = nDB.config.BindAddr
    90  	config.AdvertiseAddr = nDB.config.AdvertiseAddr
    91  
    92  	if nDB.config.BindPort != 0 {
    93  		config.BindPort = nDB.config.BindPort
    94  	}
    95  
    96  	config.ProtocolVersion = memberlist.ProtocolVersionMax
    97  	config.Delegate = &delegate{nDB: nDB}
    98  	config.Events = &eventDelegate{nDB: nDB}
    99  	config.LogOutput = &logWriter{}
   100  
   101  	var err error
   102  	if len(nDB.config.Keys) > 0 {
   103  		for i, key := range nDB.config.Keys {
   104  			logrus.Debugf("Encryption key %d: %s", i+1, hex.EncodeToString(key)[0:5])
   105  		}
   106  		nDB.keyring, err = memberlist.NewKeyring(nDB.config.Keys, nDB.config.Keys[0])
   107  		if err != nil {
   108  			return err
   109  		}
   110  		config.Keyring = nDB.keyring
   111  	}
   112  
   113  	nDB.networkBroadcasts = &memberlist.TransmitLimitedQueue{
   114  		NumNodes: func() int {
   115  			nDB.RLock()
   116  			num := len(nDB.nodes)
   117  			nDB.RUnlock()
   118  			return num
   119  		},
   120  		RetransmitMult: config.RetransmitMult,
   121  	}
   122  
   123  	nDB.nodeBroadcasts = &memberlist.TransmitLimitedQueue{
   124  		NumNodes: func() int {
   125  			nDB.RLock()
   126  			num := len(nDB.nodes)
   127  			nDB.RUnlock()
   128  			return num
   129  		},
   130  		RetransmitMult: config.RetransmitMult,
   131  	}
   132  
   133  	mlist, err := memberlist.Create(config)
   134  	if err != nil {
   135  		return fmt.Errorf("failed to create memberlist: %v", err)
   136  	}
   137  
   138  	nDB.stopCh = make(chan struct{})
   139  	nDB.memberlist = mlist
   140  	nDB.mConfig = config
   141  
   142  	for _, trigger := range []struct {
   143  		interval time.Duration
   144  		fn       func()
   145  	}{
   146  		{reapPeriod, nDB.reapState},
   147  		{config.GossipInterval, nDB.gossip},
   148  		{config.PushPullInterval, nDB.bulkSyncTables},
   149  		{retryInterval, nDB.reconnectNode},
   150  	} {
   151  		t := time.NewTicker(trigger.interval)
   152  		go nDB.triggerFunc(trigger.interval, t.C, nDB.stopCh, trigger.fn)
   153  		nDB.tickers = append(nDB.tickers, t)
   154  	}
   155  
   156  	return nil
   157  }
   158  
   159  func (nDB *NetworkDB) retryJoin(members []string, stop <-chan struct{}) {
   160  	t := time.NewTicker(retryInterval)
   161  	defer t.Stop()
   162  
   163  	for {
   164  		select {
   165  		case <-t.C:
   166  			if _, err := nDB.memberlist.Join(members); err != nil {
   167  				logrus.Errorf("Failed to join memberlist %s on retry: %v", members, err)
   168  				continue
   169  			}
   170  			if err := nDB.sendNodeEvent(NodeEventTypeJoin); err != nil {
   171  				logrus.Errorf("failed to send node join on retry: %v", err)
   172  				continue
   173  			}
   174  			return
   175  		case <-stop:
   176  			return
   177  		}
   178  	}
   179  
   180  }
   181  
   182  func (nDB *NetworkDB) clusterJoin(members []string) error {
   183  	mlist := nDB.memberlist
   184  
   185  	if _, err := mlist.Join(members); err != nil {
   186  		// Incase of failure, keep retrying join until it succeeds or the cluster is shutdown.
   187  		go nDB.retryJoin(members, nDB.stopCh)
   188  
   189  		return fmt.Errorf("could not join node to memberlist: %v", err)
   190  	}
   191  
   192  	if err := nDB.sendNodeEvent(NodeEventTypeJoin); err != nil {
   193  		return fmt.Errorf("failed to send node join: %v", err)
   194  	}
   195  
   196  	return nil
   197  }
   198  
   199  func (nDB *NetworkDB) clusterLeave() error {
   200  	mlist := nDB.memberlist
   201  
   202  	if err := nDB.sendNodeEvent(NodeEventTypeLeave); err != nil {
   203  		logrus.Errorf("failed to send node leave: %v", err)
   204  	}
   205  
   206  	if err := mlist.Leave(time.Second); err != nil {
   207  		return err
   208  	}
   209  
   210  	close(nDB.stopCh)
   211  
   212  	for _, t := range nDB.tickers {
   213  		t.Stop()
   214  	}
   215  
   216  	return mlist.Shutdown()
   217  }
   218  
   219  func (nDB *NetworkDB) triggerFunc(stagger time.Duration, C <-chan time.Time, stop <-chan struct{}, f func()) {
   220  	// Use a random stagger to avoid syncronizing
   221  	randStagger := time.Duration(uint64(rnd.Int63()) % uint64(stagger))
   222  	select {
   223  	case <-time.After(randStagger):
   224  	case <-stop:
   225  		return
   226  	}
   227  	for {
   228  		select {
   229  		case <-C:
   230  			f()
   231  		case <-stop:
   232  			return
   233  		}
   234  	}
   235  }
   236  
   237  func (nDB *NetworkDB) reconnectNode() {
   238  	nDB.RLock()
   239  	if len(nDB.failedNodes) == 0 {
   240  		nDB.RUnlock()
   241  		return
   242  	}
   243  
   244  	nodes := make([]*node, 0, len(nDB.failedNodes))
   245  	for _, n := range nDB.failedNodes {
   246  		nodes = append(nodes, n)
   247  	}
   248  	nDB.RUnlock()
   249  
   250  	node := nodes[randomOffset(len(nodes))]
   251  	addr := net.UDPAddr{IP: node.Addr, Port: int(node.Port)}
   252  
   253  	if _, err := nDB.memberlist.Join([]string{addr.String()}); err != nil {
   254  		return
   255  	}
   256  
   257  	if err := nDB.sendNodeEvent(NodeEventTypeJoin); err != nil {
   258  		logrus.Errorf("failed to send node join during reconnect: %v", err)
   259  		return
   260  	}
   261  
   262  	// Update all the local table state to a new time to
   263  	// force update on the node we are trying to rejoin, just in
   264  	// case that node has these in deleting state still. This is
   265  	// facilitate fast convergence after recovering from a gossip
   266  	// failure.
   267  	nDB.updateLocalTableTime()
   268  
   269  	logrus.Debugf("Initiating bulk sync with node %s after reconnect", node.Name)
   270  	nDB.bulkSync([]string{node.Name}, true)
   271  }
   272  
   273  func (nDB *NetworkDB) reapState() {
   274  	nDB.reapNetworks()
   275  	nDB.reapTableEntries()
   276  }
   277  
   278  func (nDB *NetworkDB) reapNetworks() {
   279  	now := time.Now()
   280  	nDB.Lock()
   281  	for name, nn := range nDB.networks {
   282  		for id, n := range nn {
   283  			if n.leaving && now.Sub(n.leaveTime) > reapInterval {
   284  				delete(nn, id)
   285  				nDB.deleteNetworkNode(id, name)
   286  			}
   287  		}
   288  	}
   289  	nDB.Unlock()
   290  }
   291  
   292  func (nDB *NetworkDB) reapTableEntries() {
   293  	var paths []string
   294  
   295  	now := time.Now()
   296  
   297  	nDB.RLock()
   298  	nDB.indexes[byTable].Walk(func(path string, v interface{}) bool {
   299  		entry, ok := v.(*entry)
   300  		if !ok {
   301  			return false
   302  		}
   303  
   304  		if !entry.deleting || now.Sub(entry.deleteTime) <= reapInterval {
   305  			return false
   306  		}
   307  
   308  		paths = append(paths, path)
   309  		return false
   310  	})
   311  	nDB.RUnlock()
   312  
   313  	nDB.Lock()
   314  	for _, path := range paths {
   315  		params := strings.Split(path[1:], "/")
   316  		tname := params[0]
   317  		nid := params[1]
   318  		key := params[2]
   319  
   320  		if _, ok := nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)); !ok {
   321  			logrus.Errorf("Could not delete entry in table %s with network id %s and key %s as it does not exist", tname, nid, key)
   322  		}
   323  
   324  		if _, ok := nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)); !ok {
   325  			logrus.Errorf("Could not delete entry in network %s with table name %s and key %s as it does not exist", nid, tname, key)
   326  		}
   327  	}
   328  	nDB.Unlock()
   329  }
   330  
   331  func (nDB *NetworkDB) gossip() {
   332  	networkNodes := make(map[string][]string)
   333  	nDB.RLock()
   334  	thisNodeNetworks := nDB.networks[nDB.config.NodeName]
   335  	for nid := range thisNodeNetworks {
   336  		networkNodes[nid] = nDB.networkNodes[nid]
   337  
   338  	}
   339  	nDB.RUnlock()
   340  
   341  	for nid, nodes := range networkNodes {
   342  		mNodes := nDB.mRandomNodes(3, nodes)
   343  		bytesAvail := udpSendBuf - compoundHeaderOverhead
   344  
   345  		nDB.RLock()
   346  		network, ok := thisNodeNetworks[nid]
   347  		nDB.RUnlock()
   348  		if !ok || network == nil {
   349  			// It is normal for the network to be removed
   350  			// between the time we collect the network
   351  			// attachments of this node and processing
   352  			// them here.
   353  			continue
   354  		}
   355  
   356  		broadcastQ := network.tableBroadcasts
   357  
   358  		if broadcastQ == nil {
   359  			logrus.Errorf("Invalid broadcastQ encountered while gossiping for network %s", nid)
   360  			continue
   361  		}
   362  
   363  		msgs := broadcastQ.GetBroadcasts(compoundOverhead, bytesAvail)
   364  		if len(msgs) == 0 {
   365  			continue
   366  		}
   367  
   368  		// Create a compound message
   369  		compound := makeCompoundMessage(msgs)
   370  
   371  		for _, node := range mNodes {
   372  			nDB.RLock()
   373  			mnode := nDB.nodes[node]
   374  			nDB.RUnlock()
   375  
   376  			if mnode == nil {
   377  				break
   378  			}
   379  
   380  			// Send the compound message
   381  			if err := nDB.memberlist.SendToUDP(&mnode.Node, compound); err != nil {
   382  				logrus.Errorf("Failed to send gossip to %s: %s", mnode.Addr, err)
   383  			}
   384  		}
   385  	}
   386  }
   387  
   388  func (nDB *NetworkDB) bulkSyncTables() {
   389  	var networks []string
   390  	nDB.RLock()
   391  	for nid, network := range nDB.networks[nDB.config.NodeName] {
   392  		if network.leaving {
   393  			continue
   394  		}
   395  		networks = append(networks, nid)
   396  	}
   397  	nDB.RUnlock()
   398  
   399  	for {
   400  		if len(networks) == 0 {
   401  			break
   402  		}
   403  
   404  		nid := networks[0]
   405  		networks = networks[1:]
   406  
   407  		nDB.RLock()
   408  		nodes := nDB.networkNodes[nid]
   409  		nDB.RUnlock()
   410  
   411  		// No peer nodes on this network. Move on.
   412  		if len(nodes) == 0 {
   413  			continue
   414  		}
   415  
   416  		completed, err := nDB.bulkSync(nodes, false)
   417  		if err != nil {
   418  			logrus.Errorf("periodic bulk sync failure for network %s: %v", nid, err)
   419  			continue
   420  		}
   421  
   422  		// Remove all the networks for which we have
   423  		// successfully completed bulk sync in this iteration.
   424  		updatedNetworks := make([]string, 0, len(networks))
   425  		for _, nid := range networks {
   426  			var found bool
   427  			for _, completedNid := range completed {
   428  				if nid == completedNid {
   429  					found = true
   430  					break
   431  				}
   432  			}
   433  
   434  			if !found {
   435  				updatedNetworks = append(updatedNetworks, nid)
   436  			}
   437  		}
   438  
   439  		networks = updatedNetworks
   440  	}
   441  }
   442  
   443  func (nDB *NetworkDB) bulkSync(nodes []string, all bool) ([]string, error) {
   444  	if !all {
   445  		// If not all, then just pick one.
   446  		nodes = nDB.mRandomNodes(1, nodes)
   447  	}
   448  
   449  	if len(nodes) == 0 {
   450  		return nil, nil
   451  	}
   452  
   453  	logrus.Debugf("%s: Initiating bulk sync with nodes %v", nDB.config.NodeName, nodes)
   454  	var err error
   455  	var networks []string
   456  	for _, node := range nodes {
   457  		if node == nDB.config.NodeName {
   458  			continue
   459  		}
   460  
   461  		networks = nDB.findCommonNetworks(node)
   462  		err = nDB.bulkSyncNode(networks, node, true)
   463  		if err != nil {
   464  			err = fmt.Errorf("bulk sync failed on node %s: %v", node, err)
   465  		}
   466  	}
   467  
   468  	if err != nil {
   469  		return nil, err
   470  	}
   471  
   472  	return networks, nil
   473  }
   474  
   475  // Bulk sync all the table entries belonging to a set of networks to a
   476  // single peer node. It can be unsolicited or can be in response to an
   477  // unsolicited bulk sync
   478  func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited bool) error {
   479  	var msgs [][]byte
   480  
   481  	var unsolMsg string
   482  	if unsolicited {
   483  		unsolMsg = "unsolicited"
   484  	}
   485  
   486  	logrus.Debugf("%s: Initiating %s bulk sync for networks %v with node %s", nDB.config.NodeName, unsolMsg, networks, node)
   487  
   488  	nDB.RLock()
   489  	mnode := nDB.nodes[node]
   490  	if mnode == nil {
   491  		nDB.RUnlock()
   492  		return nil
   493  	}
   494  
   495  	for _, nid := range networks {
   496  		nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid), func(path string, v interface{}) bool {
   497  			entry, ok := v.(*entry)
   498  			if !ok {
   499  				return false
   500  			}
   501  
   502  			eType := TableEventTypeCreate
   503  			if entry.deleting {
   504  				eType = TableEventTypeDelete
   505  			}
   506  
   507  			params := strings.Split(path[1:], "/")
   508  			tEvent := TableEvent{
   509  				Type:      eType,
   510  				LTime:     entry.ltime,
   511  				NodeName:  entry.node,
   512  				NetworkID: nid,
   513  				TableName: params[1],
   514  				Key:       params[2],
   515  				Value:     entry.value,
   516  			}
   517  
   518  			msg, err := encodeMessage(MessageTypeTableEvent, &tEvent)
   519  			if err != nil {
   520  				logrus.Errorf("Encode failure during bulk sync: %#v", tEvent)
   521  				return false
   522  			}
   523  
   524  			msgs = append(msgs, msg)
   525  			return false
   526  		})
   527  	}
   528  	nDB.RUnlock()
   529  
   530  	// Create a compound message
   531  	compound := makeCompoundMessage(msgs)
   532  
   533  	bsm := BulkSyncMessage{
   534  		LTime:       nDB.tableClock.Time(),
   535  		Unsolicited: unsolicited,
   536  		NodeName:    nDB.config.NodeName,
   537  		Networks:    networks,
   538  		Payload:     compound,
   539  	}
   540  
   541  	buf, err := encodeMessage(MessageTypeBulkSync, &bsm)
   542  	if err != nil {
   543  		return fmt.Errorf("failed to encode bulk sync message: %v", err)
   544  	}
   545  
   546  	nDB.Lock()
   547  	ch := make(chan struct{})
   548  	nDB.bulkSyncAckTbl[node] = ch
   549  	nDB.Unlock()
   550  
   551  	err = nDB.memberlist.SendToTCP(&mnode.Node, buf)
   552  	if err != nil {
   553  		nDB.Lock()
   554  		delete(nDB.bulkSyncAckTbl, node)
   555  		nDB.Unlock()
   556  
   557  		return fmt.Errorf("failed to send a TCP message during bulk sync: %v", err)
   558  	}
   559  
   560  	// Wait on a response only if it is unsolicited.
   561  	if unsolicited {
   562  		startTime := time.Now()
   563  		t := time.NewTimer(30 * time.Second)
   564  		select {
   565  		case <-t.C:
   566  			logrus.Errorf("Bulk sync to node %s timed out", node)
   567  		case <-ch:
   568  			logrus.Debugf("%s: Bulk sync to node %s took %s", nDB.config.NodeName, node, time.Now().Sub(startTime))
   569  		}
   570  		t.Stop()
   571  	}
   572  
   573  	return nil
   574  }
   575  
   576  // Returns a random offset between 0 and n
   577  func randomOffset(n int) int {
   578  	if n == 0 {
   579  		return 0
   580  	}
   581  
   582  	val, err := rand.Int(rand.Reader, big.NewInt(int64(n)))
   583  	if err != nil {
   584  		logrus.Errorf("Failed to get a random offset: %v", err)
   585  		return 0
   586  	}
   587  
   588  	return int(val.Int64())
   589  }
   590  
   591  // mRandomNodes is used to select up to m random nodes. It is possible
   592  // that less than m nodes are returned.
   593  func (nDB *NetworkDB) mRandomNodes(m int, nodes []string) []string {
   594  	n := len(nodes)
   595  	mNodes := make([]string, 0, m)
   596  OUTER:
   597  	// Probe up to 3*n times, with large n this is not necessary
   598  	// since k << n, but with small n we want search to be
   599  	// exhaustive
   600  	for i := 0; i < 3*n && len(mNodes) < m; i++ {
   601  		// Get random node
   602  		idx := randomOffset(n)
   603  		node := nodes[idx]
   604  
   605  		if node == nDB.config.NodeName {
   606  			continue
   607  		}
   608  
   609  		// Check if we have this node already
   610  		for j := 0; j < len(mNodes); j++ {
   611  			if node == mNodes[j] {
   612  				continue OUTER
   613  			}
   614  		}
   615  
   616  		// Append the node
   617  		mNodes = append(mNodes, node)
   618  	}
   619  
   620  	return mNodes
   621  }