github.com/zhuohuang-hust/src-cbuild@v0.0.0-20230105071821-c7aab3e7c840/mergeCode/libnetwork/networkdb/cluster.go (about) 1 package networkdb 2 3 import ( 4 "bytes" 5 "crypto/rand" 6 "encoding/hex" 7 "fmt" 8 "math/big" 9 rnd "math/rand" 10 "net" 11 "strings" 12 "time" 13 14 "github.com/Sirupsen/logrus" 15 "github.com/hashicorp/memberlist" 16 ) 17 18 const ( 19 reapInterval = 60 * time.Second 20 reapPeriod = 5 * time.Second 21 retryInterval = 1 * time.Second 22 ) 23 24 type logWriter struct{} 25 26 func (l *logWriter) Write(p []byte) (int, error) { 27 str := string(p) 28 29 switch { 30 case strings.Contains(str, "[WARN]"): 31 logrus.Warn(str) 32 case strings.Contains(str, "[DEBUG]"): 33 logrus.Debug(str) 34 case strings.Contains(str, "[INFO]"): 35 logrus.Info(str) 36 case strings.Contains(str, "[ERR]"): 37 logrus.Warn(str) 38 } 39 40 return len(p), nil 41 } 42 43 // SetKey adds a new key to the key ring 44 func (nDB *NetworkDB) SetKey(key []byte) { 45 logrus.Debugf("Adding key %s", hex.EncodeToString(key)[0:5]) 46 for _, dbKey := range nDB.config.Keys { 47 if bytes.Equal(key, dbKey) { 48 return 49 } 50 } 51 nDB.config.Keys = append(nDB.config.Keys, key) 52 if nDB.keyring != nil { 53 nDB.keyring.AddKey(key) 54 } 55 } 56 57 // SetPrimaryKey sets the given key as the primary key. This should have 58 // been added apriori through SetKey 59 func (nDB *NetworkDB) SetPrimaryKey(key []byte) { 60 logrus.Debugf("Primary Key %s", hex.EncodeToString(key)[0:5]) 61 for _, dbKey := range nDB.config.Keys { 62 if bytes.Equal(key, dbKey) { 63 if nDB.keyring != nil { 64 nDB.keyring.UseKey(dbKey) 65 } 66 break 67 } 68 } 69 } 70 71 // RemoveKey removes a key from the key ring. The key being removed 72 // can't be the primary key 73 func (nDB *NetworkDB) RemoveKey(key []byte) { 74 logrus.Debugf("Remove Key %s", hex.EncodeToString(key)[0:5]) 75 for i, dbKey := range nDB.config.Keys { 76 if bytes.Equal(key, dbKey) { 77 nDB.config.Keys = append(nDB.config.Keys[:i], nDB.config.Keys[i+1:]...) 78 if nDB.keyring != nil { 79 nDB.keyring.RemoveKey(dbKey) 80 } 81 break 82 } 83 } 84 } 85 86 func (nDB *NetworkDB) clusterInit() error { 87 config := memberlist.DefaultLANConfig() 88 config.Name = nDB.config.NodeName 89 config.BindAddr = nDB.config.BindAddr 90 config.AdvertiseAddr = nDB.config.AdvertiseAddr 91 92 if nDB.config.BindPort != 0 { 93 config.BindPort = nDB.config.BindPort 94 } 95 96 config.ProtocolVersion = memberlist.ProtocolVersionMax 97 config.Delegate = &delegate{nDB: nDB} 98 config.Events = &eventDelegate{nDB: nDB} 99 config.LogOutput = &logWriter{} 100 101 var err error 102 if len(nDB.config.Keys) > 0 { 103 for i, key := range nDB.config.Keys { 104 logrus.Debugf("Encryption key %d: %s", i+1, hex.EncodeToString(key)[0:5]) 105 } 106 nDB.keyring, err = memberlist.NewKeyring(nDB.config.Keys, nDB.config.Keys[0]) 107 if err != nil { 108 return err 109 } 110 config.Keyring = nDB.keyring 111 } 112 113 nDB.networkBroadcasts = &memberlist.TransmitLimitedQueue{ 114 NumNodes: func() int { 115 nDB.RLock() 116 num := len(nDB.nodes) 117 nDB.RUnlock() 118 return num 119 }, 120 RetransmitMult: config.RetransmitMult, 121 } 122 123 nDB.nodeBroadcasts = &memberlist.TransmitLimitedQueue{ 124 NumNodes: func() int { 125 nDB.RLock() 126 num := len(nDB.nodes) 127 nDB.RUnlock() 128 return num 129 }, 130 RetransmitMult: config.RetransmitMult, 131 } 132 133 mlist, err := memberlist.Create(config) 134 if err != nil { 135 return fmt.Errorf("failed to create memberlist: %v", err) 136 } 137 138 nDB.stopCh = make(chan struct{}) 139 nDB.memberlist = mlist 140 nDB.mConfig = config 141 142 for _, trigger := range []struct { 143 interval time.Duration 144 fn func() 145 }{ 146 {reapPeriod, nDB.reapState}, 147 {config.GossipInterval, nDB.gossip}, 148 {config.PushPullInterval, nDB.bulkSyncTables}, 149 {retryInterval, nDB.reconnectNode}, 150 } { 151 t := time.NewTicker(trigger.interval) 152 go nDB.triggerFunc(trigger.interval, t.C, nDB.stopCh, trigger.fn) 153 nDB.tickers = append(nDB.tickers, t) 154 } 155 156 return nil 157 } 158 159 func (nDB *NetworkDB) retryJoin(members []string, stop <-chan struct{}) { 160 t := time.NewTicker(retryInterval) 161 defer t.Stop() 162 163 for { 164 select { 165 case <-t.C: 166 if _, err := nDB.memberlist.Join(members); err != nil { 167 logrus.Errorf("Failed to join memberlist %s on retry: %v", members, err) 168 continue 169 } 170 if err := nDB.sendNodeEvent(NodeEventTypeJoin); err != nil { 171 logrus.Errorf("failed to send node join on retry: %v", err) 172 continue 173 } 174 return 175 case <-stop: 176 return 177 } 178 } 179 180 } 181 182 func (nDB *NetworkDB) clusterJoin(members []string) error { 183 mlist := nDB.memberlist 184 185 if _, err := mlist.Join(members); err != nil { 186 // Incase of failure, keep retrying join until it succeeds or the cluster is shutdown. 187 go nDB.retryJoin(members, nDB.stopCh) 188 189 return fmt.Errorf("could not join node to memberlist: %v", err) 190 } 191 192 if err := nDB.sendNodeEvent(NodeEventTypeJoin); err != nil { 193 return fmt.Errorf("failed to send node join: %v", err) 194 } 195 196 return nil 197 } 198 199 func (nDB *NetworkDB) clusterLeave() error { 200 mlist := nDB.memberlist 201 202 if err := nDB.sendNodeEvent(NodeEventTypeLeave); err != nil { 203 logrus.Errorf("failed to send node leave: %v", err) 204 } 205 206 if err := mlist.Leave(time.Second); err != nil { 207 return err 208 } 209 210 close(nDB.stopCh) 211 212 for _, t := range nDB.tickers { 213 t.Stop() 214 } 215 216 return mlist.Shutdown() 217 } 218 219 func (nDB *NetworkDB) triggerFunc(stagger time.Duration, C <-chan time.Time, stop <-chan struct{}, f func()) { 220 // Use a random stagger to avoid syncronizing 221 randStagger := time.Duration(uint64(rnd.Int63()) % uint64(stagger)) 222 select { 223 case <-time.After(randStagger): 224 case <-stop: 225 return 226 } 227 for { 228 select { 229 case <-C: 230 f() 231 case <-stop: 232 return 233 } 234 } 235 } 236 237 func (nDB *NetworkDB) reconnectNode() { 238 nDB.RLock() 239 if len(nDB.failedNodes) == 0 { 240 nDB.RUnlock() 241 return 242 } 243 244 nodes := make([]*node, 0, len(nDB.failedNodes)) 245 for _, n := range nDB.failedNodes { 246 nodes = append(nodes, n) 247 } 248 nDB.RUnlock() 249 250 node := nodes[randomOffset(len(nodes))] 251 addr := net.UDPAddr{IP: node.Addr, Port: int(node.Port)} 252 253 if _, err := nDB.memberlist.Join([]string{addr.String()}); err != nil { 254 return 255 } 256 257 if err := nDB.sendNodeEvent(NodeEventTypeJoin); err != nil { 258 logrus.Errorf("failed to send node join during reconnect: %v", err) 259 return 260 } 261 262 // Update all the local table state to a new time to 263 // force update on the node we are trying to rejoin, just in 264 // case that node has these in deleting state still. This is 265 // facilitate fast convergence after recovering from a gossip 266 // failure. 267 nDB.updateLocalTableTime() 268 269 logrus.Debugf("Initiating bulk sync with node %s after reconnect", node.Name) 270 nDB.bulkSync([]string{node.Name}, true) 271 } 272 273 func (nDB *NetworkDB) reapState() { 274 nDB.reapNetworks() 275 nDB.reapTableEntries() 276 } 277 278 func (nDB *NetworkDB) reapNetworks() { 279 now := time.Now() 280 nDB.Lock() 281 for name, nn := range nDB.networks { 282 for id, n := range nn { 283 if n.leaving && now.Sub(n.leaveTime) > reapInterval { 284 delete(nn, id) 285 nDB.deleteNetworkNode(id, name) 286 } 287 } 288 } 289 nDB.Unlock() 290 } 291 292 func (nDB *NetworkDB) reapTableEntries() { 293 var paths []string 294 295 now := time.Now() 296 297 nDB.RLock() 298 nDB.indexes[byTable].Walk(func(path string, v interface{}) bool { 299 entry, ok := v.(*entry) 300 if !ok { 301 return false 302 } 303 304 if !entry.deleting || now.Sub(entry.deleteTime) <= reapInterval { 305 return false 306 } 307 308 paths = append(paths, path) 309 return false 310 }) 311 nDB.RUnlock() 312 313 nDB.Lock() 314 for _, path := range paths { 315 params := strings.Split(path[1:], "/") 316 tname := params[0] 317 nid := params[1] 318 key := params[2] 319 320 if _, ok := nDB.indexes[byTable].Delete(fmt.Sprintf("/%s/%s/%s", tname, nid, key)); !ok { 321 logrus.Errorf("Could not delete entry in table %s with network id %s and key %s as it does not exist", tname, nid, key) 322 } 323 324 if _, ok := nDB.indexes[byNetwork].Delete(fmt.Sprintf("/%s/%s/%s", nid, tname, key)); !ok { 325 logrus.Errorf("Could not delete entry in network %s with table name %s and key %s as it does not exist", nid, tname, key) 326 } 327 } 328 nDB.Unlock() 329 } 330 331 func (nDB *NetworkDB) gossip() { 332 networkNodes := make(map[string][]string) 333 nDB.RLock() 334 thisNodeNetworks := nDB.networks[nDB.config.NodeName] 335 for nid := range thisNodeNetworks { 336 networkNodes[nid] = nDB.networkNodes[nid] 337 338 } 339 nDB.RUnlock() 340 341 for nid, nodes := range networkNodes { 342 mNodes := nDB.mRandomNodes(3, nodes) 343 bytesAvail := udpSendBuf - compoundHeaderOverhead 344 345 nDB.RLock() 346 network, ok := thisNodeNetworks[nid] 347 nDB.RUnlock() 348 if !ok || network == nil { 349 // It is normal for the network to be removed 350 // between the time we collect the network 351 // attachments of this node and processing 352 // them here. 353 continue 354 } 355 356 broadcastQ := network.tableBroadcasts 357 358 if broadcastQ == nil { 359 logrus.Errorf("Invalid broadcastQ encountered while gossiping for network %s", nid) 360 continue 361 } 362 363 msgs := broadcastQ.GetBroadcasts(compoundOverhead, bytesAvail) 364 if len(msgs) == 0 { 365 continue 366 } 367 368 // Create a compound message 369 compound := makeCompoundMessage(msgs) 370 371 for _, node := range mNodes { 372 nDB.RLock() 373 mnode := nDB.nodes[node] 374 nDB.RUnlock() 375 376 if mnode == nil { 377 break 378 } 379 380 // Send the compound message 381 if err := nDB.memberlist.SendToUDP(&mnode.Node, compound); err != nil { 382 logrus.Errorf("Failed to send gossip to %s: %s", mnode.Addr, err) 383 } 384 } 385 } 386 } 387 388 func (nDB *NetworkDB) bulkSyncTables() { 389 var networks []string 390 nDB.RLock() 391 for nid, network := range nDB.networks[nDB.config.NodeName] { 392 if network.leaving { 393 continue 394 } 395 networks = append(networks, nid) 396 } 397 nDB.RUnlock() 398 399 for { 400 if len(networks) == 0 { 401 break 402 } 403 404 nid := networks[0] 405 networks = networks[1:] 406 407 nDB.RLock() 408 nodes := nDB.networkNodes[nid] 409 nDB.RUnlock() 410 411 // No peer nodes on this network. Move on. 412 if len(nodes) == 0 { 413 continue 414 } 415 416 completed, err := nDB.bulkSync(nodes, false) 417 if err != nil { 418 logrus.Errorf("periodic bulk sync failure for network %s: %v", nid, err) 419 continue 420 } 421 422 // Remove all the networks for which we have 423 // successfully completed bulk sync in this iteration. 424 updatedNetworks := make([]string, 0, len(networks)) 425 for _, nid := range networks { 426 var found bool 427 for _, completedNid := range completed { 428 if nid == completedNid { 429 found = true 430 break 431 } 432 } 433 434 if !found { 435 updatedNetworks = append(updatedNetworks, nid) 436 } 437 } 438 439 networks = updatedNetworks 440 } 441 } 442 443 func (nDB *NetworkDB) bulkSync(nodes []string, all bool) ([]string, error) { 444 if !all { 445 // If not all, then just pick one. 446 nodes = nDB.mRandomNodes(1, nodes) 447 } 448 449 if len(nodes) == 0 { 450 return nil, nil 451 } 452 453 logrus.Debugf("%s: Initiating bulk sync with nodes %v", nDB.config.NodeName, nodes) 454 var err error 455 var networks []string 456 for _, node := range nodes { 457 if node == nDB.config.NodeName { 458 continue 459 } 460 461 networks = nDB.findCommonNetworks(node) 462 err = nDB.bulkSyncNode(networks, node, true) 463 if err != nil { 464 err = fmt.Errorf("bulk sync failed on node %s: %v", node, err) 465 } 466 } 467 468 if err != nil { 469 return nil, err 470 } 471 472 return networks, nil 473 } 474 475 // Bulk sync all the table entries belonging to a set of networks to a 476 // single peer node. It can be unsolicited or can be in response to an 477 // unsolicited bulk sync 478 func (nDB *NetworkDB) bulkSyncNode(networks []string, node string, unsolicited bool) error { 479 var msgs [][]byte 480 481 var unsolMsg string 482 if unsolicited { 483 unsolMsg = "unsolicited" 484 } 485 486 logrus.Debugf("%s: Initiating %s bulk sync for networks %v with node %s", nDB.config.NodeName, unsolMsg, networks, node) 487 488 nDB.RLock() 489 mnode := nDB.nodes[node] 490 if mnode == nil { 491 nDB.RUnlock() 492 return nil 493 } 494 495 for _, nid := range networks { 496 nDB.indexes[byNetwork].WalkPrefix(fmt.Sprintf("/%s", nid), func(path string, v interface{}) bool { 497 entry, ok := v.(*entry) 498 if !ok { 499 return false 500 } 501 502 eType := TableEventTypeCreate 503 if entry.deleting { 504 eType = TableEventTypeDelete 505 } 506 507 params := strings.Split(path[1:], "/") 508 tEvent := TableEvent{ 509 Type: eType, 510 LTime: entry.ltime, 511 NodeName: entry.node, 512 NetworkID: nid, 513 TableName: params[1], 514 Key: params[2], 515 Value: entry.value, 516 } 517 518 msg, err := encodeMessage(MessageTypeTableEvent, &tEvent) 519 if err != nil { 520 logrus.Errorf("Encode failure during bulk sync: %#v", tEvent) 521 return false 522 } 523 524 msgs = append(msgs, msg) 525 return false 526 }) 527 } 528 nDB.RUnlock() 529 530 // Create a compound message 531 compound := makeCompoundMessage(msgs) 532 533 bsm := BulkSyncMessage{ 534 LTime: nDB.tableClock.Time(), 535 Unsolicited: unsolicited, 536 NodeName: nDB.config.NodeName, 537 Networks: networks, 538 Payload: compound, 539 } 540 541 buf, err := encodeMessage(MessageTypeBulkSync, &bsm) 542 if err != nil { 543 return fmt.Errorf("failed to encode bulk sync message: %v", err) 544 } 545 546 nDB.Lock() 547 ch := make(chan struct{}) 548 nDB.bulkSyncAckTbl[node] = ch 549 nDB.Unlock() 550 551 err = nDB.memberlist.SendToTCP(&mnode.Node, buf) 552 if err != nil { 553 nDB.Lock() 554 delete(nDB.bulkSyncAckTbl, node) 555 nDB.Unlock() 556 557 return fmt.Errorf("failed to send a TCP message during bulk sync: %v", err) 558 } 559 560 // Wait on a response only if it is unsolicited. 561 if unsolicited { 562 startTime := time.Now() 563 t := time.NewTimer(30 * time.Second) 564 select { 565 case <-t.C: 566 logrus.Errorf("Bulk sync to node %s timed out", node) 567 case <-ch: 568 logrus.Debugf("%s: Bulk sync to node %s took %s", nDB.config.NodeName, node, time.Now().Sub(startTime)) 569 } 570 t.Stop() 571 } 572 573 return nil 574 } 575 576 // Returns a random offset between 0 and n 577 func randomOffset(n int) int { 578 if n == 0 { 579 return 0 580 } 581 582 val, err := rand.Int(rand.Reader, big.NewInt(int64(n))) 583 if err != nil { 584 logrus.Errorf("Failed to get a random offset: %v", err) 585 return 0 586 } 587 588 return int(val.Int64()) 589 } 590 591 // mRandomNodes is used to select up to m random nodes. It is possible 592 // that less than m nodes are returned. 593 func (nDB *NetworkDB) mRandomNodes(m int, nodes []string) []string { 594 n := len(nodes) 595 mNodes := make([]string, 0, m) 596 OUTER: 597 // Probe up to 3*n times, with large n this is not necessary 598 // since k << n, but with small n we want search to be 599 // exhaustive 600 for i := 0; i < 3*n && len(mNodes) < m; i++ { 601 // Get random node 602 idx := randomOffset(n) 603 node := nodes[idx] 604 605 if node == nDB.config.NodeName { 606 continue 607 } 608 609 // Check if we have this node already 610 for j := 0; j < len(mNodes); j++ { 611 if node == mNodes[j] { 612 continue OUTER 613 } 614 } 615 616 // Append the node 617 mNodes = append(mNodes, node) 618 } 619 620 return mNodes 621 }