github.com/FUSIONFoundation/efsn@v3.6.2-0.20200916075423-dbb5dd5d2cc7+incompatible/swarm/network/stream/stream.go (about) 1 // Copyright 2018 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package stream 18 19 import ( 20 "context" 21 "fmt" 22 "math" 23 "sync" 24 "time" 25 26 "github.com/FusionFoundation/efsn/metrics" 27 "github.com/FusionFoundation/efsn/p2p" 28 "github.com/FusionFoundation/efsn/p2p/discover" 29 "github.com/FusionFoundation/efsn/p2p/protocols" 30 "github.com/FusionFoundation/efsn/rpc" 31 "github.com/FusionFoundation/efsn/swarm/log" 32 "github.com/FusionFoundation/efsn/swarm/network" 33 "github.com/FusionFoundation/efsn/swarm/network/stream/intervals" 34 "github.com/FusionFoundation/efsn/swarm/pot" 35 "github.com/FusionFoundation/efsn/swarm/state" 36 "github.com/FusionFoundation/efsn/swarm/storage" 37 ) 38 39 const ( 40 Low uint8 = iota 41 Mid 42 High 43 Top 44 PriorityQueue = 4 // number of priority queues - Low, Mid, High, Top 45 PriorityQueueCap = 4096 // queue capacity 46 HashSize = 32 47 ) 48 49 // Registry registry for outgoing and incoming streamer constructors 50 type Registry struct { 51 api *API 52 addr *network.BzzAddr 53 skipCheck bool 54 clientMu sync.RWMutex 55 serverMu sync.RWMutex 56 peersMu sync.RWMutex 57 serverFuncs map[string]func(*Peer, string, bool) (Server, error) 58 clientFuncs map[string]func(*Peer, string, bool) (Client, error) 59 peers map[discover.NodeID]*Peer 60 delivery *Delivery 61 intervalsStore state.Store 62 doRetrieve bool 63 } 64 65 // RegistryOptions holds optional values for NewRegistry constructor. 66 type RegistryOptions struct { 67 SkipCheck bool 68 DoSync bool 69 DoRetrieve bool 70 SyncUpdateDelay time.Duration 71 } 72 73 // NewRegistry is Streamer constructor 74 func NewRegistry(addr *network.BzzAddr, delivery *Delivery, syncChunkStore storage.SyncChunkStore, intervalsStore state.Store, options *RegistryOptions) *Registry { 75 if options == nil { 76 options = &RegistryOptions{} 77 } 78 if options.SyncUpdateDelay <= 0 { 79 options.SyncUpdateDelay = 15 * time.Second 80 } 81 streamer := &Registry{ 82 addr: addr, 83 skipCheck: options.SkipCheck, 84 serverFuncs: make(map[string]func(*Peer, string, bool) (Server, error)), 85 clientFuncs: make(map[string]func(*Peer, string, bool) (Client, error)), 86 peers: make(map[discover.NodeID]*Peer), 87 delivery: delivery, 88 intervalsStore: intervalsStore, 89 doRetrieve: options.DoRetrieve, 90 } 91 streamer.api = NewAPI(streamer) 92 delivery.getPeer = streamer.getPeer 93 streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, _ bool) (Server, error) { 94 return NewSwarmChunkServer(delivery.chunkStore), nil 95 }) 96 streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { 97 return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) 98 }) 99 RegisterSwarmSyncerServer(streamer, syncChunkStore) 100 RegisterSwarmSyncerClient(streamer, syncChunkStore) 101 102 if options.DoSync { 103 // latestIntC function ensures that 104 // - receiving from the in chan is not blocked by processing inside the for loop 105 // - the latest int value is delivered to the loop after the processing is done 106 // In context of NeighbourhoodDepthC: 107 // after the syncing is done updating inside the loop, we do not need to update on the intermediate 108 // depth changes, only to the latest one 109 latestIntC := func(in <-chan int) <-chan int { 110 out := make(chan int, 1) 111 112 go func() { 113 defer close(out) 114 115 for i := range in { 116 select { 117 case <-out: 118 default: 119 } 120 out <- i 121 } 122 }() 123 124 return out 125 } 126 127 go func() { 128 // wait for kademlia table to be healthy 129 time.Sleep(options.SyncUpdateDelay) 130 131 kad := streamer.delivery.kad 132 depthC := latestIntC(kad.NeighbourhoodDepthC()) 133 addressBookSizeC := latestIntC(kad.AddrCountC()) 134 135 // initial requests for syncing subscription to peers 136 streamer.updateSyncing() 137 138 for depth := range depthC { 139 log.Debug("Kademlia neighbourhood depth change", "depth", depth) 140 141 // Prevent too early sync subscriptions by waiting until there are no 142 // new peers connecting. Sync streams updating will be done after no 143 // peers are connected for at least SyncUpdateDelay period. 144 timer := time.NewTimer(options.SyncUpdateDelay) 145 // Hard limit to sync update delay, preventing long delays 146 // on a very dynamic network 147 maxTimer := time.NewTimer(3 * time.Minute) 148 loop: 149 for { 150 select { 151 case <-maxTimer.C: 152 // force syncing update when a hard timeout is reached 153 log.Trace("Sync subscriptions update on hard timeout") 154 // request for syncing subscription to new peers 155 streamer.updateSyncing() 156 break loop 157 case <-timer.C: 158 // start syncing as no new peers has been added to kademlia 159 // for some time 160 log.Trace("Sync subscriptions update") 161 // request for syncing subscription to new peers 162 streamer.updateSyncing() 163 break loop 164 case size := <-addressBookSizeC: 165 log.Trace("Kademlia address book size changed on depth change", "size", size) 166 // new peers has been added to kademlia, 167 // reset the timer to prevent early sync subscriptions 168 if !timer.Stop() { 169 <-timer.C 170 } 171 timer.Reset(options.SyncUpdateDelay) 172 } 173 } 174 timer.Stop() 175 maxTimer.Stop() 176 } 177 }() 178 } 179 180 return streamer 181 } 182 183 // RegisterClient registers an incoming streamer constructor 184 func (r *Registry) RegisterClientFunc(stream string, f func(*Peer, string, bool) (Client, error)) { 185 r.clientMu.Lock() 186 defer r.clientMu.Unlock() 187 188 r.clientFuncs[stream] = f 189 } 190 191 // RegisterServer registers an outgoing streamer constructor 192 func (r *Registry) RegisterServerFunc(stream string, f func(*Peer, string, bool) (Server, error)) { 193 r.serverMu.Lock() 194 defer r.serverMu.Unlock() 195 196 r.serverFuncs[stream] = f 197 } 198 199 // GetClient accessor for incoming streamer constructors 200 func (r *Registry) GetClientFunc(stream string) (func(*Peer, string, bool) (Client, error), error) { 201 r.clientMu.RLock() 202 defer r.clientMu.RUnlock() 203 204 f := r.clientFuncs[stream] 205 if f == nil { 206 return nil, fmt.Errorf("stream %v not registered", stream) 207 } 208 return f, nil 209 } 210 211 // GetServer accessor for incoming streamer constructors 212 func (r *Registry) GetServerFunc(stream string) (func(*Peer, string, bool) (Server, error), error) { 213 r.serverMu.RLock() 214 defer r.serverMu.RUnlock() 215 216 f := r.serverFuncs[stream] 217 if f == nil { 218 return nil, fmt.Errorf("stream %v not registered", stream) 219 } 220 return f, nil 221 } 222 223 func (r *Registry) RequestSubscription(peerId discover.NodeID, s Stream, h *Range, prio uint8) error { 224 // check if the stream is registered 225 if _, err := r.GetServerFunc(s.Name); err != nil { 226 return err 227 } 228 229 peer := r.getPeer(peerId) 230 if peer == nil { 231 return fmt.Errorf("peer not found %v", peerId) 232 } 233 234 if _, err := peer.getServer(s); err != nil { 235 if e, ok := err.(*notFoundError); ok && e.t == "server" { 236 // request subscription only if the server for this stream is not created 237 log.Debug("RequestSubscription ", "peer", peerId, "stream", s, "history", h) 238 return peer.Send(context.TODO(), &RequestSubscriptionMsg{ 239 Stream: s, 240 History: h, 241 Priority: prio, 242 }) 243 } 244 return err 245 } 246 log.Trace("RequestSubscription: already subscribed", "peer", peerId, "stream", s, "history", h) 247 return nil 248 } 249 250 // Subscribe initiates the streamer 251 func (r *Registry) Subscribe(peerId discover.NodeID, s Stream, h *Range, priority uint8) error { 252 // check if the stream is registered 253 if _, err := r.GetClientFunc(s.Name); err != nil { 254 return err 255 } 256 257 peer := r.getPeer(peerId) 258 if peer == nil { 259 return fmt.Errorf("peer not found %v", peerId) 260 } 261 262 var to uint64 263 if !s.Live && h != nil { 264 to = h.To 265 } 266 267 err := peer.setClientParams(s, newClientParams(priority, to)) 268 if err != nil { 269 return err 270 } 271 272 if s.Live && h != nil { 273 if err := peer.setClientParams( 274 getHistoryStream(s), 275 newClientParams(getHistoryPriority(priority), h.To), 276 ); err != nil { 277 return err 278 } 279 } 280 281 msg := &SubscribeMsg{ 282 Stream: s, 283 History: h, 284 Priority: priority, 285 } 286 log.Debug("Subscribe ", "peer", peerId, "stream", s, "history", h) 287 288 return peer.SendPriority(context.TODO(), msg, priority) 289 } 290 291 func (r *Registry) Unsubscribe(peerId discover.NodeID, s Stream) error { 292 peer := r.getPeer(peerId) 293 if peer == nil { 294 return fmt.Errorf("peer not found %v", peerId) 295 } 296 297 msg := &UnsubscribeMsg{ 298 Stream: s, 299 } 300 log.Debug("Unsubscribe ", "peer", peerId, "stream", s) 301 302 if err := peer.Send(context.TODO(), msg); err != nil { 303 return err 304 } 305 return peer.removeClient(s) 306 } 307 308 // Quit sends the QuitMsg to the peer to remove the 309 // stream peer client and terminate the streaming. 310 func (r *Registry) Quit(peerId discover.NodeID, s Stream) error { 311 peer := r.getPeer(peerId) 312 if peer == nil { 313 log.Debug("stream quit: peer not found", "peer", peerId, "stream", s) 314 // if the peer is not found, abort the request 315 return nil 316 } 317 318 msg := &QuitMsg{ 319 Stream: s, 320 } 321 log.Debug("Quit ", "peer", peerId, "stream", s) 322 323 return peer.Send(context.TODO(), msg) 324 } 325 326 func (r *Registry) NodeInfo() interface{} { 327 return nil 328 } 329 330 func (r *Registry) PeerInfo(id discover.NodeID) interface{} { 331 return nil 332 } 333 334 func (r *Registry) Close() error { 335 return r.intervalsStore.Close() 336 } 337 338 func (r *Registry) getPeer(peerId discover.NodeID) *Peer { 339 r.peersMu.RLock() 340 defer r.peersMu.RUnlock() 341 342 return r.peers[peerId] 343 } 344 345 func (r *Registry) setPeer(peer *Peer) { 346 r.peersMu.Lock() 347 r.peers[peer.ID()] = peer 348 metrics.GetOrRegisterGauge("registry.peers", nil).Update(int64(len(r.peers))) 349 r.peersMu.Unlock() 350 } 351 352 func (r *Registry) deletePeer(peer *Peer) { 353 r.peersMu.Lock() 354 delete(r.peers, peer.ID()) 355 metrics.GetOrRegisterGauge("registry.peers", nil).Update(int64(len(r.peers))) 356 r.peersMu.Unlock() 357 } 358 359 func (r *Registry) peersCount() (c int) { 360 r.peersMu.Lock() 361 c = len(r.peers) 362 r.peersMu.Unlock() 363 return 364 } 365 366 // Run protocol run function 367 func (r *Registry) Run(p *network.BzzPeer) error { 368 sp := NewPeer(p.Peer, r) 369 r.setPeer(sp) 370 defer r.deletePeer(sp) 371 defer close(sp.quit) 372 defer sp.close() 373 374 if r.doRetrieve { 375 err := r.Subscribe(p.ID(), NewStream(swarmChunkServerStreamName, "", false), nil, Top) 376 if err != nil { 377 return err 378 } 379 } 380 381 return sp.Run(sp.HandleMsg) 382 } 383 384 // updateSyncing subscribes to SYNC streams by iterating over the 385 // kademlia connections and bins. If there are existing SYNC streams 386 // and they are no longer required after iteration, request to Quit 387 // them will be send to appropriate peers. 388 func (r *Registry) updateSyncing() { 389 kad := r.delivery.kad 390 // map of all SYNC streams for all peers 391 // used at the and of the function to remove servers 392 // that are not needed anymore 393 subs := make(map[discover.NodeID]map[Stream]struct{}) 394 r.peersMu.RLock() 395 for id, peer := range r.peers { 396 peer.serverMu.RLock() 397 for stream := range peer.servers { 398 if stream.Name == "SYNC" { 399 if _, ok := subs[id]; !ok { 400 subs[id] = make(map[Stream]struct{}) 401 } 402 subs[id][stream] = struct{}{} 403 } 404 } 405 peer.serverMu.RUnlock() 406 } 407 r.peersMu.RUnlock() 408 409 // request subscriptions for all nodes and bins 410 kad.EachBin(r.addr.Over(), pot.DefaultPof(256), 0, func(p *network.Peer, bin int) bool { 411 log.Debug(fmt.Sprintf("Requesting subscription by: registry %s from peer %s for bin: %d", r.addr.ID(), p.ID(), bin)) 412 413 // bin is always less then 256 and it is safe to convert it to type uint8 414 stream := NewStream("SYNC", FormatSyncBinKey(uint8(bin)), true) 415 if streams, ok := subs[p.ID()]; ok { 416 // delete live and history streams from the map, so that it won't be removed with a Quit request 417 delete(streams, stream) 418 delete(streams, getHistoryStream(stream)) 419 } 420 err := r.RequestSubscription(p.ID(), stream, NewRange(0, 0), High) 421 if err != nil { 422 log.Debug("Request subscription", "err", err, "peer", p.ID(), "stream", stream) 423 return false 424 } 425 return true 426 }) 427 428 // remove SYNC servers that do not need to be subscribed 429 for id, streams := range subs { 430 if len(streams) == 0 { 431 continue 432 } 433 peer := r.getPeer(id) 434 if peer == nil { 435 continue 436 } 437 for stream := range streams { 438 log.Debug("Remove sync server", "peer", id, "stream", stream) 439 err := r.Quit(peer.ID(), stream) 440 if err != nil && err != p2p.ErrShuttingDown { 441 log.Error("quit", "err", err, "peer", peer.ID(), "stream", stream) 442 } 443 } 444 } 445 } 446 447 func (r *Registry) runProtocol(p *p2p.Peer, rw p2p.MsgReadWriter) error { 448 peer := protocols.NewPeer(p, rw, Spec) 449 bp := network.NewBzzPeer(peer, r.addr) 450 np := network.NewPeer(bp, r.delivery.kad) 451 r.delivery.kad.On(np) 452 defer r.delivery.kad.Off(np) 453 return r.Run(bp) 454 } 455 456 // HandleMsg is the message handler that delegates incoming messages 457 func (p *Peer) HandleMsg(ctx context.Context, msg interface{}) error { 458 switch msg := msg.(type) { 459 460 case *SubscribeMsg: 461 return p.handleSubscribeMsg(ctx, msg) 462 463 case *SubscribeErrorMsg: 464 return p.handleSubscribeErrorMsg(msg) 465 466 case *UnsubscribeMsg: 467 return p.handleUnsubscribeMsg(msg) 468 469 case *OfferedHashesMsg: 470 return p.handleOfferedHashesMsg(ctx, msg) 471 472 case *TakeoverProofMsg: 473 return p.handleTakeoverProofMsg(ctx, msg) 474 475 case *WantedHashesMsg: 476 return p.handleWantedHashesMsg(ctx, msg) 477 478 case *ChunkDeliveryMsg: 479 return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, msg) 480 481 case *RetrieveRequestMsg: 482 return p.streamer.delivery.handleRetrieveRequestMsg(ctx, p, msg) 483 484 case *RequestSubscriptionMsg: 485 return p.handleRequestSubscription(ctx, msg) 486 487 case *QuitMsg: 488 return p.handleQuitMsg(msg) 489 490 default: 491 return fmt.Errorf("unknown message type: %T", msg) 492 } 493 } 494 495 type server struct { 496 Server 497 stream Stream 498 priority uint8 499 currentBatch []byte 500 } 501 502 // Server interface for outgoing peer Streamer 503 type Server interface { 504 SetNextBatch(uint64, uint64) (hashes []byte, from uint64, to uint64, proof *HandoverProof, err error) 505 GetData(context.Context, []byte) ([]byte, error) 506 Close() 507 } 508 509 type client struct { 510 Client 511 stream Stream 512 priority uint8 513 sessionAt uint64 514 to uint64 515 next chan error 516 quit chan struct{} 517 518 intervalsKey string 519 intervalsStore state.Store 520 } 521 522 func peerStreamIntervalsKey(p *Peer, s Stream) string { 523 return p.ID().String() + s.String() 524 } 525 526 func (c client) AddInterval(start, end uint64) (err error) { 527 i := &intervals.Intervals{} 528 err = c.intervalsStore.Get(c.intervalsKey, i) 529 if err != nil { 530 return err 531 } 532 i.Add(start, end) 533 return c.intervalsStore.Put(c.intervalsKey, i) 534 } 535 536 func (c client) NextInterval() (start, end uint64, err error) { 537 i := &intervals.Intervals{} 538 err = c.intervalsStore.Get(c.intervalsKey, i) 539 if err != nil { 540 return 0, 0, err 541 } 542 start, end = i.Next() 543 return start, end, nil 544 } 545 546 // Client interface for incoming peer Streamer 547 type Client interface { 548 NeedData(context.Context, []byte) func(context.Context) error 549 BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) 550 Close() 551 } 552 553 func (c *client) nextBatch(from uint64) (nextFrom uint64, nextTo uint64) { 554 if c.to > 0 && from >= c.to { 555 return 0, 0 556 } 557 if c.stream.Live { 558 return from, 0 559 } else if from >= c.sessionAt { 560 if c.to > 0 { 561 return from, c.to 562 } 563 return from, math.MaxUint64 564 } 565 nextFrom, nextTo, err := c.NextInterval() 566 if err != nil { 567 log.Error("next intervals", "stream", c.stream) 568 return 569 } 570 if nextTo > c.to { 571 nextTo = c.to 572 } 573 if nextTo == 0 { 574 nextTo = c.sessionAt 575 } 576 return 577 } 578 579 func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error { 580 if tf := c.BatchDone(req.Stream, req.From, hashes, req.Root); tf != nil { 581 tp, err := tf() 582 if err != nil { 583 return err 584 } 585 if err := p.SendPriority(context.TODO(), tp, c.priority); err != nil { 586 return err 587 } 588 if c.to > 0 && tp.Takeover.End >= c.to { 589 return p.streamer.Unsubscribe(p.Peer.ID(), req.Stream) 590 } 591 return nil 592 } 593 // TODO: make a test case for testing if the interval is added when the batch is done 594 if err := c.AddInterval(req.From, req.To); err != nil { 595 return err 596 } 597 return nil 598 } 599 600 func (c *client) close() { 601 select { 602 case <-c.quit: 603 default: 604 close(c.quit) 605 } 606 c.Close() 607 } 608 609 // clientParams store parameters for the new client 610 // between a subscription and initial offered hashes request handling. 611 type clientParams struct { 612 priority uint8 613 to uint64 614 // signal when the client is created 615 clientCreatedC chan struct{} 616 } 617 618 func newClientParams(priority uint8, to uint64) *clientParams { 619 return &clientParams{ 620 priority: priority, 621 to: to, 622 clientCreatedC: make(chan struct{}), 623 } 624 } 625 626 func (c *clientParams) waitClient(ctx context.Context) error { 627 select { 628 case <-ctx.Done(): 629 return ctx.Err() 630 case <-c.clientCreatedC: 631 return nil 632 } 633 } 634 635 func (c *clientParams) clientCreated() { 636 close(c.clientCreatedC) 637 } 638 639 // Spec is the spec of the streamer protocol 640 var Spec = &protocols.Spec{ 641 Name: "stream", 642 Version: 6, 643 MaxMsgSize: 10 * 1024 * 1024, 644 Messages: []interface{}{ 645 UnsubscribeMsg{}, 646 OfferedHashesMsg{}, 647 WantedHashesMsg{}, 648 TakeoverProofMsg{}, 649 SubscribeMsg{}, 650 RetrieveRequestMsg{}, 651 ChunkDeliveryMsg{}, 652 SubscribeErrorMsg{}, 653 RequestSubscriptionMsg{}, 654 QuitMsg{}, 655 }, 656 } 657 658 func (r *Registry) Protocols() []p2p.Protocol { 659 return []p2p.Protocol{ 660 { 661 Name: Spec.Name, 662 Version: Spec.Version, 663 Length: Spec.Length(), 664 Run: r.runProtocol, 665 // NodeInfo: , 666 // PeerInfo: , 667 }, 668 } 669 } 670 671 func (r *Registry) APIs() []rpc.API { 672 return []rpc.API{ 673 { 674 Namespace: "stream", 675 Version: "3.0", 676 Service: r.api, 677 Public: true, 678 }, 679 } 680 } 681 682 func (r *Registry) Start(server *p2p.Server) error { 683 log.Info("Streamer started") 684 return nil 685 } 686 687 func (r *Registry) Stop() error { 688 return nil 689 } 690 691 type Range struct { 692 From, To uint64 693 } 694 695 func NewRange(from, to uint64) *Range { 696 return &Range{ 697 From: from, 698 To: to, 699 } 700 } 701 702 func (r *Range) String() string { 703 return fmt.Sprintf("%v-%v", r.From, r.To) 704 } 705 706 func getHistoryPriority(priority uint8) uint8 { 707 if priority == 0 { 708 return 0 709 } 710 return priority - 1 711 } 712 713 func getHistoryStream(s Stream) Stream { 714 return NewStream(s.Name, s.Key, false) 715 } 716 717 type API struct { 718 streamer *Registry 719 } 720 721 func NewAPI(r *Registry) *API { 722 return &API{ 723 streamer: r, 724 } 725 } 726 727 func (api *API) SubscribeStream(peerId discover.NodeID, s Stream, history *Range, priority uint8) error { 728 return api.streamer.Subscribe(peerId, s, history, priority) 729 } 730 731 func (api *API) UnsubscribeStream(peerId discover.NodeID, s Stream) error { 732 return api.streamer.Unsubscribe(peerId, s) 733 }