github.com/zignig/go-ipfs@v0.0.0-20141111235910-c9e5fdf55a52/routing/dht/dht.go (about) 1 // package dht implements a distributed hash table that satisfies the ipfs routing 2 // interface. This DHT is modeled after kademlia with Coral and S/Kademlia modifications. 3 package dht 4 5 import ( 6 "bytes" 7 "crypto/rand" 8 "errors" 9 "fmt" 10 "sync" 11 "time" 12 13 inet "github.com/jbenet/go-ipfs/net" 14 msg "github.com/jbenet/go-ipfs/net/message" 15 peer "github.com/jbenet/go-ipfs/peer" 16 routing "github.com/jbenet/go-ipfs/routing" 17 pb "github.com/jbenet/go-ipfs/routing/dht/pb" 18 kb "github.com/jbenet/go-ipfs/routing/kbucket" 19 u "github.com/jbenet/go-ipfs/util" 20 ctxc "github.com/jbenet/go-ipfs/util/ctxcloser" 21 22 context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" 23 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" 24 25 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto" 26 ) 27 28 var log = u.Logger("dht") 29 30 const doPinging = false 31 32 // TODO. SEE https://github.com/jbenet/node-ipfs/blob/master/submodules/ipfs-dht/index.js 33 34 // IpfsDHT is an implementation of Kademlia with Coral and S/Kademlia modifications. 35 // It is used to implement the base IpfsRouting module. 36 type IpfsDHT struct { 37 // Array of routing tables for differently distanced nodes 38 // NOTE: (currently, only a single table is used) 39 routingTables []*kb.RoutingTable 40 41 // the network services we need 42 dialer inet.Dialer 43 sender inet.Sender 44 45 // Local peer (yourself) 46 self peer.Peer 47 48 // Other peers 49 peerstore peer.Peerstore 50 51 // Local data 52 datastore ds.Datastore 53 dslock sync.Mutex 54 55 providers *ProviderManager 56 57 // When this peer started up 58 birth time.Time 59 60 //lock to make diagnostics work better 61 diaglock sync.Mutex 62 63 ctxc.ContextCloser 64 } 65 66 // NewDHT creates a new DHT object with the given peer as the 'local' host 67 func NewDHT(ctx context.Context, p peer.Peer, ps peer.Peerstore, dialer inet.Dialer, sender inet.Sender, dstore ds.Datastore) *IpfsDHT { 68 dht := new(IpfsDHT) 69 dht.dialer = dialer 70 dht.sender = sender 71 dht.datastore = dstore 72 dht.self = p 73 dht.peerstore = ps 74 dht.ContextCloser = ctxc.NewContextCloser(ctx, nil) 75 76 dht.providers = NewProviderManager(dht.Context(), p.ID()) 77 dht.AddCloserChild(dht.providers) 78 79 dht.routingTables = make([]*kb.RoutingTable, 3) 80 dht.routingTables[0] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID()), time.Millisecond*1000) 81 dht.routingTables[1] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID()), time.Millisecond*1000) 82 dht.routingTables[2] = kb.NewRoutingTable(20, kb.ConvertPeerID(p.ID()), time.Hour) 83 dht.birth = time.Now() 84 85 if doPinging { 86 dht.Children().Add(1) 87 go dht.PingRoutine(time.Second * 10) 88 } 89 return dht 90 } 91 92 // Connect to a new peer at the given address, ping and add to the routing table 93 func (dht *IpfsDHT) Connect(ctx context.Context, npeer peer.Peer) (peer.Peer, error) { 94 log.Debugf("Connect to new peer: %s", npeer) 95 96 // TODO(jbenet,whyrusleeping) 97 // 98 // Connect should take in a Peer (with ID). In a sense, we shouldn't be 99 // allowing connections to random multiaddrs without knowing who we're 100 // speaking to (i.e. peer.ID). In terms of moving around simple addresses 101 // -- instead of an (ID, Addr) pair -- we can use: 102 // 103 // /ip4/10.20.30.40/tcp/1234/ipfs/Qxhxxchxzcncxnzcnxzcxzm 104 // 105 err := dht.dialer.DialPeer(ctx, npeer) 106 if err != nil { 107 return nil, err 108 } 109 110 // Ping new peer to register in their routing table 111 // NOTE: this should be done better... 112 err = dht.Ping(ctx, npeer) 113 if err != nil { 114 return nil, fmt.Errorf("failed to ping newly connected peer: %s\n", err) 115 } 116 117 dht.Update(npeer) 118 119 return npeer, nil 120 } 121 122 // HandleMessage implements the inet.Handler interface. 123 func (dht *IpfsDHT) HandleMessage(ctx context.Context, mes msg.NetMessage) msg.NetMessage { 124 125 mData := mes.Data() 126 if mData == nil { 127 log.Error("Message contained nil data.") 128 return nil 129 } 130 131 mPeer := mes.Peer() 132 if mPeer == nil { 133 log.Error("Message contained nil peer.") 134 return nil 135 } 136 137 // deserialize msg 138 pmes := new(pb.Message) 139 err := proto.Unmarshal(mData, pmes) 140 if err != nil { 141 log.Error("Error unmarshaling data") 142 return nil 143 } 144 145 // update the peer (on valid msgs only) 146 dht.Update(mPeer) 147 148 // Print out diagnostic 149 log.Debugf("%s got message type: '%s' from %s", 150 dht.self, pb.Message_MessageType_name[int32(pmes.GetType())], mPeer) 151 152 // get handler for this msg type. 153 handler := dht.handlerForMsgType(pmes.GetType()) 154 if handler == nil { 155 log.Error("got back nil handler from handlerForMsgType") 156 return nil 157 } 158 159 // dispatch handler. 160 rpmes, err := handler(mPeer, pmes) 161 if err != nil { 162 log.Errorf("handle message error: %s", err) 163 return nil 164 } 165 166 // if nil response, return it before serializing 167 if rpmes == nil { 168 log.Warning("Got back nil response from request.") 169 return nil 170 } 171 172 // serialize response msg 173 rmes, err := msg.FromObject(mPeer, rpmes) 174 if err != nil { 175 log.Errorf("serialze response error: %s", err) 176 return nil 177 } 178 179 return rmes 180 } 181 182 // sendRequest sends out a request using dht.sender, but also makes sure to 183 // measure the RTT for latency measurements. 184 func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.Peer, pmes *pb.Message) (*pb.Message, error) { 185 186 mes, err := msg.FromObject(p, pmes) 187 if err != nil { 188 return nil, err 189 } 190 191 start := time.Now() 192 193 // Print out diagnostic 194 log.Debugf("Sent message type: '%s' to %s", 195 pb.Message_MessageType_name[int32(pmes.GetType())], p) 196 197 rmes, err := dht.sender.SendRequest(ctx, mes) 198 if err != nil { 199 return nil, err 200 } 201 if rmes == nil { 202 return nil, errors.New("no response to request") 203 } 204 205 rtt := time.Since(start) 206 rmes.Peer().SetLatency(rtt) 207 208 rpmes := new(pb.Message) 209 if err := proto.Unmarshal(rmes.Data(), rpmes); err != nil { 210 return nil, err 211 } 212 213 return rpmes, nil 214 } 215 216 // putValueToNetwork stores the given key/value pair at the peer 'p' 217 func (dht *IpfsDHT) putValueToNetwork(ctx context.Context, p peer.Peer, 218 key string, value []byte) error { 219 220 pmes := pb.NewMessage(pb.Message_PUT_VALUE, string(key), 0) 221 pmes.Value = value 222 rpmes, err := dht.sendRequest(ctx, p, pmes) 223 if err != nil { 224 return err 225 } 226 227 if !bytes.Equal(rpmes.Value, pmes.Value) { 228 return errors.New("value not put correctly") 229 } 230 return nil 231 } 232 233 // putProvider sends a message to peer 'p' saying that the local node 234 // can provide the value of 'key' 235 func (dht *IpfsDHT) putProvider(ctx context.Context, p peer.Peer, key string) error { 236 237 pmes := pb.NewMessage(pb.Message_ADD_PROVIDER, string(key), 0) 238 239 // add self as the provider 240 pmes.ProviderPeers = pb.PeersToPBPeers([]peer.Peer{dht.self}) 241 242 rpmes, err := dht.sendRequest(ctx, p, pmes) 243 if err != nil { 244 return err 245 } 246 247 log.Debugf("%s putProvider: %s for %s", dht.self, p, u.Key(key)) 248 if rpmes.GetKey() != pmes.GetKey() { 249 return errors.New("provider not added correctly") 250 } 251 252 return nil 253 } 254 255 func (dht *IpfsDHT) getValueOrPeers(ctx context.Context, p peer.Peer, 256 key u.Key, level int) ([]byte, []peer.Peer, error) { 257 258 pmes, err := dht.getValueSingle(ctx, p, key, level) 259 if err != nil { 260 return nil, nil, err 261 } 262 263 log.Debugf("pmes.GetValue() %v", pmes.GetValue()) 264 if value := pmes.GetValue(); value != nil { 265 // Success! We were given the value 266 log.Debug("getValueOrPeers: got value") 267 return value, nil, nil 268 } 269 270 // TODO decide on providers. This probably shouldn't be happening. 271 if prv := pmes.GetProviderPeers(); prv != nil && len(prv) > 0 { 272 val, err := dht.getFromPeerList(ctx, key, prv, level) 273 if err != nil { 274 return nil, nil, err 275 } 276 log.Debug("getValueOrPeers: get from providers") 277 return val, nil, nil 278 } 279 280 // Perhaps we were given closer peers 281 var peers []peer.Peer 282 for _, pb := range pmes.GetCloserPeers() { 283 pr, err := dht.peerFromInfo(pb) 284 if err != nil { 285 log.Error(err) 286 continue 287 } 288 peers = append(peers, pr) 289 } 290 291 if len(peers) > 0 { 292 log.Debug("getValueOrPeers: peers") 293 return nil, peers, nil 294 } 295 296 log.Warning("getValueOrPeers: routing.ErrNotFound") 297 return nil, nil, routing.ErrNotFound 298 } 299 300 // getValueSingle simply performs the get value RPC with the given parameters 301 func (dht *IpfsDHT) getValueSingle(ctx context.Context, p peer.Peer, 302 key u.Key, level int) (*pb.Message, error) { 303 304 pmes := pb.NewMessage(pb.Message_GET_VALUE, string(key), level) 305 return dht.sendRequest(ctx, p, pmes) 306 } 307 308 // TODO: Im not certain on this implementation, we get a list of peers/providers 309 // from someone what do we do with it? Connect to each of them? randomly pick 310 // one to get the value from? Or just connect to one at a time until we get a 311 // successful connection and request the value from it? 312 func (dht *IpfsDHT) getFromPeerList(ctx context.Context, key u.Key, 313 peerlist []*pb.Message_Peer, level int) ([]byte, error) { 314 315 for _, pinfo := range peerlist { 316 p, err := dht.ensureConnectedToPeer(ctx, pinfo) 317 if err != nil { 318 log.Errorf("getFromPeers error: %s", err) 319 continue 320 } 321 322 pmes, err := dht.getValueSingle(ctx, p, key, level) 323 if err != nil { 324 log.Errorf("getFromPeers error: %s\n", err) 325 continue 326 } 327 328 if value := pmes.GetValue(); value != nil { 329 // Success! We were given the value 330 dht.providers.AddProvider(key, p) 331 return value, nil 332 } 333 } 334 return nil, routing.ErrNotFound 335 } 336 337 // getLocal attempts to retrieve the value from the datastore 338 func (dht *IpfsDHT) getLocal(key u.Key) ([]byte, error) { 339 dht.dslock.Lock() 340 defer dht.dslock.Unlock() 341 v, err := dht.datastore.Get(key.DsKey()) 342 if err != nil { 343 return nil, err 344 } 345 346 byt, ok := v.([]byte) 347 if !ok { 348 return nil, errors.New("value stored in datastore not []byte") 349 } 350 return byt, nil 351 } 352 353 // putLocal stores the key value pair in the datastore 354 func (dht *IpfsDHT) putLocal(key u.Key, value []byte) error { 355 return dht.datastore.Put(key.DsKey(), value) 356 } 357 358 // Update signals to all routingTables to Update their last-seen status 359 // on the given peer. 360 func (dht *IpfsDHT) Update(p peer.Peer) { 361 log.Debugf("updating peer: %s latency = %f\n", p, p.GetLatency().Seconds()) 362 removedCount := 0 363 for _, route := range dht.routingTables { 364 removed := route.Update(p) 365 // Only close the connection if no tables refer to this peer 366 if removed != nil { 367 removedCount++ 368 } 369 } 370 371 // Only close the connection if no tables refer to this peer 372 // if removedCount == len(dht.routingTables) { 373 // dht.network.ClosePeer(p) 374 // } 375 // ACTUALLY, no, let's not just close the connection. it may be connected 376 // due to other things. it seems that we just need connection timeouts 377 // after some deadline of inactivity. 378 } 379 380 // FindLocal looks for a peer with a given ID connected to this dht and returns the peer and the table it was found in. 381 func (dht *IpfsDHT) FindLocal(id peer.ID) (peer.Peer, *kb.RoutingTable) { 382 for _, table := range dht.routingTables { 383 p := table.Find(id) 384 if p != nil { 385 return p, table 386 } 387 } 388 return nil, nil 389 } 390 391 // findPeerSingle asks peer 'p' if they know where the peer with id 'id' is 392 func (dht *IpfsDHT) findPeerSingle(ctx context.Context, p peer.Peer, id peer.ID, level int) (*pb.Message, error) { 393 pmes := pb.NewMessage(pb.Message_FIND_NODE, string(id), level) 394 return dht.sendRequest(ctx, p, pmes) 395 } 396 397 func (dht *IpfsDHT) findProvidersSingle(ctx context.Context, p peer.Peer, key u.Key, level int) (*pb.Message, error) { 398 pmes := pb.NewMessage(pb.Message_GET_PROVIDERS, string(key), level) 399 return dht.sendRequest(ctx, p, pmes) 400 } 401 402 func (dht *IpfsDHT) addProviders(key u.Key, peers []*pb.Message_Peer) []peer.Peer { 403 var provArr []peer.Peer 404 for _, prov := range peers { 405 p, err := dht.peerFromInfo(prov) 406 if err != nil { 407 log.Errorf("error getting peer from info: %v", err) 408 continue 409 } 410 411 log.Debugf("%s adding provider: %s for %s", dht.self, p, key) 412 413 // Dont add outselves to the list 414 if p.ID().Equal(dht.self.ID()) { 415 continue 416 } 417 418 // TODO(jbenet) ensure providers is idempotent 419 dht.providers.AddProvider(key, p) 420 provArr = append(provArr, p) 421 } 422 return provArr 423 } 424 425 // nearestPeersToQuery returns the routing tables closest peers. 426 func (dht *IpfsDHT) nearestPeersToQuery(pmes *pb.Message, count int) []peer.Peer { 427 level := pmes.GetClusterLevel() 428 cluster := dht.routingTables[level] 429 430 key := u.Key(pmes.GetKey()) 431 closer := cluster.NearestPeers(kb.ConvertKey(key), count) 432 return closer 433 } 434 435 // betterPeerToQuery returns nearestPeersToQuery, but iff closer than self. 436 func (dht *IpfsDHT) betterPeersToQuery(pmes *pb.Message, count int) []peer.Peer { 437 closer := dht.nearestPeersToQuery(pmes, count) 438 439 // no node? nil 440 if closer == nil { 441 return nil 442 } 443 444 // == to self? thats bad 445 for _, p := range closer { 446 if p.ID().Equal(dht.self.ID()) { 447 log.Error("Attempted to return self! this shouldnt happen...") 448 return nil 449 } 450 } 451 452 var filtered []peer.Peer 453 for _, p := range closer { 454 // must all be closer than self 455 key := u.Key(pmes.GetKey()) 456 if !kb.Closer(dht.self.ID(), p.ID(), key) { 457 filtered = append(filtered, p) 458 } 459 } 460 461 // ok seems like closer nodes 462 return filtered 463 } 464 465 // getPeer searches the peerstore for a peer with the given peer ID 466 func (dht *IpfsDHT) getPeer(id peer.ID) (peer.Peer, error) { 467 p, err := dht.peerstore.Get(id) 468 if err != nil { 469 err = fmt.Errorf("Failed to get peer from peerstore: %s", err) 470 log.Error(err) 471 return nil, err 472 } 473 return p, nil 474 } 475 476 // peerFromInfo returns a peer using info in the protobuf peer struct 477 // to lookup or create a peer 478 func (dht *IpfsDHT) peerFromInfo(pbp *pb.Message_Peer) (peer.Peer, error) { 479 480 id := peer.ID(pbp.GetId()) 481 482 // bail out if it's ourselves 483 //TODO(jbenet) not sure this should be an error _here_ 484 if id.Equal(dht.self.ID()) { 485 return nil, errors.New("found self") 486 } 487 488 p, err := dht.getPeer(id) 489 if err != nil { 490 return nil, err 491 } 492 493 maddr, err := pbp.Address() 494 if err != nil { 495 return nil, err 496 } 497 p.AddAddress(maddr) 498 return p, nil 499 } 500 501 func (dht *IpfsDHT) ensureConnectedToPeer(ctx context.Context, pbp *pb.Message_Peer) (peer.Peer, error) { 502 p, err := dht.peerFromInfo(pbp) 503 if err != nil { 504 return nil, err 505 } 506 507 // dial connection 508 err = dht.dialer.DialPeer(ctx, p) 509 return p, err 510 } 511 512 //TODO: this should be smarter about which keys it selects. 513 func (dht *IpfsDHT) loadProvidableKeys() error { 514 kl, err := dht.datastore.KeyList() 515 if err != nil { 516 return err 517 } 518 for _, dsk := range kl { 519 k := u.KeyFromDsKey(dsk) 520 if len(k) == 0 { 521 log.Errorf("loadProvidableKeys error: %v", dsk) 522 } 523 524 dht.providers.AddProvider(k, dht.self) 525 } 526 return nil 527 } 528 529 // PingRoutine periodically pings nearest neighbors. 530 func (dht *IpfsDHT) PingRoutine(t time.Duration) { 531 defer dht.Children().Done() 532 533 tick := time.Tick(t) 534 for { 535 select { 536 case <-tick: 537 id := make([]byte, 16) 538 rand.Read(id) 539 peers := dht.routingTables[0].NearestPeers(kb.ConvertKey(u.Key(id)), 5) 540 for _, p := range peers { 541 ctx, _ := context.WithTimeout(dht.Context(), time.Second*5) 542 err := dht.Ping(ctx, p) 543 if err != nil { 544 log.Errorf("Ping error: %s", err) 545 } 546 } 547 case <-dht.Closing(): 548 return 549 } 550 } 551 } 552 553 // Bootstrap builds up list of peers by requesting random peer IDs 554 func (dht *IpfsDHT) Bootstrap(ctx context.Context) { 555 id := make([]byte, 16) 556 rand.Read(id) 557 p, err := dht.FindPeer(ctx, peer.ID(id)) 558 if err != nil { 559 log.Error("Bootstrap peer error: %s", err) 560 } 561 err = dht.dialer.DialPeer(ctx, p) 562 if err != nil { 563 log.Errorf("Bootstrap peer error: %s", err) 564 } 565 }