github.com/braveheart12/just@v0.8.7/network/controller/bootstrap/bootstrap.go (about) 1 /* 2 * The Clear BSD License 3 * 4 * Copyright (c) 2019 Insolar Technologies 5 * 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without modification, are permitted (subject to the limitations in the disclaimer below) provided that the following conditions are met: 9 * 10 * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 11 * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 12 * Neither the name of Insolar Technologies nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 13 * 14 * NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 15 * 16 */ 17 18 package bootstrap 19 20 import ( 21 "context" 22 "encoding/gob" 23 "fmt" 24 "math" 25 "strings" 26 "sync" 27 "time" 28 29 "github.com/insolar/insolar/component" 30 "github.com/insolar/insolar/core" 31 "github.com/insolar/insolar/instrumentation/inslogger" 32 "github.com/insolar/insolar/instrumentation/instracer" 33 "github.com/insolar/insolar/log" 34 "github.com/insolar/insolar/network" 35 "github.com/insolar/insolar/network/controller/common" 36 "github.com/insolar/insolar/network/controller/pinger" 37 "github.com/insolar/insolar/network/nodenetwork" 38 "github.com/insolar/insolar/network/transport/host" 39 "github.com/insolar/insolar/network/transport/packet/types" 40 "github.com/insolar/insolar/network/utils" 41 "github.com/insolar/insolar/platformpolicy" 42 "github.com/pkg/errors" 43 "go.opencensus.io/trace" 44 ) 45 46 var ( 47 ErrReconnectRequired = errors.New("Node should connect via consensus bootstrap") 48 ) 49 50 type DiscoveryNode struct { 51 Host *host.Host 52 Node core.DiscoveryNode 53 } 54 55 type Bootstrapper interface { 56 component.Initer 57 58 Bootstrap(ctx context.Context) (*network.BootstrapResult, *DiscoveryNode, error) 59 BootstrapDiscovery(ctx context.Context) (*network.BootstrapResult, error) 60 SetLastPulse(number core.PulseNumber) 61 GetLastPulse() core.PulseNumber 62 // GetFirstFakePulseTime() time.Time 63 } 64 65 type bootstrapper struct { 66 Certificate core.Certificate `inject:""` 67 NodeKeeper network.NodeKeeper `inject:""` 68 NetworkSwitcher core.NetworkSwitcher `inject:""` 69 70 options *common.Options 71 transport network.InternalTransport 72 pinger *pinger.Pinger 73 74 lastPulse core.PulseNumber 75 lastPulseLock sync.RWMutex 76 pulsePersisted bool 77 78 bootstrapLock chan struct{} 79 80 genesisRequestsReceived map[core.RecordRef]*GenesisRequest 81 genesisLock sync.Mutex 82 83 firstPulseTime time.Time 84 } 85 86 func (bc *bootstrapper) GetFirstFakePulseTime() time.Time { 87 return bc.firstPulseTime 88 } 89 90 func (bc *bootstrapper) getRequest(ref core.RecordRef) *GenesisRequest { 91 bc.genesisLock.Lock() 92 defer bc.genesisLock.Unlock() 93 94 return bc.genesisRequestsReceived[ref] 95 } 96 97 func (bc *bootstrapper) setRequest(ref core.RecordRef, req *GenesisRequest) { 98 bc.genesisLock.Lock() 99 defer bc.genesisLock.Unlock() 100 101 bc.genesisRequestsReceived[ref] = req 102 } 103 104 type NodeBootstrapRequest struct{} 105 106 type NodeBootstrapResponse struct { 107 Code Code 108 RedirectHost string 109 RejectReason string 110 // FirstPulseTimeUnix int64 111 } 112 113 type GenesisRequest struct { 114 LastPulse core.PulseNumber 115 Discovery *NodeStruct 116 } 117 118 type GenesisResponse struct { 119 Response GenesisRequest 120 Error string 121 } 122 123 type StartSessionRequest struct{} 124 125 type StartSessionResponse struct { 126 SessionID SessionID 127 } 128 129 type NodeStruct struct { 130 ID core.RecordRef 131 SID core.ShortNodeID 132 Role core.StaticRole 133 PK []byte 134 Address string 135 Version string 136 } 137 138 func newNode(n *NodeStruct) (core.Node, error) { 139 pk, err := platformpolicy.NewKeyProcessor().ImportPublicKeyBinary(n.PK) 140 if err != nil { 141 return nil, errors.Wrap(err, "error deserializing node public key") 142 } 143 144 result := nodenetwork.NewNode(n.ID, n.Role, pk, n.Address, n.Version) 145 mNode := result.(nodenetwork.MutableNode) 146 mNode.SetShortID(n.SID) 147 return mNode, nil 148 } 149 150 func newNodeStruct(node core.Node) (*NodeStruct, error) { 151 pk, err := platformpolicy.NewKeyProcessor().ExportPublicKeyBinary(node.PublicKey()) 152 if err != nil { 153 return nil, errors.Wrap(err, "error serializing node public key") 154 } 155 156 return &NodeStruct{ 157 ID: node.ID(), 158 SID: node.ShortID(), 159 Role: node.Role(), 160 PK: pk, 161 Address: node.Address(), 162 Version: node.Version(), 163 }, nil 164 } 165 166 type Code uint8 167 168 const ( 169 Accepted = Code(iota + 1) 170 Rejected 171 Redirected 172 ReconnectRequired 173 ) 174 175 func init() { 176 gob.Register(&NodeBootstrapRequest{}) 177 gob.Register(&NodeBootstrapResponse{}) 178 gob.Register(&StartSessionRequest{}) 179 gob.Register(&StartSessionResponse{}) 180 gob.Register(&GenesisRequest{}) 181 gob.Register(&GenesisResponse{}) 182 } 183 184 // Bootstrap on the discovery node (step 1 of the bootstrap process) 185 func (bc *bootstrapper) Bootstrap(ctx context.Context) (*network.BootstrapResult, *DiscoveryNode, error) { 186 log.Info("Bootstrapping to discovery node") 187 ctx, span := instracer.StartSpan(ctx, "Bootstrapper.Bootstrap") 188 defer span.End() 189 discoveryNodes := bc.Certificate.GetDiscoveryNodes() 190 if utils.OriginIsDiscovery(bc.Certificate) { 191 discoveryNodes = RemoveOrigin(discoveryNodes, bc.NodeKeeper.GetOrigin().ID()) 192 } 193 if len(discoveryNodes) == 0 { 194 return nil, nil, errors.New("There are 0 discovery nodes to connect to") 195 } 196 ch := bc.getDiscoveryNodesChannel(ctx, discoveryNodes, 1) 197 result := bc.waitResultFromChannel(ctx, ch) 198 if result == nil { 199 return nil, nil, errors.New("Failed to bootstrap to any of discovery nodes") 200 } 201 discovery := FindDiscovery(bc.Certificate, result.Host.NodeID) 202 return result, &DiscoveryNode{result.Host, discovery}, nil 203 } 204 205 func (bc *bootstrapper) SetLastPulse(number core.PulseNumber) { 206 _, span := instracer.StartSpan(context.Background(), "Bootstrapper.SetLastPulse wait lastPulseLock") 207 bc.lastPulseLock.Lock() 208 span.End() 209 defer bc.lastPulseLock.Unlock() 210 211 if !bc.pulsePersisted { 212 bc.lastPulse = number 213 close(bc.bootstrapLock) 214 bc.pulsePersisted = true 215 } 216 } 217 218 func (bc *bootstrapper) forceSetLastPulse(number core.PulseNumber) { 219 _, span := instracer.StartSpan(context.Background(), "Bootstrapper.forceSetLastPulse wait lastPulseLock") 220 bc.lastPulseLock.Lock() 221 span.End() 222 defer bc.lastPulseLock.Unlock() 223 224 log.Infof("Network will start from pulse %d + delta", number) 225 bc.lastPulse = number 226 } 227 228 func (bc *bootstrapper) GetLastPulse() core.PulseNumber { 229 _, span := instracer.StartSpan(context.Background(), "Bootstrapper.GetLastPulse wait lastPulseLock") 230 bc.lastPulseLock.RLock() 231 span.End() 232 defer bc.lastPulseLock.RUnlock() 233 234 return bc.lastPulse 235 } 236 237 func (bc *bootstrapper) checkActiveNode(node core.Node) error { 238 n := bc.NodeKeeper.GetActiveNode(node.ID()) 239 if n != nil { 240 return errors.Errorf("Node ID collision: %s", n.ID()) 241 } 242 n = bc.NodeKeeper.GetActiveNodeByShortID(node.ShortID()) 243 if n != nil { 244 return errors.Errorf("Short ID collision: %d", n.ShortID()) 245 } 246 if node.Version() != bc.NodeKeeper.GetOrigin().Version() { 247 return errors.Errorf("Node %s version %s does not match origin version %s", 248 node.ID(), node.Version(), bc.NodeKeeper.GetOrigin().Version()) 249 } 250 return nil 251 } 252 253 func (bc *bootstrapper) BootstrapDiscovery(ctx context.Context) (*network.BootstrapResult, error) { 254 logger := inslogger.FromContext(ctx) 255 logger.Info("[ BootstrapDiscovery ] Network bootstrap between discovery nodes") 256 ctx, span := instracer.StartSpan(ctx, "Bootstrapper.BootstrapDiscovery") 257 defer span.End() 258 discoveryNodes := RemoveOrigin(bc.Certificate.GetDiscoveryNodes(), *bc.Certificate.GetNodeRef()) 259 discoveryCount := len(discoveryNodes) 260 if discoveryCount == 0 { 261 host, err := host.NewHostN(bc.NodeKeeper.GetOrigin().Address(), bc.NodeKeeper.GetOrigin().ID()) 262 if err != nil { 263 return nil, errors.Wrap(err, "failed to create a host") 264 } 265 return &network.BootstrapResult{ 266 Host: host, 267 // FirstPulseTime: bc.firstPulseTime, 268 }, nil 269 } 270 271 var bootstrapResults []*network.BootstrapResult 272 var hosts []*host.Host 273 for { 274 ch := bc.getDiscoveryNodesChannel(ctx, discoveryNodes, discoveryCount) 275 bootstrapResults, hosts = bc.waitResultsFromChannel(ctx, ch, discoveryCount) 276 if len(hosts) == discoveryCount { 277 // we connected to all discovery nodes 278 break 279 } else { 280 logger.Infof("[ BootstrapDiscovery ] Connected to %d/%d discovery nodes", len(hosts), discoveryCount) 281 } 282 } 283 reconnectRequests := 0 284 for _, bootstrapResult := range bootstrapResults { 285 if bootstrapResult.ReconnectRequired { 286 reconnectRequests++ 287 } 288 } 289 minRequests := int(math.Floor(0.5*float64(discoveryCount))) + 1 290 if reconnectRequests >= minRequests { 291 logger.Infof("[ BootstrapDiscovery ] Need to reconnect as joiner (requested by %d/%d discovery nodes)", 292 reconnectRequests, discoveryCount) 293 return nil, ErrReconnectRequired 294 } 295 activeNodesStr := make([]string, 0) 296 297 <-bc.bootstrapLock 298 logger.Debugf("[ BootstrapDiscovery ] After bootstrap lock") 299 300 ch := bc.getGenesisRequestsChannel(ctx, hosts) 301 activeNodes, lastPulses, err := bc.waitGenesisResults(ctx, ch, len(hosts)) 302 if err != nil { 303 return nil, err 304 } 305 bc.forceSetLastPulse(bc.calculateLastIgnoredPulse(ctx, lastPulses)) 306 for _, activeNode := range activeNodes { 307 err = bc.checkActiveNode(activeNode) 308 if err != nil { 309 return nil, errors.Wrapf(err, "Discovery check of node %s failed", activeNode.ID()) 310 } 311 activeNode.(nodenetwork.MutableNode).SetState(core.NodeDiscovery) 312 activeNodesStr = append(activeNodesStr, activeNode.ID().String()) 313 } 314 bc.NodeKeeper.AddActiveNodes(activeNodes) 315 bc.NodeKeeper.GetOrigin().(nodenetwork.MutableNode).SetState(core.NodeDiscovery) 316 logger.Infof("[ BootstrapDiscovery ] Added active nodes: %s", strings.Join(activeNodesStr, ", ")) 317 return parseBotstrapResults(bootstrapResults), nil 318 } 319 320 func (bc *bootstrapper) calculateLastIgnoredPulse(ctx context.Context, lastPulses []core.PulseNumber) core.PulseNumber { 321 maxLastPulse := bc.GetLastPulse() 322 inslogger.FromContext(ctx).Debugf("Node %s (origin) LastIgnoredPulse: %d", bc.NodeKeeper.GetOrigin().ID(), maxLastPulse) 323 for _, pulse := range lastPulses { 324 if pulse > maxLastPulse { 325 maxLastPulse = pulse 326 } 327 } 328 return maxLastPulse 329 } 330 331 func (bc *bootstrapper) sendGenesisRequest(ctx context.Context, h *host.Host) (*GenesisResponse, error) { 332 ctx, span := instracer.StartSpan(ctx, "Bootstrapper.sendGenesisRequest") 333 defer span.End() 334 discovery, err := newNodeStruct(bc.NodeKeeper.GetOrigin()) 335 if err != nil { 336 return nil, errors.Wrapf(err, "Failed to prepare genesis request to address %s", h) 337 } 338 request := bc.transport.NewRequestBuilder().Type(types.Genesis).Data(&GenesisRequest{ 339 LastPulse: bc.GetLastPulse(), 340 Discovery: discovery, 341 }).Build() 342 future, err := bc.transport.SendRequestPacket(ctx, request, h) 343 if err != nil { 344 return nil, errors.Wrapf(err, "Failed to send genesis request to address %s", h) 345 } 346 response, err := future.GetResponse(bc.options.BootstrapTimeout) 347 if err != nil { 348 return nil, errors.Wrapf(err, "Failed to get response to genesis request from address %s", h) 349 } 350 data := response.GetData().(*GenesisResponse) 351 if data.Response.Discovery == nil { 352 return nil, errors.New("Error genesis response from discovery node: " + data.Error) 353 } 354 return data, nil 355 } 356 357 func (bc *bootstrapper) getDiscoveryNodesChannel(ctx context.Context, discoveryNodes []core.DiscoveryNode, needResponses int) <-chan *network.BootstrapResult { 358 // we need only one host to bootstrap 359 bootstrapResults := make(chan *network.BootstrapResult, needResponses) 360 for _, discoveryNode := range discoveryNodes { 361 go func(ctx context.Context, address string, ch chan<- *network.BootstrapResult) { 362 inslogger.FromContext(ctx).Infof("Starting bootstrap to address %s", address) 363 ctx, span := instracer.StartSpan(ctx, "Bootstrapper.getDiscoveryNodesChannel") 364 defer span.End() 365 span.AddAttributes( 366 trace.StringAttribute("Bootstrap node", address), 367 ) 368 bootstrapResult, err := bootstrap(ctx, address, bc.options, bc.startBootstrap) 369 if err != nil { 370 inslogger.FromContext(ctx).Errorf("Error bootstrapping to address %s: %s", address, err.Error()) 371 return 372 } 373 bootstrapResults <- bootstrapResult 374 }(ctx, discoveryNode.GetHost(), bootstrapResults) 375 } 376 377 return bootstrapResults 378 } 379 380 func (bc *bootstrapper) getGenesisRequestsChannel(ctx context.Context, discoveryHosts []*host.Host) chan *GenesisResponse { 381 result := make(chan *GenesisResponse) 382 for _, discoveryHost := range discoveryHosts { 383 go func(ctx context.Context, address *host.Host, ch chan<- *GenesisResponse) { 384 logger := inslogger.FromContext(ctx) 385 ctx, span := instracer.StartSpan(ctx, "Bootsytrapper.getGenesisRequestChannel") 386 span.AddAttributes( 387 trace.StringAttribute("genesis request to", address.String()), 388 ) 389 defer span.End() 390 cachedReq := bc.getRequest(address.NodeID) 391 if cachedReq != nil { 392 logger.Infof("Got genesis info of node %s from cache", address) 393 ch <- &GenesisResponse{Response: *cachedReq} 394 return 395 } 396 397 logger.Infof("Sending genesis bootstrap request to address %s", address) 398 response, err := bc.sendGenesisRequest(ctx, address) 399 if err != nil { 400 logger.Warnf("Discovery bootstrap to host %s failed: %s", address, err) 401 return 402 } 403 result <- response 404 }(ctx, discoveryHost, result) 405 } 406 return result 407 } 408 409 func (bc *bootstrapper) waitResultFromChannel(ctx context.Context, ch <-chan *network.BootstrapResult) *network.BootstrapResult { 410 for { 411 select { 412 case bootstrapHost := <-ch: 413 return bootstrapHost 414 case <-time.After(bc.options.BootstrapTimeout): 415 inslogger.FromContext(ctx).Warn("Bootstrap timeout") 416 return nil 417 } 418 } 419 } 420 421 func (bc *bootstrapper) waitResultsFromChannel(ctx context.Context, ch <-chan *network.BootstrapResult, count int) ([]*network.BootstrapResult, []*host.Host) { 422 result := make([]*network.BootstrapResult, 0) 423 hosts := make([]*host.Host, 0) 424 for { 425 select { 426 case bootstrapResult := <-ch: 427 result = append(result, bootstrapResult) 428 hosts = append(hosts, bootstrapResult.Host) 429 if len(result) == count { 430 return result, hosts 431 } 432 case <-time.After(bc.options.BootstrapTimeout): 433 inslogger.FromContext(ctx).Warnf("Bootstrap timeout, successful bootstraps: %d/%d", len(result), count) 434 return result, hosts 435 } 436 } 437 } 438 439 func (bc *bootstrapper) waitGenesisResults(ctx context.Context, ch <-chan *GenesisResponse, count int) ([]core.Node, []core.PulseNumber, error) { 440 result := make([]core.Node, 0) 441 lastPulses := make([]core.PulseNumber, 0) 442 for { 443 select { 444 case res := <-ch: 445 discovery, err := newNode(res.Response.Discovery) 446 if err != nil { 447 return nil, nil, errors.Wrap(err, "Error deserializing node from discovery node") 448 } 449 result = append(result, discovery) 450 lastPulses = append(lastPulses, res.Response.LastPulse) 451 inslogger.FromContext(ctx).Debugf("Node %s LastIgnoredPulse: %d", discovery.ID(), res.Response.LastPulse) 452 if len(result) == count { 453 return result, lastPulses, nil 454 } 455 case <-time.After(bc.options.BootstrapTimeout): 456 return nil, nil, errors.New(fmt.Sprintf("Genesis bootstrap timeout, successful genesis requests: %d/%d", len(result), count)) 457 } 458 } 459 } 460 461 func bootstrap(ctx context.Context, address string, options *common.Options, bootstrapF func(context.Context, string) (*network.BootstrapResult, error)) (*network.BootstrapResult, error) { 462 minTO := options.MinTimeout 463 if !options.InfinityBootstrap { 464 return bootstrapF(ctx, address) 465 } 466 for { 467 result, err := bootstrapF(ctx, address) 468 if err == nil { 469 return result, nil 470 } 471 time.Sleep(minTO) 472 minTO *= options.TimeoutMult 473 if minTO > options.MaxTimeout { 474 minTO = options.MaxTimeout 475 } 476 } 477 } 478 479 func (bc *bootstrapper) startBootstrap(ctx context.Context, address string) (*network.BootstrapResult, error) { 480 ctx, span := instracer.StartSpan(ctx, "Bootstrapper.startBootstrap") 481 defer span.End() 482 bootstrapHost, err := bc.pinger.Ping(ctx, address, bc.options.PingTimeout) 483 if err != nil { 484 return nil, errors.Wrapf(err, "Failed to ping address %s", address) 485 } 486 request := bc.transport.NewRequestBuilder().Type(types.Bootstrap).Data(&NodeBootstrapRequest{}).Build() 487 future, err := bc.transport.SendRequestPacket(ctx, request, bootstrapHost) 488 if err != nil { 489 return nil, errors.Wrapf(err, "Failed to send bootstrap request to address %s", address) 490 } 491 response, err := future.GetResponse(bc.options.BootstrapTimeout) 492 if err != nil { 493 return nil, errors.Wrapf(err, "Failed to get response to bootstrap request from address %s", address) 494 } 495 data := response.GetData().(*NodeBootstrapResponse) 496 switch data.Code { 497 case Rejected: 498 return nil, errors.New("Rejected: " + data.RejectReason) 499 case Redirected: 500 return bootstrap(ctx, data.RedirectHost, bc.options, bc.startBootstrap) 501 } 502 return &network.BootstrapResult{ 503 // FirstPulseTime: time.Unix(data.FirstPulseTimeUnix, 0), 504 Host: response.GetSenderHost(), 505 ReconnectRequired: data.Code == ReconnectRequired, 506 }, nil 507 } 508 509 func (bc *bootstrapper) processBootstrap(ctx context.Context, request network.Request) (network.Response, error) { 510 // TODO: redirect logic 511 var code Code 512 if bc.NetworkSwitcher.GetState() == core.CompleteNetworkState { 513 code = ReconnectRequired 514 } else { 515 code = Accepted 516 } 517 return bc.transport.BuildResponse(ctx, request, 518 &NodeBootstrapResponse{ 519 Code: code, 520 // FirstPulseTimeUnix: bc.firstPulseTime.Unix(), 521 }), nil 522 } 523 524 func (bc *bootstrapper) processGenesis(ctx context.Context, request network.Request) (network.Response, error) { 525 data := request.GetData().(*GenesisRequest) 526 discovery, err := newNodeStruct(bc.NodeKeeper.GetOrigin()) 527 if err != nil { 528 return bc.transport.BuildResponse(ctx, request, &GenesisResponse{Error: err.Error()}), nil 529 } 530 bc.SetLastPulse(data.LastPulse) 531 bc.setRequest(request.GetSender(), data) 532 return bc.transport.BuildResponse(ctx, request, &GenesisResponse{ 533 Response: GenesisRequest{Discovery: discovery, LastPulse: bc.GetLastPulse()}, 534 }), nil 535 } 536 537 func (bc *bootstrapper) Init(ctx context.Context) error { 538 bc.firstPulseTime = time.Now() 539 bc.transport.RegisterPacketHandler(types.Bootstrap, bc.processBootstrap) 540 bc.transport.RegisterPacketHandler(types.Genesis, bc.processGenesis) 541 return nil 542 } 543 544 func parseBotstrapResults(results []*network.BootstrapResult) *network.BootstrapResult { 545 minIDIndex := 0 546 minID := results[0].Host.NodeID 547 for i, result := range results { 548 if minID.Compare(result.Host.NodeID) > 0 { 549 minIDIndex = i 550 } 551 } 552 return results[minIDIndex] 553 } 554 555 func NewBootstrapper( 556 options *common.Options, 557 558 transport network.InternalTransport) Bootstrapper { 559 return &bootstrapper{ 560 options: options, 561 transport: transport, 562 pinger: pinger.NewPinger(transport), 563 bootstrapLock: make(chan struct{}), 564 565 genesisRequestsReceived: make(map[core.RecordRef]*GenesisRequest), 566 } 567 }