github.com/onflow/flow-go@v0.33.17/cmd/access/node_builder/access_node_builder.go (about) 1 package node_builder 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "math" 8 "os" 9 "path" 10 "path/filepath" 11 "strings" 12 "time" 13 14 "github.com/ipfs/boxo/bitswap" 15 badger "github.com/ipfs/go-ds-badger2" 16 "github.com/libp2p/go-libp2p/core/host" 17 "github.com/libp2p/go-libp2p/core/routing" 18 "github.com/onflow/flow/protobuf/go/flow/access" 19 "github.com/rs/zerolog" 20 "github.com/spf13/pflag" 21 "google.golang.org/grpc" 22 "google.golang.org/grpc/credentials" 23 "google.golang.org/grpc/credentials/insecure" 24 25 "github.com/onflow/flow-go/admin/commands" 26 stateSyncCommands "github.com/onflow/flow-go/admin/commands/state_synchronization" 27 storageCommands "github.com/onflow/flow-go/admin/commands/storage" 28 "github.com/onflow/flow-go/cmd" 29 "github.com/onflow/flow-go/consensus" 30 "github.com/onflow/flow-go/consensus/hotstuff" 31 "github.com/onflow/flow-go/consensus/hotstuff/committees" 32 "github.com/onflow/flow-go/consensus/hotstuff/notifications" 33 consensuspubsub "github.com/onflow/flow-go/consensus/hotstuff/notifications/pubsub" 34 "github.com/onflow/flow-go/consensus/hotstuff/signature" 35 hotstuffvalidator "github.com/onflow/flow-go/consensus/hotstuff/validator" 36 "github.com/onflow/flow-go/consensus/hotstuff/verification" 37 recovery "github.com/onflow/flow-go/consensus/recovery/protocol" 38 "github.com/onflow/flow-go/crypto" 39 "github.com/onflow/flow-go/engine" 40 "github.com/onflow/flow-go/engine/access/ingestion" 41 pingeng "github.com/onflow/flow-go/engine/access/ping" 42 "github.com/onflow/flow-go/engine/access/rest" 43 "github.com/onflow/flow-go/engine/access/rest/routes" 44 "github.com/onflow/flow-go/engine/access/rpc" 45 "github.com/onflow/flow-go/engine/access/rpc/backend" 46 rpcConnection "github.com/onflow/flow-go/engine/access/rpc/connection" 47 "github.com/onflow/flow-go/engine/access/state_stream" 48 statestreambackend "github.com/onflow/flow-go/engine/access/state_stream/backend" 49 followereng "github.com/onflow/flow-go/engine/common/follower" 50 "github.com/onflow/flow-go/engine/common/requester" 51 synceng "github.com/onflow/flow-go/engine/common/synchronization" 52 "github.com/onflow/flow-go/engine/execution/computation/query" 53 "github.com/onflow/flow-go/ledger" 54 "github.com/onflow/flow-go/ledger/complete/wal" 55 "github.com/onflow/flow-go/model/bootstrap" 56 "github.com/onflow/flow-go/model/flow" 57 "github.com/onflow/flow-go/model/flow/filter" 58 "github.com/onflow/flow-go/module" 59 "github.com/onflow/flow-go/module/blobs" 60 "github.com/onflow/flow-go/module/chainsync" 61 "github.com/onflow/flow-go/module/execution" 62 "github.com/onflow/flow-go/module/executiondatasync/execution_data" 63 execdatacache "github.com/onflow/flow-go/module/executiondatasync/execution_data/cache" 64 finalizer "github.com/onflow/flow-go/module/finalizer/consensus" 65 "github.com/onflow/flow-go/module/grpcserver" 66 "github.com/onflow/flow-go/module/id" 67 "github.com/onflow/flow-go/module/mempool/herocache" 68 "github.com/onflow/flow-go/module/mempool/stdmap" 69 "github.com/onflow/flow-go/module/metrics" 70 "github.com/onflow/flow-go/module/metrics/unstaked" 71 "github.com/onflow/flow-go/module/state_synchronization" 72 "github.com/onflow/flow-go/module/state_synchronization/indexer" 73 edrequester "github.com/onflow/flow-go/module/state_synchronization/requester" 74 "github.com/onflow/flow-go/network" 75 alspmgr "github.com/onflow/flow-go/network/alsp/manager" 76 netcache "github.com/onflow/flow-go/network/cache" 77 "github.com/onflow/flow-go/network/channels" 78 cborcodec "github.com/onflow/flow-go/network/codec/cbor" 79 "github.com/onflow/flow-go/network/p2p" 80 "github.com/onflow/flow-go/network/p2p/blob" 81 p2pbuilder "github.com/onflow/flow-go/network/p2p/builder" 82 p2pbuilderconfig "github.com/onflow/flow-go/network/p2p/builder/config" 83 "github.com/onflow/flow-go/network/p2p/cache" 84 "github.com/onflow/flow-go/network/p2p/conduit" 85 "github.com/onflow/flow-go/network/p2p/connection" 86 "github.com/onflow/flow-go/network/p2p/dht" 87 "github.com/onflow/flow-go/network/p2p/subscription" 88 "github.com/onflow/flow-go/network/p2p/translator" 89 "github.com/onflow/flow-go/network/p2p/unicast/protocols" 90 relaynet "github.com/onflow/flow-go/network/relay" 91 "github.com/onflow/flow-go/network/slashing" 92 "github.com/onflow/flow-go/network/topology" 93 "github.com/onflow/flow-go/network/underlay" 94 "github.com/onflow/flow-go/network/validator" 95 "github.com/onflow/flow-go/state/protocol" 96 badgerState "github.com/onflow/flow-go/state/protocol/badger" 97 "github.com/onflow/flow-go/state/protocol/blocktimer" 98 "github.com/onflow/flow-go/storage" 99 bstorage "github.com/onflow/flow-go/storage/badger" 100 pStorage "github.com/onflow/flow-go/storage/pebble" 101 "github.com/onflow/flow-go/utils/grpcutils" 102 ) 103 104 // AccessNodeBuilder extends cmd.NodeBuilder and declares additional functions needed to bootstrap an Access node. 105 // The private network allows the staked nodes to communicate among themselves, while the public network allows the 106 // Observers and an Access node to communicate. 107 // 108 // public network private network 109 // +------------------------+ 110 // | Observer 1 |<--------------------------| 111 // +------------------------+ v 112 // +------------------------+ +--------------------+ +------------------------+ 113 // | Observer 2 |<----------------------->| Staked Access Node |<--------------->| All other staked Nodes | 114 // +------------------------+ +--------------------+ +------------------------+ 115 // +------------------------+ ^ 116 // | Observer 3 |<--------------------------| 117 // +------------------------+ 118 119 // AccessNodeConfig defines all the user defined parameters required to bootstrap an access node 120 // For a node running as a standalone process, the config fields will be populated from the command line params, 121 // while for a node running as a library, the config fields are expected to be initialized by the caller. 122 type AccessNodeConfig struct { 123 supportsObserver bool // True if this is an Access node that supports observers and consensus follower engines 124 collectionGRPCPort uint 125 executionGRPCPort uint 126 pingEnabled bool 127 nodeInfoFile string 128 apiRatelimits map[string]int 129 apiBurstlimits map[string]int 130 rpcConf rpc.Config 131 stateStreamConf statestreambackend.Config 132 stateStreamFilterConf map[string]int 133 ExecutionNodeAddress string // deprecated 134 HistoricalAccessRPCs []access.AccessAPIClient 135 logTxTimeToFinalized bool 136 logTxTimeToExecuted bool 137 logTxTimeToFinalizedExecuted bool 138 retryEnabled bool 139 rpcMetricsEnabled bool 140 executionDataSyncEnabled bool 141 publicNetworkExecutionDataEnabled bool 142 executionDataDir string 143 executionDataStartHeight uint64 144 executionDataConfig edrequester.ExecutionDataConfig 145 PublicNetworkConfig PublicNetworkConfig 146 TxResultCacheSize uint 147 TxErrorMessagesCacheSize uint 148 executionDataIndexingEnabled bool 149 registersDBPath string 150 checkpointFile string 151 scriptExecutorConfig query.QueryConfig 152 scriptExecMinBlock uint64 153 scriptExecMaxBlock uint64 154 registerCacheType string 155 registerCacheSize uint 156 } 157 158 type PublicNetworkConfig struct { 159 // NetworkKey crypto.PublicKey // TODO: do we need a different key for the public network? 160 BindAddress string 161 Network network.EngineRegistry 162 Metrics module.NetworkMetrics 163 } 164 165 // DefaultAccessNodeConfig defines all the default values for the AccessNodeConfig 166 func DefaultAccessNodeConfig() *AccessNodeConfig { 167 homedir, _ := os.UserHomeDir() 168 return &AccessNodeConfig{ 169 supportsObserver: false, 170 collectionGRPCPort: 9000, 171 executionGRPCPort: 9000, 172 rpcConf: rpc.Config{ 173 UnsecureGRPCListenAddr: "0.0.0.0:9000", 174 SecureGRPCListenAddr: "0.0.0.0:9001", 175 HTTPListenAddr: "0.0.0.0:8000", 176 CollectionAddr: "", 177 HistoricalAccessAddrs: "", 178 BackendConfig: backend.Config{ 179 CollectionClientTimeout: 3 * time.Second, 180 ExecutionClientTimeout: 3 * time.Second, 181 ConnectionPoolSize: backend.DefaultConnectionPoolSize, 182 MaxHeightRange: backend.DefaultMaxHeightRange, 183 PreferredExecutionNodeIDs: nil, 184 FixedExecutionNodeIDs: nil, 185 CircuitBreakerConfig: rpcConnection.CircuitBreakerConfig{ 186 Enabled: false, 187 RestoreTimeout: 60 * time.Second, 188 MaxFailures: 5, 189 MaxRequests: 1, 190 }, 191 ScriptExecutionMode: backend.IndexQueryModeExecutionNodesOnly.String(), // default to ENs only for now 192 EventQueryMode: backend.IndexQueryModeExecutionNodesOnly.String(), // default to ENs only for now 193 TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly.String(), // default to ENs only for now 194 }, 195 RestConfig: rest.Config{ 196 ListenAddress: "", 197 WriteTimeout: rest.DefaultWriteTimeout, 198 ReadTimeout: rest.DefaultReadTimeout, 199 IdleTimeout: rest.DefaultIdleTimeout, 200 }, 201 MaxMsgSize: grpcutils.DefaultMaxMsgSize, 202 CompressorName: grpcutils.NoCompressor, 203 }, 204 stateStreamConf: statestreambackend.Config{ 205 MaxExecutionDataMsgSize: grpcutils.DefaultMaxMsgSize, 206 ExecutionDataCacheSize: state_stream.DefaultCacheSize, 207 ClientSendTimeout: state_stream.DefaultSendTimeout, 208 ClientSendBufferSize: state_stream.DefaultSendBufferSize, 209 MaxGlobalStreams: state_stream.DefaultMaxGlobalStreams, 210 EventFilterConfig: state_stream.DefaultEventFilterConfig, 211 ResponseLimit: state_stream.DefaultResponseLimit, 212 HeartbeatInterval: state_stream.DefaultHeartbeatInterval, 213 RegisterIDsRequestLimit: state_stream.DefaultRegisterIDsRequestLimit, 214 }, 215 stateStreamFilterConf: nil, 216 ExecutionNodeAddress: "localhost:9000", 217 logTxTimeToFinalized: false, 218 logTxTimeToExecuted: false, 219 logTxTimeToFinalizedExecuted: false, 220 pingEnabled: false, 221 retryEnabled: false, 222 rpcMetricsEnabled: false, 223 nodeInfoFile: "", 224 apiRatelimits: nil, 225 apiBurstlimits: nil, 226 TxResultCacheSize: 0, 227 TxErrorMessagesCacheSize: 1000, 228 PublicNetworkConfig: PublicNetworkConfig{ 229 BindAddress: cmd.NotSet, 230 Metrics: metrics.NewNoopCollector(), 231 }, 232 executionDataSyncEnabled: true, 233 publicNetworkExecutionDataEnabled: false, 234 executionDataDir: filepath.Join(homedir, ".flow", "execution_data"), 235 executionDataStartHeight: 0, 236 executionDataConfig: edrequester.ExecutionDataConfig{ 237 InitialBlockHeight: 0, 238 MaxSearchAhead: edrequester.DefaultMaxSearchAhead, 239 FetchTimeout: edrequester.DefaultFetchTimeout, 240 MaxFetchTimeout: edrequester.DefaultMaxFetchTimeout, 241 RetryDelay: edrequester.DefaultRetryDelay, 242 MaxRetryDelay: edrequester.DefaultMaxRetryDelay, 243 }, 244 executionDataIndexingEnabled: false, 245 registersDBPath: filepath.Join(homedir, ".flow", "execution_state"), 246 checkpointFile: cmd.NotSet, 247 scriptExecutorConfig: query.NewDefaultConfig(), 248 scriptExecMinBlock: 0, 249 scriptExecMaxBlock: math.MaxUint64, 250 registerCacheType: pStorage.CacheTypeTwoQueue.String(), 251 registerCacheSize: 0, 252 } 253 } 254 255 // FlowAccessNodeBuilder provides the common functionality needed to bootstrap a Flow access node 256 // It is composed of the FlowNodeBuilder, the AccessNodeConfig and contains all the components and modules needed for the 257 // access nodes 258 type FlowAccessNodeBuilder struct { 259 *cmd.FlowNodeBuilder 260 *AccessNodeConfig 261 262 // components 263 FollowerState protocol.FollowerState 264 SyncCore *chainsync.Core 265 RpcEng *rpc.Engine 266 FollowerDistributor *consensuspubsub.FollowerDistributor 267 CollectionRPC access.AccessAPIClient 268 TransactionTimings *stdmap.TransactionTimings 269 CollectionsToMarkFinalized *stdmap.Times 270 CollectionsToMarkExecuted *stdmap.Times 271 BlocksToMarkExecuted *stdmap.Times 272 TransactionMetrics *metrics.TransactionCollector 273 RestMetrics *metrics.RestCollector 274 AccessMetrics module.AccessMetrics 275 PingMetrics module.PingMetrics 276 Committee hotstuff.DynamicCommittee 277 Finalized *flow.Header // latest finalized block that the node knows of at startup time 278 Pending []*flow.Header 279 FollowerCore module.HotStuffFollower 280 Validator hotstuff.Validator 281 ExecutionDataDownloader execution_data.Downloader 282 PublicBlobService network.BlobService 283 ExecutionDataRequester state_synchronization.ExecutionDataRequester 284 ExecutionDataStore execution_data.ExecutionDataStore 285 ExecutionDataCache *execdatacache.ExecutionDataCache 286 ExecutionIndexer *indexer.Indexer 287 ExecutionIndexerCore *indexer.IndexerCore 288 ScriptExecutor *backend.ScriptExecutor 289 RegistersAsyncStore *execution.RegistersAsyncStore 290 EventsIndex *backend.EventsIndex 291 TxResultsIndex *backend.TransactionResultsIndex 292 IndexerDependencies *cmd.DependencyList 293 collectionExecutedMetric module.CollectionExecutedMetric 294 295 // The sync engine participants provider is the libp2p peer store for the access node 296 // which is not available until after the network has started. 297 // Hence, a factory function that needs to be called just before creating the sync engine 298 SyncEngineParticipantsProviderFactory func() module.IdentifierProvider 299 300 // engines 301 IngestEng *ingestion.Engine 302 RequestEng *requester.Engine 303 FollowerEng *followereng.ComplianceEngine 304 SyncEng *synceng.Engine 305 StateStreamEng *statestreambackend.Engine 306 307 // grpc servers 308 secureGrpcServer *grpcserver.GrpcServer 309 unsecureGrpcServer *grpcserver.GrpcServer 310 stateStreamGrpcServer *grpcserver.GrpcServer 311 312 stateStreamBackend *statestreambackend.StateStreamBackend 313 } 314 315 func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder { 316 builder.Module("mutable follower state", func(node *cmd.NodeConfig) error { 317 // For now, we only support state implementations from package badger. 318 // If we ever support different implementations, the following can be replaced by a type-aware factory 319 state, ok := node.State.(*badgerState.State) 320 if !ok { 321 return fmt.Errorf("only implementations of type badger.State are currently supported but read-only state has type %T", node.State) 322 } 323 324 followerState, err := badgerState.NewFollowerState( 325 node.Logger, 326 node.Tracer, 327 node.ProtocolEvents, 328 state, 329 node.Storage.Index, 330 node.Storage.Payloads, 331 blocktimer.DefaultBlockTimer, 332 ) 333 builder.FollowerState = followerState 334 335 return err 336 }) 337 338 return builder 339 } 340 341 func (builder *FlowAccessNodeBuilder) buildSyncCore() *FlowAccessNodeBuilder { 342 builder.Module("sync core", func(node *cmd.NodeConfig) error { 343 syncCore, err := chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector(node.RootChainID), node.RootChainID) 344 builder.SyncCore = syncCore 345 346 return err 347 }) 348 349 return builder 350 } 351 352 func (builder *FlowAccessNodeBuilder) buildCommittee() *FlowAccessNodeBuilder { 353 builder.Component("committee", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 354 // initialize consensus committee's membership state 355 // This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS committee 356 // Note: node.Me.NodeID() is not part of the consensus committee 357 committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID()) 358 node.ProtocolEvents.AddConsumer(committee) 359 builder.Committee = committee 360 361 return committee, err 362 }) 363 364 return builder 365 } 366 367 func (builder *FlowAccessNodeBuilder) buildLatestHeader() *FlowAccessNodeBuilder { 368 builder.Module("latest header", func(node *cmd.NodeConfig) error { 369 finalized, pending, err := recovery.FindLatest(node.State, node.Storage.Headers) 370 builder.Finalized, builder.Pending = finalized, pending 371 372 return err 373 }) 374 375 return builder 376 } 377 378 func (builder *FlowAccessNodeBuilder) buildFollowerCore() *FlowAccessNodeBuilder { 379 builder.Component("follower core", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 380 // create a finalizer that will handle updating the protocol 381 // state when the follower detects newly finalized blocks 382 final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, builder.FollowerState, node.Tracer) 383 384 packer := signature.NewConsensusSigDataPacker(builder.Committee) 385 // initialize the verifier for the protocol consensus 386 verifier := verification.NewCombinedVerifier(builder.Committee, packer) 387 builder.Validator = hotstuffvalidator.New(builder.Committee, verifier) 388 389 followerCore, err := consensus.NewFollower( 390 node.Logger, 391 node.Metrics.Mempool, 392 node.Storage.Headers, 393 final, 394 builder.FollowerDistributor, 395 node.FinalizedRootBlock.Header, 396 node.RootQC, 397 builder.Finalized, 398 builder.Pending, 399 ) 400 if err != nil { 401 return nil, fmt.Errorf("could not initialize follower core: %w", err) 402 } 403 builder.FollowerCore = followerCore 404 405 return builder.FollowerCore, nil 406 }) 407 408 return builder 409 } 410 411 func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuilder { 412 builder.Component("follower engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 413 var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() 414 if node.HeroCacheMetricsEnable { 415 heroCacheCollector = metrics.FollowerCacheMetrics(node.MetricsRegisterer) 416 } 417 418 core, err := followereng.NewComplianceCore( 419 node.Logger, 420 node.Metrics.Mempool, 421 heroCacheCollector, 422 builder.FollowerDistributor, 423 builder.FollowerState, 424 builder.FollowerCore, 425 builder.Validator, 426 builder.SyncCore, 427 node.Tracer, 428 ) 429 if err != nil { 430 return nil, fmt.Errorf("could not create follower core: %w", err) 431 } 432 433 builder.FollowerEng, err = followereng.NewComplianceLayer( 434 node.Logger, 435 node.EngineRegistry, 436 node.Me, 437 node.Metrics.Engine, 438 node.Storage.Headers, 439 builder.Finalized, 440 core, 441 node.ComplianceConfig, 442 ) 443 if err != nil { 444 return nil, fmt.Errorf("could not create follower engine: %w", err) 445 } 446 builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.FollowerEng.OnFinalizedBlock) 447 448 return builder.FollowerEng, nil 449 }) 450 451 return builder 452 } 453 454 func (builder *FlowAccessNodeBuilder) buildSyncEngine() *FlowAccessNodeBuilder { 455 builder.Component("sync engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 456 spamConfig, err := synceng.NewSpamDetectionConfig() 457 if err != nil { 458 return nil, fmt.Errorf("could not initialize spam detection config: %w", err) 459 } 460 sync, err := synceng.New( 461 node.Logger, 462 node.Metrics.Engine, 463 node.EngineRegistry, 464 node.Me, 465 node.State, 466 node.Storage.Blocks, 467 builder.FollowerEng, 468 builder.SyncCore, 469 builder.SyncEngineParticipantsProviderFactory(), 470 spamConfig, 471 ) 472 if err != nil { 473 return nil, fmt.Errorf("could not create synchronization engine: %w", err) 474 } 475 builder.SyncEng = sync 476 builder.FollowerDistributor.AddFinalizationConsumer(sync) 477 478 return builder.SyncEng, nil 479 }) 480 481 return builder 482 } 483 484 func (builder *FlowAccessNodeBuilder) BuildConsensusFollower() *FlowAccessNodeBuilder { 485 builder. 486 buildFollowerState(). 487 buildSyncCore(). 488 buildCommittee(). 489 buildLatestHeader(). 490 buildFollowerCore(). 491 buildFollowerEngine(). 492 buildSyncEngine() 493 494 return builder 495 } 496 497 func (builder *FlowAccessNodeBuilder) BuildExecutionSyncComponents() *FlowAccessNodeBuilder { 498 var ds *badger.Datastore 499 var bs network.BlobService 500 var processedBlockHeight storage.ConsumerProgress 501 var processedNotifications storage.ConsumerProgress 502 var bsDependable *module.ProxiedReadyDoneAware 503 var execDataDistributor *edrequester.ExecutionDataDistributor 504 var execDataCacheBackend *herocache.BlockExecutionData 505 var executionDataStoreCache *execdatacache.ExecutionDataCache 506 507 // setup dependency chain to ensure indexer starts after the requester 508 requesterDependable := module.NewProxiedReadyDoneAware() 509 builder.IndexerDependencies.Add(requesterDependable) 510 511 builder. 512 AdminCommand("read-execution-data", func(config *cmd.NodeConfig) commands.AdminCommand { 513 return stateSyncCommands.NewReadExecutionDataCommand(builder.ExecutionDataStore) 514 }). 515 Module("execution data datastore and blobstore", func(node *cmd.NodeConfig) error { 516 datastoreDir := filepath.Join(builder.executionDataDir, "blobstore") 517 err := os.MkdirAll(datastoreDir, 0700) 518 if err != nil { 519 return err 520 } 521 522 ds, err = badger.NewDatastore(datastoreDir, &badger.DefaultOptions) 523 if err != nil { 524 return err 525 } 526 527 builder.ShutdownFunc(func() error { 528 if err := ds.Close(); err != nil { 529 return fmt.Errorf("could not close execution data datastore: %w", err) 530 } 531 return nil 532 }) 533 534 return nil 535 }). 536 Module("processed block height consumer progress", func(node *cmd.NodeConfig) error { 537 // Note: progress is stored in the datastore's DB since that is where the jobqueue 538 // writes execution data to. 539 processedBlockHeight = bstorage.NewConsumerProgress(ds.DB, module.ConsumeProgressExecutionDataRequesterBlockHeight) 540 return nil 541 }). 542 Module("processed notifications consumer progress", func(node *cmd.NodeConfig) error { 543 // Note: progress is stored in the datastore's DB since that is where the jobqueue 544 // writes execution data to. 545 processedNotifications = bstorage.NewConsumerProgress(ds.DB, module.ConsumeProgressExecutionDataRequesterNotification) 546 return nil 547 }). 548 Module("blobservice peer manager dependencies", func(node *cmd.NodeConfig) error { 549 bsDependable = module.NewProxiedReadyDoneAware() 550 builder.PeerManagerDependencies.Add(bsDependable) 551 return nil 552 }). 553 Module("execution datastore", func(node *cmd.NodeConfig) error { 554 blobstore := blobs.NewBlobstore(ds) 555 builder.ExecutionDataStore = execution_data.NewExecutionDataStore(blobstore, execution_data.DefaultSerializer) 556 return nil 557 }). 558 Module("execution data cache", func(node *cmd.NodeConfig) error { 559 var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector() 560 if builder.HeroCacheMetricsEnable { 561 heroCacheCollector = metrics.AccessNodeExecutionDataCacheMetrics(builder.MetricsRegisterer) 562 } 563 564 execDataCacheBackend = herocache.NewBlockExecutionData(builder.stateStreamConf.ExecutionDataCacheSize, builder.Logger, heroCacheCollector) 565 566 // Execution Data cache that uses a blobstore as the backend (instead of a downloader) 567 // This ensures that it simply returns a not found error if the blob doesn't exist 568 // instead of attempting to download it from the network. 569 executionDataStoreCache = execdatacache.NewExecutionDataCache( 570 builder.ExecutionDataStore, 571 builder.Storage.Headers, 572 builder.Storage.Seals, 573 builder.Storage.Results, 574 execDataCacheBackend, 575 ) 576 577 return nil 578 }). 579 Component("execution data service", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 580 opts := []network.BlobServiceOption{ 581 blob.WithBitswapOptions( 582 // Only allow block requests from staked ENs and ANs 583 bitswap.WithPeerBlockRequestFilter( 584 blob.AuthorizedRequester(nil, builder.IdentityProvider, builder.Logger), 585 ), 586 bitswap.WithTracer( 587 blob.NewTracer(node.Logger.With().Str("blob_service", channels.ExecutionDataService.String()).Logger()), 588 ), 589 ), 590 } 591 592 var err error 593 bs, err = node.EngineRegistry.RegisterBlobService(channels.ExecutionDataService, ds, opts...) 594 if err != nil { 595 return nil, fmt.Errorf("could not register blob service: %w", err) 596 } 597 598 // add blobservice into ReadyDoneAware dependency passed to peer manager 599 // this starts the blob service and configures peer manager to wait for the blobservice 600 // to be ready before starting 601 bsDependable.Init(bs) 602 603 builder.ExecutionDataDownloader = execution_data.NewDownloader(bs) 604 605 return builder.ExecutionDataDownloader, nil 606 }). 607 Component("execution data requester", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 608 // Validation of the start block height needs to be done after loading state 609 if builder.executionDataStartHeight > 0 { 610 if builder.executionDataStartHeight <= builder.FinalizedRootBlock.Header.Height { 611 return nil, fmt.Errorf( 612 "execution data start block height (%d) must be greater than the root block height (%d)", 613 builder.executionDataStartHeight, builder.FinalizedRootBlock.Header.Height) 614 } 615 616 latestSeal, err := builder.State.Sealed().Head() 617 if err != nil { 618 return nil, fmt.Errorf("failed to get latest sealed height") 619 } 620 621 // Note: since the root block of a spork is also sealed in the root protocol state, the 622 // latest sealed height is always equal to the root block height. That means that at the 623 // very beginning of a spork, this check will always fail. Operators should not specify 624 // an InitialBlockHeight when starting from the beginning of a spork. 625 if builder.executionDataStartHeight > latestSeal.Height { 626 return nil, fmt.Errorf( 627 "execution data start block height (%d) must be less than or equal to the latest sealed block height (%d)", 628 builder.executionDataStartHeight, latestSeal.Height) 629 } 630 631 // executionDataStartHeight is provided as the first block to sync, but the 632 // requester expects the initial last processed height, which is the first height - 1 633 builder.executionDataConfig.InitialBlockHeight = builder.executionDataStartHeight - 1 634 } else { 635 builder.executionDataConfig.InitialBlockHeight = builder.FinalizedRootBlock.Header.Height 636 } 637 638 execDataDistributor = edrequester.NewExecutionDataDistributor() 639 640 // Execution Data cache with a downloader as the backend. This is used by the requester 641 // to download and cache execution data for each block. It shares a cache backend instance 642 // with the datastore implementation. 643 executionDataCache := execdatacache.NewExecutionDataCache( 644 builder.ExecutionDataDownloader, 645 builder.Storage.Headers, 646 builder.Storage.Seals, 647 builder.Storage.Results, 648 execDataCacheBackend, 649 ) 650 651 r, err := edrequester.New( 652 builder.Logger, 653 metrics.NewExecutionDataRequesterCollector(), 654 builder.ExecutionDataDownloader, 655 executionDataCache, 656 processedBlockHeight, 657 processedNotifications, 658 builder.State, 659 builder.Storage.Headers, 660 builder.executionDataConfig, 661 execDataDistributor, 662 ) 663 if err != nil { 664 return nil, fmt.Errorf("failed to create execution data requester: %w", err) 665 } 666 builder.ExecutionDataRequester = r 667 668 builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized) 669 670 // add requester into ReadyDoneAware dependency passed to indexer. This allows the indexer 671 // to wait for the requester to be ready before starting. 672 requesterDependable.Init(builder.ExecutionDataRequester) 673 674 return builder.ExecutionDataRequester, nil 675 }) 676 677 if builder.publicNetworkExecutionDataEnabled { 678 var publicBsDependable *module.ProxiedReadyDoneAware 679 680 builder.Module("public blobservice peer manager dependencies", func(node *cmd.NodeConfig) error { 681 publicBsDependable = module.NewProxiedReadyDoneAware() 682 builder.PeerManagerDependencies.Add(publicBsDependable) 683 return nil 684 }) 685 builder.Component("public network execution data service", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 686 opts := []network.BlobServiceOption{ 687 blob.WithBitswapOptions( 688 bitswap.WithTracer( 689 blob.NewTracer(node.Logger.With().Str("public_blob_service", channels.PublicExecutionDataService.String()).Logger()), 690 ), 691 ), 692 } 693 694 net := builder.AccessNodeConfig.PublicNetworkConfig.Network 695 696 var err error 697 builder.PublicBlobService, err = net.RegisterBlobService(channels.PublicExecutionDataService, ds, opts...) 698 if err != nil { 699 return nil, fmt.Errorf("could not register blob service: %w", err) 700 } 701 702 // add blobservice into ReadyDoneAware dependency passed to peer manager 703 // this starts the blob service and configures peer manager to wait for the blobservice 704 // to be ready before starting 705 publicBsDependable.Init(builder.PublicBlobService) 706 return &module.NoopReadyDoneAware{}, nil 707 }) 708 } 709 710 if builder.executionDataIndexingEnabled { 711 var indexedBlockHeight storage.ConsumerProgress 712 713 builder. 714 AdminCommand("execute-script", func(config *cmd.NodeConfig) commands.AdminCommand { 715 return stateSyncCommands.NewExecuteScriptCommand(builder.ScriptExecutor) 716 }). 717 Module("indexed block height consumer progress", func(node *cmd.NodeConfig) error { 718 // Note: progress is stored in the MAIN db since that is where indexed execution data is stored. 719 indexedBlockHeight = bstorage.NewConsumerProgress(builder.DB, module.ConsumeProgressExecutionDataIndexerBlockHeight) 720 return nil 721 }). 722 Module("transaction results storage", func(node *cmd.NodeConfig) error { 723 builder.Storage.LightTransactionResults = bstorage.NewLightTransactionResults(node.Metrics.Cache, node.DB, bstorage.DefaultCacheSize) 724 return nil 725 }). 726 DependableComponent("execution data indexer", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 727 // Note: using a DependableComponent here to ensure that the indexer does not block 728 // other components from starting while bootstrapping the register db since it may 729 // take hours to complete. 730 731 pdb, err := pStorage.OpenRegisterPebbleDB(builder.registersDBPath) 732 if err != nil { 733 return nil, fmt.Errorf("could not open registers db: %w", err) 734 } 735 builder.ShutdownFunc(func() error { 736 return pdb.Close() 737 }) 738 739 bootstrapped, err := pStorage.IsBootstrapped(pdb) 740 if err != nil { 741 return nil, fmt.Errorf("could not check if registers db is bootstrapped: %w", err) 742 } 743 744 if !bootstrapped { 745 checkpointFile := builder.checkpointFile 746 if checkpointFile == cmd.NotSet { 747 checkpointFile = path.Join(builder.BootstrapDir, bootstrap.PathRootCheckpoint) 748 } 749 750 // currently, the checkpoint must be from the root block. 751 // read the root hash from the provided checkpoint and verify it matches the 752 // state commitment from the root snapshot. 753 err := wal.CheckpointHasRootHash( 754 node.Logger, 755 "", // checkpoint file already full path 756 checkpointFile, 757 ledger.RootHash(node.RootSeal.FinalState), 758 ) 759 if err != nil { 760 return nil, fmt.Errorf("could not verify checkpoint file: %w", err) 761 } 762 763 checkpointHeight := builder.SealedRootBlock.Header.Height 764 765 if builder.SealedRootBlock.ID() != builder.RootSeal.BlockID { 766 return nil, fmt.Errorf("mismatching sealed root block and root seal: %v != %v", 767 builder.SealedRootBlock.ID(), builder.RootSeal.BlockID) 768 } 769 770 rootHash := ledger.RootHash(builder.RootSeal.FinalState) 771 bootstrap, err := pStorage.NewRegisterBootstrap(pdb, checkpointFile, checkpointHeight, rootHash, builder.Logger) 772 if err != nil { 773 return nil, fmt.Errorf("could not create registers bootstrap: %w", err) 774 } 775 776 // TODO: find a way to hook a context up to this to allow a graceful shutdown 777 workerCount := 10 778 err = bootstrap.IndexCheckpointFile(context.Background(), workerCount) 779 if err != nil { 780 return nil, fmt.Errorf("could not load checkpoint file: %w", err) 781 } 782 } 783 784 registers, err := pStorage.NewRegisters(pdb) 785 if err != nil { 786 return nil, fmt.Errorf("could not create registers storage: %w", err) 787 } 788 789 if builder.registerCacheSize > 0 { 790 cacheType, err := pStorage.ParseCacheType(builder.registerCacheType) 791 if err != nil { 792 return nil, fmt.Errorf("could not parse register cache type: %w", err) 793 } 794 cacheMetrics := metrics.NewCacheCollector(builder.RootChainID) 795 registersCache, err := pStorage.NewRegistersCache(registers, cacheType, builder.registerCacheSize, cacheMetrics) 796 if err != nil { 797 return nil, fmt.Errorf("could not create registers cache: %w", err) 798 } 799 builder.Storage.RegisterIndex = registersCache 800 } else { 801 builder.Storage.RegisterIndex = registers 802 } 803 804 indexerCore, err := indexer.New( 805 builder.Logger, 806 metrics.NewExecutionStateIndexerCollector(), 807 builder.DB, 808 builder.Storage.RegisterIndex, 809 builder.Storage.Headers, 810 builder.Storage.Events, 811 builder.Storage.Collections, 812 builder.Storage.Transactions, 813 builder.Storage.LightTransactionResults, 814 builder.collectionExecutedMetric, 815 ) 816 if err != nil { 817 return nil, err 818 } 819 builder.ExecutionIndexerCore = indexerCore 820 821 // execution state worker uses a jobqueue to process new execution data and indexes it by using the indexer. 822 builder.ExecutionIndexer, err = indexer.NewIndexer( 823 builder.Logger, 824 registers.FirstHeight(), 825 registers, 826 indexerCore, 827 executionDataStoreCache, 828 builder.ExecutionDataRequester.HighestConsecutiveHeight, 829 indexedBlockHeight, 830 ) 831 if err != nil { 832 return nil, err 833 } 834 835 // setup requester to notify indexer when new execution data is received 836 execDataDistributor.AddOnExecutionDataReceivedConsumer(builder.ExecutionIndexer.OnExecutionData) 837 838 // create script execution module, this depends on the indexer being initialized and the 839 // having the register storage bootstrapped 840 scripts, err := execution.NewScripts( 841 builder.Logger, 842 metrics.NewExecutionCollector(builder.Tracer), 843 builder.RootChainID, 844 query.NewProtocolStateWrapper(builder.State), 845 builder.Storage.Headers, 846 builder.ExecutionIndexerCore.RegisterValue, 847 builder.scriptExecutorConfig, 848 ) 849 if err != nil { 850 return nil, err 851 } 852 853 err = builder.ScriptExecutor.Initialize(builder.ExecutionIndexer, scripts) 854 if err != nil { 855 return nil, err 856 } 857 858 err = builder.EventsIndex.Initialize(builder.ExecutionIndexer) 859 if err != nil { 860 return nil, err 861 } 862 863 err = builder.TxResultsIndex.Initialize(builder.ExecutionIndexer) 864 if err != nil { 865 return nil, err 866 } 867 868 err = builder.RegistersAsyncStore.Initialize(registers) 869 if err != nil { 870 return nil, err 871 } 872 873 return builder.ExecutionIndexer, nil 874 }, builder.IndexerDependencies) 875 } 876 877 if builder.stateStreamConf.ListenAddr != "" { 878 builder.Component("exec state stream engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 879 for key, value := range builder.stateStreamFilterConf { 880 switch key { 881 case "EventTypes": 882 builder.stateStreamConf.MaxEventTypes = value 883 case "Addresses": 884 builder.stateStreamConf.MaxAddresses = value 885 case "Contracts": 886 builder.stateStreamConf.MaxContracts = value 887 } 888 } 889 builder.stateStreamConf.RpcMetricsEnabled = builder.rpcMetricsEnabled 890 891 highestAvailableHeight, err := builder.ExecutionDataRequester.HighestConsecutiveHeight() 892 if err != nil { 893 return nil, fmt.Errorf("could not get highest consecutive height: %w", err) 894 } 895 broadcaster := engine.NewBroadcaster() 896 897 eventQueryMode, err := backend.ParseIndexQueryMode(builder.rpcConf.BackendConfig.EventQueryMode) 898 if err != nil { 899 return nil, fmt.Errorf("could not parse event query mode: %w", err) 900 } 901 902 // use the events index for events if enabled and the node is configured to use it for 903 // regular event queries 904 useIndex := builder.executionDataIndexingEnabled && 905 eventQueryMode != backend.IndexQueryModeExecutionNodesOnly 906 907 builder.stateStreamBackend, err = statestreambackend.New( 908 node.Logger, 909 builder.stateStreamConf, 910 node.State, 911 node.Storage.Headers, 912 node.Storage.Seals, 913 node.Storage.Results, 914 builder.ExecutionDataStore, 915 executionDataStoreCache, 916 broadcaster, 917 builder.executionDataConfig.InitialBlockHeight, 918 highestAvailableHeight, 919 builder.RegistersAsyncStore, 920 builder.EventsIndex, 921 useIndex, 922 ) 923 if err != nil { 924 return nil, fmt.Errorf("could not create state stream backend: %w", err) 925 } 926 927 stateStreamEng, err := statestreambackend.NewEng( 928 node.Logger, 929 builder.stateStreamConf, 930 executionDataStoreCache, 931 node.Storage.Headers, 932 node.RootChainID, 933 builder.stateStreamGrpcServer, 934 builder.stateStreamBackend, 935 broadcaster, 936 ) 937 if err != nil { 938 return nil, fmt.Errorf("could not create state stream engine: %w", err) 939 } 940 builder.StateStreamEng = stateStreamEng 941 942 execDataDistributor.AddOnExecutionDataReceivedConsumer(builder.StateStreamEng.OnExecutionData) 943 944 return builder.StateStreamEng, nil 945 }) 946 } 947 948 return builder 949 } 950 951 func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder { 952 dist := consensuspubsub.NewFollowerDistributor() 953 dist.AddProposalViolationConsumer(notifications.NewSlashingViolationsConsumer(nodeBuilder.Logger)) 954 return &FlowAccessNodeBuilder{ 955 AccessNodeConfig: DefaultAccessNodeConfig(), 956 FlowNodeBuilder: nodeBuilder, 957 FollowerDistributor: dist, 958 IndexerDependencies: cmd.NewDependencyList(), 959 } 960 } 961 962 func (builder *FlowAccessNodeBuilder) ParseFlags() error { 963 builder.BaseFlags() 964 965 builder.extraFlags() 966 967 return builder.ParseAndPrintFlags() 968 } 969 970 func (builder *FlowAccessNodeBuilder) extraFlags() { 971 builder.ExtraFlags(func(flags *pflag.FlagSet) { 972 defaultConfig := DefaultAccessNodeConfig() 973 974 flags.UintVar(&builder.collectionGRPCPort, "collection-ingress-port", defaultConfig.collectionGRPCPort, "the grpc ingress port for all collection nodes") 975 flags.UintVar(&builder.executionGRPCPort, "execution-ingress-port", defaultConfig.executionGRPCPort, "the grpc ingress port for all execution nodes") 976 flags.StringVarP(&builder.rpcConf.UnsecureGRPCListenAddr, 977 "rpc-addr", 978 "r", 979 defaultConfig.rpcConf.UnsecureGRPCListenAddr, 980 "the address the unsecured gRPC server listens on") 981 flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, 982 "secure-rpc-addr", 983 defaultConfig.rpcConf.SecureGRPCListenAddr, 984 "the address the secure gRPC server listens on") 985 flags.StringVar(&builder.stateStreamConf.ListenAddr, 986 "state-stream-addr", 987 defaultConfig.stateStreamConf.ListenAddr, 988 "the address the state stream server listens on (if empty the server will not be started)") 989 flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on") 990 flags.StringVar(&builder.rpcConf.RestConfig.ListenAddress, 991 "rest-addr", 992 defaultConfig.rpcConf.RestConfig.ListenAddress, 993 "the address the REST server listens on (if empty the REST server will not be started)") 994 flags.DurationVar(&builder.rpcConf.RestConfig.WriteTimeout, 995 "rest-write-timeout", 996 defaultConfig.rpcConf.RestConfig.WriteTimeout, 997 "timeout to use when writing REST response") 998 flags.DurationVar(&builder.rpcConf.RestConfig.ReadTimeout, 999 "rest-read-timeout", 1000 defaultConfig.rpcConf.RestConfig.ReadTimeout, 1001 "timeout to use when reading REST request headers") 1002 flags.DurationVar(&builder.rpcConf.RestConfig.IdleTimeout, "rest-idle-timeout", defaultConfig.rpcConf.RestConfig.IdleTimeout, "idle timeout for REST connections") 1003 flags.StringVarP(&builder.rpcConf.CollectionAddr, 1004 "static-collection-ingress-addr", 1005 "", 1006 defaultConfig.rpcConf.CollectionAddr, 1007 "the address (of the collection node) to send transactions to") 1008 flags.StringVarP(&builder.ExecutionNodeAddress, 1009 "script-addr", 1010 "s", 1011 defaultConfig.ExecutionNodeAddress, 1012 "the address (of the execution node) forward the script to") 1013 flags.StringVarP(&builder.rpcConf.HistoricalAccessAddrs, 1014 "historical-access-addr", 1015 "", 1016 defaultConfig.rpcConf.HistoricalAccessAddrs, 1017 "comma separated rpc addresses for historical access nodes") 1018 flags.DurationVar(&builder.rpcConf.BackendConfig.CollectionClientTimeout, 1019 "collection-client-timeout", 1020 defaultConfig.rpcConf.BackendConfig.CollectionClientTimeout, 1021 "grpc client timeout for a collection node") 1022 flags.DurationVar(&builder.rpcConf.BackendConfig.ExecutionClientTimeout, 1023 "execution-client-timeout", 1024 defaultConfig.rpcConf.BackendConfig.ExecutionClientTimeout, 1025 "grpc client timeout for an execution node") 1026 flags.UintVar(&builder.rpcConf.BackendConfig.ConnectionPoolSize, 1027 "connection-pool-size", 1028 defaultConfig.rpcConf.BackendConfig.ConnectionPoolSize, 1029 "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size") 1030 flags.UintVar(&builder.rpcConf.MaxMsgSize, 1031 "rpc-max-message-size", 1032 grpcutils.DefaultMaxMsgSize, 1033 "the maximum message size in bytes for messages sent or received over grpc") 1034 flags.UintVar(&builder.rpcConf.BackendConfig.MaxHeightRange, 1035 "rpc-max-height-range", 1036 defaultConfig.rpcConf.BackendConfig.MaxHeightRange, 1037 "maximum size for height range requests") 1038 flags.StringSliceVar(&builder.rpcConf.BackendConfig.PreferredExecutionNodeIDs, 1039 "preferred-execution-node-ids", 1040 defaultConfig.rpcConf.BackendConfig.PreferredExecutionNodeIDs, 1041 "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.") 1042 flags.StringSliceVar(&builder.rpcConf.BackendConfig.FixedExecutionNodeIDs, 1043 "fixed-execution-node-ids", 1044 defaultConfig.rpcConf.BackendConfig.FixedExecutionNodeIDs, 1045 "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.") 1046 flags.StringVar(&builder.rpcConf.CompressorName, 1047 "grpc-compressor", 1048 defaultConfig.rpcConf.CompressorName, 1049 "name of grpc compressor that will be used for requests to other nodes. One of (gzip, snappy, deflate)") 1050 flags.BoolVar(&builder.logTxTimeToFinalized, "log-tx-time-to-finalized", defaultConfig.logTxTimeToFinalized, "log transaction time to finalized") 1051 flags.BoolVar(&builder.logTxTimeToExecuted, "log-tx-time-to-executed", defaultConfig.logTxTimeToExecuted, "log transaction time to executed") 1052 flags.BoolVar(&builder.logTxTimeToFinalizedExecuted, 1053 "log-tx-time-to-finalized-executed", 1054 defaultConfig.logTxTimeToFinalizedExecuted, 1055 "log transaction time to finalized and executed") 1056 flags.BoolVar(&builder.pingEnabled, 1057 "ping-enabled", 1058 defaultConfig.pingEnabled, 1059 "whether to enable the ping process that pings all other peers and report the connectivity to metrics") 1060 flags.BoolVar(&builder.retryEnabled, "retry-enabled", defaultConfig.retryEnabled, "whether to enable the retry mechanism at the access node level") 1061 flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics") 1062 flags.UintVar(&builder.TxResultCacheSize, "transaction-result-cache-size", defaultConfig.TxResultCacheSize, "transaction result cache size.(Disabled by default i.e 0)") 1063 flags.UintVar(&builder.TxErrorMessagesCacheSize, "transaction-error-messages-cache-size", defaultConfig.TxErrorMessagesCacheSize, "transaction error messages cache size.(By default 1000)") 1064 flags.StringVarP(&builder.nodeInfoFile, 1065 "node-info-file", 1066 "", 1067 defaultConfig.nodeInfoFile, 1068 "full path to a json file which provides more details about nodes when reporting its reachability metrics") 1069 flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") 1070 flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") 1071 flags.BoolVar(&builder.supportsObserver, "supports-observer", defaultConfig.supportsObserver, "true if this staked access node supports observer or follower connections") 1072 flags.StringVar(&builder.PublicNetworkConfig.BindAddress, "public-network-address", defaultConfig.PublicNetworkConfig.BindAddress, "staked access node's public network bind address") 1073 flags.BoolVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, 1074 "circuit-breaker-enabled", 1075 defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled, 1076 "specifies whether the circuit breaker is enabled for collection and execution API clients.") 1077 flags.DurationVar(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, 1078 "circuit-breaker-restore-timeout", 1079 defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout, 1080 "duration after which the circuit breaker will restore the connection to the client after closing it due to failures. Default value is 60s") 1081 flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, 1082 "circuit-breaker-max-failures", 1083 defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures, 1084 "maximum number of failed calls to the client that will cause the circuit breaker to close the connection. Default value is 5") 1085 flags.Uint32Var(&builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, 1086 "circuit-breaker-max-requests", 1087 defaultConfig.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests, 1088 "maximum number of requests to check if connection restored after timeout. Default value is 1") 1089 // ExecutionDataRequester config 1090 flags.BoolVar(&builder.executionDataSyncEnabled, 1091 "execution-data-sync-enabled", 1092 defaultConfig.executionDataSyncEnabled, 1093 "whether to enable the execution data sync protocol") 1094 flags.BoolVar(&builder.publicNetworkExecutionDataEnabled, 1095 "public-network-execution-data-sync-enabled", 1096 defaultConfig.publicNetworkExecutionDataEnabled, 1097 "[experimental] whether to enable the execution data sync protocol on public network") 1098 flags.StringVar(&builder.executionDataDir, "execution-data-dir", defaultConfig.executionDataDir, "directory to use for Execution Data database") 1099 flags.Uint64Var(&builder.executionDataStartHeight, 1100 "execution-data-start-height", 1101 defaultConfig.executionDataStartHeight, 1102 "height of first block to sync execution data from when starting with an empty Execution Data database") 1103 flags.Uint64Var(&builder.executionDataConfig.MaxSearchAhead, 1104 "execution-data-max-search-ahead", 1105 defaultConfig.executionDataConfig.MaxSearchAhead, 1106 "max number of heights to search ahead of the lowest outstanding execution data height") 1107 flags.DurationVar(&builder.executionDataConfig.FetchTimeout, 1108 "execution-data-fetch-timeout", 1109 defaultConfig.executionDataConfig.FetchTimeout, 1110 "initial timeout to use when fetching execution data from the network. timeout increases using an incremental backoff until execution-data-max-fetch-timeout. e.g. 30s") 1111 flags.DurationVar(&builder.executionDataConfig.MaxFetchTimeout, 1112 "execution-data-max-fetch-timeout", 1113 defaultConfig.executionDataConfig.MaxFetchTimeout, 1114 "maximum timeout to use when fetching execution data from the network e.g. 300s") 1115 flags.DurationVar(&builder.executionDataConfig.RetryDelay, 1116 "execution-data-retry-delay", 1117 defaultConfig.executionDataConfig.RetryDelay, 1118 "initial delay for exponential backoff when fetching execution data fails e.g. 10s") 1119 flags.DurationVar(&builder.executionDataConfig.MaxRetryDelay, 1120 "execution-data-max-retry-delay", 1121 defaultConfig.executionDataConfig.MaxRetryDelay, 1122 "maximum delay for exponential backoff when fetching execution data fails e.g. 5m") 1123 1124 // Execution State Streaming API 1125 flags.Uint32Var(&builder.stateStreamConf.ExecutionDataCacheSize, "execution-data-cache-size", defaultConfig.stateStreamConf.ExecutionDataCacheSize, "block execution data cache size") 1126 flags.Uint32Var(&builder.stateStreamConf.MaxGlobalStreams, "state-stream-global-max-streams", defaultConfig.stateStreamConf.MaxGlobalStreams, "global maximum number of concurrent streams") 1127 flags.UintVar(&builder.stateStreamConf.MaxExecutionDataMsgSize, 1128 "state-stream-max-message-size", 1129 defaultConfig.stateStreamConf.MaxExecutionDataMsgSize, 1130 "maximum size for a gRPC message containing block execution data") 1131 flags.StringToIntVar(&builder.stateStreamFilterConf, 1132 "state-stream-event-filter-limits", 1133 defaultConfig.stateStreamFilterConf, 1134 "event filter limits for ExecutionData SubscribeEvents API e.g. EventTypes=100,Addresses=100,Contracts=100 etc.") 1135 flags.DurationVar(&builder.stateStreamConf.ClientSendTimeout, 1136 "state-stream-send-timeout", 1137 defaultConfig.stateStreamConf.ClientSendTimeout, 1138 "maximum wait before timing out while sending a response to a streaming client e.g. 30s") 1139 flags.UintVar(&builder.stateStreamConf.ClientSendBufferSize, 1140 "state-stream-send-buffer-size", 1141 defaultConfig.stateStreamConf.ClientSendBufferSize, 1142 "maximum number of responses to buffer within a stream") 1143 flags.Float64Var(&builder.stateStreamConf.ResponseLimit, 1144 "state-stream-response-limit", 1145 defaultConfig.stateStreamConf.ResponseLimit, 1146 "max number of responses per second to send over streaming endpoints. this helps manage resources consumed by each client querying data not in the cache e.g. 3 or 0.5. 0 means no limit") 1147 flags.Uint64Var(&builder.stateStreamConf.HeartbeatInterval, 1148 "state-stream-heartbeat-interval", 1149 defaultConfig.stateStreamConf.HeartbeatInterval, 1150 "default interval in blocks at which heartbeat messages should be sent. applied when client did not specify a value.") 1151 flags.Uint32Var(&builder.stateStreamConf.RegisterIDsRequestLimit, 1152 "state-stream-max-register-values", 1153 defaultConfig.stateStreamConf.RegisterIDsRequestLimit, 1154 "maximum number of register ids to include in a single request to the GetRegisters endpoint") 1155 1156 // Execution Data Indexer 1157 flags.BoolVar(&builder.executionDataIndexingEnabled, 1158 "execution-data-indexing-enabled", 1159 defaultConfig.executionDataIndexingEnabled, 1160 "whether to enable the execution data indexing") 1161 flags.StringVar(&builder.registersDBPath, "execution-state-dir", defaultConfig.registersDBPath, "directory to use for execution-state database") 1162 flags.StringVar(&builder.checkpointFile, "execution-state-checkpoint", defaultConfig.checkpointFile, "execution-state checkpoint file") 1163 1164 flags.StringVar(&builder.rpcConf.BackendConfig.EventQueryMode, 1165 "event-query-mode", 1166 defaultConfig.rpcConf.BackendConfig.EventQueryMode, 1167 "mode to use when querying events. one of [local-only, execution-nodes-only(default), failover]") 1168 1169 flags.StringVar(&builder.rpcConf.BackendConfig.TxResultQueryMode, 1170 "tx-result-query-mode", 1171 defaultConfig.rpcConf.BackendConfig.TxResultQueryMode, 1172 "mode to use when querying transaction results. one of [local-only, execution-nodes-only(default), failover]") 1173 1174 // Script Execution 1175 flags.StringVar(&builder.rpcConf.BackendConfig.ScriptExecutionMode, 1176 "script-execution-mode", 1177 defaultConfig.rpcConf.BackendConfig.ScriptExecutionMode, 1178 "mode to use when executing scripts. one of (local-only, execution-nodes-only, failover, compare)") 1179 flags.Uint64Var(&builder.scriptExecutorConfig.ComputationLimit, 1180 "script-execution-computation-limit", 1181 defaultConfig.scriptExecutorConfig.ComputationLimit, 1182 "maximum number of computation units a locally executed script can use. default: 100000") 1183 flags.IntVar(&builder.scriptExecutorConfig.MaxErrorMessageSize, 1184 "script-execution-max-error-length", 1185 defaultConfig.scriptExecutorConfig.MaxErrorMessageSize, 1186 "maximum number characters to include in error message strings. additional characters are truncated. default: 1000") 1187 flags.DurationVar(&builder.scriptExecutorConfig.LogTimeThreshold, 1188 "script-execution-log-time-threshold", 1189 defaultConfig.scriptExecutorConfig.LogTimeThreshold, 1190 "emit a log for any scripts that take over this threshold. default: 1s") 1191 flags.DurationVar(&builder.scriptExecutorConfig.ExecutionTimeLimit, 1192 "script-execution-timeout", 1193 defaultConfig.scriptExecutorConfig.ExecutionTimeLimit, 1194 "timeout value for locally executed scripts. default: 10s") 1195 flags.Uint64Var(&builder.scriptExecMinBlock, 1196 "script-execution-min-height", 1197 defaultConfig.scriptExecMinBlock, 1198 "lowest block height to allow for script execution. default: no limit") 1199 flags.Uint64Var(&builder.scriptExecMaxBlock, 1200 "script-execution-max-height", 1201 defaultConfig.scriptExecMaxBlock, 1202 "highest block height to allow for script execution. default: no limit") 1203 1204 flags.StringVar(&builder.registerCacheType, 1205 "register-cache-type", 1206 defaultConfig.registerCacheType, 1207 "type of backend cache to use for registers (lru or 2q)") 1208 flags.UintVar(&builder.registerCacheSize, 1209 "register-cache-size", 1210 defaultConfig.registerCacheSize, 1211 "number of registers to cache for script execution. default: 0 (no cache)") 1212 1213 }).ValidateFlags(func() error { 1214 if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") { 1215 return errors.New("public-network-address must be set if supports-observer is true") 1216 } 1217 if builder.executionDataSyncEnabled { 1218 if builder.executionDataConfig.FetchTimeout <= 0 { 1219 return errors.New("execution-data-fetch-timeout must be greater than 0") 1220 } 1221 if builder.executionDataConfig.MaxFetchTimeout < builder.executionDataConfig.FetchTimeout { 1222 return errors.New("execution-data-max-fetch-timeout must be greater than execution-data-fetch-timeout") 1223 } 1224 if builder.executionDataConfig.RetryDelay <= 0 { 1225 return errors.New("execution-data-retry-delay must be greater than 0") 1226 } 1227 if builder.executionDataConfig.MaxRetryDelay < builder.executionDataConfig.RetryDelay { 1228 return errors.New("execution-data-max-retry-delay must be greater than or equal to execution-data-retry-delay") 1229 } 1230 if builder.executionDataConfig.MaxSearchAhead == 0 { 1231 return errors.New("execution-data-max-search-ahead must be greater than 0") 1232 } 1233 } 1234 if builder.stateStreamConf.ListenAddr != "" { 1235 if builder.stateStreamConf.ExecutionDataCacheSize == 0 { 1236 return errors.New("execution-data-cache-size must be greater than 0") 1237 } 1238 if builder.stateStreamConf.ClientSendBufferSize == 0 { 1239 return errors.New("state-stream-send-buffer-size must be greater than 0") 1240 } 1241 if len(builder.stateStreamFilterConf) > 3 { 1242 return errors.New("state-stream-event-filter-limits must have at most 3 keys (EventTypes, Addresses, Contracts)") 1243 } 1244 for key, value := range builder.stateStreamFilterConf { 1245 switch key { 1246 case "EventTypes", "Addresses", "Contracts": 1247 if value <= 0 { 1248 return fmt.Errorf("state-stream-event-filter-limits %s must be greater than 0", key) 1249 } 1250 default: 1251 return errors.New("state-stream-event-filter-limits may only contain the keys EventTypes, Addresses, Contracts") 1252 } 1253 } 1254 if builder.stateStreamConf.ResponseLimit < 0 { 1255 return errors.New("state-stream-response-limit must be greater than or equal to 0") 1256 } 1257 if builder.stateStreamConf.RegisterIDsRequestLimit <= 0 { 1258 return errors.New("state-stream-max-register-values must be greater than 0") 1259 } 1260 } 1261 if builder.rpcConf.BackendConfig.CircuitBreakerConfig.Enabled { 1262 if builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxFailures == 0 { 1263 return errors.New("circuit-breaker-max-failures must be greater than 0") 1264 } 1265 if builder.rpcConf.BackendConfig.CircuitBreakerConfig.MaxRequests == 0 { 1266 return errors.New("circuit-breaker-max-requests must be greater than 0") 1267 } 1268 if builder.rpcConf.BackendConfig.CircuitBreakerConfig.RestoreTimeout <= 0 { 1269 return errors.New("circuit-breaker-restore-timeout must be greater than 0") 1270 } 1271 } 1272 if builder.TxErrorMessagesCacheSize == 0 { 1273 return errors.New("transaction-error-messages-cache-size must be greater than 0") 1274 } 1275 1276 return nil 1277 }) 1278 } 1279 1280 func publicNetworkMsgValidators(log zerolog.Logger, idProvider module.IdentityProvider, selfID flow.Identifier) []network.MessageValidator { 1281 return []network.MessageValidator{ 1282 // filter out messages sent by this node itself 1283 validator.ValidateNotSender(selfID), 1284 validator.NewAnyValidator( 1285 // message should be either from a valid staked node 1286 validator.NewOriginValidator( 1287 id.NewIdentityFilterIdentifierProvider(filter.IsValidCurrentEpochParticipant, idProvider), 1288 ), 1289 // or the message should be specifically targeted for this node 1290 validator.ValidateTarget(log, selfID), 1291 ), 1292 } 1293 } 1294 1295 func (builder *FlowAccessNodeBuilder) InitIDProviders() { 1296 builder.Module("id providers", func(node *cmd.NodeConfig) error { 1297 idCache, err := cache.NewProtocolStateIDCache(node.Logger, node.State, node.ProtocolEvents) 1298 if err != nil { 1299 return fmt.Errorf("could not initialize ProtocolStateIDCache: %w", err) 1300 } 1301 builder.IDTranslator = translator.NewHierarchicalIDTranslator(idCache, translator.NewPublicNetworkIDTranslator()) 1302 1303 // The following wrapper allows to disallow-list byzantine nodes via an admin command: 1304 // the wrapper overrides the 'Ejected' flag of disallow-listed nodes to true 1305 disallowListWrapper, err := cache.NewNodeDisallowListWrapper(idCache, node.DB, func() network.DisallowListNotificationConsumer { 1306 return builder.NetworkUnderlay 1307 }) 1308 if err != nil { 1309 return fmt.Errorf("could not initialize NodeBlockListWrapper: %w", err) 1310 } 1311 builder.IdentityProvider = disallowListWrapper 1312 1313 // register the wrapper for dynamic configuration via admin command 1314 err = node.ConfigManager.RegisterIdentifierListConfig("network-id-provider-blocklist", 1315 disallowListWrapper.GetDisallowList, disallowListWrapper.Update) 1316 if err != nil { 1317 return fmt.Errorf("failed to register disallow-list wrapper with config manager: %w", err) 1318 } 1319 1320 builder.SyncEngineParticipantsProviderFactory = func() module.IdentifierProvider { 1321 return id.NewIdentityFilterIdentifierProvider( 1322 filter.And( 1323 filter.HasRole(flow.RoleConsensus), 1324 filter.Not(filter.HasNodeID(node.Me.NodeID())), 1325 underlay.NotEjectedFilter, 1326 ), 1327 builder.IdentityProvider, 1328 ) 1329 } 1330 return nil 1331 }) 1332 } 1333 1334 func (builder *FlowAccessNodeBuilder) Initialize() error { 1335 builder.InitIDProviders() 1336 1337 builder.EnqueueResolver() 1338 1339 // enqueue the regular network 1340 builder.EnqueueNetworkInit() 1341 1342 builder.AdminCommand("get-transactions", func(conf *cmd.NodeConfig) commands.AdminCommand { 1343 return storageCommands.NewGetTransactionsCommand(conf.State, conf.Storage.Payloads, conf.Storage.Collections) 1344 }) 1345 1346 // if this is an access node that supports public followers, enqueue the public network 1347 if builder.supportsObserver { 1348 builder.enqueuePublicNetworkInit() 1349 builder.enqueueRelayNetwork() 1350 } 1351 1352 builder.EnqueuePingService() 1353 1354 builder.EnqueueMetricsServerInit() 1355 1356 if err := builder.RegisterBadgerMetrics(); err != nil { 1357 return err 1358 } 1359 1360 builder.EnqueueTracer() 1361 builder.PreInit(cmd.DynamicStartPreInit) 1362 builder.ValidateRootSnapshot(badgerState.ValidRootSnapshotContainsEntityExpiryRange) 1363 1364 return nil 1365 } 1366 1367 func (builder *FlowAccessNodeBuilder) enqueueRelayNetwork() { 1368 builder.Component("relay network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1369 relayNet := relaynet.NewRelayNetwork( 1370 node.EngineRegistry, 1371 builder.AccessNodeConfig.PublicNetworkConfig.Network, 1372 node.Logger, 1373 map[channels.Channel]channels.Channel{ 1374 channels.ReceiveBlocks: channels.PublicReceiveBlocks, 1375 }, 1376 ) 1377 node.EngineRegistry = relayNet 1378 return relayNet, nil 1379 }) 1380 } 1381 1382 func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { 1383 if builder.executionDataSyncEnabled { 1384 builder.BuildExecutionSyncComponents() 1385 } 1386 1387 ingestionDependable := module.NewProxiedReadyDoneAware() 1388 builder.IndexerDependencies.Add(ingestionDependable) 1389 1390 builder. 1391 BuildConsensusFollower(). 1392 Module("collection node client", func(node *cmd.NodeConfig) error { 1393 // collection node address is optional (if not specified, collection nodes will be chosen at random) 1394 if strings.TrimSpace(builder.rpcConf.CollectionAddr) == "" { 1395 node.Logger.Info().Msg("using a dynamic collection node address") 1396 return nil 1397 } 1398 1399 node.Logger.Info(). 1400 Str("collection_node", builder.rpcConf.CollectionAddr). 1401 Msg("using the static collection node address") 1402 1403 collectionRPCConn, err := grpc.Dial( 1404 builder.rpcConf.CollectionAddr, 1405 grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(builder.rpcConf.MaxMsgSize))), 1406 grpc.WithTransportCredentials(insecure.NewCredentials()), 1407 rpcConnection.WithClientTimeoutOption(builder.rpcConf.BackendConfig.CollectionClientTimeout)) 1408 if err != nil { 1409 return err 1410 } 1411 builder.CollectionRPC = access.NewAccessAPIClient(collectionRPCConn) 1412 return nil 1413 }). 1414 Module("historical access node clients", func(node *cmd.NodeConfig) error { 1415 addrs := strings.Split(builder.rpcConf.HistoricalAccessAddrs, ",") 1416 for _, addr := range addrs { 1417 if strings.TrimSpace(addr) == "" { 1418 continue 1419 } 1420 node.Logger.Info().Str("access_nodes", addr).Msg("historical access node addresses") 1421 1422 historicalAccessRPCConn, err := grpc.Dial( 1423 addr, 1424 grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(builder.rpcConf.MaxMsgSize))), 1425 grpc.WithTransportCredentials(insecure.NewCredentials())) 1426 if err != nil { 1427 return err 1428 } 1429 builder.HistoricalAccessRPCs = append(builder.HistoricalAccessRPCs, access.NewAccessAPIClient(historicalAccessRPCConn)) 1430 } 1431 return nil 1432 }). 1433 Module("transaction timing mempools", func(node *cmd.NodeConfig) error { 1434 var err error 1435 builder.TransactionTimings, err = stdmap.NewTransactionTimings(1500 * 300) // assume 1500 TPS * 300 seconds 1436 if err != nil { 1437 return err 1438 } 1439 1440 builder.CollectionsToMarkFinalized, err = stdmap.NewTimes(50 * 300) // assume 50 collection nodes * 300 seconds 1441 if err != nil { 1442 return err 1443 } 1444 1445 builder.CollectionsToMarkExecuted, err = stdmap.NewTimes(50 * 300) // assume 50 collection nodes * 300 seconds 1446 if err != nil { 1447 return err 1448 } 1449 1450 builder.BlocksToMarkExecuted, err = stdmap.NewTimes(1 * 300) // assume 1 block per second * 300 seconds 1451 1452 return err 1453 }). 1454 Module("transaction metrics", func(node *cmd.NodeConfig) error { 1455 builder.TransactionMetrics = metrics.NewTransactionCollector( 1456 node.Logger, 1457 builder.TransactionTimings, 1458 builder.logTxTimeToFinalized, 1459 builder.logTxTimeToExecuted, 1460 builder.logTxTimeToFinalizedExecuted, 1461 ) 1462 return nil 1463 }). 1464 Module("rest metrics", func(node *cmd.NodeConfig) error { 1465 m, err := metrics.NewRestCollector(routes.URLToRoute, node.MetricsRegisterer) 1466 if err != nil { 1467 return err 1468 } 1469 builder.RestMetrics = m 1470 return nil 1471 }). 1472 Module("access metrics", func(node *cmd.NodeConfig) error { 1473 builder.AccessMetrics = metrics.NewAccessCollector( 1474 metrics.WithTransactionMetrics(builder.TransactionMetrics), 1475 metrics.WithBackendScriptsMetrics(builder.TransactionMetrics), 1476 metrics.WithRestMetrics(builder.RestMetrics), 1477 ) 1478 return nil 1479 }). 1480 Module("collection metrics", func(node *cmd.NodeConfig) error { 1481 var err error 1482 builder.collectionExecutedMetric, err = indexer.NewCollectionExecutedMetricImpl( 1483 builder.Logger, 1484 builder.AccessMetrics, 1485 builder.CollectionsToMarkFinalized, 1486 builder.CollectionsToMarkExecuted, 1487 builder.BlocksToMarkExecuted, 1488 builder.Storage.Collections, 1489 builder.Storage.Blocks, 1490 ) 1491 if err != nil { 1492 return err 1493 } 1494 1495 return nil 1496 }). 1497 Module("ping metrics", func(node *cmd.NodeConfig) error { 1498 builder.PingMetrics = metrics.NewPingCollector() 1499 return nil 1500 }). 1501 Module("server certificate", func(node *cmd.NodeConfig) error { 1502 // generate the server certificate that will be served by the GRPC server 1503 x509Certificate, err := grpcutils.X509Certificate(node.NetworkKey) 1504 if err != nil { 1505 return err 1506 } 1507 tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate) 1508 builder.rpcConf.TransportCredentials = credentials.NewTLS(tlsConfig) 1509 return nil 1510 }). 1511 Module("creating grpc servers", func(node *cmd.NodeConfig) error { 1512 builder.secureGrpcServer = grpcserver.NewGrpcServerBuilder( 1513 node.Logger, 1514 builder.rpcConf.SecureGRPCListenAddr, 1515 builder.rpcConf.MaxMsgSize, 1516 builder.rpcMetricsEnabled, 1517 builder.apiRatelimits, 1518 builder.apiBurstlimits, 1519 grpcserver.WithTransportCredentials(builder.rpcConf.TransportCredentials)).Build() 1520 1521 builder.stateStreamGrpcServer = grpcserver.NewGrpcServerBuilder( 1522 node.Logger, 1523 builder.stateStreamConf.ListenAddr, 1524 builder.stateStreamConf.MaxExecutionDataMsgSize, 1525 builder.rpcMetricsEnabled, 1526 builder.apiRatelimits, 1527 builder.apiBurstlimits, 1528 grpcserver.WithStreamInterceptor()).Build() 1529 1530 if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { 1531 builder.unsecureGrpcServer = grpcserver.NewGrpcServerBuilder(node.Logger, 1532 builder.rpcConf.UnsecureGRPCListenAddr, 1533 builder.rpcConf.MaxMsgSize, 1534 builder.rpcMetricsEnabled, 1535 builder.apiRatelimits, 1536 builder.apiBurstlimits).Build() 1537 } else { 1538 builder.unsecureGrpcServer = builder.stateStreamGrpcServer 1539 } 1540 1541 return nil 1542 }). 1543 Module("backend script executor", func(node *cmd.NodeConfig) error { 1544 builder.ScriptExecutor = backend.NewScriptExecutor(builder.Logger, builder.scriptExecMinBlock, builder.scriptExecMaxBlock) 1545 return nil 1546 }). 1547 Module("async register store", func(node *cmd.NodeConfig) error { 1548 builder.RegistersAsyncStore = execution.NewRegistersAsyncStore() 1549 return nil 1550 }). 1551 Module("events storage", func(node *cmd.NodeConfig) error { 1552 builder.Storage.Events = bstorage.NewEvents(node.Metrics.Cache, node.DB) 1553 return nil 1554 }). 1555 Module("events index", func(node *cmd.NodeConfig) error { 1556 builder.EventsIndex = backend.NewEventsIndex(builder.Storage.Events) 1557 return nil 1558 }). 1559 Module("transaction result index", func(node *cmd.NodeConfig) error { 1560 builder.TxResultsIndex = backend.NewTransactionResultsIndex(builder.Storage.LightTransactionResults) 1561 return nil 1562 }). 1563 Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1564 config := builder.rpcConf 1565 backendConfig := config.BackendConfig 1566 accessMetrics := builder.AccessMetrics 1567 cacheSize := int(backendConfig.ConnectionPoolSize) 1568 1569 var connBackendCache *rpcConnection.Cache 1570 var err error 1571 if cacheSize > 0 { 1572 connBackendCache, err = rpcConnection.NewCache(node.Logger, accessMetrics, cacheSize) 1573 if err != nil { 1574 return nil, fmt.Errorf("could not initialize connection cache: %w", err) 1575 } 1576 } 1577 1578 connFactory := &rpcConnection.ConnectionFactoryImpl{ 1579 CollectionGRPCPort: builder.collectionGRPCPort, 1580 ExecutionGRPCPort: builder.executionGRPCPort, 1581 CollectionNodeGRPCTimeout: backendConfig.CollectionClientTimeout, 1582 ExecutionNodeGRPCTimeout: backendConfig.ExecutionClientTimeout, 1583 AccessMetrics: accessMetrics, 1584 Log: node.Logger, 1585 Manager: rpcConnection.NewManager( 1586 node.Logger, 1587 accessMetrics, 1588 connBackendCache, 1589 config.MaxMsgSize, 1590 backendConfig.CircuitBreakerConfig, 1591 config.CompressorName, 1592 ), 1593 } 1594 1595 scriptExecMode, err := backend.ParseIndexQueryMode(config.BackendConfig.ScriptExecutionMode) 1596 if err != nil { 1597 return nil, fmt.Errorf("could not parse script execution mode: %w", err) 1598 } 1599 1600 eventQueryMode, err := backend.ParseIndexQueryMode(config.BackendConfig.EventQueryMode) 1601 if err != nil { 1602 return nil, fmt.Errorf("could not parse event query mode: %w", err) 1603 } 1604 if eventQueryMode == backend.IndexQueryModeCompare { 1605 return nil, fmt.Errorf("event query mode 'compare' is not supported") 1606 } 1607 1608 txResultQueryMode, err := backend.ParseIndexQueryMode(config.BackendConfig.TxResultQueryMode) 1609 if err != nil { 1610 return nil, fmt.Errorf("could not parse transaction result query mode: %w", err) 1611 } 1612 if txResultQueryMode == backend.IndexQueryModeCompare { 1613 return nil, fmt.Errorf("transaction result query mode 'compare' is not supported") 1614 } 1615 1616 nodeBackend, err := backend.New(backend.Params{ 1617 State: node.State, 1618 CollectionRPC: builder.CollectionRPC, 1619 HistoricalAccessNodes: builder.HistoricalAccessRPCs, 1620 Blocks: node.Storage.Blocks, 1621 Headers: node.Storage.Headers, 1622 Collections: node.Storage.Collections, 1623 Transactions: node.Storage.Transactions, 1624 ExecutionReceipts: node.Storage.Receipts, 1625 ExecutionResults: node.Storage.Results, 1626 ChainID: node.RootChainID, 1627 AccessMetrics: builder.AccessMetrics, 1628 ConnFactory: connFactory, 1629 RetryEnabled: builder.retryEnabled, 1630 MaxHeightRange: backendConfig.MaxHeightRange, 1631 PreferredExecutionNodeIDs: backendConfig.PreferredExecutionNodeIDs, 1632 FixedExecutionNodeIDs: backendConfig.FixedExecutionNodeIDs, 1633 Log: node.Logger, 1634 SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, 1635 Communicator: backend.NewNodeCommunicator(backendConfig.CircuitBreakerConfig.Enabled), 1636 TxResultCacheSize: builder.TxResultCacheSize, 1637 TxErrorMessagesCacheSize: builder.TxErrorMessagesCacheSize, 1638 ScriptExecutor: builder.ScriptExecutor, 1639 ScriptExecutionMode: scriptExecMode, 1640 EventQueryMode: eventQueryMode, 1641 EventsIndex: builder.EventsIndex, 1642 TxResultQueryMode: txResultQueryMode, 1643 TxResultsIndex: builder.TxResultsIndex, 1644 }) 1645 if err != nil { 1646 return nil, fmt.Errorf("could not initialize backend: %w", err) 1647 } 1648 1649 engineBuilder, err := rpc.NewBuilder( 1650 node.Logger, 1651 node.State, 1652 config, 1653 node.RootChainID, 1654 builder.AccessMetrics, 1655 builder.rpcMetricsEnabled, 1656 builder.Me, 1657 nodeBackend, 1658 nodeBackend, 1659 builder.secureGrpcServer, 1660 builder.unsecureGrpcServer, 1661 builder.stateStreamBackend, 1662 builder.stateStreamConf, 1663 ) 1664 if err != nil { 1665 return nil, err 1666 } 1667 1668 builder.RpcEng, err = engineBuilder. 1669 WithLegacy(). 1670 WithBlockSignerDecoder(signature.NewBlockSignerDecoder(builder.Committee)). 1671 Build() 1672 if err != nil { 1673 return nil, err 1674 } 1675 builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.RpcEng.OnFinalizedBlock) 1676 1677 return builder.RpcEng, nil 1678 }). 1679 Component("ingestion engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1680 var err error 1681 1682 builder.RequestEng, err = requester.New( 1683 node.Logger, 1684 node.Metrics.Engine, 1685 node.EngineRegistry, 1686 node.Me, 1687 node.State, 1688 channels.RequestCollections, 1689 filter.HasRole(flow.RoleCollection), 1690 func() flow.Entity { return &flow.Collection{} }, 1691 ) 1692 if err != nil { 1693 return nil, fmt.Errorf("could not create requester engine: %w", err) 1694 } 1695 1696 builder.IngestEng, err = ingestion.New( 1697 node.Logger, 1698 node.EngineRegistry, 1699 node.State, 1700 node.Me, 1701 builder.RequestEng, 1702 node.Storage.Blocks, 1703 node.Storage.Headers, 1704 node.Storage.Collections, 1705 node.Storage.Transactions, 1706 node.Storage.Results, 1707 node.Storage.Receipts, 1708 builder.collectionExecutedMetric, 1709 ) 1710 if err != nil { 1711 return nil, err 1712 } 1713 ingestionDependable.Init(builder.IngestEng) 1714 builder.RequestEng.WithHandle(builder.IngestEng.OnCollection) 1715 builder.FollowerDistributor.AddOnBlockFinalizedConsumer(builder.IngestEng.OnFinalizedBlock) 1716 1717 return builder.IngestEng, nil 1718 }). 1719 Component("requester engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1720 // We initialize the requester engine inside the ingestion engine due to the mutual dependency. However, in 1721 // order for it to properly start and shut down, we should still return it as its own engine here, so it can 1722 // be handled by the scaffold. 1723 return builder.RequestEng, nil 1724 }) 1725 1726 if builder.supportsObserver { 1727 builder.Component("public sync request handler", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1728 syncRequestHandler, err := synceng.NewRequestHandlerEngine( 1729 node.Logger.With().Bool("public", true).Logger(), 1730 unstaked.NewUnstakedEngineCollector(node.Metrics.Engine), 1731 builder.AccessNodeConfig.PublicNetworkConfig.Network, 1732 node.Me, 1733 node.State, 1734 node.Storage.Blocks, 1735 builder.SyncCore, 1736 ) 1737 if err != nil { 1738 return nil, fmt.Errorf("could not create public sync request handler: %w", err) 1739 } 1740 builder.FollowerDistributor.AddFinalizationConsumer(syncRequestHandler) 1741 1742 return syncRequestHandler, nil 1743 }) 1744 } 1745 1746 builder.Component("secure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1747 return builder.secureGrpcServer, nil 1748 }) 1749 1750 builder.Component("state stream unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1751 return builder.stateStreamGrpcServer, nil 1752 }) 1753 1754 if builder.rpcConf.UnsecureGRPCListenAddr != builder.stateStreamConf.ListenAddr { 1755 builder.Component("unsecure grpc server", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1756 return builder.unsecureGrpcServer, nil 1757 }) 1758 } 1759 1760 builder.Component("ping engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1761 ping, err := pingeng.New( 1762 node.Logger, 1763 node.IdentityProvider, 1764 node.IDTranslator, 1765 node.Me, 1766 builder.PingMetrics, 1767 builder.pingEnabled, 1768 builder.nodeInfoFile, 1769 node.PingService, 1770 ) 1771 if err != nil { 1772 return nil, fmt.Errorf("could not create ping engine: %w", err) 1773 } 1774 1775 return ping, nil 1776 }) 1777 1778 return builder.FlowNodeBuilder.Build() 1779 } 1780 1781 // enqueuePublicNetworkInit enqueues the public network component initialized for the staked node 1782 func (builder *FlowAccessNodeBuilder) enqueuePublicNetworkInit() { 1783 var publicLibp2pNode p2p.LibP2PNode 1784 builder. 1785 Module("public network metrics", func(node *cmd.NodeConfig) error { 1786 builder.PublicNetworkConfig.Metrics = metrics.NewNetworkCollector(builder.Logger, metrics.WithNetworkPrefix("public")) 1787 return nil 1788 }). 1789 Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1790 var err error 1791 publicLibp2pNode, err = builder.initPublicLibp2pNode( 1792 builder.NodeConfig.NetworkKey, 1793 builder.PublicNetworkConfig.BindAddress, 1794 builder.PublicNetworkConfig.Metrics) 1795 if err != nil { 1796 return nil, fmt.Errorf("could not create public libp2p node: %w", err) 1797 } 1798 1799 return publicLibp2pNode, nil 1800 }). 1801 Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1802 msgValidators := publicNetworkMsgValidators(node.Logger.With().Bool("public", true).Logger(), node.IdentityProvider, builder.NodeID) 1803 receiveCache := netcache.NewHeroReceiveCache(builder.FlowConfig.NetworkConfig.NetworkReceivedMessageCacheSize, 1804 builder.Logger, 1805 metrics.NetworkReceiveCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork)) 1806 1807 err := node.Metrics.Mempool.Register(metrics.PrependPublicPrefix(metrics.ResourceNetworkingReceiveCache), receiveCache.Size) 1808 if err != nil { 1809 return nil, fmt.Errorf("could not register networking receive cache metric: %w", err) 1810 } 1811 1812 net, err := underlay.NewNetwork(&underlay.NetworkConfig{ 1813 Logger: builder.Logger.With().Str("module", "public-network").Logger(), 1814 Libp2pNode: publicLibp2pNode, 1815 Codec: cborcodec.NewCodec(), 1816 Me: builder.Me, 1817 Topology: topology.EmptyTopology{}, // topology returns empty list since peers are not known upfront 1818 Metrics: builder.PublicNetworkConfig.Metrics, 1819 BitSwapMetrics: builder.Metrics.Bitswap, 1820 IdentityProvider: builder.IdentityProvider, 1821 ReceiveCache: receiveCache, 1822 ConduitFactory: conduit.NewDefaultConduitFactory(), 1823 SporkId: builder.SporkID, 1824 UnicastMessageTimeout: underlay.DefaultUnicastTimeout, 1825 IdentityTranslator: builder.IDTranslator, 1826 AlspCfg: &alspmgr.MisbehaviorReportManagerConfig{ 1827 Logger: builder.Logger, 1828 SpamRecordCacheSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamRecordCacheSize, 1829 SpamReportQueueSize: builder.FlowConfig.NetworkConfig.AlspConfig.SpamReportQueueSize, 1830 DisablePenalty: builder.FlowConfig.NetworkConfig.AlspConfig.DisablePenalty, 1831 HeartBeatInterval: builder.FlowConfig.NetworkConfig.AlspConfig.HearBeatInterval, 1832 AlspMetrics: builder.Metrics.Network, 1833 NetworkType: network.PublicNetwork, 1834 HeroCacheMetricsFactory: builder.HeroCacheMetricsFactory(), 1835 }, 1836 SlashingViolationConsumerFactory: func(adapter network.ConduitAdapter) network.ViolationsConsumer { 1837 return slashing.NewSlashingViolationsConsumer(builder.Logger, builder.Metrics.Network, adapter) 1838 }, 1839 }, underlay.WithMessageValidators(msgValidators...)) 1840 if err != nil { 1841 return nil, fmt.Errorf("could not initialize network: %w", err) 1842 } 1843 1844 builder.NetworkUnderlay = net 1845 builder.AccessNodeConfig.PublicNetworkConfig.Network = net 1846 1847 node.Logger.Info().Msgf("network will run on address: %s", builder.PublicNetworkConfig.BindAddress) 1848 return net, nil 1849 }). 1850 Component("public peer manager", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { 1851 return publicLibp2pNode.PeerManagerComponent(), nil 1852 }) 1853 } 1854 1855 // initPublicLibp2pNode initializes the public libp2p node for the public (unstaked) network. 1856 // The LibP2P host is created with the following options: 1857 // - DHT as server 1858 // - The address from the node config or the specified bind address as the listen address 1859 // - The passed in private key as the libp2p key 1860 // - No connection gater 1861 // - Default Flow libp2p pubsub options 1862 // 1863 // Args: 1864 // - networkKey: The private key to use for the libp2p node 1865 // 1866 // - bindAddress: The address to bind the libp2p node to. 1867 // - networkMetrics: The metrics collector for the network 1868 // Returns: 1869 // - The libp2p node instance for the public network. 1870 // - Any error encountered during initialization. Any error should be considered fatal. 1871 func (builder *FlowAccessNodeBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) (p2p.LibP2PNode, 1872 error, 1873 ) { 1874 connManager, err := connection.NewConnManager(builder.Logger, networkMetrics, &builder.FlowConfig.NetworkConfig.ConnectionManager) 1875 if err != nil { 1876 return nil, fmt.Errorf("could not create connection manager: %w", err) 1877 } 1878 1879 libp2pNode, err := p2pbuilder.NewNodeBuilder(builder.Logger, &builder.FlowConfig.NetworkConfig.GossipSub, &p2pbuilderconfig.MetricsConfig{ 1880 HeroCacheFactory: builder.HeroCacheMetricsFactory(), 1881 Metrics: networkMetrics, 1882 }, 1883 network.PublicNetwork, 1884 bindAddress, 1885 networkKey, 1886 builder.SporkID, 1887 builder.IdentityProvider, 1888 &builder.FlowConfig.NetworkConfig.ResourceManager, 1889 &p2pbuilderconfig.PeerManagerConfig{ 1890 // TODO: eventually, we need pruning enabled even on public network. However, it needs a modified version of 1891 // the peer manager that also operate on the public identities. 1892 ConnectionPruning: connection.PruningDisabled, 1893 UpdateInterval: builder.FlowConfig.NetworkConfig.PeerUpdateInterval, 1894 ConnectorFactory: connection.DefaultLibp2pBackoffConnectorFactory(), 1895 }, 1896 &p2p.DisallowListCacheConfig{ 1897 MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, 1898 Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), 1899 }, 1900 &p2pbuilderconfig.UnicastConfig{ 1901 Unicast: builder.FlowConfig.NetworkConfig.Unicast, 1902 }). 1903 SetBasicResolver(builder.Resolver). 1904 SetSubscriptionFilter(subscription.NewRoleBasedFilter(flow.RoleAccess, builder.IdentityProvider)). 1905 SetConnectionManager(connManager). 1906 SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { 1907 return dht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), builder.Logger, networkMetrics, dht.AsServer()) 1908 }). 1909 Build() 1910 if err != nil { 1911 return nil, fmt.Errorf("could not build libp2p node for staked access node: %w", err) 1912 } 1913 1914 return libp2pNode, nil 1915 }