github.com/koko1123/flow-go-1@v0.29.6/cmd/access/node_builder/access_node_builder.go (about)

     1  package node_builder
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"os"
     8  	"path/filepath"
     9  	"strings"
    10  	"time"
    11  
    12  	badger "github.com/ipfs/go-ds-badger2"
    13  	"github.com/libp2p/go-libp2p/core/host"
    14  	"github.com/libp2p/go-libp2p/core/routing"
    15  	"github.com/rs/zerolog"
    16  	"github.com/spf13/pflag"
    17  	"google.golang.org/grpc"
    18  	"google.golang.org/grpc/credentials"
    19  
    20  	"github.com/onflow/flow/protobuf/go/flow/access"
    21  	"github.com/onflow/go-bitswap"
    22  
    23  	"github.com/koko1123/flow-go-1/admin/commands"
    24  	stateSyncCommands "github.com/koko1123/flow-go-1/admin/commands/state_synchronization"
    25  	storageCommands "github.com/koko1123/flow-go-1/admin/commands/storage"
    26  	"github.com/koko1123/flow-go-1/cmd"
    27  	"github.com/koko1123/flow-go-1/consensus"
    28  	"github.com/koko1123/flow-go-1/consensus/hotstuff"
    29  	"github.com/koko1123/flow-go-1/consensus/hotstuff/committees"
    30  	consensuspubsub "github.com/koko1123/flow-go-1/consensus/hotstuff/notifications/pubsub"
    31  	"github.com/koko1123/flow-go-1/consensus/hotstuff/signature"
    32  	"github.com/koko1123/flow-go-1/consensus/hotstuff/verification"
    33  	recovery "github.com/koko1123/flow-go-1/consensus/recovery/protocol"
    34  	"github.com/onflow/flow-go/crypto"
    35  	"github.com/koko1123/flow-go-1/engine/access/ingestion"
    36  	pingeng "github.com/koko1123/flow-go-1/engine/access/ping"
    37  	"github.com/koko1123/flow-go-1/engine/access/rpc"
    38  	"github.com/koko1123/flow-go-1/engine/access/rpc/backend"
    39  	"github.com/koko1123/flow-go-1/engine/access/state_stream"
    40  	"github.com/koko1123/flow-go-1/engine/common/follower"
    41  	followereng "github.com/koko1123/flow-go-1/engine/common/follower"
    42  	"github.com/koko1123/flow-go-1/engine/common/requester"
    43  	synceng "github.com/koko1123/flow-go-1/engine/common/synchronization"
    44  	"github.com/koko1123/flow-go-1/model/flow"
    45  	"github.com/koko1123/flow-go-1/model/flow/filter"
    46  	"github.com/koko1123/flow-go-1/module"
    47  	"github.com/koko1123/flow-go-1/module/blobs"
    48  	"github.com/koko1123/flow-go-1/module/buffer"
    49  	"github.com/koko1123/flow-go-1/module/chainsync"
    50  	"github.com/koko1123/flow-go-1/module/compliance"
    51  	"github.com/koko1123/flow-go-1/module/executiondatasync/execution_data"
    52  	finalizer "github.com/koko1123/flow-go-1/module/finalizer/consensus"
    53  	"github.com/koko1123/flow-go-1/module/id"
    54  	"github.com/koko1123/flow-go-1/module/mempool/stdmap"
    55  	"github.com/koko1123/flow-go-1/module/metrics"
    56  	"github.com/koko1123/flow-go-1/module/metrics/unstaked"
    57  	"github.com/koko1123/flow-go-1/module/state_synchronization"
    58  	edrequester "github.com/koko1123/flow-go-1/module/state_synchronization/requester"
    59  	"github.com/koko1123/flow-go-1/network"
    60  	netcache "github.com/koko1123/flow-go-1/network/cache"
    61  	"github.com/koko1123/flow-go-1/network/channels"
    62  	cborcodec "github.com/koko1123/flow-go-1/network/codec/cbor"
    63  	"github.com/koko1123/flow-go-1/network/p2p"
    64  	"github.com/koko1123/flow-go-1/network/p2p/blob"
    65  	"github.com/koko1123/flow-go-1/network/p2p/cache"
    66  	"github.com/koko1123/flow-go-1/network/p2p/connection"
    67  	"github.com/koko1123/flow-go-1/network/p2p/dht"
    68  	"github.com/koko1123/flow-go-1/network/p2p/middleware"
    69  	"github.com/koko1123/flow-go-1/network/p2p/p2pbuilder"
    70  	"github.com/koko1123/flow-go-1/network/p2p/subscription"
    71  	"github.com/koko1123/flow-go-1/network/p2p/translator"
    72  	"github.com/koko1123/flow-go-1/network/p2p/unicast"
    73  	relaynet "github.com/koko1123/flow-go-1/network/relay"
    74  	"github.com/koko1123/flow-go-1/network/slashing"
    75  	"github.com/koko1123/flow-go-1/network/topology"
    76  	"github.com/koko1123/flow-go-1/network/validator"
    77  	"github.com/koko1123/flow-go-1/state/protocol"
    78  	badgerState "github.com/koko1123/flow-go-1/state/protocol/badger"
    79  	"github.com/koko1123/flow-go-1/state/protocol/blocktimer"
    80  	"github.com/koko1123/flow-go-1/storage"
    81  	bstorage "github.com/koko1123/flow-go-1/storage/badger"
    82  	"github.com/koko1123/flow-go-1/utils/grpcutils"
    83  )
    84  
    85  // AccessNodeBuilder extends cmd.NodeBuilder and declares additional functions needed to bootstrap an Access node.
    86  // The private network allows the staked nodes to communicate among themselves, while the public network allows the
    87  // Observers and an Access node to communicate.
    88  //
    89  //                                 public network                           private network
    90  //  +------------------------+
    91  //  | Observer             1 |<--------------------------|
    92  //  +------------------------+                           v
    93  //  +------------------------+                         +--------------------+                 +------------------------+
    94  //  | Observer             2 |<----------------------->| Staked Access Node |<--------------->| All other staked Nodes |
    95  //  +------------------------+                         +--------------------+                 +------------------------+
    96  //  +------------------------+                           ^
    97  //  | Observer             3 |<--------------------------|
    98  //  +------------------------+
    99  
   100  // AccessNodeConfig defines all the user defined parameters required to bootstrap an access node
   101  // For a node running as a standalone process, the config fields will be populated from the command line params,
   102  // while for a node running as a library, the config fields are expected to be initialized by the caller.
   103  type AccessNodeConfig struct {
   104  	supportsObserver             bool // True if this is an Access node that supports observers and consensus follower engines
   105  	collectionGRPCPort           uint
   106  	executionGRPCPort            uint
   107  	pingEnabled                  bool
   108  	nodeInfoFile                 string
   109  	apiRatelimits                map[string]int
   110  	apiBurstlimits               map[string]int
   111  	rpcConf                      rpc.Config
   112  	ExecutionNodeAddress         string // deprecated
   113  	HistoricalAccessRPCs         []access.AccessAPIClient
   114  	logTxTimeToFinalized         bool
   115  	logTxTimeToExecuted          bool
   116  	logTxTimeToFinalizedExecuted bool
   117  	retryEnabled                 bool
   118  	rpcMetricsEnabled            bool
   119  	executionDataSyncEnabled     bool
   120  	executionDataDir             string
   121  	executionDataStartHeight     uint64
   122  	executionDataConfig          edrequester.ExecutionDataConfig
   123  	PublicNetworkConfig          PublicNetworkConfig
   124  }
   125  
   126  type PublicNetworkConfig struct {
   127  	// NetworkKey crypto.PublicKey // TODO: do we need a different key for the public network?
   128  	BindAddress string
   129  	Network     network.Network
   130  	Metrics     module.NetworkMetrics
   131  }
   132  
   133  // DefaultAccessNodeConfig defines all the default values for the AccessNodeConfig
   134  func DefaultAccessNodeConfig() *AccessNodeConfig {
   135  	homedir, _ := os.UserHomeDir()
   136  	return &AccessNodeConfig{
   137  		supportsObserver:   false,
   138  		collectionGRPCPort: 9000,
   139  		executionGRPCPort:  9000,
   140  		rpcConf: rpc.Config{
   141  			UnsecureGRPCListenAddr:    "0.0.0.0:9000",
   142  			SecureGRPCListenAddr:      "0.0.0.0:9001",
   143  			StateStreamListenAddr:     "",
   144  			HTTPListenAddr:            "0.0.0.0:8000",
   145  			RESTListenAddr:            "",
   146  			CollectionAddr:            "",
   147  			HistoricalAccessAddrs:     "",
   148  			CollectionClientTimeout:   3 * time.Second,
   149  			ExecutionClientTimeout:    3 * time.Second,
   150  			ConnectionPoolSize:        backend.DefaultConnectionPoolSize,
   151  			MaxHeightRange:            backend.DefaultMaxHeightRange,
   152  			PreferredExecutionNodeIDs: nil,
   153  			FixedExecutionNodeIDs:     nil,
   154  			MaxExecutionDataMsgSize:   grpcutils.DefaultMaxMsgSize,
   155  		},
   156  		ExecutionNodeAddress:         "localhost:9000",
   157  		logTxTimeToFinalized:         false,
   158  		logTxTimeToExecuted:          false,
   159  		logTxTimeToFinalizedExecuted: false,
   160  		pingEnabled:                  false,
   161  		retryEnabled:                 false,
   162  		rpcMetricsEnabled:            false,
   163  		nodeInfoFile:                 "",
   164  		apiRatelimits:                nil,
   165  		apiBurstlimits:               nil,
   166  		PublicNetworkConfig: PublicNetworkConfig{
   167  			BindAddress: cmd.NotSet,
   168  			Metrics:     metrics.NewNoopCollector(),
   169  		},
   170  		executionDataSyncEnabled: false,
   171  		executionDataDir:         filepath.Join(homedir, ".flow", "execution_data"),
   172  		executionDataStartHeight: 0,
   173  		executionDataConfig: edrequester.ExecutionDataConfig{
   174  			InitialBlockHeight: 0,
   175  			MaxSearchAhead:     edrequester.DefaultMaxSearchAhead,
   176  			FetchTimeout:       edrequester.DefaultFetchTimeout,
   177  			MaxFetchTimeout:    edrequester.DefaultMaxFetchTimeout,
   178  			RetryDelay:         edrequester.DefaultRetryDelay,
   179  			MaxRetryDelay:      edrequester.DefaultMaxRetryDelay,
   180  		},
   181  	}
   182  }
   183  
   184  // FlowAccessNodeBuilder provides the common functionality needed to bootstrap a Flow access node
   185  // It is composed of the FlowNodeBuilder, the AccessNodeConfig and contains all the components and modules needed for the
   186  // access nodes
   187  type FlowAccessNodeBuilder struct {
   188  	*cmd.FlowNodeBuilder
   189  	*AccessNodeConfig
   190  
   191  	// components
   192  	FollowerState              protocol.MutableState
   193  	SyncCore                   *chainsync.Core
   194  	RpcEng                     *rpc.Engine
   195  	FinalizationDistributor    *consensuspubsub.FinalizationDistributor
   196  	FinalizedHeader            *synceng.FinalizedHeaderCache
   197  	CollectionRPC              access.AccessAPIClient
   198  	TransactionTimings         *stdmap.TransactionTimings
   199  	CollectionsToMarkFinalized *stdmap.Times
   200  	CollectionsToMarkExecuted  *stdmap.Times
   201  	BlocksToMarkExecuted       *stdmap.Times
   202  	TransactionMetrics         module.TransactionMetrics
   203  	AccessMetrics              module.AccessMetrics
   204  	PingMetrics                module.PingMetrics
   205  	Committee                  hotstuff.Committee
   206  	Finalized                  *flow.Header
   207  	Pending                    []*flow.Header
   208  	FollowerCore               module.HotStuffFollower
   209  	ExecutionDataDownloader    execution_data.Downloader
   210  	ExecutionDataRequester     state_synchronization.ExecutionDataRequester
   211  	ExecutionDataStore         execution_data.ExecutionDataStore
   212  
   213  	// The sync engine participants provider is the libp2p peer store for the access node
   214  	// which is not available until after the network has started.
   215  	// Hence, a factory function that needs to be called just before creating the sync engine
   216  	SyncEngineParticipantsProviderFactory func() module.IdentifierProvider
   217  
   218  	// engines
   219  	IngestEng      *ingestion.Engine
   220  	RequestEng     *requester.Engine
   221  	FollowerEng    *followereng.Engine
   222  	SyncEng        *synceng.Engine
   223  	StateStreamEng *state_stream.Engine
   224  }
   225  
   226  func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder {
   227  	builder.Module("mutable follower state", func(node *cmd.NodeConfig) error {
   228  		// For now, we only support state implementations from package badger.
   229  		// If we ever support different implementations, the following can be replaced by a type-aware factory
   230  		state, ok := node.State.(*badgerState.State)
   231  		if !ok {
   232  			return fmt.Errorf("only implementations of type badger.State are currently supported but read-only state has type %T", node.State)
   233  		}
   234  
   235  		followerState, err := badgerState.NewFollowerState(
   236  			state,
   237  			node.Storage.Index,
   238  			node.Storage.Payloads,
   239  			node.Tracer,
   240  			node.ProtocolEvents,
   241  			blocktimer.DefaultBlockTimer,
   242  		)
   243  		builder.FollowerState = followerState
   244  
   245  		return err
   246  	})
   247  
   248  	return builder
   249  }
   250  
   251  func (builder *FlowAccessNodeBuilder) buildSyncCore() *FlowAccessNodeBuilder {
   252  	builder.Module("sync core", func(node *cmd.NodeConfig) error {
   253  		syncCore, err := chainsync.New(node.Logger, node.SyncCoreConfig, metrics.NewChainSyncCollector())
   254  		builder.SyncCore = syncCore
   255  
   256  		return err
   257  	})
   258  
   259  	return builder
   260  }
   261  
   262  func (builder *FlowAccessNodeBuilder) buildCommittee() *FlowAccessNodeBuilder {
   263  	builder.Module("committee", func(node *cmd.NodeConfig) error {
   264  		// initialize consensus committee's membership state
   265  		// This committee state is for the HotStuff follower, which follows the MAIN CONSENSUS Committee
   266  		// Note: node.Me.NodeID() is not part of the consensus committee
   267  		committee, err := committees.NewConsensusCommittee(node.State, node.Me.NodeID())
   268  		builder.Committee = committee
   269  
   270  		return err
   271  	})
   272  
   273  	return builder
   274  }
   275  
   276  func (builder *FlowAccessNodeBuilder) buildLatestHeader() *FlowAccessNodeBuilder {
   277  	builder.Module("latest header", func(node *cmd.NodeConfig) error {
   278  		finalized, pending, err := recovery.FindLatest(node.State, node.Storage.Headers)
   279  		builder.Finalized, builder.Pending = finalized, pending
   280  
   281  		return err
   282  	})
   283  
   284  	return builder
   285  }
   286  
   287  func (builder *FlowAccessNodeBuilder) buildFollowerCore() *FlowAccessNodeBuilder {
   288  	builder.Component("follower core", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   289  		// create a finalizer that will handle updating the protocol
   290  		// state when the follower detects newly finalized blocks
   291  		final := finalizer.NewFinalizer(node.DB, node.Storage.Headers, builder.FollowerState, node.Tracer)
   292  
   293  		packer := signature.NewConsensusSigDataPacker(builder.Committee)
   294  		// initialize the verifier for the protocol consensus
   295  		verifier := verification.NewCombinedVerifier(builder.Committee, packer)
   296  
   297  		followerCore, err := consensus.NewFollower(
   298  			node.Logger,
   299  			builder.Committee,
   300  			node.Storage.Headers,
   301  			final,
   302  			verifier,
   303  			builder.FinalizationDistributor,
   304  			node.RootBlock.Header,
   305  			node.RootQC,
   306  			builder.Finalized,
   307  			builder.Pending,
   308  		)
   309  		if err != nil {
   310  			return nil, fmt.Errorf("could not initialize follower core: %w", err)
   311  		}
   312  		builder.FollowerCore = followerCore
   313  
   314  		return builder.FollowerCore, nil
   315  	})
   316  
   317  	return builder
   318  }
   319  
   320  func (builder *FlowAccessNodeBuilder) buildFollowerEngine() *FlowAccessNodeBuilder {
   321  	builder.Component("follower engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   322  		// initialize cleaner for DB
   323  		cleaner := bstorage.NewCleaner(node.Logger, node.DB, builder.Metrics.CleanCollector, flow.DefaultValueLogGCFrequency)
   324  		conCache := buffer.NewPendingBlocks()
   325  
   326  		followerEng, err := follower.New(
   327  			node.Logger,
   328  			node.Network,
   329  			node.Me,
   330  			node.Metrics.Engine,
   331  			node.Metrics.Mempool,
   332  			cleaner,
   333  			node.Storage.Headers,
   334  			node.Storage.Payloads,
   335  			builder.FollowerState,
   336  			conCache,
   337  			builder.FollowerCore,
   338  			builder.SyncCore,
   339  			node.Tracer,
   340  			follower.WithComplianceOptions(compliance.WithSkipNewProposalsThreshold(builder.ComplianceConfig.SkipNewProposalsThreshold)),
   341  		)
   342  		if err != nil {
   343  			return nil, fmt.Errorf("could not create follower engine: %w", err)
   344  		}
   345  		builder.FollowerEng = followerEng
   346  
   347  		return builder.FollowerEng, nil
   348  	})
   349  
   350  	return builder
   351  }
   352  
   353  func (builder *FlowAccessNodeBuilder) buildFinalizedHeader() *FlowAccessNodeBuilder {
   354  	builder.Component("finalized snapshot", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   355  		finalizedHeader, err := synceng.NewFinalizedHeaderCache(node.Logger, node.State, builder.FinalizationDistributor)
   356  		if err != nil {
   357  			return nil, fmt.Errorf("could not create finalized snapshot cache: %w", err)
   358  		}
   359  		builder.FinalizedHeader = finalizedHeader
   360  
   361  		return builder.FinalizedHeader, nil
   362  	})
   363  
   364  	return builder
   365  }
   366  
   367  func (builder *FlowAccessNodeBuilder) buildSyncEngine() *FlowAccessNodeBuilder {
   368  	builder.Component("sync engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   369  		sync, err := synceng.New(
   370  			node.Logger,
   371  			node.Metrics.Engine,
   372  			node.Network,
   373  			node.Me,
   374  			node.Storage.Blocks,
   375  			builder.FollowerEng,
   376  			builder.SyncCore,
   377  			builder.FinalizedHeader,
   378  			builder.SyncEngineParticipantsProviderFactory(),
   379  		)
   380  		if err != nil {
   381  			return nil, fmt.Errorf("could not create synchronization engine: %w", err)
   382  		}
   383  		builder.SyncEng = sync
   384  
   385  		return builder.SyncEng, nil
   386  	})
   387  
   388  	return builder
   389  }
   390  
   391  func (builder *FlowAccessNodeBuilder) BuildConsensusFollower() *FlowAccessNodeBuilder {
   392  	builder.
   393  		buildFollowerState().
   394  		buildSyncCore().
   395  		buildCommittee().
   396  		buildLatestHeader().
   397  		buildFollowerCore().
   398  		buildFollowerEngine().
   399  		buildFinalizedHeader().
   400  		buildSyncEngine()
   401  
   402  	return builder
   403  }
   404  
   405  func (builder *FlowAccessNodeBuilder) BuildExecutionDataRequester() *FlowAccessNodeBuilder {
   406  	var ds *badger.Datastore
   407  	var bs network.BlobService
   408  	var processedBlockHeight storage.ConsumerProgress
   409  	var processedNotifications storage.ConsumerProgress
   410  	var bsDependable *module.ProxiedReadyDoneAware
   411  
   412  	builder.
   413  		AdminCommand("read-execution-data", func(config *cmd.NodeConfig) commands.AdminCommand {
   414  			return stateSyncCommands.NewReadExecutionDataCommand(builder.ExecutionDataStore)
   415  		}).
   416  		Module("execution data datastore and blobstore", func(node *cmd.NodeConfig) error {
   417  			datastoreDir := filepath.Join(builder.executionDataDir, "blobstore")
   418  			err := os.MkdirAll(datastoreDir, 0700)
   419  			if err != nil {
   420  				return err
   421  			}
   422  
   423  			ds, err = badger.NewDatastore(datastoreDir, &badger.DefaultOptions)
   424  			if err != nil {
   425  				return err
   426  			}
   427  
   428  			builder.ShutdownFunc(func() error {
   429  				if err := ds.Close(); err != nil {
   430  					return fmt.Errorf("could not close execution data datastore: %w", err)
   431  				}
   432  				return nil
   433  			})
   434  
   435  			return nil
   436  		}).
   437  		Module("processed block height consumer progress", func(node *cmd.NodeConfig) error {
   438  			// uses the datastore's DB
   439  			processedBlockHeight = bstorage.NewConsumerProgress(ds.DB, module.ConsumeProgressExecutionDataRequesterBlockHeight)
   440  			return nil
   441  		}).
   442  		Module("processed notifications consumer progress", func(node *cmd.NodeConfig) error {
   443  			// uses the datastore's DB
   444  			processedNotifications = bstorage.NewConsumerProgress(ds.DB, module.ConsumeProgressExecutionDataRequesterNotification)
   445  			return nil
   446  		}).
   447  		Module("blobservice peer manager dependencies", func(node *cmd.NodeConfig) error {
   448  			bsDependable = module.NewProxiedReadyDoneAware()
   449  			builder.PeerManagerDependencies.Add(bsDependable)
   450  			return nil
   451  		}).
   452  		Module("execution datastore", func(node *cmd.NodeConfig) error {
   453  			blobstore := blobs.NewBlobstore(ds)
   454  			builder.ExecutionDataStore = execution_data.NewExecutionDataStore(blobstore, execution_data.DefaultSerializer)
   455  			return nil
   456  		}).
   457  		Component("execution data service", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   458  
   459  			opts := []network.BlobServiceOption{
   460  				blob.WithBitswapOptions(
   461  					// Only allow block requests from staked ENs and ANs
   462  					bitswap.WithPeerBlockRequestFilter(
   463  						blob.AuthorizedRequester(nil, builder.IdentityProvider, builder.Logger),
   464  					),
   465  					bitswap.WithTracer(
   466  						blob.NewTracer(node.Logger.With().Str("blob_service", channels.ExecutionDataService.String()).Logger()),
   467  					),
   468  				),
   469  			}
   470  
   471  			var err error
   472  			bs, err = node.Network.RegisterBlobService(channels.ExecutionDataService, ds, opts...)
   473  			if err != nil {
   474  				return nil, fmt.Errorf("could not register blob service: %w", err)
   475  			}
   476  
   477  			// add blobservice into ReadyDoneAware dependency passed to peer manager
   478  			// this starts the blob service and configures peer manager to wait for the blobservice
   479  			// to be ready before starting
   480  			bsDependable.Init(bs)
   481  
   482  			builder.ExecutionDataDownloader = execution_data.NewDownloader(bs)
   483  
   484  			return builder.ExecutionDataDownloader, nil
   485  		}).
   486  		Component("execution data requester", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   487  			// Validation of the start block height needs to be done after loading state
   488  			if builder.executionDataStartHeight > 0 {
   489  				if builder.executionDataStartHeight <= builder.RootBlock.Header.Height {
   490  					return nil, fmt.Errorf(
   491  						"execution data start block height (%d) must be greater than the root block height (%d)",
   492  						builder.executionDataStartHeight, builder.RootBlock.Header.Height)
   493  				}
   494  
   495  				latestSeal, err := builder.State.Sealed().Head()
   496  				if err != nil {
   497  					return nil, fmt.Errorf("failed to get latest sealed height")
   498  				}
   499  
   500  				// Note: since the root block of a spork is also sealed in the root protocol state, the
   501  				// latest sealed height is always equal to the root block height. That means that at the
   502  				// very beginning of a spork, this check will always fail. Operators should not specify
   503  				// an InitialBlockHeight when starting from the beginning of a spork.
   504  				if builder.executionDataStartHeight > latestSeal.Height {
   505  					return nil, fmt.Errorf(
   506  						"execution data start block height (%d) must be less than or equal to the latest sealed block height (%d)",
   507  						builder.executionDataStartHeight, latestSeal.Height)
   508  				}
   509  
   510  				// executionDataStartHeight is provided as the first block to sync, but the
   511  				// requester expects the initial last processed height, which is the first height - 1
   512  				builder.executionDataConfig.InitialBlockHeight = builder.executionDataStartHeight - 1
   513  			} else {
   514  				builder.executionDataConfig.InitialBlockHeight = builder.RootBlock.Header.Height
   515  			}
   516  
   517  			builder.ExecutionDataRequester = edrequester.New(
   518  				builder.Logger,
   519  				metrics.NewExecutionDataRequesterCollector(),
   520  				builder.ExecutionDataDownloader,
   521  				processedBlockHeight,
   522  				processedNotifications,
   523  				builder.State,
   524  				builder.Storage.Headers,
   525  				builder.Storage.Results,
   526  				builder.Storage.Seals,
   527  				builder.executionDataConfig,
   528  			)
   529  
   530  			builder.FinalizationDistributor.AddOnBlockFinalizedConsumer(builder.ExecutionDataRequester.OnBlockFinalized)
   531  
   532  			return builder.ExecutionDataRequester, nil
   533  		})
   534  
   535  	if builder.rpcConf.StateStreamListenAddr != "" {
   536  		builder.Component("exec state stream engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   537  			conf := state_stream.Config{
   538  				ListenAddr:              builder.rpcConf.StateStreamListenAddr,
   539  				MaxExecutionDataMsgSize: builder.rpcConf.MaxExecutionDataMsgSize,
   540  				RpcMetricsEnabled:       builder.rpcMetricsEnabled,
   541  			}
   542  
   543  			builder.StateStreamEng = state_stream.NewEng(
   544  				conf,
   545  				builder.ExecutionDataStore,
   546  				node.Storage.Headers,
   547  				node.Storage.Seals,
   548  				node.Storage.Results,
   549  				node.Logger,
   550  				node.RootChainID,
   551  				builder.apiRatelimits,
   552  				builder.apiBurstlimits,
   553  			)
   554  			return builder.StateStreamEng, nil
   555  		})
   556  	}
   557  
   558  	return builder
   559  }
   560  
   561  func FlowAccessNode(nodeBuilder *cmd.FlowNodeBuilder) *FlowAccessNodeBuilder {
   562  	return &FlowAccessNodeBuilder{
   563  		AccessNodeConfig:        DefaultAccessNodeConfig(),
   564  		FlowNodeBuilder:         nodeBuilder,
   565  		FinalizationDistributor: consensuspubsub.NewFinalizationDistributor(),
   566  	}
   567  }
   568  
   569  func (builder *FlowAccessNodeBuilder) ParseFlags() error {
   570  
   571  	builder.BaseFlags()
   572  
   573  	builder.extraFlags()
   574  
   575  	return builder.ParseAndPrintFlags()
   576  }
   577  
   578  func (builder *FlowAccessNodeBuilder) extraFlags() {
   579  	builder.ExtraFlags(func(flags *pflag.FlagSet) {
   580  		defaultConfig := DefaultAccessNodeConfig()
   581  
   582  		flags.UintVar(&builder.collectionGRPCPort, "collection-ingress-port", defaultConfig.collectionGRPCPort, "the grpc ingress port for all collection nodes")
   583  		flags.UintVar(&builder.executionGRPCPort, "execution-ingress-port", defaultConfig.executionGRPCPort, "the grpc ingress port for all execution nodes")
   584  		flags.StringVarP(&builder.rpcConf.UnsecureGRPCListenAddr, "rpc-addr", "r", defaultConfig.rpcConf.UnsecureGRPCListenAddr, "the address the unsecured gRPC server listens on")
   585  		flags.StringVar(&builder.rpcConf.SecureGRPCListenAddr, "secure-rpc-addr", defaultConfig.rpcConf.SecureGRPCListenAddr, "the address the secure gRPC server listens on")
   586  		flags.StringVar(&builder.rpcConf.StateStreamListenAddr, "state-stream-addr", defaultConfig.rpcConf.StateStreamListenAddr, "the address the state stream server listens on (if empty the server will not be started)")
   587  		flags.StringVarP(&builder.rpcConf.HTTPListenAddr, "http-addr", "h", defaultConfig.rpcConf.HTTPListenAddr, "the address the http proxy server listens on")
   588  		flags.StringVar(&builder.rpcConf.RESTListenAddr, "rest-addr", defaultConfig.rpcConf.RESTListenAddr, "the address the REST server listens on (if empty the REST server will not be started)")
   589  		flags.StringVarP(&builder.rpcConf.CollectionAddr, "static-collection-ingress-addr", "", defaultConfig.rpcConf.CollectionAddr, "the address (of the collection node) to send transactions to")
   590  		flags.StringVarP(&builder.ExecutionNodeAddress, "script-addr", "s", defaultConfig.ExecutionNodeAddress, "the address (of the execution node) forward the script to")
   591  		flags.StringVarP(&builder.rpcConf.HistoricalAccessAddrs, "historical-access-addr", "", defaultConfig.rpcConf.HistoricalAccessAddrs, "comma separated rpc addresses for historical access nodes")
   592  		flags.DurationVar(&builder.rpcConf.CollectionClientTimeout, "collection-client-timeout", defaultConfig.rpcConf.CollectionClientTimeout, "grpc client timeout for a collection node")
   593  		flags.DurationVar(&builder.rpcConf.ExecutionClientTimeout, "execution-client-timeout", defaultConfig.rpcConf.ExecutionClientTimeout, "grpc client timeout for an execution node")
   594  		flags.UintVar(&builder.rpcConf.ConnectionPoolSize, "connection-pool-size", defaultConfig.rpcConf.ConnectionPoolSize, "maximum number of connections allowed in the connection pool, size of 0 disables the connection pooling, and anything less than the default size will be overridden to use the default size")
   595  		flags.UintVar(&builder.rpcConf.MaxHeightRange, "rpc-max-height-range", defaultConfig.rpcConf.MaxHeightRange, "maximum size for height range requests")
   596  		flags.IntVar(&builder.rpcConf.MaxExecutionDataMsgSize, "max-block-msg-size", defaultConfig.rpcConf.MaxExecutionDataMsgSize, "maximum size for a gRPC message containing block execution data")
   597  		flags.StringSliceVar(&builder.rpcConf.PreferredExecutionNodeIDs, "preferred-execution-node-ids", defaultConfig.rpcConf.PreferredExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call e.g. b4a4dbdcd443d...,fb386a6a... etc.")
   598  		flags.StringSliceVar(&builder.rpcConf.FixedExecutionNodeIDs, "fixed-execution-node-ids", defaultConfig.rpcConf.FixedExecutionNodeIDs, "comma separated list of execution nodes ids to choose from when making an upstream call if no matching preferred execution id is found e.g. b4a4dbdcd443d...,fb386a6a... etc.")
   599  		flags.BoolVar(&builder.logTxTimeToFinalized, "log-tx-time-to-finalized", defaultConfig.logTxTimeToFinalized, "log transaction time to finalized")
   600  		flags.BoolVar(&builder.logTxTimeToExecuted, "log-tx-time-to-executed", defaultConfig.logTxTimeToExecuted, "log transaction time to executed")
   601  		flags.BoolVar(&builder.logTxTimeToFinalizedExecuted, "log-tx-time-to-finalized-executed", defaultConfig.logTxTimeToFinalizedExecuted, "log transaction time to finalized and executed")
   602  		flags.BoolVar(&builder.pingEnabled, "ping-enabled", defaultConfig.pingEnabled, "whether to enable the ping process that pings all other peers and report the connectivity to metrics")
   603  		flags.BoolVar(&builder.retryEnabled, "retry-enabled", defaultConfig.retryEnabled, "whether to enable the retry mechanism at the access node level")
   604  		flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics")
   605  		flags.StringVarP(&builder.nodeInfoFile, "node-info-file", "", defaultConfig.nodeInfoFile, "full path to a json file which provides more details about nodes when reporting its reachability metrics")
   606  		flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.")
   607  		flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.")
   608  		flags.BoolVar(&builder.supportsObserver, "supports-observer", defaultConfig.supportsObserver, "true if this staked access node supports observer or follower connections")
   609  		flags.StringVar(&builder.PublicNetworkConfig.BindAddress, "public-network-address", defaultConfig.PublicNetworkConfig.BindAddress, "staked access node's public network bind address")
   610  
   611  		// ExecutionDataRequester config
   612  		flags.BoolVar(&builder.executionDataSyncEnabled, "execution-data-sync-enabled", defaultConfig.executionDataSyncEnabled, "whether to enable the execution data sync protocol")
   613  		flags.StringVar(&builder.executionDataDir, "execution-data-dir", defaultConfig.executionDataDir, "directory to use for Execution Data database")
   614  		flags.Uint64Var(&builder.executionDataStartHeight, "execution-data-start-height", defaultConfig.executionDataStartHeight, "height of first block to sync execution data from when starting with an empty Execution Data database")
   615  		flags.Uint64Var(&builder.executionDataConfig.MaxSearchAhead, "execution-data-max-search-ahead", defaultConfig.executionDataConfig.MaxSearchAhead, "max number of heights to search ahead of the lowest outstanding execution data height")
   616  		flags.DurationVar(&builder.executionDataConfig.FetchTimeout, "execution-data-fetch-timeout", defaultConfig.executionDataConfig.FetchTimeout, "initial timeout to use when fetching execution data from the network. timeout increases using an incremental backoff until execution-data-max-fetch-timeout. e.g. 30s")
   617  		flags.DurationVar(&builder.executionDataConfig.MaxFetchTimeout, "execution-data-max-fetch-timeout", defaultConfig.executionDataConfig.MaxFetchTimeout, "maximum timeout to use when fetching execution data from the network e.g. 300s")
   618  		flags.DurationVar(&builder.executionDataConfig.RetryDelay, "execution-data-retry-delay", defaultConfig.executionDataConfig.RetryDelay, "initial delay for exponential backoff when fetching execution data fails e.g. 10s")
   619  		flags.DurationVar(&builder.executionDataConfig.MaxRetryDelay, "execution-data-max-retry-delay", defaultConfig.executionDataConfig.MaxRetryDelay, "maximum delay for exponential backoff when fetching execution data fails e.g. 5m")
   620  	}).ValidateFlags(func() error {
   621  		if builder.supportsObserver && (builder.PublicNetworkConfig.BindAddress == cmd.NotSet || builder.PublicNetworkConfig.BindAddress == "") {
   622  			return errors.New("public-network-address must be set if supports-observer is true")
   623  		}
   624  		if builder.executionDataSyncEnabled {
   625  			if builder.executionDataConfig.FetchTimeout <= 0 {
   626  				return errors.New("execution-data-fetch-timeout must be greater than 0")
   627  			}
   628  			if builder.executionDataConfig.MaxFetchTimeout < builder.executionDataConfig.FetchTimeout {
   629  				return errors.New("execution-data-max-fetch-timeout must be greater than execution-data-fetch-timeout")
   630  			}
   631  			if builder.executionDataConfig.RetryDelay <= 0 {
   632  				return errors.New("execution-data-retry-delay must be greater than 0")
   633  			}
   634  			if builder.executionDataConfig.MaxRetryDelay < builder.executionDataConfig.RetryDelay {
   635  				return errors.New("execution-data-max-retry-delay must be greater than or equal to execution-data-retry-delay")
   636  			}
   637  			if builder.executionDataConfig.MaxSearchAhead == 0 {
   638  				return errors.New("execution-data-max-search-ahead must be greater than 0")
   639  			}
   640  		}
   641  
   642  		return nil
   643  	})
   644  }
   645  
   646  // initNetwork creates the network.Network implementation with the given metrics, middleware, initial list of network
   647  // participants and topology used to choose peers from the list of participants. The list of participants can later be
   648  // updated by calling network.SetIDs.
   649  func (builder *FlowAccessNodeBuilder) initNetwork(nodeID module.Local,
   650  	networkMetrics module.NetworkCoreMetrics,
   651  	middleware network.Middleware,
   652  	topology network.Topology,
   653  	receiveCache *netcache.ReceiveCache,
   654  ) (*p2p.Network, error) {
   655  
   656  	// creates network instance
   657  	net, err := p2p.NewNetwork(&p2p.NetworkParameters{
   658  		Logger:              builder.Logger,
   659  		Codec:               cborcodec.NewCodec(),
   660  		Me:                  nodeID,
   661  		MiddlewareFactory:   func() (network.Middleware, error) { return builder.Middleware, nil },
   662  		Topology:            topology,
   663  		SubscriptionManager: subscription.NewChannelSubscriptionManager(middleware),
   664  		Metrics:             networkMetrics,
   665  		IdentityProvider:    builder.IdentityProvider,
   666  		ReceiveCache:        receiveCache,
   667  	})
   668  	if err != nil {
   669  		return nil, fmt.Errorf("could not initialize network: %w", err)
   670  	}
   671  
   672  	return net, nil
   673  }
   674  
   675  func publicNetworkMsgValidators(log zerolog.Logger, idProvider module.IdentityProvider, selfID flow.Identifier) []network.MessageValidator {
   676  	return []network.MessageValidator{
   677  		// filter out messages sent by this node itself
   678  		validator.ValidateNotSender(selfID),
   679  		validator.NewAnyValidator(
   680  			// message should be either from a valid staked node
   681  			validator.NewOriginValidator(
   682  				id.NewIdentityFilterIdentifierProvider(filter.IsValidCurrentEpochParticipant, idProvider),
   683  			),
   684  			// or the message should be specifically targeted for this node
   685  			validator.ValidateTarget(log, selfID),
   686  		),
   687  	}
   688  }
   689  
   690  func (builder *FlowAccessNodeBuilder) InitIDProviders() {
   691  	builder.Module("id providers", func(node *cmd.NodeConfig) error {
   692  		idCache, err := cache.NewProtocolStateIDCache(node.Logger, node.State, node.ProtocolEvents)
   693  		if err != nil {
   694  			return fmt.Errorf("could not initialize ProtocolStateIDCache: %w", err)
   695  		}
   696  		builder.IDTranslator = translator.NewHierarchicalIDTranslator(idCache, translator.NewPublicNetworkIDTranslator())
   697  
   698  		// The following wrapper allows to black-list byzantine nodes via an admin command:
   699  		// the wrapper overrides the 'Ejected' flag of blocked nodes to true
   700  		blocklistWrapper, err := cache.NewNodeBlocklistWrapper(idCache, node.DB)
   701  		if err != nil {
   702  			return fmt.Errorf("could not initialize NodeBlocklistWrapper: %w", err)
   703  		}
   704  		builder.IdentityProvider = blocklistWrapper
   705  
   706  		// register the blocklist for dynamic configuration via admin command
   707  		err = node.ConfigManager.RegisterIdentifierListConfig("network-id-provider-blocklist",
   708  			blocklistWrapper.GetBlocklist, blocklistWrapper.Update)
   709  		if err != nil {
   710  			return fmt.Errorf("failed to register blocklist with config manager: %w", err)
   711  		}
   712  
   713  		builder.SyncEngineParticipantsProviderFactory = func() module.IdentifierProvider {
   714  			return id.NewIdentityFilterIdentifierProvider(
   715  				filter.And(
   716  					filter.HasRole(flow.RoleConsensus),
   717  					filter.Not(filter.HasNodeID(node.Me.NodeID())),
   718  					p2p.NotEjectedFilter,
   719  				),
   720  				builder.IdentityProvider,
   721  			)
   722  		}
   723  		return nil
   724  	})
   725  }
   726  
   727  func (builder *FlowAccessNodeBuilder) Initialize() error {
   728  	builder.InitIDProviders()
   729  
   730  	builder.EnqueueResolver()
   731  
   732  	// enqueue the regular network
   733  	builder.EnqueueNetworkInit()
   734  
   735  	builder.AdminCommand("get-transactions", func(conf *cmd.NodeConfig) commands.AdminCommand {
   736  		return storageCommands.NewGetTransactionsCommand(conf.State, conf.Storage.Payloads, conf.Storage.Collections)
   737  	})
   738  
   739  	// if this is an access node that supports public followers, enqueue the public network
   740  	if builder.supportsObserver {
   741  		builder.enqueuePublicNetworkInit()
   742  		builder.enqueueRelayNetwork()
   743  	}
   744  
   745  	builder.EnqueuePingService()
   746  
   747  	builder.EnqueueMetricsServerInit()
   748  
   749  	if err := builder.RegisterBadgerMetrics(); err != nil {
   750  		return err
   751  	}
   752  
   753  	builder.EnqueueTracer()
   754  	builder.PreInit(cmd.DynamicStartPreInit)
   755  	return nil
   756  }
   757  
   758  func (builder *FlowAccessNodeBuilder) enqueueRelayNetwork() {
   759  	builder.Component("relay network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   760  		relayNet := relaynet.NewRelayNetwork(
   761  			node.Network,
   762  			builder.AccessNodeConfig.PublicNetworkConfig.Network,
   763  			node.Logger,
   764  			map[channels.Channel]channels.Channel{
   765  				channels.ReceiveBlocks: channels.PublicReceiveBlocks,
   766  			},
   767  		)
   768  		node.Network = relayNet
   769  		return relayNet, nil
   770  	})
   771  }
   772  
   773  func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) {
   774  	builder.
   775  		BuildConsensusFollower().
   776  		Module("collection node client", func(node *cmd.NodeConfig) error {
   777  			// collection node address is optional (if not specified, collection nodes will be chosen at random)
   778  			if strings.TrimSpace(builder.rpcConf.CollectionAddr) == "" {
   779  				node.Logger.Info().Msg("using a dynamic collection node address")
   780  				return nil
   781  			}
   782  
   783  			node.Logger.Info().
   784  				Str("collection_node", builder.rpcConf.CollectionAddr).
   785  				Msg("using the static collection node address")
   786  
   787  			collectionRPCConn, err := grpc.Dial(
   788  				builder.rpcConf.CollectionAddr,
   789  				grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)),
   790  				grpc.WithInsecure(), //nolint:staticcheck
   791  				backend.WithClientUnaryInterceptor(builder.rpcConf.CollectionClientTimeout))
   792  			if err != nil {
   793  				return err
   794  			}
   795  			builder.CollectionRPC = access.NewAccessAPIClient(collectionRPCConn)
   796  			return nil
   797  		}).
   798  		Module("historical access node clients", func(node *cmd.NodeConfig) error {
   799  			addrs := strings.Split(builder.rpcConf.HistoricalAccessAddrs, ",")
   800  			for _, addr := range addrs {
   801  				if strings.TrimSpace(addr) == "" {
   802  					continue
   803  				}
   804  				node.Logger.Info().Str("access_nodes", addr).Msg("historical access node addresses")
   805  
   806  				historicalAccessRPCConn, err := grpc.Dial(
   807  					addr,
   808  					grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(grpcutils.DefaultMaxMsgSize)),
   809  					grpc.WithInsecure()) //nolint:staticcheck
   810  				if err != nil {
   811  					return err
   812  				}
   813  				builder.HistoricalAccessRPCs = append(builder.HistoricalAccessRPCs, access.NewAccessAPIClient(historicalAccessRPCConn))
   814  			}
   815  			return nil
   816  		}).
   817  		Module("transaction timing mempools", func(node *cmd.NodeConfig) error {
   818  			var err error
   819  			builder.TransactionTimings, err = stdmap.NewTransactionTimings(1500 * 300) // assume 1500 TPS * 300 seconds
   820  			if err != nil {
   821  				return err
   822  			}
   823  
   824  			builder.CollectionsToMarkFinalized, err = stdmap.NewTimes(50 * 300) // assume 50 collection nodes * 300 seconds
   825  			if err != nil {
   826  				return err
   827  			}
   828  
   829  			builder.CollectionsToMarkExecuted, err = stdmap.NewTimes(50 * 300) // assume 50 collection nodes * 300 seconds
   830  			if err != nil {
   831  				return err
   832  			}
   833  
   834  			builder.BlocksToMarkExecuted, err = stdmap.NewTimes(1 * 300) // assume 1 block per second * 300 seconds
   835  			return err
   836  		}).
   837  		Module("transaction metrics", func(node *cmd.NodeConfig) error {
   838  			builder.TransactionMetrics = metrics.NewTransactionCollector(builder.TransactionTimings, node.Logger, builder.logTxTimeToFinalized,
   839  				builder.logTxTimeToExecuted, builder.logTxTimeToFinalizedExecuted)
   840  			return nil
   841  		}).
   842  		Module("access metrics", func(node *cmd.NodeConfig) error {
   843  			builder.AccessMetrics = metrics.NewAccessCollector()
   844  			return nil
   845  		}).
   846  		Module("ping metrics", func(node *cmd.NodeConfig) error {
   847  			builder.PingMetrics = metrics.NewPingCollector()
   848  			return nil
   849  		}).
   850  		Module("server certificate", func(node *cmd.NodeConfig) error {
   851  			// generate the server certificate that will be served by the GRPC server
   852  			x509Certificate, err := grpcutils.X509Certificate(node.NetworkKey)
   853  			if err != nil {
   854  				return err
   855  			}
   856  			tlsConfig := grpcutils.DefaultServerTLSConfig(x509Certificate)
   857  			builder.rpcConf.TransportCredentials = credentials.NewTLS(tlsConfig)
   858  			return nil
   859  		}).
   860  		Component("RPC engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   861  			engineBuilder, err := rpc.NewBuilder(
   862  				node.Logger,
   863  				node.State,
   864  				builder.rpcConf,
   865  				builder.CollectionRPC,
   866  				builder.HistoricalAccessRPCs,
   867  				node.Storage.Blocks,
   868  				node.Storage.Headers,
   869  				node.Storage.Collections,
   870  				node.Storage.Transactions,
   871  				node.Storage.Receipts,
   872  				node.Storage.Results,
   873  				node.RootChainID,
   874  				builder.TransactionMetrics,
   875  				builder.AccessMetrics,
   876  				builder.collectionGRPCPort,
   877  				builder.executionGRPCPort,
   878  				builder.retryEnabled,
   879  				builder.rpcMetricsEnabled,
   880  				builder.apiRatelimits,
   881  				builder.apiBurstlimits,
   882  			)
   883  			if err != nil {
   884  				return nil, err
   885  			}
   886  
   887  			builder.RpcEng, err = engineBuilder.
   888  				WithLegacy().
   889  				WithBlockSignerDecoder(signature.NewBlockSignerDecoder(builder.Committee)).
   890  				Build()
   891  			if err != nil {
   892  				return nil, err
   893  			}
   894  
   895  			return builder.RpcEng, nil
   896  		}).
   897  		Component("ingestion engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   898  			var err error
   899  
   900  			builder.RequestEng, err = requester.New(
   901  				node.Logger,
   902  				node.Metrics.Engine,
   903  				node.Network,
   904  				node.Me,
   905  				node.State,
   906  				channels.RequestCollections,
   907  				filter.HasRole(flow.RoleCollection),
   908  				func() flow.Entity { return &flow.Collection{} },
   909  			)
   910  			if err != nil {
   911  				return nil, fmt.Errorf("could not create requester engine: %w", err)
   912  			}
   913  
   914  			builder.IngestEng, err = ingestion.New(
   915  				node.Logger,
   916  				node.Network,
   917  				node.State,
   918  				node.Me,
   919  				builder.RequestEng,
   920  				node.Storage.Blocks,
   921  				node.Storage.Headers,
   922  				node.Storage.Collections,
   923  				node.Storage.Transactions,
   924  				node.Storage.Results,
   925  				node.Storage.Receipts,
   926  				builder.TransactionMetrics,
   927  				builder.CollectionsToMarkFinalized,
   928  				builder.CollectionsToMarkExecuted,
   929  				builder.BlocksToMarkExecuted,
   930  				builder.RpcEng,
   931  			)
   932  			if err != nil {
   933  				return nil, err
   934  			}
   935  			builder.RequestEng.WithHandle(builder.IngestEng.OnCollection)
   936  			builder.FinalizationDistributor.AddConsumer(builder.IngestEng)
   937  
   938  			return builder.IngestEng, nil
   939  		}).
   940  		Component("requester engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   941  			// We initialize the requester engine inside the ingestion engine due to the mutual dependency. However, in
   942  			// order for it to properly start and shut down, we should still return it as its own engine here, so it can
   943  			// be handled by the scaffold.
   944  			return builder.RequestEng, nil
   945  		})
   946  
   947  	if builder.supportsObserver {
   948  		builder.Component("public sync request handler", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   949  			syncRequestHandler, err := synceng.NewRequestHandlerEngine(
   950  				node.Logger.With().Bool("public", true).Logger(),
   951  				unstaked.NewUnstakedEngineCollector(node.Metrics.Engine),
   952  				builder.AccessNodeConfig.PublicNetworkConfig.Network,
   953  				node.Me,
   954  				node.Storage.Blocks,
   955  				builder.SyncCore,
   956  				builder.FinalizedHeader,
   957  			)
   958  
   959  			if err != nil {
   960  				return nil, fmt.Errorf("could not create public sync request handler: %w", err)
   961  			}
   962  
   963  			return syncRequestHandler, nil
   964  		})
   965  	}
   966  
   967  	if builder.executionDataSyncEnabled {
   968  		builder.BuildExecutionDataRequester()
   969  	}
   970  
   971  	builder.Component("ping engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
   972  		ping, err := pingeng.New(
   973  			node.Logger,
   974  			node.IdentityProvider,
   975  			node.IDTranslator,
   976  			node.Me,
   977  			builder.PingMetrics,
   978  			builder.pingEnabled,
   979  			builder.nodeInfoFile,
   980  			node.PingService,
   981  		)
   982  
   983  		if err != nil {
   984  			return nil, fmt.Errorf("could not create ping engine: %w", err)
   985  		}
   986  
   987  		return ping, nil
   988  	})
   989  
   990  	return builder.FlowNodeBuilder.Build()
   991  }
   992  
   993  // enqueuePublicNetworkInit enqueues the public network component initialized for the staked node
   994  func (builder *FlowAccessNodeBuilder) enqueuePublicNetworkInit() {
   995  	var libp2pNode p2p.LibP2PNode
   996  	builder.
   997  		Module("public network metrics", func(node *cmd.NodeConfig) error {
   998  			builder.PublicNetworkConfig.Metrics = metrics.NewNetworkCollector(builder.Logger, metrics.WithNetworkPrefix("public"))
   999  			return nil
  1000  		}).
  1001  		Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
  1002  
  1003  			libP2PFactory := builder.initLibP2PFactory(builder.NodeConfig.NetworkKey, builder.PublicNetworkConfig.BindAddress, builder.PublicNetworkConfig.Metrics)
  1004  
  1005  			var err error
  1006  			libp2pNode, err = libP2PFactory()
  1007  			if err != nil {
  1008  				return nil, fmt.Errorf("could not create public libp2p node: %w", err)
  1009  			}
  1010  
  1011  			return libp2pNode, nil
  1012  		}).
  1013  		Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
  1014  			msgValidators := publicNetworkMsgValidators(node.Logger.With().Bool("public", true).Logger(), node.IdentityProvider, builder.NodeID)
  1015  
  1016  			middleware := builder.initMiddleware(builder.NodeID, builder.PublicNetworkConfig.Metrics, libp2pNode, msgValidators...)
  1017  
  1018  			// topology returns empty list since peers are not known upfront
  1019  			top := topology.EmptyTopology{}
  1020  
  1021  			var heroCacheCollector module.HeroCacheMetrics = metrics.NewNoopCollector()
  1022  			if builder.HeroCacheMetricsEnable {
  1023  				heroCacheCollector = metrics.PublicNetworkReceiveCacheMetricsFactory(builder.MetricsRegisterer)
  1024  			}
  1025  			receiveCache := netcache.NewHeroReceiveCache(builder.NetworkReceivedMessageCacheSize,
  1026  				builder.Logger,
  1027  				heroCacheCollector)
  1028  
  1029  			err := node.Metrics.Mempool.Register(metrics.ResourcePublicNetworkingReceiveCache, receiveCache.Size)
  1030  			if err != nil {
  1031  				return nil, fmt.Errorf("could not register networking receive cache metric: %w", err)
  1032  			}
  1033  
  1034  			net, err := builder.initNetwork(builder.Me, builder.PublicNetworkConfig.Metrics, middleware, top, receiveCache)
  1035  			if err != nil {
  1036  				return nil, err
  1037  			}
  1038  
  1039  			builder.AccessNodeConfig.PublicNetworkConfig.Network = net
  1040  
  1041  			node.Logger.Info().Msgf("network will run on address: %s", builder.PublicNetworkConfig.BindAddress)
  1042  			return net, nil
  1043  		}).
  1044  		Component("public peer manager", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
  1045  			return libp2pNode.PeerManagerComponent(), nil
  1046  		})
  1047  }
  1048  
  1049  // initLibP2PFactory creates the LibP2P factory function for the given node ID and network key.
  1050  // The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance
  1051  // The LibP2P host is created with the following options:
  1052  //   - DHT as server
  1053  //   - The address from the node config or the specified bind address as the listen address
  1054  //   - The passed in private key as the libp2p key
  1055  //   - No connection gater
  1056  //   - Default Flow libp2p pubsub options
  1057  func (builder *FlowAccessNodeBuilder) initLibP2PFactory(networkKey crypto.PrivateKey, bindAddress string, networkMetrics module.LibP2PMetrics) p2pbuilder.LibP2PFactoryFunc {
  1058  	return func() (p2p.LibP2PNode, error) {
  1059  		connManager := connection.NewConnManager(builder.Logger, networkMetrics)
  1060  
  1061  		libp2pNode, err := p2pbuilder.NewNodeBuilder(
  1062  			builder.Logger,
  1063  			networkMetrics,
  1064  			bindAddress,
  1065  			networkKey,
  1066  			builder.SporkID,
  1067  			builder.LibP2PResourceManagerConfig).
  1068  			SetBasicResolver(builder.Resolver).
  1069  			SetSubscriptionFilter(
  1070  				subscription.NewRoleBasedFilter(
  1071  					flow.RoleAccess, builder.IdentityProvider,
  1072  				),
  1073  			).
  1074  			SetConnectionManager(connManager).
  1075  			SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) {
  1076  				return dht.NewDHT(
  1077  					ctx,
  1078  					h,
  1079  					unicast.FlowPublicDHTProtocolID(builder.SporkID),
  1080  					builder.Logger,
  1081  					networkMetrics,
  1082  					dht.AsServer(),
  1083  				)
  1084  			}).
  1085  			// disable connection pruning for the access node which supports the observer
  1086  			SetPeerManagerOptions(connection.ConnectionPruningDisabled, builder.PeerUpdateInterval).
  1087  			Build()
  1088  
  1089  		if err != nil {
  1090  			return nil, fmt.Errorf("could not build libp2p node for staked access node: %w", err)
  1091  		}
  1092  
  1093  		return libp2pNode, nil
  1094  	}
  1095  }
  1096  
  1097  // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update
  1098  // interval, and validators. The network.Middleware is then passed into the initNetwork function.
  1099  func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier,
  1100  	networkMetrics module.NetworkSecurityMetrics,
  1101  	libp2pNode p2p.LibP2PNode,
  1102  	validators ...network.MessageValidator) network.Middleware {
  1103  
  1104  	logger := builder.Logger.With().Bool("staked", false).Logger()
  1105  
  1106  	slashingViolationsConsumer := slashing.NewSlashingViolationsConsumer(logger, networkMetrics)
  1107  
  1108  	builder.Middleware = middleware.NewMiddleware(
  1109  		logger,
  1110  		libp2pNode,
  1111  		nodeID,
  1112  		builder.Metrics.Bitswap,
  1113  		builder.SporkID,
  1114  		middleware.DefaultUnicastTimeout,
  1115  		builder.IDTranslator,
  1116  		builder.CodecFactory(),
  1117  		slashingViolationsConsumer,
  1118  		middleware.WithMessageValidators(validators...))
  1119  
  1120  	return builder.Middleware
  1121  }