diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 36516e393..9ec6b94cc 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -37,13 +37,21 @@ jobs: cargo-lint: runs-on: arc-public-8xlarge-amd64-runner timeout-minutes: 20 + permissions: + contents: write name: lint steps: - uses: actions/checkout@v6 - - uses: taiki-e/install-action@just + with: + ref: ${{ github.head_ref }} + persist-credentials: true - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt, clippy + + - uses: foundry-rs/foundry-toolchain@v1 + + - uses: taiki-e/install-action@just - name: Cache uses: actions/cache@v5 continue-on-error: false @@ -54,9 +62,17 @@ jobs: ~/.cargo/git/db/ key: cargo-test-${{ hashFiles('**/Cargo.lock') }} restore-keys: cargo-test- - - name: fmt + lint - run: cargo +nightly fmt --all -- --check + - name: Run Formatter + run: just fmt + - name: Commit changes + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git diff --staged --quiet || git commit -m "chore: auto-format" + git push + cargo-clippy: runs-on: arc-public-8xlarge-amd64-runner timeout-minutes: 20 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 297eb0f4c..132b3ee55 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -9,7 +9,7 @@ on: push: tags: - v* - + pull_request: env: CARGO_TERM_COLOR: always diff --git a/Cargo.lock b/Cargo.lock index 3ae8afc12..7277dfeb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3752,7 +3752,6 @@ dependencies = [ "alloy-signer-local", "alloy-sol-types", "alloy-trie", - "backon", "bon", "color-eyre", "crossbeam-channel", @@ -3763,6 +3762,7 @@ dependencies = [ "futures", "lazy_static", "metrics", + "metrics-derive", "op-alloy-consensus", "op-alloy-network", "parking_lot", @@ -3772,6 +3772,7 @@ dependencies = [ "reth-basic-payload-builder", "reth-chain-state", "reth-chainspec", + "reth-engine-tree", "reth-evm", "reth-node-api", "reth-node-builder", @@ -3795,6 +3796,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", + "tracing-subscriber 0.3.22", ] [[package]] @@ -3804,7 +3806,6 @@ dependencies = [ "clap", "color-eyre", "ed25519-dalek", - "flashblocks-builder", "hex", ] @@ -3890,14 +3891,20 @@ dependencies = [ "alloy-rlp", "chrono", "ed25519-dalek", + "enr", + "flashblocks-cli", "flashblocks-primitives", "futures", "metrics", "parking_lot", + "pin-project", + "rand 0.9.2", "reth", "reth-eth-wire", "reth-ethereum", "reth-network", + "reth-network-api", + "reth-network-peers", "reth-tasks", "thiserror 2.0.18", "tokio", @@ -6230,7 +6237,7 @@ source = "git+https://github.com/0xForerunner/optimism?rev=79c9153#79c91536e0713 dependencies = [ "op-alloy-consensus", "op-alloy-network", - "op-alloy-provider", + "op-alloy-provider 0.23.1 (git+https://github.com/0xForerunner/optimism?rev=79c9153)", "op-alloy-rpc-types", "op-alloy-rpc-types-engine", ] @@ -6275,6 +6282,21 @@ dependencies = [ "op-alloy-rpc-types", ] +[[package]] +name = "op-alloy-provider" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6753d90efbaa8ea8bcb89c1737408ca85fa60d7adb875049d3f382c063666f86" +dependencies = [ + "alloy-network", + "alloy-primitives", + "alloy-provider", + "alloy-rpc-types-engine", + "alloy-transport", + "async-trait", + "op-alloy-rpc-types-engine", +] + [[package]] name = "op-alloy-provider" version = "0.23.1" @@ -14198,6 +14220,8 @@ dependencies = [ "hex", "jsonrpsee", "op-alloy-consensus", + "op-alloy-network", + "op-alloy-provider 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", "op-alloy-rpc-types", "op-alloy-rpc-types-engine", "parking_lot", @@ -14375,6 +14399,7 @@ dependencies = [ "alloy-sol-types", "bon", "chrono", + "flashblocks-builder", "flashblocks-cli", "flashblocks-primitives", "futures", diff --git a/Cargo.toml b/Cargo.toml index fa2aeaad9..a6a4ffb0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ reth = { git = "https://github.com/worldcoin/reth", rev = "15d0a8f" } reth-cli-util = { git = "https://github.com/worldcoin/reth", rev = "15d0a8f" } reth-cli = { git = "https://github.com/worldcoin/reth", rev = "15d0a8f" } reth-engine-primitives = { git = "https://github.com/worldcoin/reth", rev = "15d0a8f" } +reth-engine-tree = { git = "https://github.com/worldcoin/reth", rev = "15d0a8f" } reth-evm = { git = "https://github.com/worldcoin/reth", rev = "15d0a8f", features = [ "op", ] } @@ -144,6 +145,7 @@ op-alloy-rpc-types = { version = "0.23.1", default-features = false } op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false } op-alloy-network = { version = "0.23.1", default-features = false } alloy-op-hardforks = { version = "0.4.4", default-features = false } +op-alloy-provider = { version = "0.23.1", default-features = false } # alloy alloy = { version = "1.1.2" } @@ -199,7 +201,6 @@ revm-inspectors = "0.34" alloy-op-evm = { version = "0.27", default-features = false } alloy-evm = { version = "0.27", default-features = false } - # rpc jsonrpsee = { version = "0.26.0", features = ["server", "client", "macros"] } jsonrpsee-core = { version = "0.26.0" } @@ -256,6 +257,7 @@ url = "2.5.7" brotli = "8.0.2" once_cell = "1.19" either = { version = "1.15.0", default-features = false } +pin-project = "1.0.1" # Test testcontainers = "0.27" diff --git a/Justfile b/Justfile index 475e2963f..4ed5775c7 100644 --- a/Justfile +++ b/Justfile @@ -24,8 +24,10 @@ devnet-down: test *args='': RUST_LOG="info" cargo nextest run --workspace $@ +fmt: fmt-fix fmt-check contracts-fmt + # Formats the whole workspace -fmt: devnet-fmt contracts-fmt fmt-fix fmt-check +fmt-all: devnet-fmt contracts-fmt fmt-fix fmt-check devnet-fmt: @just ./devnet/fmt diff --git a/crates/flashblocks/builder/Cargo.toml b/crates/flashblocks/builder/Cargo.toml index c48276fa3..d314f9a91 100644 --- a/crates/flashblocks/builder/Cargo.toml +++ b/crates/flashblocks/builder/Cargo.toml @@ -21,6 +21,7 @@ reth-payload-util.workspace = true reth-evm.workspace = true reth-node-api.workspace = true reth-node-builder.workspace = true +reth-engine-tree.workspace = true # op-reth reth-optimism-forks.workspace = true @@ -60,8 +61,8 @@ dashmap.workspace = true thiserror.workspace = true either.workspace = true metrics.workspace = true +metrics-derive.workspace = true serde.workspace = true -backon.workspace = true [dev-dependencies] serde.workspace = true @@ -70,6 +71,7 @@ eyre.workspace = true lazy_static.workspace = true proptest.workspace = true reth-tracing.workspace = true +tracing-subscriber.workspace = true alloy-genesis.workspace = true op-alloy-network.workspace = true alloy-signer-local.workspace = true diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 7f9342fcb..ac3a5981b 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -1,17 +1,22 @@ use alloy_eips::{Decodable2718, eip2718::WithEncoded, eip4895::Withdrawals}; use alloy_op_evm::OpBlockExecutionCtx; use eyre::eyre::eyre; -use flashblocks_p2p::protocol::handler::FlashblocksHandle; +use flashblocks_p2p::protocol::{ + event::{ChainEvent, WorldChainEvent, WorldChainEventsStream}, + handler::FlashblocksHandle, +}; use flashblocks_primitives::{p2p::AuthorizedPayload, primitives::FlashblocksPayloadV1}; -use futures::StreamExt as _; +use futures::StreamExt; use op_alloy_consensus::{OpTxEnvelope, encode_holocene_extra_data}; use parking_lot::RwLock; use reth::{ payload::EthPayloadBuilderAttributes, revm::{cancelled::CancelOnDrop, database::StateProviderDatabase}, + rpc::types::BlockNumHash, }; use reth_basic_payload_builder::PayloadConfig; -use reth_chain_state::ExecutedBlock; +use reth_chain_state::{DeferredTrieData, ExecutedBlock}; +use reth_engine_tree::tree::executor::WorkloadExecutor; use reth_evm::ConfigureEvm; use reth_node_api::{BuiltPayload as _, Events, FullNodeTypes, NodeTypes}; use reth_node_builder::BuilderContext; @@ -21,29 +26,37 @@ use reth_optimism_node::{OpBuiltPayload, OpEngineTypes, OpEvmConfig, OpPayloadBu use reth_optimism_primitives::OpPrimitives; use reth_payload_util::BestPayloadTransactions; -use reth_provider::{ChainSpecProvider, HeaderProvider, StateProviderFactory}; +use reth_provider::{ + CanonStateSubscriptions, ChainSpecProvider, HeaderProvider, StateProviderFactory, +}; use reth_transaction_pool::{EthPooledTransaction, noop::NoopTransactionPool}; -use std::{ - sync::Arc, - time::{Duration, Instant}, +use std::sync::Arc; +use tokio::sync::{ + Semaphore, SemaphorePermit, + broadcast::{self, Sender}, + oneshot, }; -use tokio::sync::broadcast; -use tracing::{error, trace, warn}; +use tracing::{debug, error, trace}; + +/// Placeholder for future task handle variants. Currently unused — the +/// hook updates P2P state directly via the flushed cursor. +#[derive(Clone, Debug)] +pub enum TrieTaskHandle {} use crate::{ bal_executor::CommittedState, bal_validator::{FlashblocksBlockValidator, decode_transactions_with_indices}, + metrics::EXECUTION, payload_builder::build, + spawn_blocking_io_with_shutdown_signal, traits::{context::OpPayloadBuilderCtxBuilder, context_builder::PayloadBuilderCtxBuilder}, }; -use backon::BlockingRetryable; use flashblocks_primitives::flashblocks::{Flashblock, Flashblocks}; -/// The maximum backoff duration when waiting for the parent header to be available in the database when processing a flashblock. -const FETCH_PARENT_HEADER_MAX_DELAY: Duration = Duration::from_millis(2000); +/// Semaphore locking the [`WorkloadExecutor`] thread pool for flashblock processing tasks. +/// Ensures the Pending Block is always in sync when a concurrent task is spawned. +static PENDING_BLOCK_WRITE_PERMIT: Semaphore = Semaphore::const_new(1); -/// The minimum backoff duration when waiting for the parent header to be available in the database when processing a flashblock. -const FETCH_PARENT_HEADER_MIN_DELAY: Duration = Duration::from_millis(100); /// The current state of all known pre confirmations received over the P2P layer /// or generated from the payload building job of this node. /// @@ -64,7 +77,10 @@ pub struct FlashblocksExecutionCoordinatorInner { /// The latest built payload with its associated flashblock index latest_payload: Option<(OpBuiltPayload, u64)>, /// Broadcast channel for built payload events - payload_events: Option>>, + payload_events: Option>>, + /// Deferred trie handles from prior flashblocks in the current epoch. + /// Used as ancestors for the next flashblock's [`DeferredTrieData`]. + ancestor_handles: Vec, } impl FlashblocksExecutionCoordinator { @@ -79,6 +95,7 @@ impl FlashblocksExecutionCoordinator { flashblocks: Default::default(), latest_payload: None, payload_events: None, + ancestor_handles: Vec::new(), })); Self { @@ -92,34 +109,187 @@ impl FlashblocksExecutionCoordinator { pub fn launch(&self, ctx: &BuilderContext, evm_config: OpEvmConfig) where Node: FullNodeTypes, - Node::Provider: StateProviderFactory + HeaderProvider
, + Node::Provider: StateProviderFactory + + HeaderProvider
+ + CanonStateSubscriptions, Node::Types: NodeTypes, { - let mut stream = self.p2p_handle.live_flashblock_stream(); - let this = self.clone(); let provider = ctx.provider().clone(); + let p2p_state = self.p2p_handle.state.clone(); + let mut stream: WorldChainEventsStream = + self.p2p_handle + .event_stream(provider.clone(), move |event| { + if let WorldChainEvent::Chain(ChainEvent::Pending(fb)) = event { + let mut state = p2p_state.lock(); + state.flushed_payload_id = Some(fb.payload_id); + state.flushed_index = fb.index; + } + None + }); + + let this = self.clone(); let chain_spec = ctx.chain_spec().clone(); let pending_block = self.pending_block.clone(); + let workload = WorkloadExecutor::default(); + + let database_permit = &PENDING_BLOCK_WRITE_PERMIT; + ctx.task_executor() .spawn_critical("flashblocks executor", async move { - while let Some(flashblock) = stream.next().await { - let provider = provider.clone(); - if let Err(e) = process_flashblock( - provider, - &evm_config, - &this, - chain_spec.clone(), - flashblock, - pending_block.clone(), - ) { - error!("error processing flashblock: {e:#?}") + // Tracks the in-flight shutdown signal and current epoch block number. + let mut inflight_shutdown: Option> = None; + let mut epoch_block_number: Option = None; + + while let Some(event) = stream.next().await { + match event { + WorldChainEvent::Chain(ChainEvent::Pending(flashblock)) => { + let flashblock = + Arc::try_unwrap(flashblock).unwrap_or_else(|arc| (*arc).clone()); + + trace!( + target: "flashblocks::coordinator", + payload_id = %flashblock.payload_id, + index = %flashblock.index, + is_base = flashblock.base.is_some(), + "received pending flashblock" + ); + + // Track epoch block number from base flashblocks + if let Some(base) = &flashblock.base { + epoch_block_number = Some(base.block_number); + } + + this.on_flashblock( + flashblock, + &mut inflight_shutdown, + database_permit, + &workload, + &provider, + &evm_config, + &chain_spec, + &pending_block, + ) + .await; + } + WorldChainEvent::Chain(ChainEvent::Canon(tip)) => { + trace!( + target: "flashblocks::coordinator", + tip_number = tip.number, + tip_hash = %tip.hash, + "received canonical tip" + ); + + this.on_canon( + tip, + &mut inflight_shutdown, + &mut epoch_block_number, + &pending_block, + ); + } + WorldChainEvent::Event(_) => {} } } }); } + /// Handles a new pending flashblock event. Cancels any previous in-flight + /// task, acquires a thread pool permit, and spawns processing on the + /// [`WorkloadExecutor`]. + async fn on_flashblock( + &self, + flashblock: FlashblocksPayloadV1, + shutdown_tx: &mut Option>, + database_permit: &'static Semaphore, + workload: &WorkloadExecutor, + provider: &Provider, + evm_config: &OpEvmConfig, + chain_spec: &Arc, + pending_block: &tokio::sync::watch::Sender>>, + ) where + Provider: StateProviderFactory + + HeaderProvider
+ + ChainSpecProvider + + Clone + + 'static, + { + // Cancel any previous in-flight task. Ancestor handles are NOT cleared + // here — the new flashblock is typically in the same epoch and needs them. + // New epoch clearing is handled inside process_flashblock when is_new_payload. + shutdown_tx.take(); + + let (tx, rx) = oneshot::channel::<()>(); + *shutdown_tx = Some(tx); + + let provider = provider.clone(); + let evm_config = evm_config.clone(); + let this = self.clone(); + let chain_spec = chain_spec.clone(); + let pending_block = pending_block.clone(); + + let payload_id = flashblock.payload_id; + let index = flashblock.index; + + spawn_blocking_io_with_shutdown_signal(workload, rx, database_permit, move |permit| { + if let Err(e) = process_flashblock( + permit, + provider, + &evm_config, + &this, + chain_spec, + flashblock, + pending_block, + ) { + error!( + target: "flashblocks::coordinator", + %payload_id, + index, + "error processing flashblock: {e:#?}" + ); + } + }); + } + + /// Handles a canonical chain tip update. Cancels any in-flight task, + /// clears stale ancestor trie handles, and clears the pending block if + /// it was built on the now-canonical tip. + fn on_canon( + &self, + tip: BlockNumHash, + inflight_shutdown: &mut Option>, + epoch_block_number: &mut Option, + pending_block: &tokio::sync::watch::Sender>>, + ) { + // Only cancel in-flight work and clear ancestor handles if the current + // epoch is at or behind the canonical tip (stale). If the epoch is + // ahead of the tip, the work is still valid. + let is_stale = epoch_block_number.is_some_and(|n| n <= tip.number); + + if is_stale { + debug!( + target: "flashblocks::coordinator", + epoch_block_number = *epoch_block_number, + "stale epoch — cancelling inflight and clearing ancestors" + ); + EXECUTION.stale_resets.increment(1); + inflight_shutdown.take(); + self.inner.write().ancestor_handles.clear(); + *epoch_block_number = None; + } + + pending_block.send_if_modified(|block| { + let matches = block + .as_ref() // We want to remove the pending block immediately when the canonical tip matches + .is_some_and(|b| b.recovered_block().hash() == tip.hash); + + if matches { + *block = None; + } + matches + }); + } + pub fn publish_built_payload( &self, authorized_payload: AuthorizedPayload, @@ -182,6 +352,7 @@ impl FlashblocksExecutionCoordinator { } fn process_flashblock( + database_permit: SemaphorePermit<'static>, provider: Provider, evm_config: &OpEvmConfig, coordinator: &FlashblocksExecutionCoordinator, @@ -196,71 +367,64 @@ where + Clone + 'static, { - let FlashblocksExecutionCoordinatorInner { - ref mut flashblocks, - ref mut latest_payload, - ref mut payload_events, - } = *coordinator.inner.write(); - let flashblock = Flashblock { flashblock }; - if let Some(latest_payload) = latest_payload - && latest_payload.0.id() == flashblock.flashblock.payload_id - && latest_payload.1 >= flashblock.flashblock.index - { - // Already processed this flashblock. This happens when set directly - // from publish_build_payload. Since we already built the payload, no need - // to do it again. - pending_block.send_replace( - latest_payload - .0 - .executed_block() - .map(|p| p.into_executed_payload()), - ); - return Ok(()); - } + // --- Short read: check if already processed, extract base info --- + let (base, is_new_epoch) = { + let inner = coordinator.inner.read(); - let diff = flashblock.diff().clone(); - let index = flashblock.flashblock.index; + if let Some(latest_payload) = &inner.latest_payload + && latest_payload.0.id() == flashblock.flashblock.payload_id + && latest_payload.1 >= flashblock.flashblock.index + { + // Already processed — send current pending block and return + if let Some(executed) = latest_payload.0.executed_block() { + let block = ExecutedBlock::with_deferred_trie_data( + executed.recovered_block.clone(), + executed.execution_output.clone(), + DeferredTrieData::ready(Default::default()), + ); + pending_block.send_replace(Some(block)); + } + return Ok(()); + } - // If for whatever reason we are not processing flashblocks in order - // we will error and return here. - let base = if flashblocks.is_new_payload(&flashblock)? { - *latest_payload = None; - // safe unwrap from check in is_new_payload - flashblock.base().unwrap() - } else { - flashblocks.base() - }; + let is_new = inner.flashblocks.is_new_payload(&flashblock)?; + let base = if is_new { + flashblock.base().unwrap().clone() + } else { + inner.flashblocks.base().clone() + }; - let f = || { - provider - .sealed_header_by_hash(base.parent_hash)? - .ok_or(eyre!("failed to fetch sealed header {}", base.parent_hash)) + (base, is_new) }; - let sealed_header = f - .retry( - backon::ExponentialBuilder::default() - .with_min_delay(FETCH_PARENT_HEADER_MIN_DELAY) - .with_max_delay(FETCH_PARENT_HEADER_MAX_DELAY) - .with_max_times(10), + // Clear ancestor handles on new epoch + if is_new_epoch { + let mut inner = coordinator.inner.write(); + inner.latest_payload = None; + inner.ancestor_handles.clear(); + } + + // Accumulate committed state from latest payload (brief read lock) + let committed_state = { + let inner = coordinator.inner.read(); + CommittedState::::try_from( + inner.latest_payload.as_ref().map(|(p, _)| p), ) - .notify(|e, duration| { - warn!( - "waiting for parent header {}: {e:#?}. waited {:#?} so far", - base.parent_hash, duration - ) - }) - .call() - .inspect_err(|e| { - error!( - flashblock_index = index, - parent_hash = %base.parent_hash, - error = %e, - "failed to fetch parent header after multiple attempts" - ) - })?; + .map_err(|e| eyre!("Failed to construct committed state {:#?}", e))? + }; + + let diff = flashblock.diff().clone(); + let index = flashblock.flashblock.index; + + // this should never fail. if it does there's a bug in our streaming. + let sealed_header = provider + .sealed_header_by_hash(base.parent_hash) + .inspect_err(|e| error!("failed to fetch sealed header {}: {e:#?}", base.parent_hash))? + .ok_or_else(|| eyre!("sealed header not found for hash {}", base.parent_hash))?; + + let anchor_hash = sealed_header.hash(); let execution_context = OpBlockExecutionCtx { parent_hash: base.parent_hash, @@ -277,27 +441,24 @@ where extra_data: base.extra_data.clone(), }; - trace!( - target: "flashblocks::coordinator", - id = %flashblock.flashblock().payload_id, - index = %flashblock.flashblock().index, - min_tx_index = %flashblock.flashblock().diff.access_list_data.as_ref().map_or("None".to_string(), |d| d.access_list.min_tx_index.to_string()), - max_tx_index = %flashblock.flashblock().diff.access_list_data.as_ref().map_or("None".to_string(), |d| d.access_list.max_tx_index.to_string()), - execution_context = ?execution_context, - next_block_context = ?next_block_context, - "processing flashblock" - ); - let evm_env = evm_config.next_evm_env(sealed_header.header(), &next_block_context)?; - - let committed_state = - CommittedState::::try_from(latest_payload.as_ref().map(|(p, _)| p)) - .map_err(|e| eyre!("Failed to construct committed state {:#?}", e))?; - let transactions_offset = committed_state.transactions.len() + 1; - let start = Instant::now(); + let has_bal = flashblock.diff().access_list_data.is_some(); + + let _validate_span = crate::metrics::MetricsSpan::new( + tracing::trace_span!( + target: "flashblocks::coordinator", + "validate", + id = %flashblock.flashblock().payload_id, + index, + path = if has_bal { "bal" } else { "legacy" }, + tx_count = flashblock.diff().transactions.len(), + duration_ms = tracing::field::Empty, + ), + EXECUTION.validate_duration.clone(), + ); - let payload = if flashblock.diff().access_list_data.is_some() { + let payload = if has_bal { let sealed_header = Arc::new(sealed_header); let executor_transactions = decode_transactions_with_indices( @@ -359,13 +520,20 @@ where let config = PayloadConfig::new(Arc::new(sealed_header), attributes); let cancel = CancelOnDrop::default(); + let prev_payload = coordinator + .inner + .read() + .latest_payload + .as_ref() + .map(|(p, _)| p.clone()); + let builder_ctx = OpPayloadBuilderCtxBuilder.build( provider.clone(), evm_config.clone(), Default::default(), config, &cancel, - latest_payload.as_ref().map(|p| p.0.clone()), + prev_payload.clone(), ); let best = |_| BestPayloadTransactions::new(vec![].into_iter()); @@ -377,7 +545,7 @@ where Option::>::None, db, &builder_ctx, - latest_payload.as_ref().map(|p| &p.0), + prev_payload.as_ref(), false, )?; @@ -388,16 +556,62 @@ where } }; - let duration = Instant::now().duration_since(start); - metrics::histogram!("flashblocks.validate", "access_list" => flashblock.diff().access_list_data.is_some().to_string()) - .record(duration.as_nanos() as f64 / 1_000_000_000.0); + drop(_validate_span); + + // Build ExecutedBlock with deferred trie data — sorting happens in background + let deferred = if let Some(executed) = payload.executed_block() { + let (hashed_state, trie_updates) = match (&executed.hashed_state, &executed.trie_updates) { + (either::Left(hs), either::Left(tu)) => (hs.clone(), tu.clone()), + _ => unreachable!("payload builder always produces unsorted (Left) variants"), + }; + + let ancestors = coordinator.inner.read().ancestor_handles.clone(); + + let deferred = + DeferredTrieData::pending(hashed_state, trie_updates, anchor_hash, ancestors); + + let block = ExecutedBlock::with_deferred_trie_data( + executed.recovered_block.clone(), + executed.execution_output.clone(), + deferred.clone(), + ); + + pending_block.send_replace(Some(block)); - // construct the full payload - *latest_payload = Some((payload.clone(), index)); + Some(deferred) + } else { + None + }; - flashblocks.push(flashblock)?; + // --- Brief write lock: update state, then release database permit --- + { + let mut inner = coordinator.inner.write(); + inner.latest_payload = Some((payload.clone(), index)); + inner.flashblocks.push(flashblock)?; + if let Some(ref deferred) = deferred { + inner.ancestor_handles.push(deferred.clone()); + } + } + // Release database permit immediately after state update. + // Everything after this point (trie sort, broadcast) can run concurrently. + drop(database_permit); + + // Spawn background trie sort after releasing the permit. + // Link the rayon span back to the processing span for trace correlation. + if let Some(deferred) = deferred { + let trie_span = tracing::trace_span!( + target: "flashblocks::coordinator", + "trie_sort", + id = %payload.id(), + index, + ); + trie_span.follows_from(tracing::Span::current()); - pending_block.send_replace(payload.executed_block().map(|p| p.into_executed_payload())); + rayon::spawn(move || { + let _enter = trie_span.enter(); + deferred.wait_cloned(); + }); + } trace!( target: "flashblocks::state_executor", @@ -407,7 +621,9 @@ where "built payload from flashblock" ); - coordinator.broadcast_payload(Events::BuiltPayload(payload), payload_events.clone())?; + let payload_events = coordinator.inner.read().payload_events.clone(); + + coordinator.broadcast_payload(Events::BuiltPayload(payload), payload_events)?; Ok(()) } diff --git a/crates/flashblocks/builder/src/lib.rs b/crates/flashblocks/builder/src/lib.rs index a628c8d1d..a10b464db 100644 --- a/crates/flashblocks/builder/src/lib.rs +++ b/crates/flashblocks/builder/src/lib.rs @@ -138,6 +138,9 @@ //! [`BalBlockBuilder`]: executor::BalBlockBuilder //! [`TemporalDb`]: database::temporal_db::TemporalDb +use std::{panic::AssertUnwindSafe, time::Instant}; + +use reth_engine_tree::tree::executor::WorkloadExecutor; use reth_evm::{ block::BlockExecutionError, execute::{BlockBuilder, BlockBuilderOutcome}, @@ -145,6 +148,10 @@ use reth_evm::{ use reth_optimism_payload_builder::config::OpBuilderConfig; use reth_provider::StateProvider; use revm_database::BundleState; +use tokio::sync::{Semaphore, SemaphorePermit, oneshot}; +use tracing::{error, trace}; + +use crate::metrics::EXECUTION; /// Utilities for constructing and serializing Block Access Lists (BAL). pub mod access_list; @@ -186,6 +193,9 @@ pub mod executor; /// Block building utilities pub mod utils; +/// Metric name constants. +pub mod metrics; + /// Configuration for the flashblocks payload builder. #[derive(Default, Debug, Clone)] pub struct FlashblocksPayloadBuilderConfig { @@ -212,3 +222,59 @@ pub trait BlockBuilderExt: BlockBuilder { state_provider: impl StateProvider, ) -> Result<(BlockBuilderOutcome, BundleState), BlockExecutionError>; } + +/// Spawns a blocking task on the [`WorkloadExecutor`] thread pool, racing it +/// against a shutdown signal. If the shutdown receiver resolves first (sender +/// dropped), the task result is discarded. Acquires the `database_permit` +/// before running `f` to serialize pending block writes. +/// +/// The current tracing span is captured and re-entered on the blocking thread +/// so that all events inside `f` are nested under the caller's span. +#[track_caller] +pub(crate) fn spawn_blocking_io_with_shutdown_signal( + executor: &WorkloadExecutor, + shutdown_rx: oneshot::Receiver<()>, + database_permit: &'static Semaphore, + f: F, +) where + F: FnOnce(SemaphorePermit<'static>) + Send + 'static, +{ + let parent_span = tracing::Span::current(); + + let task = executor.spawn_blocking(move || { + let _enter = parent_span.enter(); + + let unwind = AssertUnwindSafe(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build runtime for permit acquisition"); + + let permit = rt + .block_on(database_permit.acquire()) + .expect("database semaphore closed"); + + f(permit); + }); + + if let Err(e) = std::panic::catch_unwind(unwind) { + error!("flashblock processing panicked: {e:?}"); + } + }); + + // Race the blocking task against the shutdown signal. + // If shutdown fires first (sender dropped by on_flashblock), the + // blocking result is discarded — the newer flashblock takes priority. + tokio::spawn(async move { + match futures::future::select(task, shutdown_rx).await { + futures::future::Either::Left((result, _)) => { + if let Err(e) = result { + error!("flashblock thread pool task panicked: {e:#?}"); + } + } + futures::future::Either::Right(_) => { + trace!("flashblock processing cancelled by shutdown signal"); + } + } + }); +} diff --git a/crates/flashblocks/builder/src/metrics.rs b/crates/flashblocks/builder/src/metrics.rs new file mode 100644 index 000000000..c06fa4119 --- /dev/null +++ b/crates/flashblocks/builder/src/metrics.rs @@ -0,0 +1,74 @@ +//! Metrics and instrumentation for the flashblocks builder. + +use metrics::{Counter, Histogram}; +use metrics_derive::Metrics; +use std::{sync::LazyLock, time::Instant}; + +/// Execution coordinator metrics, auto-registered under `flashblocks.coordinator.*`. +#[derive(Clone, Metrics)] +#[metrics(scope = "flashblocks.coordinator")] +pub struct ExecutionMetrics { + // -- Latency -- + /// Validation / build phase duration (seconds). + pub validate_duration: Histogram, + /// Flashblocks processed in a single epoch (recorded at epoch boundary). + pub flashblocks_per_epoch: Histogram, + + // -- Issues -- + /// Epoch invalidated by a newer canonical tip. + pub stale_resets: Counter, + /// Invalid payload received from P2P (decode error, bad structure). + pub invalid_payload: Counter, + /// Broadcast of built payload to in-memory tree failed. + pub broadcast_failed: Counter, + /// newPayloadV3/V4 cache hit — payload already built for this id+index. + pub payload_cache_hits: Counter, +} + +/// Global singleton — zero lookup cost per call site. +pub static EXECUTION: LazyLock = LazyLock::new(ExecutionMetrics::default); + +/// RAII guard that enters a [`tracing::Span`] on creation. On drop it: +/// 1. Records `duration_ms` on the tracing span +/// 2. Records elapsed seconds to the provided [`Histogram`] +pub struct MetricsSpan { + inner: tracing::span::EnteredSpan, + start: Instant, + histogram: Histogram, +} + +impl MetricsSpan { + /// Enter `span` and start the timer. `histogram` receives elapsed seconds on drop. + pub fn new(span: tracing::Span, histogram: Histogram) -> Self { + Self { + inner: span.entered(), + start: Instant::now(), + histogram, + } + } + + /// Record a field on the underlying tracing span. + pub fn record(&self, field: &str, value: V) { + self.inner.record(field, value); + } +} + +impl Drop for MetricsSpan { + fn drop(&mut self) { + let elapsed = self.start.elapsed(); + self.inner.record("duration_ms", elapsed.as_millis() as u64); + self.histogram.record(elapsed.as_secs_f64()); + } +} + +/// Execute `f` inside a metered tracing span. The span is entered before `f` +/// runs and duration is recorded (both on the span and as a histogram) on +/// completion. `f` receives a [`MetricsSpan`] reference for recording dynamic +/// span fields mid-execution. +pub fn metered_fn(span: tracing::Span, histogram: Histogram, f: F) -> R +where + F: FnOnce(&MetricsSpan) -> R, +{ + let guard = MetricsSpan::new(span, histogram); + f(&guard) +} diff --git a/crates/flashblocks/cli/Cargo.toml b/crates/flashblocks/cli/Cargo.toml index 097cfac68..dcf53ada4 100644 --- a/crates/flashblocks/cli/Cargo.toml +++ b/crates/flashblocks/cli/Cargo.toml @@ -5,8 +5,6 @@ edition.workspace = true license.workspace = true [dependencies] -flashblocks-builder.workspace = true - ed25519-dalek.workspace = true clap.workspace = true eyre.workspace = true diff --git a/crates/flashblocks/cli/src/lib.rs b/crates/flashblocks/cli/src/lib.rs index 51b355fa9..1ae889b29 100644 --- a/crates/flashblocks/cli/src/lib.rs +++ b/crates/flashblocks/cli/src/lib.rs @@ -2,11 +2,67 @@ use clap::ArgGroup; use ed25519_dalek::{SigningKey, VerifyingKey}; use hex::FromHex; +pub const DEFAULT_MAX_SEND_PEERS: usize = 10; +pub const DEFAULT_MAX_RECEIVE_PEERS: usize = 3; +pub const DEFAULT_ROTATION_INTERVAL: u64 = 30; +pub const DEFAULT_SCORE_SAMPLES: i64 = 1000; + /// Flashblocks configuration #[derive(Debug, Clone, PartialEq, Eq, clap::Args)] -#[command(next_help_heading = "Flashblocks", - group = ArgGroup::new("authorizer") - .multiple(false) +pub struct FanoutArgs { + /// Override the flashblocks send-set size. + #[arg( + long = "flashblocks.max_send_peers", + env = "FLASHBLOCKS_MAX_SEND_PEERS", + required = false, + default_value_t = DEFAULT_MAX_SEND_PEERS + )] + pub max_send_peers: usize, + + /// Override the number of receive peers maintained for flashblocks fanout. + #[arg( + long = "flashblocks.max_receive_peers", + env = "FLASHBLOCKS_MAX_RECEIVE_PEERS", + required = false, + default_value_t = DEFAULT_MAX_RECEIVE_PEERS + )] + pub max_receive_peers: usize, + + /// Override the flashblocks rotation interval in seconds. + #[arg( + long = "flashblocks.rotation_interval", + env = "FLASHBLOCKS_ROTATION_INTERVAL", + required = false, + default_value_t = DEFAULT_ROTATION_INTERVAL + )] + pub rotation_interval: u64, + + /// Override the number of latency samples retained for receive-peer scoring. + #[arg( + long = "flashblocks.score_samples", + env = "FLASHBLOCKS_SCORE_SAMPLES", + required = false, + default_value_t = DEFAULT_SCORE_SAMPLES + )] + pub score_samples: i64, +} + +impl Default for FanoutArgs { + fn default() -> Self { + Self { + max_send_peers: DEFAULT_MAX_SEND_PEERS, + max_receive_peers: DEFAULT_MAX_RECEIVE_PEERS, + rotation_interval: DEFAULT_ROTATION_INTERVAL, + score_samples: DEFAULT_SCORE_SAMPLES, + } + } +} + +/// Flashblocks configuration +#[derive(Debug, Clone, PartialEq, Eq, clap::Args)] +#[command( + next_help_heading = "Flashblocks", + group = ArgGroup::new("authorizer").multiple(false) )] #[group(requires = "flashblocks.enabled")] pub struct FlashblocksArgs { @@ -21,7 +77,7 @@ pub struct FlashblocksArgs { /// used to verify flashblock authenticity. #[arg( long = "flashblocks.authorizer_vk", - env = "FLASHBLOCKS_AUTHORIZER_VK", + env = "FLASHBLOCKS_AUTHORIZER_VK", group = "authorizer", value_parser = parse_vk, required = false, @@ -31,8 +87,8 @@ pub struct FlashblocksArgs { /// Flashblocks signing key /// used to sign authorized flashblocks payloads. #[arg( - long = "flashblocks.builder_sk", - env = "FLASHBLOCKS_BUILDER_SK", + long = "flashblocks.builder_sk", + env = "FLASHBLOCKS_BUILDER_SK", required = false, value_parser = parse_sk, )] @@ -94,6 +150,9 @@ pub struct FlashblocksArgs { default_value_t = false )] pub access_list: bool, + + #[command(flatten)] + pub fanout: FanoutArgs, } pub fn parse_sk(s: &str) -> eyre::Result { @@ -106,8 +165,6 @@ pub fn parse_vk(s: &str) -> eyre::Result { Ok(VerifyingKey::from_bytes(&bytes)?) } -pub use flashblocks_builder::FlashblocksPayloadBuilderConfig; - #[cfg(test)] mod tests { use super::*; @@ -116,7 +173,7 @@ mod tests { #[derive(Debug, Parser)] struct CommandParser { #[command(flatten)] - flashblocks: Option, + flashblocks: FlashblocksArgs, } #[test] @@ -130,6 +187,7 @@ mod tests { recommit_interval: 200, flashblocks_interval: 200, access_list: true, + fanout: FanoutArgs::default(), }; let args = CommandParser::parse_from([ @@ -146,7 +204,7 @@ mod tests { "200", ]); - assert_eq!(args.flashblocks.unwrap(), flashblocks); + assert_eq!(args.flashblocks, flashblocks); } #[test] @@ -160,6 +218,7 @@ mod tests { recommit_interval: 200, flashblocks_interval: 200, access_list: false, + fanout: FanoutArgs::default(), }; let args = CommandParser::parse_from([ @@ -169,7 +228,7 @@ mod tests { "0000000000000000000000000000000000000000000000000000000000000000", ]); - assert_eq!(args.flashblocks.unwrap(), flashblocks); + assert_eq!(args.flashblocks, flashblocks); } #[test] diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index a438537b7..c205c5a18 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -8,7 +8,10 @@ use eyre::eyre::eyre; use flashblocks_cli::FlashblocksArgs; use flashblocks_p2p::{ monitor, - protocol::handler::{FlashblocksHandle, PeerMsg}, + protocol::{ + connection::ReceiveStatus, + handler::{FlashblocksHandle, PublishingStatus}, + }, }; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, @@ -38,10 +41,12 @@ use serde::{Deserialize, Serialize}; use std::{ any::Any, collections::HashMap, + fmt, io::Write, net::{IpAddr, SocketAddr}, path::PathBuf, - sync::Arc, + sync::{Arc, Mutex}, + time::{SystemTime, UNIX_EPOCH}, }; use tempfile::NamedTempFile; use tokio::time::{Duration, Instant, sleep}; @@ -58,6 +63,46 @@ use world_chain_test::{ utils::{account, eip1559, raw_tx, signer}, }; +/// Thread-safe log buffer for capturing tracing output across threads. +#[derive(Clone, Default)] +struct SharedLogBuffer(Arc>>); + +impl SharedLogBuffer { + fn logs(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +impl tracing_subscriber::Layer for SharedLogBuffer { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = LogVisitor(String::new()); + visitor + .0 + .push_str(&format!("{} ", event.metadata().level())); + visitor + .0 + .push_str(&format!("{}: ", event.metadata().target())); + event.record(&mut visitor); + self.0.lock().unwrap().push(visitor.0); + } +} + +struct LogVisitor(String); + +impl tracing::field::Visit for LogVisitor { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) { + if field.name() == "message" { + self.0.push_str(&format!("{:?}", value)); + } else { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } +} + #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct Metadata { pub receipts: HashMap, @@ -114,6 +159,116 @@ impl NodeContext { } } +async fn wait_for_pending_block( + node: &NodeContext, + expected_number: u64, + expected_txs: usize, +) -> eyre::Result<()> { + let provider = node.provider().await?; + let timeout = Duration::from_secs(10); + let poll_interval = Duration::from_millis(50); + let start = Instant::now(); + let mut last_observed = "no pending block".to_string(); + + loop { + let pending_block = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) + .await?; + + if let Some(pending_block) = pending_block { + let observed_number = pending_block.number(); + let observed_txs = pending_block.transactions.hashes().len(); + if observed_number == expected_number && observed_txs == expected_txs { + return Ok(()); + } + + last_observed = format!("number {observed_number}, txs {observed_txs}"); + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for pending block state: expected number {expected_number}, txs {expected_txs}; last observed {last_observed}" + )); + } + + sleep(poll_interval).await; + } +} + +async fn wait_for_trusted_peers( + node: &NodeContext, + expected_connections: usize, +) -> eyre::Result<()> { + let timeout = Duration::from_secs(10); + let poll_interval = Duration::from_millis(100); + let start = Instant::now(); + + loop { + let trusted_peers = node.network_handle.get_trusted_peers().await?; + if trusted_peers.len() == expected_connections { + return Ok(()); + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for trusted peers: expected {expected_connections}, last observed {}", + trusted_peers.len() + )); + } + + sleep(poll_interval).await; + } +} + +#[expect(clippy::await_holding_lock)] // lock is explicitly dropped before await +async fn wait_for_flashblocks_topology( + node: &NodeContext, + expected_connections: usize, + expected_receive_peers: usize, +) -> eyre::Result<(Vec, Vec)> { + let timeout = Duration::from_secs(10); + let poll_interval = Duration::from_millis(100); + let start = Instant::now(); + + loop { + let state = node.p2p_handle.state.lock(); + if state.connections.len() == expected_connections { + let receive_peers: Vec<_> = state + .connections + .iter() + .filter_map(|(peer_id, conn)| { + matches!(conn.receive_status, ReceiveStatus::Receiving { .. }) + .then_some(*peer_id) + }) + .collect(); + let candidate_peers: Vec<_> = state + .connections + .iter() + .filter_map(|(peer_id, conn)| { + (conn.receive_status == ReceiveStatus::NotReceiving).then_some(*peer_id) + }) + .collect(); + drop(state); + + if receive_peers.len() == expected_receive_peers + && receive_peers.len() + candidate_peers.len() == expected_connections + { + return Ok((receive_peers, candidate_peers)); + } + } else { + drop(state); + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for flashblocks topology: expected {expected_connections} connections with {expected_receive_peers} receive peers" + )); + } + + sleep(poll_interval).await; + } +} + fn init_tracing(filter: &str) -> tracing::subscriber::DefaultGuard { let sub = tracing_subscriber::fmt() .with_env_filter(filter) @@ -125,13 +280,18 @@ fn init_tracing(filter: &str) -> tracing::subscriber::DefaultGuard { Dispatch::new(sub).set_default() } -async fn setup_node( - exec: TaskExecutor, - authorizer_sk: SigningKey, - builder_sk: SigningKey, - peers: Vec<(PeerId, SocketAddr)>, -) -> eyre::Result { - setup_node_extended_cfg(exec, authorizer_sk, builder_sk, peers, None, None).await +fn test_flashblocks_args(authorizer_sk: &SigningKey, builder_sk: &SigningKey) -> FlashblocksArgs { + FlashblocksArgs { + enabled: true, + authorizer_vk: Some(authorizer_sk.verifying_key()), + builder_sk: Some(builder_sk.clone()), + force_publish: false, + override_authorizer_sk: None, + flashblocks_interval: 200, + recommit_interval: 200, + access_list: true, + fanout: Default::default(), + } } async fn setup_node_extended_cfg( @@ -141,6 +301,7 @@ async fn setup_node_extended_cfg( peers: Vec<(PeerId, SocketAddr)>, port: Option, p2p_secret_key: Option, + flashblocks_args: Option, ) -> eyre::Result { let genesis: Genesis = serde_json::from_str(include_str!("assets/genesis.json")).unwrap(); let chain_spec = Arc::new( @@ -208,16 +369,10 @@ async fn setup_node_extended_cfg( rollup: Default::default(), builder, pbh, - flashblocks: Some(FlashblocksArgs { - enabled: true, - authorizer_vk: Some(authorizer_sk.verifying_key()), - builder_sk: Some(builder_sk.clone()), - force_publish: false, - override_authorizer_sk: None, - flashblocks_interval: 200, - recommit_interval: 200, - access_list: true, - }), + flashblocks: Some( + flashblocks_args + .unwrap_or_else(|| test_flashblocks_args(&authorizer_sk, &builder_sk)), + ), tx_peers: None, disable_bootnodes: true, }, @@ -341,7 +496,73 @@ async fn next_payload(payload_id: PayloadId, index: u64) -> FlashblocksPayloadV1 } } +async fn publish_flashblock_with_latency( + sender: &NodeContext, + authorizer: &SigningKey, + payload_id: PayloadId, + authorization_timestamp: u64, + simulated_latency: Duration, +) -> eyre::Result<()> { + let latest_block = sender + .provider() + .await? + .get_block_by_number(alloy_eips::BlockNumberOrTag::Latest) + .await? + .expect("latest block expected"); + let mut payload = base_payload( + 0, + payload_id, + 0, + latest_block.hash(), + authorization_timestamp, + ); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_nanos() as i64; + payload.metadata.flashblock_timestamp = Some(now - simulated_latency.as_nanos() as i64); + let authorization = Authorization::new( + payload.payload_id, + authorization_timestamp, + authorizer, + sender.p2p_handle.builder_sk()?.verifying_key(), + ); + let authorized = + AuthorizedPayload::new(sender.p2p_handle.builder_sk()?, authorization, payload); + + { + let state = sender.p2p_handle.state.lock(); + state + .publishing_status + .send_replace(PublishingStatus::Publishing { authorization }); + } + sender.p2p_handle.publish_new(authorized)?; + { + let state = sender.p2p_handle.state.lock(); + state + .publishing_status + .send_replace(PublishingStatus::NotPublishing { + active_publishers: Vec::new(), + }); + } + + Ok(()) +} + async fn setup_nodes(n: u8) -> eyre::Result { + setup_nodes_with_flashblocks_args(n, |_, authorizer, builder| { + test_flashblocks_args(authorizer, builder) + }) + .await +} + +async fn setup_nodes_with_flashblocks_args( + n: u8, + mut make_flashblocks_args: F, +) -> eyre::Result +where + F: FnMut(u8, &SigningKey, &SigningKey) -> FlashblocksArgs, +{ let mut nodes = Vec::new(); let mut peers = Vec::new(); let tasks = TaskManager::new(tokio::runtime::Handle::current()); @@ -350,14 +571,25 @@ async fn setup_nodes(n: u8) -> eyre::Result { for i in 0..n { let builder = SigningKey::from_bytes(&[(i + 1) % n; 32]); - let node = setup_node(exec.clone(), authorizer.clone(), builder, peers.clone()).await?; + let flashblocks_args = make_flashblocks_args(i, &authorizer, &builder); + let node = setup_node_extended_cfg( + exec.clone(), + authorizer.clone(), + builder, + peers.clone(), + None, + None, + Some(flashblocks_args), + ) + .await?; + if !peers.is_empty() { + wait_for_trusted_peers(&node, peers.len()).await?; + } let enr = node.local_node_record; peers.push((enr.id, enr.tcp_addr())); nodes.push(node); } - sleep(Duration::from_millis(6000)).await; - Ok(NodeTestFixture { nodes, authorizer, @@ -365,8 +597,7 @@ async fn setup_nodes(n: u8) -> eyre::Result { }) } -#[tokio::test] -#[ignore] +#[tokio::test(flavor = "multi_thread")] async fn test_double_failover() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -457,7 +688,7 @@ async fn test_double_failover() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_force_race_condition() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -506,18 +737,9 @@ async fn test_force_race_condition() -> eyre::Result<()> { let authorized = AuthorizedPayload::new(nodes[0].p2p_handle.builder_sk()?, authorization, msg); nodes[0].p2p_handle.start_publishing(authorization)?; nodes[0].p2p_handle.publish_new(authorized).unwrap(); - sleep(Duration::from_millis(100)).await; // Query pending block after sending the base payload with an empty delta - let pending_block = nodes[1] - .provider() - .await? - .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) - .await? - .expect("pending block expected"); - - assert_eq!(pending_block.number(), expected_pending_number); - assert_eq!(pending_block.transactions.hashes().len(), 0); + wait_for_pending_block(&nodes[0], expected_pending_number, 0).await?; info!("Sending payload 0, index 1"); let payload_1 = next_payload(payload_0.payload_id, 1).await; @@ -533,18 +755,9 @@ async fn test_force_race_condition() -> eyre::Result<()> { payload_1.clone(), ); nodes[0].p2p_handle.publish_new(authorized).unwrap(); - sleep(Duration::from_millis(100)).await; // Query pending block after sending the second payload with two transactions - let block = nodes[1] - .provider() - .await? - .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) - .await? - .expect("pending block expected"); - - assert_eq!(block.number(), expected_pending_number); - assert_eq!(block.transactions.hashes().len(), 0); + wait_for_pending_block(&nodes[0], expected_pending_number, 0).await?; // Send a new block, this time from node 1 let payload_2 = base_payload(1, test_payload_id(21), 0, latest_block.hash(), AUTH_TS_NEXT); @@ -591,7 +804,97 @@ async fn test_force_race_condition() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] +async fn test_receive_peer_rotation_uses_latency_scores() -> eyre::Result<()> { + let _tracing = init_tracing("warn,flashblocks=trace"); + + let fixture = setup_nodes_with_flashblocks_args(4, |_, authorizer, builder| { + let mut args = test_flashblocks_args(authorizer, builder); + args.fanout.max_receive_peers = 2; + args.fanout.rotation_interval = 1; + args.fanout.score_samples = 4; + args + }) + .await?; + let nodes = fixture.nodes(); + let authorizer = fixture.authorizer(); + + let (receive_peers, candidate_peers) = wait_for_flashblocks_topology(&nodes[0], 3, 2).await?; + assert_eq!( + candidate_peers.len(), + 1, + "expected one spare candidate peer" + ); + + let slow_peer = receive_peers[0]; + let fast_peer = receive_peers[1]; + let replacement_peer = candidate_peers[0]; + + let peer_map: HashMap<_, _> = nodes + .iter() + .skip(1) + .map(|node| (node.local_node_record.id, node)) + .collect(); + + let fast_node = peer_map + .get(&fast_peer) + .copied() + .expect("fast peer should map to a node"); + let slow_node = peer_map + .get(&slow_peer) + .copied() + .expect("slow peer should map to a node"); + + for (payload_suffix, authorization_timestamp) in [(41, 41_u64), (42, 42), (43, 43), (44, 44)] { + publish_flashblock_with_latency( + fast_node, + authorizer, + test_payload_id(payload_suffix), + authorization_timestamp, + Duration::from_millis(10), + ) + .await?; + sleep(Duration::from_millis(50)).await; + + publish_flashblock_with_latency( + slow_node, + authorizer, + test_payload_id(payload_suffix + 10), + authorization_timestamp + 10, + Duration::from_millis(300), + ) + .await?; + sleep(Duration::from_millis(50)).await; + } + + let timeout = Duration::from_secs(5); + let poll_interval = Duration::from_millis(100); + let start = Instant::now(); + + loop { + let (current_receive_peers, _) = wait_for_flashblocks_topology(&nodes[0], 3, 2).await?; + if current_receive_peers.contains(&fast_peer) + && current_receive_peers.contains(&replacement_peer) + && !current_receive_peers.contains(&slow_peer) + { + break; + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for peer rotation: fast={fast_peer}, slow={slow_peer}, replacement={replacement_peer}, current_receive_peers={current_receive_peers:?}" + )); + } + + sleep(poll_interval).await; + } + + drop(fixture); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] async fn test_get_block_by_number_pending() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -670,7 +973,7 @@ async fn test_get_block_by_number_pending() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_peer_reputation() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -701,31 +1004,50 @@ async fn test_peer_reputation() -> eyre::Result<()> { authorized_msg, ); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StartPublishing(p2p_msg.encode()); + let bytes = p2p_msg.encode(); let peers = nodes[1].network_handle.get_all_peers().await?; let peer_0 = &peers[0].remote_id; + let mut reputation_was_negative = false; + let mut peer_banned = false; for _ in 0..100 { - nodes[0].p2p_handle.ctx.peer_tx.send(peer_msg.clone()).ok(); + nodes[0] + .p2p_handle + .send_serialized_to_all_peers(bytes.clone()); sleep(Duration::from_millis(10)).await; let rep_0 = nodes[1].network_handle.reputation_by_id(*peer_0).await?; - if let Some(rep) = rep_0 { - assert!(rep < 0, "Peer reputation should be negative"); + if let Some(rep) = rep_0 + && rep < 0 + { + reputation_was_negative = true; + } + if nodes[1].network_handle.get_all_peers().await?.is_empty() { + peer_banned = true; + break; } } - // Assert that the peer is banned - assert!(nodes[1].network_handle.get_all_peers().await?.is_empty()); + // Assert that the peer reputation became negative and peer was banned + assert!( + reputation_was_negative, + "Peer reputation should have become negative" + ); + assert!(peer_banned, "Peer should have been banned"); drop(fixture); Ok(()) } -#[tokio::test] -#[tracing_test::traced_test] +#[tokio::test(flavor = "multi_thread")] async fn test_peer_monitoring() -> eyre::Result<()> { + use tracing_subscriber::layer::SubscriberExt; + + let log_buffer = SharedLogBuffer::default(); + let subscriber = tracing_subscriber::registry().with(log_buffer.clone()); + tracing::subscriber::set_global_default(subscriber).expect("failed to set global subscriber"); + let authorizer = SigningKey::from_bytes(&[0; 32]); // Create a temporary P2P secret key file for node1 to ensure consistent peer ID across restarts @@ -747,6 +1069,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { vec![], // No peers initially None, // Use random port (we'll capture it) Some(p2p_key_path.clone()), // Use deterministic P2P key + None, ) .await?; @@ -768,6 +1091,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { vec![(peer1_id, peer1_addr)], // Node1 as trusted peer None, // Use random port None, // No deterministic P2P key needed + None, ) .await?; @@ -804,18 +1128,17 @@ async fn test_peer_monitoring() -> eyre::Result<()> { sleep(Duration::from_millis(500)).await; // Check that disconnection was logged by the event listener (immediate detection) - logs_assert(|logs: &[&str]| { + { + let logs = log_buffer.logs(); let disconnect_log_exists = logs.iter().any(|log| { log.contains("trusted peer disconnected") && log.contains(&peer1_id.to_string()) }); - assert!( disconnect_log_exists, "Should have logged 'trusted peer disconnected' for peer {} from event listener", peer1_id ); - Ok(()) - }); + } // Wait for PeerMonitor periodic checks to detect the disconnection and emit multiple warning logs // Wait for at least 2 periodic ticks to ensure we get multiple log outputs (1s * 3 + 1s buffer for safety) @@ -848,6 +1171,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { )], // Configure node2 as trusted peer Some(peer1_port), // Reuse the same port Some(p2p_key_path.clone()), // Reuse the same P2P key + None, ) .await?; let peer1_id_new = node1_restarted.local_node_record.id; @@ -892,75 +1216,63 @@ async fn test_peer_monitoring() -> eyre::Result<()> { } // Assert that the "connection to trusted peer established" log appears for node1 - logs_assert(|logs: &[&str]| { + { + let logs = log_buffer.logs(); let reconnection_log_exists = logs.iter().any(|log| { log.contains("connection to trusted peer established") && log.contains(&peer1_id.to_string()) }); - assert!( reconnection_log_exists, "Should have logged 'connection to trusted peer established' for peer {} after restart", peer1_id ); - Ok(()) - }); + } // Wait for at least one more monitor tick to verify warnings stopped (1s interval + 1s buffer) sleep(monitor::PEER_MONITOR_INTERVAL + Duration::from_secs(1)).await; // Count the number of warning logs before and after reconnection to ensure they stopped - logs_assert(|logs: &[&str]| { - // Find the index where reconnection happened (use rposition to find the LAST occurrence) + { + let logs = log_buffer.logs(); let reconnection_log_idx = logs .iter() .rposition(|log| log.contains("connection to trusted peer established")) - .ok_or_else(|| { - "Could not find 'connection to trusted peer established' log".to_string() - })?; + .expect("Could not find 'connection to trusted peer established' log"); - // Split logs at the reconnection point let (logs_before_reconnect, logs_after_reconnect) = logs.split_at(reconnection_log_idx); - // Filter for disconnect warnings in logs before reconnection - let warnings_before_reconnect: Vec<&str> = logs_before_reconnect + let warnings_before_reconnect: Vec<&String> = logs_before_reconnect .iter() .filter(|log| { log.contains(&peer1_id.to_string()) && log.contains("WARN") && log.contains("trusted peer disconnected") }) - .copied() .collect(); - // We should have seen at least 2 warnings before reconnection assert!( warnings_before_reconnect.len() >= 2, "Should have had at least 2 warnings before reconnection, found {}", warnings_before_reconnect.len() ); - // Filter for disconnect warnings in logs after reconnection - let warnings_after_reconnect: Vec<&str> = logs_after_reconnect + let warnings_after_reconnect: Vec<&String> = logs_after_reconnect .iter() .filter(|log| { log.contains(&peer1_id.to_string()) && log.contains("WARN") && log.contains("trusted peer disconnected") }) - .copied() .collect(); - // There should be no warnings after reconnection assert!( warnings_after_reconnect.is_empty(), "Should have no warnings after reconnection, found {}: {:?}", warnings_after_reconnect.len(), warnings_after_reconnect ); - - Ok(()) - }); + } Ok(()) } diff --git a/crates/flashblocks/p2p/Cargo.toml b/crates/flashblocks/p2p/Cargo.toml index 2bc32cc6a..05073168f 100644 --- a/crates/flashblocks/p2p/Cargo.toml +++ b/crates/flashblocks/p2p/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true test-utils = [] [dependencies] +flashblocks-cli.workspace = true flashblocks-primitives.workspace = true reth.workspace = true @@ -26,5 +27,12 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true thiserror.workspace = true parking_lot.workspace = true +pin-project.workspace = true chrono.workspace = true reth-tasks = { workspace = true } +rand.workspace = true + +[dev-dependencies] +reth-network-api.workspace = true +reth-network-peers.workspace = true +enr = { version = "0.13.0", default-features = false, features = ["rust-secp256k1"] } diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index cafeb3ca8..25181132e 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -1,6 +1,6 @@ -use crate::protocol::handler::{ - FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, MAX_FLASHBLOCK_INDEX, PeerMsg, - PublishingStatus, +use crate::protocol::{ + event::MAX_FLASHBLOCKS, + handler::{FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, PublishingStatus}, }; use alloy_primitives::bytes::BytesMut; use chrono::Utc; @@ -12,30 +12,75 @@ use flashblocks_primitives::{ }; use futures::{Stream, StreamExt}; use metrics::gauge; -use reth::payload::PayloadId; use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnection}; -use reth_network::{cache::LruMap, types::ReputationChangeKind}; +use reth_network::types::ReputationChangeKind; use std::{ pin::Pin, task::{Context, Poll, ready}, + time::Instant, }; -use tokio_stream::wrappers::BroadcastStream; +use tokio::sync::mpsc; use tracing::{info, trace}; /// Grace period for authorization timestamp checks to reduce false positives from /// minor skew/races between peers. const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; -/// Number of payload receive-sets cached per peer. -/// -/// This should be large enough to retain entries across the grace window. -const RECEIVED_CACHE_LEN: u32 = AUTHORIZATION_TIMESTAMP_GRACE_SEC as u32 * 20; +/// Represents the current flashblocks receive status for a peer connection. +#[derive(Clone, Debug, Default, PartialEq)] +pub enum ReceiveStatus { + /// We are not currently receiving flashblocks from this peer. + #[default] + NotReceiving, + /// We are currently receiving flashblocks from this peer. + /// + /// Score used for adaptive timeouts and peer selection. + /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks + /// counting as 10s. + Receiving { score: Score }, + /// We have sent a request for flashblocks to this peer and are awaiting their response. + Requesting, +} + +/// Shared connection metadata for a single peer connection. +#[derive(Clone, Debug)] +pub struct FlashblocksConnectionState { + /// Whether this peer is marked as trusted or not. + pub trusted: bool, + /// Whether we are currently sending flashblocks to this peer. + pub send_enabled: bool, + /// Current status of receiving flashblocks from this peer. + pub receive_status: ReceiveStatus, + /// Timestamp of the last receive-side state transition for this peer. + /// Used for late-message grace checks and receive retry cooldown. + pub receive_status_timestamp: u64, + /// Per-peer channel for sending serialized protocol messages to this peer. + pub outbound_tx: Option>, + /// Number of control messages received in the current rate-limit window. + pub control_msg_count: u32, + /// Start of the current rate-limit window. + pub control_msg_window_start: Instant, +} + +impl FlashblocksConnectionState { + pub(crate) fn new() -> Self { + Self { + trusted: false, + send_enabled: false, + receive_status: ReceiveStatus::NotReceiving, + receive_status_timestamp: 0, + outbound_tx: None, + control_msg_count: 0, + control_msg_window_start: Instant::now(), + } + } +} /// Represents a single P2P connection for the flashblocks protocol. /// /// This struct manages the bidirectional communication with a single peer in the flashblocks /// P2P network. It handles incoming messages from the peer, validates and processes them, -/// and also streams outgoing messages that need to be broadcast. +/// and also streams serialized outgoing messages queued for this peer. /// /// The connection implements the `Stream` trait to provide outgoing message bytes that /// should be sent to the connected peer over the underlying protocol connection. @@ -46,12 +91,8 @@ pub struct FlashblocksConnection { conn: ProtocolConnection, /// The unique identifier of the connected peer. peer_id: PeerId, - /// Receiver for peer messages to be sent to all peers. - /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. - peer_rx: BroadcastStream, - /// Per-peer tracking of flashblocks this peer has already sent us. - /// Uses `peek` for lookups to avoid LRU promotion, giving FIFO eviction semantics. - received_cache: LruMap<(PayloadId, usize), ()>, + /// Receiver for already serialized protocol messages targeted at this specific peer. + outbound_rx: mpsc::UnboundedReceiver, } impl FlashblocksConnection { @@ -61,46 +102,28 @@ impl FlashblocksConnection { /// * `protocol` - The flashblocks protocol handler managing the connection. /// * `conn` - The underlying protocol connection for sending and receiving messages. /// * `peer_id` - The unique identifier of the connected peer. - /// * `peer_rx` - Receiver for peer messages to be sent to all peers. - pub fn new( + pub(crate) fn new( protocol: FlashblocksP2PProtocol, conn: ProtocolConnection, peer_id: PeerId, - peer_rx: BroadcastStream, ) -> Self { + let (outbound_tx, outbound_rx) = mpsc::unbounded_channel(); + + protocol + .handle + .on_peer_connected(protocol.network.clone(), peer_id, outbound_tx); + gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); Self { protocol, conn, peer_id, - peer_rx, - received_cache: LruMap::new(RECEIVED_CACHE_LEN), + outbound_rx, } } } -impl FlashblocksConnection { - /// Insert a `(payload_id, flashblock_index)` into the received cache. - /// - /// Uses [`LruMap::peek`] before insert to avoid promoting duplicates, - /// giving FIFO eviction semantics instead of LRU. - /// - /// Returns `true` if the key was newly inserted, `false` if it already existed. - fn received_cache_insert(&mut self, key: (PayloadId, usize)) -> bool { - if self.received_cache.peek(&key).is_some() { - return false; - } - self.received_cache.insert(key, ()) - } - - /// Check if a `(payload_id, flashblock_index)` exists in the received cache - /// without promoting it (preserves FIFO eviction order). - fn received_cache_contains(&self, key: &(PayloadId, usize)) -> bool { - self.received_cache.peek(key).is_some() - } -} - impl Drop for FlashblocksConnection { fn drop(&mut self) { info!( @@ -109,6 +132,8 @@ impl Drop for FlashblocksConnection { "dropping flashblocks connection" ); + self.protocol.handle.on_peer_disconnected(self.peer_id); + gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).decrement(1); } } @@ -120,57 +145,13 @@ impl Stream for FlashblocksConnection { let this = self.get_mut(); loop { - // Check if there are any flashblocks ready to broadcast to our peers. - if let Poll::Ready(Some(res)) = this.peer_rx.poll_next_unpin(cx) { - match res { - Ok(peer_msg) => { - match peer_msg { - PeerMsg::FlashblocksPayloadV1(( - payload_id, - flashblock_index, - bytes, - )) => { - // Check if this flashblock actually originated from this peer. - if !this.received_cache_contains(&(payload_id, flashblock_index)) { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - %payload_id, - %flashblock_index, - "Broadcasting `FlashblocksPayloadV1` message to peer" - ); - metrics::counter!("flashblocks.bandwidth_outbound") - .increment(bytes.len() as u64); - - return Poll::Ready(Some(bytes)); - } - } - PeerMsg::StartPublishing(bytes_mut) => { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Broadcasting `StartPublishing` to peer" - ); - return Poll::Ready(Some(bytes_mut)); - } - PeerMsg::StopPublishing(bytes_mut) => { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Broadcasting `StopPublishing` to peer" - ); - return Poll::Ready(Some(bytes_mut)); - } - } - } - Err(error) => { - tracing::error!( - target: "flashblocks::p2p", - %error, - "failed to receive flashblocks message from peer_rx" - ); - } - } + if let Poll::Ready(Some(bytes)) = this.outbound_rx.poll_recv(cx) { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "Sending serialized flashblocks protocol message to peer" + ); + return Poll::Ready(Some(bytes)); } // Check if there are any messages from the peer. @@ -234,6 +215,74 @@ impl Stream for FlashblocksConnection { } } } + FlashblocksP2PMsg::RequestFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received RequestFlashblocks from peer", + ); + if this + .protocol + .handle + .handle_request_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } + FlashblocksP2PMsg::AcceptFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received AcceptFlashblocks from peer", + ); + if this + .protocol + .handle + .handle_accept_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } + FlashblocksP2PMsg::RejectFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received RejectFlashblocks from peer", + ); + if this + .protocol + .handle + .handle_reject_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } + FlashblocksP2PMsg::CancelFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received CancelFlashblocks from peer", + ); + if this + .protocol + .handle + .handle_cancel_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } } } } @@ -259,22 +308,22 @@ impl FlashblocksConnection { &mut self, authorized_payload: AuthorizedPayload, ) { - let state_handle = self.protocol.handle.state.clone(); - let mut state = state_handle.lock(); let authorization = &authorized_payload.authorized.authorization; let msg = authorized_payload.msg(); + let flashblock_timestamp = msg.metadata.flashblock_timestamp; + let mut p2p_state = self.protocol.handle.state.lock(); // Check if this payload is older than our current view by more than the allowed // grace window. if authorization.timestamp - < state + < p2p_state .payload_timestamp .saturating_sub(AUTHORIZATION_TIMESTAMP_GRACE_SEC) { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = state.payload_timestamp, + current_timestamp = p2p_state.payload_timestamp, timestamp = authorization.timestamp, grace_sec = AUTHORIZATION_TIMESTAMP_GRACE_SEC, "received flashblock with outdated timestamp", @@ -286,22 +335,55 @@ impl FlashblocksConnection { } // Check if the payload index is within the allowed range - if msg.index as usize > MAX_FLASHBLOCK_INDEX { + if msg.index as usize >= MAX_FLASHBLOCKS { tracing::error!( target: "flashblocks::p2p", peer_id = %self.peer_id, index = msg.index, payload_id = %msg.payload_id, - max_index = MAX_FLASHBLOCK_INDEX, + max = MAX_FLASHBLOCKS, "Received flashblocks payload with index exceeding maximum" ); return; } - // Check if this peer is spamming us with the same payload index - if !self.received_cache_insert((msg.payload_id, msg.index as usize)) { - // We've already seen this index from this peer. - // They could be trying to DOS us. + let Some(conn_state) = p2p_state.connection_state(&self.peer_id) else { + return; + }; + match &conn_state.receive_status { + ReceiveStatus::Requesting => { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock before request was accepted", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + return; + } + ReceiveStatus::NotReceiving => { + if conn_state.receive_status_timestamp + 2 < authorization.timestamp { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock from peer outside receive window", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + } + return; + } + ReceiveStatus::Receiving { .. } => {} + } + + // Check if this peer is spamming us with the same payload index. + if !p2p_state.note_peer_received_flashblock(authorization, msg, self.peer_id) { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -315,11 +397,9 @@ impl FlashblocksConnection { return; } - state.publishing_status.send_modify(|status| { + p2p_state.publishing_status.send_modify(|status| { let active_publishers = match status { PublishingStatus::Publishing { .. } => { - // We are currently building, so we should not be seeing any new flashblocks - // over the p2p network. tracing::error!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -333,38 +413,38 @@ impl FlashblocksConnection { PublishingStatus::NotPublishing { active_publishers } => active_publishers, }; - // Update the list of active publishers if let Some((_, timestamp)) = active_publishers .iter_mut() .find(|(publisher, _)| *publisher == authorization.builder_vk) { - // This is an existing publisher, we should update their block number *timestamp = authorization.timestamp; } else { - // This is a new publisher, we should add them to the list of active publishers active_publishers.push((authorization.builder_vk, authorization.timestamp)); } }); - let now = Utc::now() - .timestamp_nanos_opt() - .expect("time went backwards"); - - if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { + if let Some(flashblock_timestamp) = flashblock_timestamp { + let now = Utc::now() + .timestamp_nanos_opt() + .expect("time went backwards"); let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); + if let Some(ReceiveStatus::Receiving { score }) = p2p_state + .connection_state_mut(&self.peer_id) + .map(|peer_state| &mut peer_state.receive_status) + { + score.record(latency); + } } self.protocol .handle .ctx - .publish(&mut state, authorized_payload); + .publish(&mut p2p_state, authorized_payload); } /// Handles incoming `StartPublish` messages from a peer. /// - /// TODO: handle propogating this if we care. For now we assume direct peering. - /// /// # Arguments /// * `authorized_payload` - The authorized `StartPublish` message received from the peer /// @@ -375,11 +455,11 @@ impl FlashblocksConnection { /// - If we are waiting to publish, updates the list of active publishers /// - If we are not publishing, adds the new publisher to the list of active publishers fn handle_start_publish(&mut self, authorized_payload: AuthorizedPayload) { - let state = self.protocol.handle.state.lock(); let Ok(builder_sk) = self.protocol.handle.builder_sk() else { return; }; let authorization = &authorized_payload.authorized.authorization; + let state = self.protocol.handle.state.lock(); // Check if the request is expired for dos protection. // It's important to ensure that this `StartPublish` request @@ -392,6 +472,7 @@ impl FlashblocksConnection { timestamp = authorized_payload.authorized.authorization.timestamp, "received initiate build request with outdated timestamp", ); + drop(state); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); @@ -412,8 +493,7 @@ impl FlashblocksConnection { let authorized = Authorized::new(builder_sk, *our_authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.protocol.handle.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: vec![( @@ -456,8 +536,6 @@ impl FlashblocksConnection { /// Handles incoming `StopPublish` messages from a peer. /// - /// TODO: handle propogating this if we care. For now we assume direct peering. - /// /// # Arguments /// * `authorized_payload` - The authorized `StopPublish` message received from the peer /// @@ -468,11 +546,11 @@ impl FlashblocksConnection { /// - If we are waiting to publish, removes the publisher from the list of active publishers and checks if we can start publishing /// - If we are not publishing, removes the publisher from the list of active publishers fn handle_stop_publish(&mut self, authorized_payload: AuthorizedPayload) { - let state = self.protocol.handle.state.lock(); let authorization = &authorized_payload.authorized.authorization; + let state = self.protocol.handle.state.lock(); // Check if the request is expired for dos protection. - // It's important to ensure that this `StartPublish` request + // It's important to ensure that this `StopPublish` request // is very recent, or it could be used in a replay attack. if state.payload_timestamp > authorization.timestamp { tracing::warn!( @@ -482,6 +560,7 @@ impl FlashblocksConnection { timestamp = authorized_payload.authorized.authorization.timestamp, "Received initiate build response with outdated timestamp", ); + drop(state); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); @@ -558,3 +637,30 @@ impl FlashblocksConnection { }); } } + +/// A lightweight moving average with a configurable smoothing window. +#[derive(Clone, Debug, PartialEq)] +pub struct Score { + value: Option, + window: i64, +} + +impl Score { + pub(crate) fn new(window: i64) -> Self { + Self { + value: None, + window: window.max(1), + } + } + + pub(crate) fn record(&mut self, sample: i64) { + self.value = Some(match self.value { + Some(current) => (current * (self.window - 1) + sample) / self.window, + None => sample, + }); + } + + pub(crate) fn value(&self) -> Option { + self.value + } +} diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs new file mode 100644 index 000000000..ad1be62bd --- /dev/null +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -0,0 +1,553 @@ +//! Canon-aware flashblock event stream. +//! +//! Merges a raw flashblock stream with canonical chain notifications, yielding +//! [`ChainEvent::Pending`] only when the flashblock's epoch parent matches +//! the current canonical tip, and [`ChainEvent::Canon`] whenever the tip +//! changes. + +use flashblocks_primitives::primitives::FlashblocksPayloadV1; +use futures::{ + Stream, StreamExt, + stream::{self, PollNext}, +}; +use reth::{payload::PayloadId, rpc::types::BlockNumHash}; +use std::{ + collections::VecDeque, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +#[derive(Clone, Debug)] +pub enum ChainEvent { + /// A new canonical tip has been observed. Consumers should clear any + /// pending flashblocks that are stale relative to the new tip. + Canon(BlockNumHash), + /// A flashblock has been received whose epoch parent matches the current + /// canonical tip. Zero-copy via [`Arc`] — no payload cloning through the + /// buffer or downstream consumers. + Pending(Arc), +} + +/// Events yielded by [`WorldChainEventsStream`]. +#[derive(Clone, Debug)] +pub enum WorldChainEvent { + /// An event emitted when executable pending flashblocks are observed. + Chain(ChainEvent), + /// An event emitted by any source. + Event(T), +} + +/// A stream of [`WorldChainEvent`]s that merges flashblocks with canonical +/// chain notifications, reducing them through a [`BufferedFlashblocks`] +/// state machine. +/// +/// A [`ChainEvent::Pending`] is emitted only when the flashblock's epoch parent +/// matches the canonical tip. Stale flashblocks are silently discarded. +/// Flashblocks are buffered when the epoch parent is not yet canonical, but the +/// [`PayloadId`] is fresh. A [`ChainEvent::Canon`] is emitted on every +/// canonical tip change so consumers can clear pending state. +pub type WorldChainEventsStream = Pin> + Send>>; + +/// Constructs a [`WorldChainEventsStream`] by merging a flashblock stream with +/// canonical chain notifications, reducing through [`BufferedFlashblocks`], and +/// applying `hook` to each yielded event. +#[must_use] +pub fn world_chain_events_stream( + flashblocks: Pin + Send>>, + canon: Pin + Send>>, + mut hook: F, +) -> WorldChainEventsStream +where + T: Send + Unpin + 'static, + F: FnMut(&WorldChainEvent) -> Option> + Send + 'static, +{ + let merged = + futures::stream::select_with_strategy(flashblocks, canon, |_: &mut ()| PollNext::Left); + + BufferedStream::new(merged) + .map(WorldChainEvent::Chain) + .flat_map(move |event| { + let extra = hook(&event); + stream::iter(std::iter::once(event).chain(extra)) + }) + .boxed() +} + +// --------------------------------------------------------------------------- +// BufferedStream — zero-allocation stream adapter +// --------------------------------------------------------------------------- + +/// Stream adapter that wraps a merged `ChainEvent` stream and map reduces it +/// into a [`BufferedFlashblocks`]. +#[pin_project::pin_project] +struct BufferedStream { + #[pin] + inner: S, + state: BufferedFlashblocks, +} + +impl BufferedStream { + fn new(inner: S) -> Self { + Self { + inner, + state: BufferedFlashblocks::default(), + } + } +} + +impl> Stream for BufferedStream { + type Item = ChainEvent; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + + // Drain buffered output first. + if let Some(event) = this.state.output.pop_front() { + return Poll::Ready(Some(event)); + } + + // Poll inner stream, reduce through state machine, yield first output. + match this.inner.poll_next(cx) { + Poll::Ready(Some(event)) => { + this.state.step(event); + Poll::Ready(this.state.output.pop_front()) + } + // Inner exhausted — drain any remaining buffered output before closing. + Poll::Ready(None) => Poll::Ready(this.state.output.pop_front()), + Poll::Pending => Poll::Pending, + } + } +} + +// --------------------------------------------------------------------------- +// Epoch — scoped state for a single flashblock epoch +// --------------------------------------------------------------------------- + +/// Maximum number of flashblocks per epoch. +pub(crate) const MAX_FLASHBLOCKS: usize = 12; + +/// State for a single flashblock epoch: the parent block it builds on, +/// its payload identifier, and a fixed-size buffer of received flashblocks. +pub(crate) struct BlockEpochState { + /// The parent block this epoch builds on. + parent: BlockNumHash, + /// Payload identifier for this epoch. + payload_id: PayloadId, + /// Drain watermark — next index to yield. + cursor: usize, + /// Fixed-size sparse buffer indexed by flashblock sequence number. + /// 96 bytes inline — no heap allocation. + buffer: [Option>; MAX_FLASHBLOCKS], +} + +impl BlockEpochState { + /// Create a new epoch from a base flashblock. Returns `None` if the base + /// is stale (parent behind the canonical tip) or missing its base field. + fn try_new(fb: Arc, canon_tip: Option) -> Option { + let base = fb.base.as_ref()?; + + // Stale check: reject if the epoch's parent is behind the canon tip. + let parent_number = base.block_number.saturating_sub(1); + if canon_tip.is_some_and(|tip| parent_number < tip.number) { + tracing::trace!( + target: "flashblocks::event_stream", + payload_id = %fb.payload_id, + parent_number, + canon_tip_number = canon_tip.map(|t| t.number), + "stale epoch rejected" + ); + metrics::counter!("flashblocks.event_stream.epochs_stale").increment(1); + return None; + } + + let parent = BlockNumHash { + number: parent_number, + hash: base.parent_hash, + }; + + let mut epoch = Self { + parent, + payload_id: fb.payload_id, + cursor: 0, + buffer: Default::default(), + }; + epoch.insert(fb); + Some(epoch) + } + + /// Insert a flashblock at its sequence index. Returns `false` if the + /// payload_id doesn't match, the index is out of bounds, or the slot + /// is already occupied. + fn insert(&mut self, fb: Arc) -> bool { + let idx = fb.index as usize; + if fb.payload_id != self.payload_id || idx >= MAX_FLASHBLOCKS || self.buffer[idx].is_some() + { + return false; + } + self.buffer[idx] = Some(fb); + true + } +} + +// --------------------------------------------------------------------------- +// BufferedFlashblocks — stateful reducer with Extend + Iterator +// --------------------------------------------------------------------------- + +/// Buffers flashblocks for the current epoch, gating output on the canonical +/// tip. Phase is derived from state — not tracked separately: +/// +/// - `epoch.is_none()` → no active epoch +/// - `epoch.is_some() && canon_tip != epoch.parent` → pending (buffering) +/// - `epoch.is_some() && canon_tip == epoch.parent` → executable (draining) +/// +/// Implements [`Extend`] to accept input events and +/// [`Iterator`] to drain output events. +#[derive(Default)] +pub struct BufferedFlashblocks { + /// Current epoch, if any. `None` means no active epoch. + epoch: Option, + /// Most recent canonical tip. + canon_tip: Option, + /// Output events ready to be yielded by the iterator. + output: VecDeque, +} + +impl BufferedFlashblocks { + /// Process a single input event, updating state and buffering output. + fn step(&mut self, event: ChainEvent) { + match event { + ChainEvent::Canon(tip) => { + self.canon_tip = Some(tip); + self.output.push_back(ChainEvent::Canon(tip)); + } + ChainEvent::Pending(ref fb) if fb.base.is_some() => { + self.epoch = BlockEpochState::try_new(Arc::clone(fb), self.canon_tip); + } + ChainEvent::Pending(fb) => { + if let Some(epoch) = &mut self.epoch { + epoch.insert(fb); + } + } + } + self.drain(); + } + + /// Drain contiguous flashblocks from the cursor into the output queue, + /// but only if the epoch is anchored to the canonical tip. + fn drain(&mut self) { + let canon_tip = self.canon_tip; + let Some(ref mut epoch) = self.epoch else { + return; + }; + if !canon_tip.is_some_and(|tip| tip == epoch.parent) { + return; + } + + while let Some(Some(_)) = epoch.buffer.get(epoch.cursor) { + let fb = epoch.buffer[epoch.cursor].take().unwrap(); + epoch.cursor += 1; + self.output.push_back(ChainEvent::Pending(fb)); + } + } +} + +impl Iterator for BufferedFlashblocks { + type Item = ChainEvent; + + fn next(&mut self) -> Option { + self.output.pop_front() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use flashblocks_primitives::primitives::{ + ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, + }; + + fn canon(number: u64, hash: B256) -> ChainEvent { + ChainEvent::Canon(BlockNumHash { number, hash }) + } + + fn base_fb( + payload_id: PayloadId, + index: u64, + parent_hash: B256, + block_number: u64, + ) -> ChainEvent { + ChainEvent::Pending(Arc::new(FlashblocksPayloadV1 { + payload_id, + index, + base: Some(ExecutionPayloadBaseV1 { + parent_hash, + block_number, + timestamp: block_number + 1000, // well above any block number + ..Default::default() + }), + diff: ExecutionPayloadFlashblockDeltaV1::default(), + metadata: Default::default(), + })) + } + + fn delta_fb(payload_id: PayloadId, index: u64) -> ChainEvent { + ChainEvent::Pending(Arc::new(FlashblocksPayloadV1 { + payload_id, + index, + base: None, + diff: ExecutionPayloadFlashblockDeltaV1::default(), + metadata: Default::default(), + })) + } + + fn pid(b: u8) -> PayloadId { + PayloadId::new([b; 8]) + } + + fn hash(b: u8) -> B256 { + B256::with_last_byte(b) + } + + fn collect_pending(buf: &mut BufferedFlashblocks) -> Vec { + buf.by_ref() + .filter_map(|e| match e { + ChainEvent::Pending(fb) => Some(fb.index), + _ => None, + }) + .collect() + } + + fn collect_all(buf: &mut BufferedFlashblocks) -> Vec { + buf.by_ref().collect() + } + + // ----------------------------------------------------------------------- + // Core state machine tests + // ----------------------------------------------------------------------- + + #[test] + fn no_output_before_canon_tip() { + let mut buf = BufferedFlashblocks::default(); + + // Send a base flashblock — no canon tip yet, goes to Pending + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert!( + collect_pending(&mut buf).is_empty(), + "should not yield without canon tip" + ); + } + + #[test] + fn canon_tip_triggers_drain() { + let mut buf = BufferedFlashblocks::default(); + + // Canon tip first, then base flashblock whose parent matches + buf.step(canon(0, hash(0))); + let events = collect_all(&mut buf); + assert_eq!(events.len(), 1); // just the canon event + assert!(matches!(events[0], ChainEvent::Canon(_))); + + // Now a base flashblock building on block 1 with parent hash(0) + buf.step(base_fb(pid(1), 0, hash(0), 1)); + let indices = collect_pending(&mut buf); + assert_eq!( + indices, + vec![0], + "should drain immediately when parent matches canon tip" + ); + } + + #[test] + fn canon_tip_after_buffered_flashblock_flushes() { + let mut buf = BufferedFlashblocks::default(); + + // Base flashblock arrives first — parent hash(5), block_number 6 + buf.step(base_fb(pid(1), 0, hash(5), 6)); + assert!(collect_pending(&mut buf).is_empty(), "no canon tip yet"); + + // Delta flashblock for same epoch + buf.step(delta_fb(pid(1), 1)); + assert!(collect_pending(&mut buf).is_empty(), "still no canon tip"); + + // Now canon tip arrives matching the parent + buf.step(canon(5, hash(5))); + let events: Vec<_> = collect_all(&mut buf); + + // Should yield: Canon(5), Pending(0), Pending(1) + assert!(matches!(events[0], ChainEvent::Canon(_))); + assert_eq!(events.len(), 3); + + let indices: Vec<_> = events + .iter() + .filter_map(|e| match e { + ChainEvent::Pending(fb) => Some(fb.index), + _ => None, + }) + .collect(); + assert_eq!(indices, vec![0, 1]); + } + + #[test] + fn out_of_order_flashblocks_buffered_until_contiguous() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); // drain canon + + // Base at index 0 + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert_eq!(collect_pending(&mut buf), vec![0]); + + // Index 2 arrives before 1 — gap, can't drain + buf.step(delta_fb(pid(1), 2)); + assert!(collect_pending(&mut buf).is_empty(), "gap at index 1"); + + // Index 1 fills the gap — both 1 and 2 should drain + buf.step(delta_fb(pid(1), 1)); + assert_eq!(collect_pending(&mut buf), vec![1, 2]); + } + + #[test] + fn stale_base_flashblock_discarded() { + let mut buf = BufferedFlashblocks::default(); + + // Canon tip is at block 10 + buf.step(canon(10, hash(10))); + collect_all(&mut buf); + + // Base flashblock building on block 5 (parent_number=4 < tip=10) — stale + buf.step(base_fb(pid(1), 0, hash(4), 5)); + assert!( + collect_pending(&mut buf).is_empty(), + "stale flashblock should be discarded" + ); + } + + #[test] + fn new_epoch_resets_buffer() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + // Epoch A + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert_eq!(collect_pending(&mut buf), vec![0]); + buf.step(delta_fb(pid(1), 1)); + assert_eq!(collect_pending(&mut buf), vec![1]); + + // Epoch B — new base with different payload_id, same parent + buf.step(base_fb(pid(2), 0, hash(0), 1)); + let indices = collect_pending(&mut buf); + assert_eq!(indices, vec![0], "new epoch should reset and yield base"); + } + + #[test] + fn canon_event_always_yielded() { + let mut buf = BufferedFlashblocks::default(); + + // Multiple canon events should all be yielded + buf.step(canon(0, hash(0))); + buf.step(canon(1, hash(1))); + buf.step(canon(2, hash(2))); + + let events = collect_all(&mut buf); + let canon_numbers: Vec<_> = events + .iter() + .filter_map(|e| match e { + ChainEvent::Canon(tip) => Some(tip.number), + _ => None, + }) + .collect(); + assert_eq!(canon_numbers, vec![0, 1, 2]); + } + + #[test] + fn non_base_flashblock_ignored_when_uninitialized() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + // Delta without any base — should be silently ignored + buf.step(delta_fb(pid(1), 5)); + assert!(collect_pending(&mut buf).is_empty()); + } + + #[test] + fn canon_tip_not_matching_parent_does_not_drain() { + let mut buf = BufferedFlashblocks::default(); + + // Base building on hash(5) at block 6 + buf.step(base_fb(pid(1), 0, hash(5), 6)); + assert!(collect_pending(&mut buf).is_empty()); + + // Canon tip at block 3, hash(3) — doesn't match parent hash(5) + buf.step(canon(3, hash(3))); + let events = collect_all(&mut buf); + + // Canon event is yielded, but no pending drained + assert_eq!(events.len(), 1); + assert!(matches!(events[0], ChainEvent::Canon(_))); + } + + #[test] + fn batch_processes_multiple_events() { + let mut buf = BufferedFlashblocks::default(); + for e in [ + canon(0, hash(0)), + base_fb(pid(1), 0, hash(0), 1), + delta_fb(pid(1), 1), + delta_fb(pid(1), 2), + ] { + buf.step(e); + } + + let events = collect_all(&mut buf); + + // Canon(0), Pending(0), Pending(1), Pending(2) + assert_eq!(events.len(), 4); + assert!(matches!(events[0], ChainEvent::Canon(_))); + + let indices: Vec<_> = events + .iter() + .filter_map(|e| match e { + ChainEvent::Pending(fb) => Some(fb.index), + _ => None, + }) + .collect(); + assert_eq!(indices, vec![0, 1, 2]); + } + + #[test] + fn duplicate_index_ignored() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert_eq!(collect_pending(&mut buf), vec![0]); + + // Same index again — ignored + buf.step(delta_fb(pid(1), 0)); + assert!(collect_pending(&mut buf).is_empty()); + } + + #[test] + fn wrong_payload_id_ignored() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + buf.step(base_fb(pid(1), 0, hash(0), 1)); + collect_pending(&mut buf); + + // Delta with wrong payload_id — ignored + buf.step(delta_fb(pid(99), 1)); + assert!(collect_pending(&mut buf).is_empty()); + } +} diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 721278e9c..7f7f57afe 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,7 +1,12 @@ -use crate::protocol::{connection::FlashblocksConnection, error::FlashblocksP2PError}; +use crate::protocol::{ + connection::{FlashblocksConnection, FlashblocksConnectionState, ReceiveStatus, Score}, + error::FlashblocksP2PError, + event::{ChainEvent, WorldChainEvent, WorldChainEventsStream, world_chain_events_stream}, +}; use alloy_rlp::BytesMut; use chrono::Utc; use ed25519_dalek::{SigningKey, VerifyingKey}; +use flashblocks_cli::FanoutArgs; use flashblocks_primitives::{ p2p::{ Authorization, Authorized, AuthorizedMsg, AuthorizedPayload, FlashblocksP2PMsg, @@ -9,16 +14,24 @@ use flashblocks_primitives::{ }, primitives::FlashblocksPayloadV1, }; -use futures::{Stream, StreamExt, stream}; -use metrics::histogram; +use futures::{Stream, StreamExt as _}; use parking_lot::Mutex; -use reth::payload::PayloadId; +use rand::Rng; +use reth::{payload::PayloadId, rpc::types::BlockNumHash}; + use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; use reth_network::Peers; -use std::{net::SocketAddr, sync::Arc}; -use tokio::sync::{broadcast, watch}; -use tokio_stream::wrappers::BroadcastStream; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{ + sync::{broadcast, mpsc, watch}, + time, +}; use tracing::{debug, info, warn}; use reth_ethereum::network::{ @@ -26,14 +39,11 @@ use reth_ethereum::network::{ eth_wire::{capability::SharedCapabilities, multiplex::ProtocolConnection, protocol::Protocol}, protocol::{ConnectionHandler, OnNotSupported}, }; +use tokio_stream::wrappers::BroadcastStream; + /// Maximum frame size for rlpx messages. const MAX_FRAME: usize = 1 << 24; // 16 MiB -/// Maximum index for flashblocks payloads. -/// Not intended to ever be hit. Since we resize the flashblocks vector dynamically, -/// this is just a sanity check to prevent excessive memory usage. -pub(crate) const MAX_FLASHBLOCK_INDEX: usize = 100; - /// The maximum number of seconds we will wait for a previous publisher to stop /// before continueing anyways. const MAX_PUBLISH_WAIT_SEC: u64 = 2; @@ -42,6 +52,29 @@ const MAX_PUBLISH_WAIT_SEC: u64 = 2; /// before dropping them. In practice, we should rarely need to buffer any messages. const BROADCAST_BUFFER_CAPACITY: usize = 100; +/// A missed flashblock should dominate modest latency differences when rotating receive peers. +const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; +/// Grace window in number of flashblocks to receive late flashblocks from peers before scoring them for missing flashblocks. +/// +/// This must be at least long enough to cover AUTHORIZATION_TIMESTAMP_GRACE_SEC to prevent a spam +/// attack. +pub(crate) const RECEIVE_FLASHBLOCK_GRACE_WINDOW: usize = 50; + +/// Maximum number of control messages (Request/Accept/Reject/Cancel) a peer may send +/// within a sliding window before being penalized. +const MAX_CONTROL_MSGS_PER_WINDOW: u32 = 10; + +/// Duration of the per-peer control-message rate-limit window. +const CONTROL_MSG_WINDOW: Duration = Duration::from_secs(30); +/// Maximum time to wait for a peer to answer a `RequestFlashblocks` message. +const RECEIVE_REQUEST_TIMEOUT_SECS: u64 = 2; + +/// Maximum time to wait for the network manager to expose the newly connected peer's trust info. +const PEER_INFO_LOOKUP_TIMEOUT: Duration = Duration::from_secs(1); + +/// Poll interval while waiting for connected peer metadata to become available. +const PEER_INFO_LOOKUP_RETRY_INTERVAL: Duration = Duration::from_millis(10); + /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -50,20 +83,6 @@ pub trait FlashblocksP2PNetworkHandle: Clone + Unpin + Peers + std::fmt::Debug + impl FlashblocksP2PNetworkHandle for N {} -/// Messages that can be broadcast over a channel to each internal peer connection. -/// -/// These messages are used internally to coordinate the broadcasting of flashblocks -/// and publishing status changes to all connected peers. -#[derive(Clone, Debug)] -pub enum PeerMsg { - /// Send an already serialized flashblock to all peers. - FlashblocksPayloadV1((PayloadId, usize, BytesMut)), - /// Send a previously serialized StartPublish message to all peers. - StartPublishing(BytesMut), - /// Send a previously serialized StopPublish message to all peers. - StopPublishing(BytesMut), -} - /// The current publishing status of this node in the flashblocks P2P network. /// /// This enum tracks whether we are actively publishing flashblocks, waiting to publish, @@ -100,12 +119,24 @@ impl Default for PublishingStatus { } } +/// Tracked information about a flashblock payload observed from the network. +#[derive(Clone, Debug)] +pub struct ObservedPayload { + payload_id: PayloadId, + timestamp: u64, + flashblock_index: u64, + /// Peers from which we've received this flashblock. + received_peers: HashSet, + /// Peers who we have sent this flashblock to. + send_peers: HashSet, +} + /// Protocol state that stores the flashblocks P2P protocol events and coordination data. /// /// This struct maintains the current state of flashblock publishing, including coordination /// with other publishers, payload buffering, and ordering information. It serves as the /// central state management for the flashblocks P2P protocol handler. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct FlashblocksP2PState { /// Current publishing status indicating whether we're publishing, waiting, or not publishing. pub publishing_status: watch::Sender, @@ -115,22 +146,477 @@ pub struct FlashblocksP2PState { pub payload_timestamp: u64, /// Timestamp at which the most recent flashblock was received in ns since the unix epoch. pub flashblock_timestamp: i64, - /// The index of the next flashblock to emit over the flashblocks stream. - /// Used to maintain strict ordering of flashblock delivery. - pub flashblock_index: usize, - /// Buffer of flashblocks for the current payload, indexed by flashblock sequence number. - /// Contains `None` for flashblocks not yet received, enabling out-of-order receipt - /// while maintaining in-order delivery. - pub flashblocks: Vec>, + /// Most recent canonical tip. Updated by the stream hook. + /// Used to reject stale flashblocks in `publish()`. + pub canon_tip: Option, + /// Last flashblock flushed through the stream to the coordinator. + /// Only flashblocks at or ahead of this cursor should be peered. + pub flushed_payload_id: Option, + pub flushed_index: u64, + /// Flashblocks observed from network peers, tracked until their receive grace windows expire. + pub observed_payloads: VecDeque, + /// All currently connected peers and their connection state. + pub connections: HashMap, +} + +impl Default for FlashblocksP2PState { + fn default() -> Self { + let (publishing_status, _) = watch::channel(PublishingStatus::default()); + + Self { + publishing_status, + payload_id: PayloadId::default(), + payload_timestamp: 0, + flashblock_timestamp: 0, + canon_tip: None, + flushed_payload_id: None, + flushed_index: 0, + observed_payloads: VecDeque::new(), + connections: HashMap::new(), + } + } } impl FlashblocksP2PState { - /// Returns the current publishing status of this node. + /// Returns the connection state of a peer. + pub(crate) fn connection_state(&self, peer_id: &PeerId) -> Option<&FlashblocksConnectionState> { + self.connections.get(peer_id) + } + + pub(crate) fn connection_state_mut( + &mut self, + peer_id: &PeerId, + ) -> Option<&mut FlashblocksConnectionState> { + self.connections.get_mut(peer_id) + } + + /// Marks receiving a flashblock from a peer and returns whether this is the first time we've observed this peer receive this flashblock. /// - /// This indicates whether the node is actively publishing flashblocks, - /// waiting to publish, or not publishing at all. - pub fn publishing_status(&self) -> PublishingStatus { - self.publishing_status.borrow().clone() + /// Called when a flashblock is received from any peer. + pub(crate) fn note_peer_received_flashblock( + &mut self, + authorization: &Authorization, + flashblock: &FlashblocksPayloadV1, + peer_id: PeerId, + ) -> bool { + if let Some(observed_payload) = self.observed_payloads.iter_mut().find(|observed_payload| { + observed_payload.payload_id == flashblock.payload_id + && observed_payload.flashblock_index == flashblock.index + }) { + return observed_payload.received_peers.insert(peer_id); + } + + if self.observed_payloads.len() >= RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let evicted = self.observed_payloads.pop_front().unwrap(); + for (peer_id, connection) in &mut self.connections { + if connection.receive_status_timestamp + 2 <= evicted.timestamp + && !evicted.received_peers.contains(peer_id) + && !evicted.send_peers.contains(peer_id) + && let ReceiveStatus::Receiving { score } = &mut connection.receive_status + { + debug!( + target: "flashblocks::p2p", + %peer_id, + payload_id = %evicted.payload_id, + flashblock_index = evicted.flashblock_index, + "scoring peer for missed flashblock", + ); + score.record(MISSED_FLASHBLOCK_PENALTY_NS); + } + } + } + + self.observed_payloads.push_back(ObservedPayload { + payload_id: flashblock.payload_id, + timestamp: authorization.timestamp, + flashblock_index: flashblock.index, + received_peers: HashSet::from([peer_id]), + send_peers: HashSet::new(), + }); + + true + } + + /// Returns whether we've seen a given flashblock from a given peer. + pub(crate) fn peer_received_flashblock( + &self, + peer_id: PeerId, + payload_id: PayloadId, + index: u64, + ) -> bool { + self.observed_payloads + .iter() + .find(|observed_payload| { + observed_payload.payload_id == payload_id + && observed_payload.flashblock_index == index + }) + .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) + } + + /// Sends an already serialized message to all connected peers. + pub(crate) fn send_to_all_peers(&self, bytes: &BytesMut) { + for conn in self.connections.values() { + if let Some(tx) = &conn.outbound_tx { + tx.send(bytes.clone()).ok(); + } + } + } + + /// Sends a serialized flashblock to peers in the current send set that have not + /// already delivered that flashblock to us. + fn send_flashblock_to_send_set( + &mut self, + payload_id: PayloadId, + flashblock_index: u64, + bytes: &BytesMut, + ) { + for (peer_id, conn) in &self.connections { + if !conn.send_enabled + || self.peer_received_flashblock(*peer_id, payload_id, flashblock_index) + { + continue; + } + self.observed_payloads + .iter_mut() + .find(|observed_payload| { + observed_payload.payload_id == payload_id + && observed_payload.flashblock_index == flashblock_index + }) + .map(|observed_payload| observed_payload.send_peers.insert(*peer_id)); + + if let Some(tx) = &conn.outbound_tx + && tx.send(bytes.clone()).is_ok() + { + metrics::counter!("flashblocks.bandwidth_outbound").increment(bytes.len() as u64); + } + } + } + + /// Sends a control message directly to a specific peer. + fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { + let bytes: &BytesMut = &msg.encode(); + if let Some(conn) = self.connections.get(&peer_id) + && let Some(tx) = &conn.outbound_tx + { + tx.send(bytes.clone()).ok(); + } + } + + /// Returns `true` if the peer has exceeded the control-message rate limit. + fn check_control_rate_limit(&mut self, peer_id: &PeerId) -> bool { + let Some(peer_state) = self.connections.get_mut(peer_id) else { + return true; + }; + let now = Instant::now(); + if now.duration_since(peer_state.control_msg_window_start) > CONTROL_MSG_WINDOW { + peer_state.control_msg_count = 0; + peer_state.control_msg_window_start = now; + } + peer_state.control_msg_count += 1; + peer_state.control_msg_count > MAX_CONTROL_MSGS_PER_WINDOW + } + + fn num_receive_peers(&self) -> usize { + self.connections + .values() + .filter(|peer_state| { + matches!(peer_state.receive_status, ReceiveStatus::Receiving { .. }) + }) + .count() + } + + fn receive_retry_cooldown_secs(ctx: &FlashblocksP2PCtx) -> u64 { + Duration::from_secs(ctx.fanout_args.rotation_interval) + .as_secs() + .max(1) + } + + fn clear_receive_state( + peer_state: &mut FlashblocksConnectionState, + receive_status_timestamp: u64, + ) { + peer_state.receive_status = ReceiveStatus::NotReceiving; + peer_state.receive_status_timestamp = receive_status_timestamp; + } + + fn available_receive_candidates(&self, ctx: &FlashblocksP2PCtx) -> Vec<(PeerId, bool)> { + let now = Utc::now().timestamp() as u64; + let retry_cooldown = Self::receive_retry_cooldown_secs(ctx); + self.connections + .iter() + .filter_map(|(peer_id, peer_state)| { + if peer_state.receive_status == ReceiveStatus::NotReceiving + && (peer_state.receive_status_timestamp == 0 + || peer_state.receive_status_timestamp + retry_cooldown <= now) + { + Some((*peer_id, peer_state.trusted)) + } else { + None + } + }) + .collect() + } + + fn begin_requesting_peer(&mut self, peer_id: PeerId) { + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return; + }; + let timestamp = Utc::now().timestamp() as u64; + peer_state.receive_status = ReceiveStatus::Requesting; + peer_state.receive_status_timestamp = timestamp; + debug!( + target: "flashblocks::p2p", + %peer_id, + "sending RequestFlashblocks to peer", + ); + self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + } + + fn num_receive_or_requesting_peers(&self) -> usize { + self.connections + .values() + .filter(|peer_state| { + matches!( + peer_state.receive_status, + ReceiveStatus::Receiving { .. } | ReceiveStatus::Requesting + ) + }) + .count() + } + + pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { + while self.num_receive_or_requesting_peers() < ctx.fanout_args.max_receive_peers { + let candidates = self.available_receive_candidates(ctx); + if candidates.is_empty() { + return; + } + let rand = rand::rng().random_range(0..candidates.len()); + self.begin_requesting_peer(candidates[rand].0); + } + } + + fn expire_stale_receive_requests(&mut self, ctx: &FlashblocksP2PCtx) { + let now = Utc::now().timestamp() as u64; + let mut cleared_any = false; + + for (peer_id, peer_state) in &mut self.connections { + if matches!(peer_state.receive_status, ReceiveStatus::Requesting) + && peer_state.receive_status_timestamp + RECEIVE_REQUEST_TIMEOUT_SECS <= now + { + debug!( + target: "flashblocks::p2p", + %peer_id, + "receive request timed out, clearing peer", + ); + Self::clear_receive_state(peer_state, now); + cleared_any = true; + } + } + + if cleared_any { + self.maybe_request_receive_peers(ctx); + } + } + + fn worst_receive_peer(&self) -> Option { + self.connections + .iter() + .filter_map(|(peer_id, peer_state)| { + let ReceiveStatus::Receiving { score } = &peer_state.receive_status else { + return None; + }; + Some((*peer_id, score.value())) + }) + .max_by( + |(_, lhs_score), (_, rhs_score)| match (lhs_score, rhs_score) { + (None, None) => std::cmp::Ordering::Equal, + (None, Some(_)) => std::cmp::Ordering::Greater, + (Some(_), None) => std::cmp::Ordering::Less, + (Some(lhs), Some(rhs)) => lhs.cmp(rhs), + }, + ) + .map(|(peer_id, _)| peer_id) + } + + fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { + if self.num_receive_peers() < ctx.fanout_args.max_receive_peers { + return; + } + + let Some(evict) = self.worst_receive_peer() else { + return; + }; + + let candidates = self.available_receive_candidates(ctx); + if candidates.is_empty() { + return; + } + + let rand = rand::rng().random_range(0..candidates.len()); + let candidate = candidates[rand].0; + + debug!( + target: "flashblocks::p2p", + evicted_peer = %evict, + new_peer = %candidate, + "rotating receive peer", + ); + + let evict_timestamp = Utc::now().timestamp() as u64; + if let Some(evict_state) = self.connection_state_mut(&evict) { + Self::clear_receive_state(evict_state, evict_timestamp); + } + self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + + self.begin_requesting_peer(candidate); + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + warn!( + target: "flashblocks::p2p", + %peer_id, + "rejecting RequestFlashblocks: rate limit exceeded", + ); + return Err(()); + } + + let Some(peer_state) = self.connection_state(&peer_id) else { + return Ok(()); + }; + + if peer_state.send_enabled { + warn!( + target: "flashblocks::p2p", + %peer_id, + "rejecting RequestFlashblocks: already sending to peer", + ); + return Err(()); + } + let peer_is_trusted = peer_state.trusted; + let send_count = self.connections.values().filter(|s| s.send_enabled).count(); + + if !peer_is_trusted && send_count >= ctx.fanout_args.max_send_peers { + debug!( + target: "flashblocks::p2p", + %peer_id, + send_count, + max_send_peers = ctx.fanout_args.max_send_peers, + "rejecting RequestFlashblocks: send set full", + ); + self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); + return Ok(()); + } + + info!( + target: "flashblocks::p2p", + %peer_id, + trusted = peer_is_trusted, + send_count = send_count + 1, + "accepted RequestFlashblocks, adding peer to send set", + ); + let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); + peer_state.send_enabled = true; + self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + Ok(()) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return Ok(()); + }; + + match peer_state.receive_status { + ReceiveStatus::Requesting => { + info!( + target: "flashblocks::p2p", + %peer_id, + "peer accepted our receive request, now receiving flashblocks", + ); + peer_state.receive_status = ReceiveStatus::Receiving { + score: Score::new(ctx.fanout_args.score_samples), + }; + Ok(()) + } + // Unsolicited accept — we never asked this peer. + _ => { + warn!( + target: "flashblocks::p2p", + %peer_id, + status = ?peer_state.receive_status, + "received unsolicited AcceptFlashblocks", + ); + Err(()) + } + } + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return Ok(()); + }; + + match peer_state.receive_status { + ReceiveStatus::Requesting => { + info!( + target: "flashblocks::p2p", + %peer_id, + "peer rejected our receive request, will try another peer", + ); + Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); + self.maybe_request_receive_peers(ctx); + Ok(()) + } + // Unsolicited reject — we never asked this peer. + _ => { + warn!( + target: "flashblocks::p2p", + %peer_id, + status = ?peer_state.receive_status, + "received unsolicited RejectFlashblocks", + ); + Err(()) + } + } + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_cancel(&mut self, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return Ok(()); + }; + + if !peer_state.send_enabled { + warn!( + target: "flashblocks::p2p", + %peer_id, + "received CancelFlashblocks from peer we are not sending to", + ); + return Err(()); + } + + info!( + target: "flashblocks::p2p", + %peer_id, + "peer cancelled flashblocks, removing from send set", + ); + peer_state.send_enabled = false; + Ok(()) } } @@ -143,11 +629,8 @@ impl FlashblocksP2PState { pub struct FlashblocksP2PCtx { /// Authorizer's verifying key used to verify authorization signatures from rollup-boost. pub authorizer_vk: VerifyingKey, - /// Builder's signing key used to sign outgoing authorized P2P messages. - pub builder_sk: Option, - /// Broadcast sender for peer messages that will be sent to all connected peers. - /// Messages may not be strictly ordered due to network conditions. - pub peer_tx: broadcast::Sender, + /// Flashblocks configuration including signing keys and fanout args. + pub fanout_args: FanoutArgs, /// Broadcast sender for verified and strictly ordered flashblock payloads. /// Used by RPC overlays and other consumers of flashblock data. pub flashblock_tx: broadcast::Sender, @@ -161,6 +644,8 @@ pub struct FlashblocksP2PCtx { pub struct FlashblocksHandle { /// Shared context containing network handle, keys, and communication channels. pub ctx: FlashblocksP2PCtx, + /// Builder signing key used to sign outgoing authorized P2P messages. + pub builder_sk: Option, /// Thread-safe mutable state of the flashblocks protocol. /// Protected by a mutex to allow concurrent access from multiple connections. pub state: Arc>, @@ -168,28 +653,204 @@ pub struct FlashblocksHandle { impl FlashblocksHandle { pub fn new(authorizer_vk: VerifyingKey, builder_sk: Option) -> Self { + Self::with_fanout_args(authorizer_vk, builder_sk, FanoutArgs::default()) + } + + pub fn with_fanout_args( + authorizer_vk: VerifyingKey, + builder_sk: Option, + fanout_args: FanoutArgs, + ) -> Self { let flashblock_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); - let peer_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); let state = Arc::new(Mutex::new(FlashblocksP2PState::default())); let ctx = FlashblocksP2PCtx { authorizer_vk, - builder_sk, - peer_tx, + fanout_args, flashblock_tx, }; + let handle = Self { + ctx, + builder_sk, + state, + }; + let moved_handle = handle.clone(); + + tokio::spawn(async move { + let mut rotation_interval = time::interval(Duration::from_secs( + moved_handle.ctx.fanout_args.rotation_interval, + )); + rotation_interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); + rotation_interval.tick().await; + + loop { + rotation_interval.tick().await; + let mut state = moved_handle.state.lock(); + state.expire_stale_receive_requests(&moved_handle.ctx); + state.maybe_request_receive_peers(&moved_handle.ctx); + state.maybe_start_rotation(&moved_handle.ctx); + } + }); + + handle + } + + /// Returns a [`WorldChainEventsStream`] merging flashblocks from the P2P + /// broadcast channel with canonical chain notifications from `provider`. + /// + /// Canon events automatically update the P2P state's `canon_tip` so + /// `publish()` rejects stale flashblocks. The caller's `hook` is applied + /// after the canon_tip update. + pub fn event_stream(&self, provider: P, hook: F) -> WorldChainEventsStream + where + T: Send + Clone + Unpin + 'static, + P: reth::providers::CanonStateSubscriptions + Clone + Send + Sync + 'static, + N: reth::api::NodePrimitives, + F: FnMut(&WorldChainEvent) -> Option> + Send + 'static, + { + let state = self.state.clone(); + let mut user_hook = hook; + + let combined_hook = move |event: &WorldChainEvent| { + if let WorldChainEvent::Chain(ChainEvent::Canon(tip)) = event { + state.lock().canon_tip = Some(*tip); + } + user_hook(event) + }; + + world_chain_events_stream( + BroadcastStream::new(self.ctx.flashblock_tx.subscribe()) + .filter_map(|x| { + futures::future::ready(match x { + Ok(fb) => Some(fb), + Err(tokio_stream::wrappers::errors::BroadcastStreamRecvError::Lagged( + n, + )) => { + tracing::warn!(missed = n, "flashblocks broadcast receiver lagged"); + None + } + }) + }) + .map(|fb| ChainEvent::Pending(Arc::new(fb))) + .boxed(), + provider + .canonical_state_stream() + .map(|n| ChainEvent::Canon(n.tip().num_hash())) + .boxed(), + combined_hook, + ) + } + + pub(crate) fn on_peer_connected( + &self, + network: N, + peer_id: PeerId, + outbound_tx: mpsc::UnboundedSender, + ) { + let trusted = tokio::task::block_in_place(|| { + let network = network.clone(); + tokio::runtime::Handle::current().block_on(async move { + let deadline = Instant::now() + PEER_INFO_LOOKUP_TIMEOUT; + + loop { + match network.get_peer_by_id(peer_id).await { + Ok(Some(peer_info)) => return Ok(peer_info.kind.is_trusted()), + Ok(None) if Instant::now() < deadline => { + time::sleep(PEER_INFO_LOOKUP_RETRY_INTERVAL).await; + } + Ok(None) => { + return Err( + "timed out waiting for peer info after connection".to_owned() + ); + } + Err(error) if Instant::now() < deadline => { + time::sleep(PEER_INFO_LOOKUP_RETRY_INTERVAL).await; + tracing::debug!( + target: "flashblocks::p2p", + %peer_id, + %error, + "retrying peer info lookup for flashblocks fanout" + ); + } + Err(error) => { + return Err(format!( + "failed to load peer info for flashblocks fanout: {error}" + )); + } + } + } + }) + }); + + let trusted = match trusted { + Ok(trusted) => trusted, + Err(error) => { + warn!( + target: "flashblocks::p2p", + %peer_id, + %error, + "failed to classify peer for flashblocks fanout; defaulting to untrusted" + ); + false + } + }; + + let mut state = self.state.lock(); + let mut conn_state = FlashblocksConnectionState::new(); + conn_state.outbound_tx = Some(outbound_tx); + conn_state.trusted = trusted; + state.connections.insert(peer_id, conn_state); + + info!( + target: "flashblocks::p2p", + %peer_id, + trusted, + total_peers = state.connections.len(), + "flashblocks peer connected", + ); - Self { ctx, state } + state.maybe_request_receive_peers(&self.ctx); } - pub fn flashblocks_tx(&self) -> broadcast::Sender { - self.ctx.flashblock_tx.clone() + pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + let removed = state.connections.remove(&peer_id); + + if let Some(conn_state) = &removed { + info!( + target: "flashblocks::p2p", + %peer_id, + was_sending = conn_state.send_enabled, + receive_status = ?conn_state.receive_status, + remaining_peers = state.connections.len(), + "flashblocks peer disconnected", + ); + } + + state.maybe_request_receive_peers(&self.ctx); } - pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { - self.ctx - .builder_sk - .as_ref() - .ok_or(FlashblocksP2PError::MissingBuilderSk) + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_request_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_request(&self.ctx, peer_id) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_accept_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_accept(&self.ctx, peer_id) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_reject_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_reject(&self.ctx, peer_id) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_cancel(peer_id) } } @@ -230,42 +891,28 @@ impl FlashblocksP2PProtocol { } impl FlashblocksP2PProtocol { - /// Returns the P2P capability for the flashblocks v1 protocol. + /// Returns the P2P capability for the flashblocks v2 protocol. /// /// This capability is used during devp2p handshake to advertise support - /// for the flashblocks protocol with protocol name "flblk" and version 1. + /// for the flashblocks protocol with protocol name "flblk" and version 2. pub fn capability() -> Capability { - Capability::new_static("flblk", 1) + Capability::new_static("flblk", 2) } } impl FlashblocksHandle { - /// Retrieves the next flashblock from the protocol state based on the provided cursor. - /// - /// Will return the flashblock at the cursor if it exists. - /// Will return the first flashblock if the cursor points to a different payload or is None. - /// Returns None if the flashblock at the cursor or the first flashblock does not exist. - fn next_flashblock_from_state( - state: &FlashblocksP2PState, - cursor: Option<&(PayloadId, usize)>, - ) -> Option { - match cursor { - Some((payload_id, next_index)) if *payload_id == state.payload_id => state - .flashblocks - .get(*next_index) - .and_then(|flashblock| flashblock.clone()), - _ => state - .flashblocks - .first() - .and_then(|flashblock| flashblock.clone()), - } + /// Returns the builder signing key if configured. + pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { + self.builder_sk + .as_ref() + .ok_or(FlashblocksP2PError::MissingBuilderSk) } /// Publishes a newly created flashblock from the payload builder to the P2P network. /// /// This method validates that the builder has authorization to publish and that /// the authorization matches the current publishing session. The flashblock is - /// then processed, cached, and broadcast to all connected peers. + /// then processed, cached, and forwarded to peers in the current send set. /// /// # Arguments /// * `authorized_payload` - The signed flashblock payload with authorization @@ -294,6 +941,11 @@ impl FlashblocksHandle { Ok(()) } + /// Sends an already serialized protocol message to all currently connected peers. + pub fn send_serialized_to_all_peers(&self, bytes: BytesMut) { + self.state.lock().send_to_all_peers(&bytes); + } + /// Returns the current publishing status of this node. /// /// The status indicates whether the node is actively publishing flashblocks, @@ -381,13 +1033,12 @@ impl FlashblocksHandle { } } PublishingStatus::NotPublishing { active_publishers } => { - // Send an authorized `StartPublish` message to the network + // Send an authorized `StartPublish` message to direct peers. let authorized_msg = AuthorizedMsg::StartPublish(StartPublish); let authorized_payload = Authorized::new(builder_sk, new_authorization, authorized_msg); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StartPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); if active_publishers.is_empty() { // If we have no previous publishers, we can start publishing immediately. @@ -440,8 +1091,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, *authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: Vec::new(), }; @@ -461,8 +1111,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, *authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: active_publishers.clone(), }; @@ -477,64 +1126,13 @@ impl FlashblocksHandle { /// Returns a stream of ordered flashblocks starting from the beginning of the current payload. /// /// # Behavior - /// The stream will continue to yield flashblocks for consecutive payloads. + /// The stream will continue to yield flashblocks for consecutive payloads as well, so + /// consumers should take care to handle the stream appropriately. + /// Returns a raw stream of flashblock payloads from the broadcast channel. + /// For ordered, canon-gated delivery, use [`Self::event_stream`] instead. pub fn flashblock_stream(&self) -> impl Stream + Send + 'static { - // Seed the stream with already-buffered contiguous flashblocks, then rely on the broadcast - // channel for future ones so ordering stays strict even if inserts arrive out of order. - let flashblocks = self - .state - .lock() - .flashblocks - .clone() - .into_iter() - .map_while(|x| x); - - let receiver = self.ctx.flashblock_tx.subscribe(); - - let current = stream::iter(flashblocks); - let future = tokio_stream::StreamExt::map_while(BroadcastStream::new(receiver), |x| x.ok()); - current.chain(future) - } - - /// Returns a stream of ordered flashblocks starting from the beginning of the current payload. - /// - /// # Behavior - /// - /// The stream will continue to yield flashblocks for consecutive payloads. - /// - /// Items not consumed from the stream by the time the next payload starts will be skipped. - pub fn live_flashblock_stream( - &self, - ) -> impl Stream + Send + Unpin + 'static { - let state = self.state.clone(); let receiver = self.ctx.flashblock_tx.subscribe(); - - Box::pin(stream::unfold( - (state, receiver, None::<(PayloadId, usize)>), - |(state, mut receiver, mut cursor)| async move { - loop { - if let Some(flashblock) = { - let state = state.lock(); - Self::next_flashblock_from_state(&state, cursor.as_ref()) - } { - cursor = Some((flashblock.payload_id, flashblock.index as usize + 1)); - return Some((flashblock, (state, receiver, cursor))); - } - - match receiver.recv().await { - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - warn!( - target: "flashblocks::p2p", - skipped, - "flashblock stream lagged; resyncing from protocol state" - ); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => return None, - } - } - }, - )) + tokio_stream::StreamExt::map_while(BroadcastStream::new(receiver), |x| x.ok()) } } @@ -553,7 +1151,14 @@ impl FlashblocksP2PCtx { /// - Validates payload consistency with authorization /// - Updates global state for new payloads with newer timestamps /// - Caches flashblocks and maintains ordering for sequential delivery - /// - Broadcasts to peers and publishes ordered flashblocks to the stream + /// - Forwards flashblocks to peers in the current send set and publishes ordered + /// flashblocks to the local stream + /// + /// Publishes a verified flashblock payload to peers and the local broadcast channel. + /// + /// Ordering, buffering, and canon-gating are handled downstream by + /// [`BufferedFlashblocks`](crate::protocol::event::BufferedFlashblocks) inside + /// the [`WorldChainEventsStream`](crate::protocol::event::WorldChainEventsStream). pub fn publish( &self, state: &mut FlashblocksP2PState, @@ -562,9 +1167,7 @@ impl FlashblocksP2PCtx { let payload = authorized_payload.msg(); let authorization = authorized_payload.authorized.authorization; - // Do some basic validation if authorization.payload_id != payload.payload_id { - // Since the builders are trusted, the only reason this should happen is a bug. tracing::error!( target: "flashblocks::p2p", authorization_payload_id = %authorization.payload_id, @@ -574,102 +1177,49 @@ impl FlashblocksP2PCtx { return; } - // Check if this is a globally new payload + // Reject flashblocks for epochs that have already been canonicalized. + if let Some(canon_tip) = &state.canon_tip + && let Some(base) = &payload.base + && base.block_number.saturating_sub(1) <= canon_tip.number + { + return; + } + if authorization.timestamp > state.payload_timestamp { state.payload_id = authorization.payload_id; state.payload_timestamp = authorization.timestamp; - state.flashblock_index = 0; - state.flashblocks.fill(None); } - // Resize our array if needed - if payload.index as usize > MAX_FLASHBLOCK_INDEX { + let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); + let bytes = p2p_msg.encode(); + let len = bytes.len(); + + if len > MAX_FRAME { tracing::error!( target: "flashblocks::p2p", - index = payload.index, - max_index = MAX_FLASHBLOCK_INDEX, - "Received flashblocks payload with index exceeding maximum" + size = len, + max_size = MAX_FRAME, + "FlashblocksP2PMsg too large", ); return; } - let len = state.flashblocks.len(); - state - .flashblocks - .resize_with(len.max(payload.index as usize + 1), || None); - let flashblock = &mut state.flashblocks[payload.index as usize]; - - // If we've already seen this index, skip it - // Otherwise, add it to the list - if flashblock.is_none() { - // We haven't seen this index yet - // Add the flashblock to our cache - - *flashblock = Some(payload.clone()); - tracing::trace!( + if len > MAX_FRAME / 2 { + tracing::warn!( target: "flashblocks::p2p", - payload_id = %payload.payload_id, - flashblock_index = payload.index, - "queueing flashblock", + size = len, + max_size = MAX_FRAME, + "FlashblocksP2PMsg almost too large", ); + } - let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); - let bytes = p2p_msg.encode(); - let len = bytes.len(); - - if len > MAX_FRAME { - tracing::error!( - target: "flashblocks::p2p", - size = bytes.len(), - max_size = MAX_FRAME, - "FlashblocksP2PMsg too large", - ); - return; - } - if len > MAX_FRAME / 2 { - tracing::warn!( - target: "flashblocks::p2p", - size = bytes.len(), - max_size = MAX_FRAME, - "FlashblocksP2PMsg almost too large", - ); - } - - metrics::histogram!("flashblocks.size").record(len as f64); - metrics::histogram!("flashblocks.gas_used").record(payload.diff.gas_used as f64); - metrics::histogram!("flashblocks.tx_count") - .record(payload.diff.transactions.len() as f64); - - let peer_msg = - PeerMsg::FlashblocksPayloadV1((payload.payload_id, payload.index as usize, bytes)); - - self.peer_tx.send(peer_msg).ok(); - - let now = Utc::now() - .timestamp_nanos_opt() - .expect("time went backwards"); - - // Broadcast any flashblocks in the cache that are in order - while let Some(Some(flashblock_event)) = state.flashblocks.get(state.flashblock_index) { - // Publish the flashblock - debug!( - target: "flashblocks::p2p", - payload_id = %flashblock_event.payload_id, - flashblock_index = %state.flashblock_index, - "publishing flashblock" - ); - self.flashblock_tx.send(flashblock_event.clone()).ok(); + metrics::histogram!("flashblocks.size").record(len as f64); + metrics::histogram!("flashblocks.gas_used").record(payload.diff.gas_used as f64); + metrics::histogram!("flashblocks.tx_count").record(payload.diff.transactions.len() as f64); - // Don't measure the interval at the block boundary - if state.flashblock_index != 0 { - let interval = now - state.flashblock_timestamp; - histogram!("flashblocks.interval").record(interval as f64 / 1_000_000_000.0); - } + state.send_flashblock_to_send_set(payload.payload_id, payload.index, &bytes); - // Update the index and timestamp - state.flashblock_timestamp = now; - state.flashblock_index += 1; - } - } + // Broadcast to local subscribers — ordering handled by WorldChainEventsStream + self.flashblock_tx.send(payload.clone()).ok(); } } @@ -693,7 +1243,7 @@ impl ConnectionHandler for FlashblocksP2PProtoco type Connection = FlashblocksConnection; fn protocol(&self) -> Protocol { - Protocol::new(Self::capability(), 1) + Protocol::new(Self::capability(), 5) } fn on_unsupported_by_peer( @@ -722,8 +1272,882 @@ impl ConnectionHandler for FlashblocksP2PProtoco "new flashblocks connection" ); - let peer_rx = self.handle.ctx.peer_tx.subscribe(); + FlashblocksConnection::new(self, conn, peer_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::SigningKey; + use enr::{Enr, secp256k1::SecretKey}; + use reth_eth_wire::{Capabilities, EthVersion, Status, StatusMessage, UnifiedStatus}; + use reth_network::{ + PeerInfo, PeersInfo, + types::{PeerKind, Reputation, ReputationChangeKind}, + }; + use reth_network_api::{NetworkError, noop::NoopNetwork}; + use reth_network_peers::NodeRecord; + use std::{ + collections::VecDeque, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + #[derive(Clone, Debug, Default)] + struct MockNetwork { + noop: NoopNetwork, + peer_lookup_responses: Arc>>>, + lookup_calls: Arc, + disconnected_peers: Arc>>, + } + + impl MockNetwork { + fn with_peer_lookup_responses(peer_lookup_responses: Vec>) -> Self { + Self { + peer_lookup_responses: Arc::new(Mutex::new(peer_lookup_responses.into())), + ..Default::default() + } + } + + fn lookup_calls(&self) -> usize { + self.lookup_calls.load(Ordering::SeqCst) + } + + fn disconnected_peers(&self) -> Vec { + self.disconnected_peers.lock().clone() + } + } + + impl PeersInfo for MockNetwork { + fn num_connected_peers(&self) -> usize { + self.noop.num_connected_peers() + } + + fn local_node_record(&self) -> NodeRecord { + self.noop.local_node_record() + } + + fn local_enr(&self) -> Enr { + self.noop.local_enr() + } + } + + impl Peers for MockNetwork { + fn add_trusted_peer_id(&self, _peer: PeerId) {} + + fn add_peer_kind( + &self, + _peer: PeerId, + _kind: PeerKind, + _tcp_addr: SocketAddr, + _udp_addr: Option, + ) { + } + + async fn get_peers_by_kind(&self, _kind: PeerKind) -> Result, NetworkError> { + Ok(vec![]) + } + + async fn get_all_peers(&self) -> Result, NetworkError> { + Ok(vec![]) + } + + async fn get_peer_by_id(&self, _peer_id: PeerId) -> Result, NetworkError> { + self.lookup_calls.fetch_add(1, Ordering::SeqCst); + Ok(self.peer_lookup_responses.lock().pop_front().flatten()) + } + + async fn get_peers_by_id( + &self, + _peer_ids: Vec, + ) -> Result, NetworkError> { + Ok(vec![]) + } + + fn remove_peer(&self, _peer: PeerId, _kind: PeerKind) {} + + fn disconnect_peer(&self, peer: PeerId) { + self.disconnected_peers.lock().push(peer); + } + + fn disconnect_peer_with_reason( + &self, + peer: PeerId, + _reason: reth_eth_wire::DisconnectReason, + ) { + self.disconnect_peer(peer); + } + + fn connect_peer_kind( + &self, + _peer: PeerId, + _kind: PeerKind, + _tcp_addr: SocketAddr, + _udp_addr: Option, + ) { + } + + fn reputation_change(&self, _peer_id: PeerId, _kind: ReputationChangeKind) {} + + async fn reputation_by_id( + &self, + _peer_id: PeerId, + ) -> Result, NetworkError> { + Ok(None) + } + } + + fn test_fanout_args() -> FanoutArgs { + FanoutArgs::default() + } + + fn test_ctx(fanout_args: FanoutArgs) -> FlashblocksP2PCtx { + let authorizer = SigningKey::from_bytes(&[7; 32]); + + FlashblocksP2PCtx { + authorizer_vk: authorizer.verifying_key(), + fanout_args, + flashblock_tx: broadcast::Sender::new(16), + } + } + + fn test_peer_state(trusted: bool) -> FlashblocksConnectionState { + let mut state = FlashblocksConnectionState::new(); + state.trusted = trusted; + state + } + + /// Creates a peer state with a per-peer outbound channel for message assertions. + fn test_peer_state_with_channel( + trusted: bool, + ) -> ( + FlashblocksConnectionState, + mpsc::UnboundedReceiver, + ) { + let (tx, rx) = mpsc::unbounded_channel(); + let mut state = FlashblocksConnectionState::new(); + state.trusted = trusted; + state.outbound_tx = Some(tx); + (state, rx) + } + + /// Receives and decodes a direct control message from a per-peer channel. + fn recv_direct(rx: &mut mpsc::UnboundedReceiver) -> FlashblocksP2PMsg { + let bytes = rx.try_recv().expect("expected a direct message"); + FlashblocksP2PMsg::decode(&mut &bytes[..]).expect("valid message") + } + + fn peer_state(fanout: &FlashblocksP2PState, peer_id: PeerId) -> &FlashblocksConnectionState { + fanout.connection_state(&peer_id).expect("peer exists") + } + + fn apply_observation( + fanout: &mut FlashblocksP2PState, + authorization: &Authorization, + flashblock: &FlashblocksPayloadV1, + peer_id: PeerId, + ) { + fanout.note_peer_received_flashblock(authorization, flashblock, peer_id); + } + + fn test_peer_info(peer_id: PeerId, trusted: bool) -> PeerInfo { + PeerInfo { + capabilities: Arc::new(Capabilities::new(vec![])), + remote_id: peer_id, + client_version: Arc::::from("mock"), + enode: "enode://mock".to_owned(), + enr: None, + remote_addr: SocketAddr::from((IpAddr::V4(Ipv4Addr::LOCALHOST), 30303)), + local_addr: None, + direction: Direction::Incoming, + eth_version: EthVersion::Eth67, + status: Arc::new(UnifiedStatus::from_message(StatusMessage::Legacy(Status { + version: EthVersion::Eth67, + ..Status::default() + }))), + session_established: Instant::now(), + kind: if trusted { + PeerKind::Trusted + } else { + PeerKind::Basic + }, + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn on_peer_connected_retries_until_peer_info_is_available() { + let authorizer = SigningKey::from_bytes(&[7; 32]); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let handle = FlashblocksHandle::with_fanout_args( + authorizer.verifying_key(), + Some(SigningKey::from_bytes(&[8; 32])), + fanout_args, + ); + let peer_id = PeerId::random(); + let network = MockNetwork::with_peer_lookup_responses(vec![ + None, + Some(test_peer_info(peer_id, true)), + ]); + let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel(); + + handle.on_peer_connected(network.clone(), peer_id, outbound_tx); + + assert!(network.lookup_calls() >= 2); + assert!(network.disconnected_peers().is_empty()); + let state = handle.state.lock(); + assert!( + state + .connection_state(&peer_id) + .expect("peer exists") + .trusted + ); + drop(state); + assert_eq!( + recv_direct(&mut outbound_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn on_peer_connected_defaults_to_untrusted_when_peer_info_never_arrives() { + let authorizer = SigningKey::from_bytes(&[7; 32]); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let handle = FlashblocksHandle::with_fanout_args( + authorizer.verifying_key(), + Some(SigningKey::from_bytes(&[8; 32])), + fanout_args, + ); + let peer_id = PeerId::random(); + let network = MockNetwork::default(); + let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel(); + + handle.on_peer_connected(network.clone(), peer_id, outbound_tx); + + assert!(network.lookup_calls() > 1); + assert!(network.disconnected_peers().is_empty()); + let state = handle.state.lock(); + assert!( + !state + .connection_state(&peer_id) + .expect("peer exists") + .trusted + ); + drop(state); + assert_eq!( + recv_direct(&mut outbound_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn publish_sends_flashblocks_only_to_send_enabled_peers() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + let payload_id = PayloadId::new([1; 8]); + let authorization = Authorization::new(payload_id, 1, &authorizer, builder.verifying_key()); + let flashblock = FlashblocksPayloadV1 { + payload_id, + index: 0, + ..Default::default() + }; + let authorized_payload = + AuthorizedPayload::new(&builder, authorization, flashblock.clone()); + let expected = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); + + let source_peer = PeerId::random(); + let send_peer = PeerId::random(); + let non_send_peer = PeerId::random(); + + let (mut source_state, mut source_rx) = test_peer_state_with_channel(false); + source_state.send_enabled = true; + let (mut send_state, mut send_rx) = test_peer_state_with_channel(false); + send_state.send_enabled = true; + let (non_send_state, mut non_send_rx) = test_peer_state_with_channel(false); + + fanout.connections.insert(source_peer, source_state); + fanout.connections.insert(send_peer, send_state); + fanout.connections.insert(non_send_peer, non_send_state); + apply_observation(&mut fanout, &authorization, &flashblock, source_peer); + + ctx.publish(&mut fanout, authorized_payload); + + assert_eq!(recv_direct(&mut send_rx), expected); + assert!(source_rx.try_recv().is_err()); + assert!(non_send_rx.try_recv().is_err()); + } + + #[test] + fn trusted_peers_are_requested_first() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let trusted_peer = PeerId::random(); + let untrusted_peer = PeerId::random(); + let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true); + let untrusted_state = test_peer_state(false); + fanout.connections.insert(trusted_peer, trusted_state); + fanout.connections.insert(untrusted_peer, untrusted_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, trusted_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + peer_state(&fanout, untrusted_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + recv_direct(&mut trusted_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn trusted_request_bypasses_non_trusted_limit() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_send_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let victim = PeerId::random(); + let trusted_requester = PeerId::random(); + let (mut victim_state, mut victim_rx) = test_peer_state_with_channel(false); + let (requester_state, mut requester_rx) = test_peer_state_with_channel(true); + victim_state.send_enabled = true; + fanout.connections.insert(victim, victim_state); + fanout + .connections + .insert(trusted_requester, requester_state); + + assert!(fanout.handle_request(&ctx, trusted_requester).is_ok()); + + assert!(peer_state(&fanout, victim).send_enabled); + assert!(peer_state(&fanout, trusted_requester).send_enabled); + assert!(victim_rx.try_recv().is_err()); + assert_eq!( + recv_direct(&mut requester_rx), + FlashblocksP2PMsg::AcceptFlashblocks + ); + } + + #[test] + fn rotation_replaces_peer_before_requesting_candidate() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let current_peer = PeerId::random(); + let candidate_peer = PeerId::random(); + let (mut current_state, mut current_rx) = test_peer_state_with_channel(false); + let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false); + let mut score = Score::new(score_samples); + score.record(42); + current_state.receive_status = ReceiveStatus::Receiving { score }; + fanout.connections.insert(current_peer, current_state); + fanout.connections.insert(candidate_peer, candidate_state); + + fanout.maybe_start_rotation(&ctx); + + assert_eq!( + peer_state(&fanout, current_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::Requesting + ); + + assert_eq!( + recv_direct(&mut current_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut candidate_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(fanout.handle_accept(&ctx, candidate_peer).is_ok()); + + assert!(matches!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + } + + #[test] + fn multiple_pending_requests_clear_independently() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let first_peer = PeerId::random(); + let second_peer = PeerId::random(); + let (first_state, mut first_rx) = test_peer_state_with_channel(false); + let (second_state, mut second_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(first_peer, first_state); + fanout.connections.insert(second_peer, second_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, first_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + peer_state(&fanout, second_peer).receive_status, + ReceiveStatus::Requesting + ); + + assert_eq!( + recv_direct(&mut first_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + assert_eq!( + recv_direct(&mut second_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(fanout.handle_accept(&ctx, first_peer).is_ok()); + assert!(fanout.handle_accept(&ctx, second_peer).is_ok()); + + assert!(matches!( + peer_state(&fanout, first_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + assert!(matches!( + peer_state(&fanout, second_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + } + + #[test] + fn rejected_peer_is_not_immediately_retried() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (candidate_state, mut peer_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(peer, candidate_state); + + fanout.maybe_request_receive_peers(&ctx); + assert_eq!( + recv_direct(&mut peer_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(fanout.handle_reject(&ctx, peer).is_ok()); + + assert_eq!( + peer_state(&fanout, peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert!(peer_rx.try_recv().is_err()); + + fanout.maybe_request_receive_peers(&ctx); + assert!(peer_rx.try_recv().is_err()); + + fanout + .connection_state_mut(&peer) + .expect("peer exists") + .receive_status_timestamp = + Utc::now().timestamp() as u64 - ctx.fanout_args.rotation_interval.max(1); + + fanout.maybe_request_receive_peers(&ctx); + assert_eq!( + recv_direct(&mut peer_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn timed_out_request_is_cleared_and_replaced() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let stale_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + let (stale_state, mut stale_rx) = test_peer_state_with_channel(true); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(stale_peer, stale_state); + fanout + .connections + .insert(replacement_peer, replacement_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, stale_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + recv_direct(&mut stale_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + fanout + .connection_state_mut(&stale_peer) + .expect("peer exists") + .receive_status_timestamp = + Utc::now().timestamp() as u64 - RECEIVE_REQUEST_TIMEOUT_SECS; + + fanout.expire_stale_receive_requests(&ctx); + + assert_eq!( + peer_state(&fanout, stale_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + assert!(stale_rx.try_recv().is_err()); + } + + #[test] + fn silent_receive_peer_can_be_rotated_out_without_samples() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let silent_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + + let (mut silent_state, mut silent_rx) = test_peer_state_with_channel(false); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); + + silent_state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; + + fanout.connections.insert(silent_peer, silent_state); + fanout + .connections + .insert(replacement_peer, replacement_state); + + fanout.maybe_start_rotation(&ctx); + + assert_eq!( + peer_state(&fanout, silent_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + + assert_eq!( + recv_direct(&mut silent_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn peer_score_penalizes_missed_flashblocks() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let mut fanout = FlashblocksP2PState::default(); + + let steady_peer = PeerId::random(); + let lagging_peer = PeerId::random(); + let mut steady_state = test_peer_state(false); + let mut lagging_state = test_peer_state(false); + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + + let mut steady_score = Score::new(score_samples); + steady_score.record(10); + steady_state.receive_status = ReceiveStatus::Receiving { + score: steady_score, + }; + let mut lagging_score = Score::new(score_samples); + lagging_score.record(100); + lagging_state.receive_status = ReceiveStatus::Receiving { + score: lagging_score, + }; + + fanout.connections.insert(steady_peer, steady_state); + fanout.connections.insert(lagging_peer, lagging_state); + + // Use timestamps starting well after receive_status_timestamp (0) so the grace + // check `receive_status_timestamp + 2 <= evicted.timestamp` is satisfied. + let ts_offset = 10_u64; + for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let authorization = Authorization::new( + PayloadId::default(), + ts_offset + index as u64, + &authorizer, + builder.verifying_key(), + ); + let flashblock = FlashblocksPayloadV1 { + payload_id: PayloadId::default(), + index: index as u64, + ..Default::default() + }; + apply_observation(&mut fanout, &authorization, &flashblock, steady_peer); + } + + assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); + let ReceiveStatus::Receiving { + score: steady_score, + } = &peer_state(&fanout, steady_peer).receive_status + else { + panic!("expected Receiving"); + }; + assert_eq!(steady_score.value(), Some(10)); + let ReceiveStatus::Receiving { + score: lagging_score, + } = &peer_state(&fanout, lagging_peer).receive_status + else { + panic!("expected Receiving"); + }; + assert_eq!( + lagging_score.value(), + Some((100 * (score_samples - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / score_samples) + ); + } + + #[test] + fn pending_candidate_is_rotated_out_after_missing_blocks() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let steady_peer = PeerId::random(); + let rotating_peer = PeerId::random(); + let candidate_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + + let mut steady_state = test_peer_state(false); + let mut rotating_state = test_peer_state(false); + let candidate_state = test_peer_state(true); + let replacement_state = test_peer_state(true); + + let mut steady_score = Score::new(score_samples); + steady_score.record(10); + steady_state.receive_status = ReceiveStatus::Receiving { + score: steady_score, + }; + let mut rotating_score = Score::new(score_samples); + rotating_score.record(100); + rotating_state.receive_status = ReceiveStatus::Receiving { + score: rotating_score, + }; + + fanout.connections.insert(steady_peer, steady_state); + fanout.connections.insert(rotating_peer, rotating_state); + fanout.connections.insert(candidate_peer, candidate_state); + + fanout.maybe_start_rotation(&ctx); + + // Accept the candidate so it transitions to Receiving. + assert!(fanout.handle_accept(&ctx, candidate_peer).is_ok()); + + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + // Use timestamps well after the candidate's receive_status_timestamp so the + // grace check `receive_status_timestamp + 2 <= evicted.timestamp` is satisfied. + let ts_base = Utc::now().timestamp() as u64 + 10; + for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let authorization = Authorization::new( + PayloadId::default(), + ts_base + index as u64, + &authorizer, + builder.verifying_key(), + ); + let flashblock = FlashblocksPayloadV1 { + payload_id: PayloadId::default(), + index: index as u64, + ..Default::default() + }; + apply_observation(&mut fanout, &authorization, &flashblock, steady_peer); + } + + assert_eq!(fanout.worst_receive_peer(), Some(candidate_peer)); + + fanout + .connections + .insert(replacement_peer, replacement_state); + fanout.maybe_start_rotation(&ctx); + + assert_eq!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + } + + #[test] + fn unsolicited_accept_is_penalized() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false); + fanout.connections.insert(peer, state); - FlashblocksConnection::new(self, conn, peer_id, BroadcastStream::new(peer_rx)) + // Accept without a prior request should be penalized. + assert!(fanout.handle_accept(&ctx, peer).is_err()); + } + + #[test] + fn unsolicited_reject_is_penalized() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false); + fanout.connections.insert(peer, state); + + // Reject without a prior request should be penalized. + assert!(fanout.handle_reject(&ctx, peer).is_err()); + } + + #[test] + fn cancel_without_relationship_is_penalized() { + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false); + fanout.connections.insert(peer, state); + + // Cancel with no send/receive relationship should be penalized. + assert!(fanout.handle_cancel(peer).is_err()); + } + + #[test] + fn cancel_only_clears_send_direction() { + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.send_enabled = true; + state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_cancel(peer).is_ok()); + assert!(!peer_state(&fanout, peer).send_enabled); + assert!(matches!( + peer_state(&fanout, peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + } + + #[test] + fn cancel_from_sender_is_penalized() { + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_cancel(peer).is_err()); + } + + #[test] + fn duplicate_request_when_already_sending_is_penalized() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.send_enabled = true; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_request(&ctx, peer).is_err()); + } + + #[test] + fn receive_retry_cooldown_does_not_penalize_inbound_request() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (mut state, mut rx) = test_peer_state_with_channel(false); + state.receive_status_timestamp = Utc::now().timestamp() as u64; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_request(&ctx, peer).is_ok()); + assert!(peer_state(&fanout, peer).send_enabled); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::AcceptFlashblocks); + } + + #[test] + fn repeated_rejected_requests_are_rate_limited() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_send_peers = 0; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (state, mut rx) = test_peer_state_with_channel(false); + fanout.connections.insert(peer, state); + + for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { + assert!(fanout.handle_request(&ctx, peer).is_ok()); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::RejectFlashblocks); + } + assert!(fanout.handle_request(&ctx, peer).is_err()); + } + + #[test] + fn control_message_rate_limit_triggers_penalty() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.send_enabled = true; + fanout.connections.insert(peer, state); + + // Spam requests to exceed the rate limit. + for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { + // These return Err because send_enabled is already set (duplicate request), + // but the rate limit hasn't been hit yet. + assert!(fanout.handle_request(&ctx, peer).is_err()); + } + // The next one should hit the rate limit. + assert!(fanout.handle_request(&ctx, peer).is_err()); } } diff --git a/crates/flashblocks/p2p/src/protocol/mod.rs b/crates/flashblocks/p2p/src/protocol/mod.rs index a83f17231..fab9f9b55 100644 --- a/crates/flashblocks/p2p/src/protocol/mod.rs +++ b/crates/flashblocks/p2p/src/protocol/mod.rs @@ -1,3 +1,4 @@ pub mod connection; pub mod error; +pub mod event; pub mod handler; diff --git a/crates/flashblocks/p2p/tests/protocol.rs b/crates/flashblocks/p2p/tests/protocol.rs index 6ea3e9f69..ab3070df4 100644 --- a/crates/flashblocks/p2p/tests/protocol.rs +++ b/crates/flashblocks/p2p/tests/protocol.rs @@ -119,7 +119,7 @@ async fn flashblock_stream_is_ordered() { handle.publish_new(signed).unwrap(); } - let mut flashblock_stream = handle.live_flashblock_stream(); + let mut flashblock_stream = handle.flashblock_stream(); // Expect to receive 0, then 1 over the ordered broadcast. let first = flashblock_stream.next().await.unwrap(); @@ -232,7 +232,7 @@ async fn flashblock_stream_buffers_and_live() { handle.publish_new(signed0).unwrap(); // now create the combined stream - let mut stream = handle.live_flashblock_stream(); + let mut stream = handle.flashblock_stream(); // first item comes from the cached vector let first = stream.next().await.unwrap(); @@ -247,122 +247,6 @@ async fn flashblock_stream_buffers_and_live() { assert_eq!(second.index, 1); } -#[tokio::test] -async fn flashblock_stream_recovers_after_receiver_lag() { - let timestamp = 1000; - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let pid = PayloadId::new([8; 8]); - let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); - handle.start_publishing(auth).unwrap(); - - // Create the stream first, then publish more messages than the broadcast buffer can retain - // before polling it. The stream must resync from protocol state instead of terminating. - let mut stream = handle.live_flashblock_stream(); - - for idx in 0..=200 { - let signed = AuthorizedPayload::new(builder_sk, auth, payload(pid, idx)); - handle.publish_new(signed).unwrap(); - } - - for expected in 0..=100u64 { - let flashblock = stream.next().await.unwrap(); - assert_eq!(flashblock.index, expected); - } - - // We actually fail to continue publishing here - // but this is an acceptable edge case - assert!( - tokio::time::timeout(Duration::from_millis(10), stream.next()) - .await - .is_err(), - ); -} - -#[tokio::test] -async fn live_flashblock_stream_skips_stale_flashblocks() { - let timestamp = 1000; - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let pid_a = PayloadId::new([8; 8]); - let auth_a = Authorization::new( - pid_a, - timestamp, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth_a).unwrap(); - - // Create the stream first, then partially consume payload A before payload B starts. - // The stream should skip the unread remainder of payload A once protocol state rolls over. - let mut stream = handle.live_flashblock_stream(); - - for idx in 0..=10u64 { - let signed = AuthorizedPayload::new(builder_sk, auth_a, payload(pid_a, idx)); - handle.publish_new(signed).unwrap(); - } - - let first = stream.next().await.unwrap(); - assert_eq!(first.payload_id, pid_a); - assert_eq!(first.index, 0); - - let pid_b = PayloadId::new([9; 8]); - let auth_b = Authorization::new( - pid_b, - timestamp + 1, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth_b).unwrap(); - let signed = AuthorizedPayload::new(builder_sk, auth_b, payload(pid_b, 0)); - handle.publish_new(signed).unwrap(); - - let flashblock = stream.next().await.unwrap(); - assert_eq!(flashblock.payload_id, pid_b); - assert_eq!(flashblock.index, 0); -} - -#[tokio::test] -async fn live_flashblock_stream_handles_out_of_order() { - let timestamp = 1000; - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let pid = PayloadId::new([8; 8]); - let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); - handle.start_publishing(auth).unwrap(); - - // Create the stream first, then publish more messages than the broadcast buffer can retain - // before polling it. The stream must resync from protocol state instead of terminating. - let mut stream = handle.live_flashblock_stream(); - - handle - .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 0))) - .unwrap(); - - assert_eq!(stream.next().await.unwrap().index, 0); - - handle - .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 2))) - .unwrap(); - - // Assert not ready - assert!( - tokio::time::timeout(Duration::from_millis(10), stream.next()) - .await - .is_err() - ); - - handle - .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 1))) - .unwrap(); - - assert_eq!(stream.next().await.unwrap().index, 1); - assert_eq!(stream.next().await.unwrap().index, 2); -} - #[tokio::test] async fn await_clearance_unblocks_on_publish() { let handle = fresh_handle(); diff --git a/crates/flashblocks/payload/src/generator.rs b/crates/flashblocks/payload/src/generator.rs index c4e49b28a..9faf9164f 100644 --- a/crates/flashblocks/payload/src/generator.rs +++ b/crates/flashblocks/payload/src/generator.rs @@ -258,7 +258,7 @@ where let authorization = match ( &self.override_authorizer_sk, - &self.p2p_handler.ctx.builder_sk, + self.p2p_handler.builder_sk().ok(), can_override, ) { (Some(override_authorizer_sk), Some(builder_sk), true) => Some(Authorization::new( diff --git a/crates/flashblocks/primitives/src/p2p.rs b/crates/flashblocks/primitives/src/p2p.rs index 3ef29e158..ae3a06977 100644 --- a/crates/flashblocks/primitives/src/p2p.rs +++ b/crates/flashblocks/primitives/src/p2p.rs @@ -42,11 +42,20 @@ pub struct StopPublish; /// This enum represents the top-level message types that can be transmitted /// over the P2P network. Currently all messages are wrapped in authorization to ensure /// only authorized builders can create new messages. +#[allow(clippy::large_enum_variant)] #[repr(u8)] #[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Eq)] pub enum FlashblocksP2PMsg { /// An authorized message containing a signed and authorized payload Authorized(Authorized) = 0x00, + /// Requests that the remote peer begin forwarding flashblocks to us. + RequestFlashblocks = 0x01, + /// Accepts a previously sent [`Self::RequestFlashblocks`] request. + AcceptFlashblocks = 0x02, + /// Rejects a previously sent [`Self::RequestFlashblocks`] request. + RejectFlashblocks = 0x03, + /// Sent by a receiver to terminate an active flashblocks feed from a sender. + CancelFlashblocks = 0x04, } /// The different types of authorized messages that can be sent over the Flashblocks P2P network. @@ -443,6 +452,10 @@ impl FlashblocksP2PMsg { buf.put_u8(0x00); payload.encode(&mut buf); } + FlashblocksP2PMsg::RequestFlashblocks => buf.put_u8(0x01), + FlashblocksP2PMsg::AcceptFlashblocks => buf.put_u8(0x02), + FlashblocksP2PMsg::RejectFlashblocks => buf.put_u8(0x03), + FlashblocksP2PMsg::CancelFlashblocks => buf.put_u8(0x04), } buf } @@ -458,6 +471,10 @@ impl FlashblocksP2PMsg { let payload = Authorized::decode(buf)?; Ok(FlashblocksP2PMsg::Authorized(payload)) } + 0x01 => Ok(FlashblocksP2PMsg::RequestFlashblocks), + 0x02 => Ok(FlashblocksP2PMsg::AcceptFlashblocks), + 0x03 => Ok(FlashblocksP2PMsg::RejectFlashblocks), + 0x04 => Ok(FlashblocksP2PMsg::CancelFlashblocks), _ => Err(FlashblocksError::UnknownMessageType), } } @@ -815,6 +832,25 @@ mod tests { match decoded { FlashblocksP2PMsg::Authorized(inner) => assert_eq!(inner, authorized), + _ => panic!("decoded wrong message variant"), + } + } + + #[test] + fn p2p_control_msg_roundtrip() { + let variants = [ + FlashblocksP2PMsg::RequestFlashblocks, + FlashblocksP2PMsg::AcceptFlashblocks, + FlashblocksP2PMsg::RejectFlashblocks, + FlashblocksP2PMsg::CancelFlashblocks, + ]; + + for msg in variants { + let encoded = msg.encode(); + let mut view: &[u8] = &encoded; + let decoded = FlashblocksP2PMsg::decode(&mut view).expect("decoding succeeds"); + assert!(view.is_empty(), "all bytes consumed"); + assert_eq!(decoded, msg); } } diff --git a/crates/flashblocks/rpc/src/eth/pending_block.rs b/crates/flashblocks/rpc/src/eth/pending_block.rs index d253151a9..8a5995b47 100644 --- a/crates/flashblocks/rpc/src/eth/pending_block.rs +++ b/crates/flashblocks/rpc/src/eth/pending_block.rs @@ -1,7 +1,6 @@ //! Loads OP pending block for a RPC response. use alloy_eips::BlockNumberOrTag; -use alloy_primitives::{B256, BlockNumber}; use reth_optimism_primitives::OpPrimitives; use reth_optimism_rpc::{OpEthApi, OpEthApiError}; use reth_provider::{BlockReader, BlockReaderIdExt, ReceiptProvider}; @@ -13,15 +12,6 @@ use reth_rpc_eth_types::{EthApiError, PendingBlock, block::BlockAndReceipts}; use crate::eth::FlashblocksEthApi; -fn is_pending_block_fresh( - pending_number: BlockNumber, - pending_parent_hash: B256, - latest_number: BlockNumber, - latest_hash: B256, -) -> bool { - pending_number > latest_number && pending_parent_hash == latest_hash -} - impl LoadPendingBlock for FlashblocksEthApi where N: RpcNodeCore, @@ -50,40 +40,32 @@ where async fn local_pending_block( &self, ) -> Result::Primitives>>, Self::Error> { - let latest = self - .provider() - .latest_header()? - .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; - // check the pending block from the executor if let Some(pending_block) = self.pending_block.as_ref() { let pending_block = pending_block.borrow().clone(); if let Some(pending_block) = pending_block { let block = pending_block.recovered_block; - if is_pending_block_fresh( - block.header().number, - block.header().parent_hash, - latest.number, - latest.hash(), - ) { - let receipts = pending_block - .execution_output - .receipts - .clone() - .into_iter() - .collect::>(); // always a single block executed through the state executor - - let block_and_receipts = BlockAndReceipts { - block, - receipts: receipts.into(), - }; - return Ok(Some(block_and_receipts)); - } + let receipts = pending_block + .execution_output + .receipts + .clone() + .into_iter() + .collect::>(); // always a single block executed through the state executor + + let block_and_receipts = BlockAndReceipts { + block, + receipts: receipts.into(), + }; + return Ok(Some(block_and_receipts)); } } // See: + let latest = self + .provider() + .latest_header()? + .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; let block_id = latest.hash().into(); let block = self .provider() @@ -107,23 +89,3 @@ where self.inner.pending_block_kind() } } - -#[cfg(test)] -mod tests { - use super::is_pending_block_fresh; - use alloy_primitives::B256; - - #[test] - fn fresh_pending_block_must_be_ahead_of_latest_and_build_on_latest_hash() { - let latest_hash = B256::from([1; 32]); - - assert!(is_pending_block_fresh(11, latest_hash, 10, latest_hash)); - assert!(!is_pending_block_fresh(10, latest_hash, 10, latest_hash)); - assert!(!is_pending_block_fresh( - 12, - B256::from([2; 32]), - 10, - latest_hash - )); - } -} diff --git a/crates/world/node/Cargo.toml b/crates/world/node/Cargo.toml index b84c891da..088c078a4 100644 --- a/crates/world/node/Cargo.toml +++ b/crates/world/node/Cargo.toml @@ -43,6 +43,7 @@ alloy-primitives.workspace = true alloy-rpc-types-eth.workspace = true alloy-signer-local.workspace = true + op-alloy-consensus.workspace = true tokio.workspace = true @@ -59,6 +60,9 @@ world-chain-pool.workspace = true world-chain-test.workspace = true world-chain-node.workspace = true +op-alloy-network.workspace = true +op-alloy-provider.workspace = true + reth-db.workspace = true reth-e2e-test-utils.workspace = true reth-engine-primitives.workspace = true @@ -71,7 +75,7 @@ reth-primitives.workspace = true reth-tracing.workspace = true reth-network-api.workspace = true reth-eth-wire.workspace = true -alloy-rpc-types.workspace = true +alloy-rpc-types = { workspace = true } alloy-genesis.workspace = true alloy-network.workspace = true diff --git a/crates/world/node/src/args.rs b/crates/world/node/src/args.rs index 19cb80b4e..a1c640a9f 100644 --- a/crates/world/node/src/args.rs +++ b/crates/world/node/src/args.rs @@ -3,7 +3,8 @@ use alloy_primitives::Address; use alloy_signer_local::PrivateKeySigner; use clap::value_parser; use ed25519_dalek::{SigningKey, VerifyingKey}; -use flashblocks_cli::{FlashblocksArgs, FlashblocksPayloadBuilderConfig}; +use flashblocks_builder::FlashblocksPayloadBuilderConfig; +use flashblocks_cli::FlashblocksArgs; use hex::FromHex; use reth::chainspec::NamedChain; use reth_network_peers::{PeerId, TrustedPeer}; diff --git a/crates/world/node/src/context.rs b/crates/world/node/src/context.rs index cfc909351..9a78c9651 100644 --- a/crates/world/node/src/context.rs +++ b/crates/world/node/src/context.rs @@ -396,6 +396,7 @@ impl From for FlashblocksComponentsContext { let authorizer_vk = flashblocks.authorizer_vk.unwrap_or_else(|| { flashblocks .override_authorizer_sk + .as_ref() .expect("flashblocks authorizer_vk or override_authorizer_sk required") .verifying_key() }); @@ -405,8 +406,11 @@ impl From for FlashblocksComponentsContext { authorizer_vk.as_bytes().encode_hex::() ); - let builder_sk = flashblocks.builder_sk.clone(); - let flashblocks_handle = FlashblocksHandle::new(authorizer_vk, builder_sk.clone()); + let flashblocks_handle = FlashblocksHandle::with_fanout_args( + authorizer_vk, + flashblocks.builder_sk.clone(), + flashblocks.fanout.clone(), + ); let (pending_block, _) = tokio::sync::watch::channel(None); diff --git a/crates/world/node/tests/e2e-testsuite/actions.rs b/crates/world/node/tests/e2e-testsuite/actions.rs index c15794db9..ead030d5f 100644 --- a/crates/world/node/tests/e2e-testsuite/actions.rs +++ b/crates/world/node/tests/e2e-testsuite/actions.rs @@ -17,8 +17,7 @@ use futures::{ stream::{self, FuturesUnordered}, }; use op_alloy_rpc_types::OpTransactionReceipt; -use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4}; -use parking_lot::RwLock; +use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelopeV4; use reth::rpc::api::{EngineApiClient, EthApiClient}; use reth_e2e_test_utils::testsuite::{Environment, actions::Action}; use reth_node_api::{ConsensusEngineHandle, EngineApiMessageVersion}; @@ -28,11 +27,130 @@ use reth_optimism_primitives::OpTransactionSigned; use reth_primitives::TransactionSigned; use revm_primitives::{Address, B256, Bytes, U256}; use std::{pin::Pin, sync::Arc, time::Duration}; -use tokio::sync::{mpsc, watch}; +use tokio::sync::mpsc; use tracing::{error, info}; use crate::setup::execution_data_from_from_reduced_flashblock; +// --------------------------------------------------------------------------- +// Test helper macros for Eth API queries +// --------------------------------------------------------------------------- + +/// Create an `alloy_provider::RootProvider` from a node's RPC URL. +/// +/// ```ignore +/// let provider = provider!(nodes[0]); +/// ``` +#[macro_export] +macro_rules! provider { + ($node:expr) => {{ + let url = $node.node.rpc_url(); + alloy_provider::ProviderBuilder::new().connect_http(url) + }}; +} + +/// Fetch a block by tag (`Pending`, `Latest`, etc.) from a node. +/// +/// ```ignore +/// let block = fetch_block!(nodes[0], Pending); +/// let block = fetch_block!(nodes[0], Latest, true); // full txs +/// ``` +#[macro_export] +macro_rules! fetch_block { + ($node:expr, $tag:ident) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_block_by_number( + &provider, + alloy_eips::BlockNumberOrTag::$tag, + false, + ) + .await + }}; + ($node:expr, $tag:ident, $full_txs:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_block_by_number( + &provider, + alloy_eips::BlockNumberOrTag::$tag, + $full_txs, + ) + .await + }}; +} + +/// Fetch a transaction receipt by hash. +/// +/// ```ignore +/// let receipt = fetch_receipt!(nodes[0], tx_hash); +/// ``` +#[macro_export] +macro_rules! fetch_receipt { + ($node:expr, $tx_hash:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_transaction_receipt(&provider, $tx_hash).await + }}; +} + +/// Fetch a transaction by hash. +/// +/// ```ignore +/// let tx = fetch_tx!(nodes[0], tx_hash); +/// ``` +#[macro_export] +macro_rules! fetch_tx { + ($node:expr, $tx_hash:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_transaction_by_hash(&provider, $tx_hash).await + }}; +} + +/// Perform an `eth_call` against a node. +/// +/// ```ignore +/// let result = eth_call!(nodes[0], tx_request); +/// let result = eth_call!(nodes[0], tx_request, Pending); +/// ``` +#[macro_export] +macro_rules! eth_call { + ($node:expr, $tx:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::call(&provider, &$tx).await + }}; + ($node:expr, $tx:expr, $tag:ident) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::call(&provider, &$tx) + .block(alloy_eips::BlockId::Number( + alloy_eips::BlockNumberOrTag::$tag, + )) + .await + }}; +} + +/// Fetch logs matching a filter. +/// +/// ```ignore +/// let logs = fetch_logs!(nodes[0], filter); +/// ``` +#[macro_export] +macro_rules! fetch_logs { + ($node:expr, $filter:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_logs(&provider, &$filter).await + }}; +} + +/// Subscribe to new block headers (uses polling via `watch_blocks`). +/// +/// ```ignore +/// let poller = stream_blocks!(nodes[0]); +/// ``` +#[macro_export] +macro_rules! stream_blocks { + ($node:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::watch_blocks(&provider).await + }}; +} + pub type Hook = Arc Result<()> + Send + Sync>; pub fn hook(f: F) -> Hook @@ -1030,951 +1148,349 @@ where } } -// ============================================================================ -// Shared State and Advanced Composition Actions -// ============================================================================ - -/// Shared state for communication between parallel actions during block production -#[derive(Clone)] -pub struct BlockProductionState { - /// Current payload being produced - pub payload: Arc>>, - /// Block hashes from validated flashblocks (for GetBlockByHash) - pub validated_block_hashes: Arc>>, - /// Transaction hashes submitted by spammer (for GetReceipts) - pub submitted_tx_hashes: Arc>>, - /// Signal when final flashblock is validated - pub final_validated: watch::Sender, - pub final_validated_rx: watch::Receiver, +/// Sleep for a duration - useful between block cycles +pub struct Sleep { + pub duration: Duration, } -impl BlockProductionState { - pub fn new() -> Self { - let (final_validated, final_validated_rx) = watch::channel(false); - Self { - payload: Arc::new(RwLock::new(None)), - validated_block_hashes: Arc::new(RwLock::new(Vec::new())), - submitted_tx_hashes: Arc::new(RwLock::new(Vec::new())), - final_validated, - final_validated_rx, - } - } - - pub fn set_payload(&self, payload: OpExecutionPayloadEnvelopeV3) { - *self.payload.write() = Some(payload); - } - - pub fn get_payload(&self) -> Option { - self.payload.read().clone() - } - - pub fn add_validated_hash(&self, hash: B256) { - self.validated_block_hashes.write().push(hash); +impl Sleep { + pub fn new(duration: Duration) -> Self { + Self { duration } } - pub fn add_tx_hash(&self, hash: B256) { - self.submitted_tx_hashes.write().push(hash); + pub fn millis(ms: u64) -> Self { + Self::new(Duration::from_millis(ms)) } +} - pub fn get_tx_hashes(&self) -> Vec { - self.submitted_tx_hashes.read().clone() +impl Action for Sleep { + fn execute<'a>( + &'a mut self, + _env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + tokio::time::sleep(self.duration).await; + Ok(()) + }) } +} - pub fn signal_final(&self) { - let _ = self.final_validated.send(true); - } +// --------------------------------------------------------------------------- +// EngineDriver — drives the consensus engine through N block-building cycles +// --------------------------------------------------------------------------- - pub fn reset(&self) { - *self.payload.write() = None; - self.validated_block_hashes.write().clear(); - self.submitted_tx_hashes.write().clear(); - let _ = self.final_validated.send(false); - } -} +/// Callback invoked after each block is built and canonicalized. +pub type BlockCallback = Box< + dyn Fn( + usize, + &OpExecutionPayloadEnvelopeV4, + ) -> std::pin::Pin> + Send>> + + Send + + Sync, +>; -/// Canonicalize a block by sending new_payload + fork_choice_updated to follower nodes -pub struct Canonicalize { - pub node_idxs: Vec, - pub state: BlockProductionState, -} +/// Callback invoked during the build interval (no payload available yet). +pub type MidBuildCallback = Box< + dyn Fn(usize) -> std::pin::Pin> + Send>> + + Send + + Sync, +>; -impl Canonicalize { - pub fn new(node_idxs: Vec, state: BlockProductionState) -> Self { - Self { node_idxs, state } - } +/// Drives the consensus engine through `num_blocks` block-building cycles. +/// +/// Each cycle: +/// 1. Generates payload attributes for the next block +/// 2. Sends `forkchoiceUpdatedV3` with attributes to start building +/// 3. Waits for `block_interval` (the build deadline) +/// 4. Calls `getPayloadV4` to retrieve the built payload +/// 5. Sends `newPayloadV4` + `forkchoiceUpdated` on all follower nodes +/// 6. Invokes the optional `on_block` callback +/// 7. Advances to the next cycle with the new block as head +pub struct EngineDriver { + /// Index of the builder node in the environment's node_clients. + pub builder_idx: usize, + /// Indices of follower nodes that receive `newPayload` + FCU. + pub follower_idxs: Vec, + /// Initial parent hash (genesis). If None, fetched from latest block. + pub initial_parent_hash: Option, + /// Number of blocks to build. + pub num_blocks: usize, + /// Time to wait between FCU (start building) and getPayload (retrieve). + pub block_interval: Duration, + /// Whether to use flashblocks FCU with authorization. + pub flashblocks: bool, + /// Generates `Authorization` from `(parent_hash, OpPayloadAttributes)`. + pub authorization_gen: A, + /// Generates attributes for the next block given (block_number, parent_timestamp). + pub attributes_gen: Box Result + Send + Sync>, + /// Optional callback during the build interval (between FCU and getPayload). + /// Called while the payload builder is actively working. + pub during_build: Option, + /// Optional callback after each block is built and canonicalized. + pub on_block: Option, } -impl Action for Canonicalize { +impl EngineDriver +where + A: Fn(B256, OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, +{ fn execute<'a>( &'a mut self, env: &'a mut Environment, ) -> BoxFuture<'a, Result<()>> { Box::pin(async move { - use alloy_rpc_types_engine::CancunPayloadFields; - use op_alloy_rpc_types_engine::{OpExecutionData, OpExecutionPayloadSidecar}; - - let payload = self - .state - .get_payload() - .ok_or_else(|| eyre!("No payload to canonicalize"))?; - - let block_hash = payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - - let parent_hash = payload - .execution_payload - .payload_inner - .payload_inner - .parent_hash; - - info!( - target: "actions", - block_hash = ?block_hash, - "Canonicalizing block" - ); + let builder = &env.node_clients[self.builder_idx]; + let engine = builder.engine.http_client(); - // Construct OpExecutionData from the envelope - use op_alloy_rpc_types_engine::OpExecutionPayload; - let execution_data = OpExecutionData { - payload: OpExecutionPayload::V3(payload.execution_payload.clone()), - sidecar: OpExecutionPayloadSidecar::v3(CancunPayloadFields::new( - payload.parent_beacon_block_root, - vec![], - )), + // Get the initial head + let mut parent_hash = if let Some(hash) = self.initial_parent_hash { + hash + } else { + let latest: Option = + EthApiClient::< + TransactionRequest, + Transaction, + alloy_rpc_types_eth::Block, + alloy_consensus::Receipt, + Header, + TransactionSigned, + >::block_by_number( + &builder.rpc, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await?; + latest + .ok_or_else(|| eyre!("No latest block"))? + .header + .hash_slow() }; - for &node_idx in &self.node_idxs { - // Use beacon engine handle which accepts OpExecutionData directly - if let Some(beacon_handle) = - env.node_clients[node_idx].beacon_engine_handle.as_ref() - { - // First: update forkchoice to parent so node can accept the new block - let parent_fcu = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - beacon_handle - .fork_choice_updated(parent_fcu, None, EngineApiMessageVersion::V3) - .await - .map_err(|e| eyre!("fork_choice_updated to parent failed: {:?}", e))?; - - // Second: send new_payload with the block - let status = beacon_handle - .new_payload(execution_data.clone()) - .await - .map_err(|e| eyre!("new_payload failed: {:?}", e))?; - - if !matches!(status.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "new_payload failed for node {}: {:?}", - node_idx, - status - )); - } - - // Third: send fork_choice_updated to make it canonical - let fcu_state = ForkchoiceState { - head_block_hash: block_hash, - safe_block_hash: block_hash, - finalized_block_hash: block_hash, - }; - - beacon_handle - .fork_choice_updated(fcu_state, None, EngineApiMessageVersion::V3) - .await - .map_err(|e| eyre!("fork_choice_updated failed: {:?}", e))?; + let mut parent_timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + + for block_num in 0..self.num_blocks { + // 1. Generate attributes + let block_number = block_num as u64 + 1; + parent_timestamp += self.block_interval.as_secs().max(1); + let attributes = (self.attributes_gen)(block_number, parent_timestamp)?; + + // 2. FCU with attributes → start building + let fcu_state = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; - info!( - target: "actions", - node_idx = node_idx, - block_hash = ?block_hash, - "Block canonicalized successfully via beacon handle" - ); - } else { - // Fallback: use RPC engine client - let engine = env.node_clients[node_idx].engine.http_client(); - - // First: update forkchoice to parent so node can accept the new block - let parent_fcu = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - let _ = EngineApiClient::::fork_choice_updated_v3( - &engine, parent_fcu, None, + let fcu_result = if self.flashblocks { + FlashblocksEngineApiExtClient::::flashblocks_fork_choice_updated_v3( + &engine, + fcu_state, + Some(attributes.clone()), + Some((self.authorization_gen)(parent_hash, attributes.clone())), ) - .await?; - - // Second: send new_payload with the block via RPC - let np_result = EngineApiClient::::new_payload_v3( + .await? + } else { + EngineApiClient::::fork_choice_updated_v3( &engine, - payload.execution_payload.clone(), - vec![], - payload.parent_beacon_block_root, + fcu_state, + Some(attributes.clone()), ) - .await?; + .await? + }; - if !matches!(np_result.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "new_payload failed for node {}: {:?}", - node_idx, - np_result - )); - } + if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { + return Err(eyre!( + "block {block_num}: FCU status not valid: {:?}", + fcu_result.payload_status + )); + } + + let payload_id = fcu_result + .payload_id + .ok_or_else(|| eyre!("block {block_num}: No payload ID returned"))?; + + info!( + target: "engine_driver", + block = block_num, + %payload_id, + "building block" + ); + + // 3. Wait for build deadline + tokio::time::sleep(self.block_interval).await; + + // 3.5. Mid-build callback (payload builder is still working) + if let Some(ref during_build) = self.during_build { + during_build(block_num).await?; + } + + // 4. getPayloadV4 + let payload = + EngineApiClient::::get_payload_v4(&engine, payload_id).await?; + + let block_hash = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .block_hash; + + let tx_count = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .transactions + .len(); + + info!( + target: "engine_driver", + block = block_num, + %block_hash, + tx_count, + "payload retrieved" + ); + + // 5. Canonicalize: FCU(parent) → newPayload → FCU(head) + // on builder AND all follower nodes + use alloy_rpc_types_engine::CancunPayloadFields; + use op_alloy_rpc_types_engine::{ + OpExecutionData, OpExecutionPayload, OpExecutionPayloadSidecar, + }; - // Third: send fork_choice_updated to make it canonical - let fcu_state = ForkchoiceState { + // Canonicalize on builder via FCU only (it already has the payload) + { + let builder_engine = env.node_clients[self.builder_idx].engine.http_client(); + let head_fcu = ForkchoiceState { head_block_hash: block_hash, safe_block_hash: block_hash, finalized_block_hash: block_hash, }; - let fcu_result = EngineApiClient::::fork_choice_updated_v3( - &engine, fcu_state, None, + &builder_engine, + head_fcu, + None, ) .await?; if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { return Err(eyre!( - "fork_choice_updated failed for node {}: {:?}", - node_idx, + "block {block_num}: builder FCU to head failed: {:?}", fcu_result.payload_status )); } - - info!( - target: "actions", - node_idx = node_idx, - block_hash = ?block_hash, - "Block canonicalized successfully via RPC" - ); } - } - - Ok(()) - }) - } -} - -/// Mine a block and store the payload in shared state -pub struct MineBlockWithState { - pub node_idx: usize, - pub attributes: OpPayloadAttributes, - pub authorization_gen: A, - pub block_interval: Duration, - pub flashblocks: bool, - pub state: BlockProductionState, -} - -impl MineBlockWithState -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync, -{ - pub fn new( - node_idx: usize, - attributes: OpPayloadAttributes, - authorization_gen: A, - state: BlockProductionState, - ) -> Self { - Self { - node_idx, - attributes, - authorization_gen, - block_interval: Duration::from_millis(2000), - flashblocks: true, - state, - } - } - pub fn with_interval(mut self, interval: Duration) -> Self { - self.block_interval = interval; - self - } -} + // Canonicalize on follower nodes: FCU(parent) → newPayload → FCU(head) + for follower_idx in self.follower_idxs.iter().copied() { + if let Some(beacon_handle) = + env.node_clients[follower_idx].beacon_engine_handle.as_ref() + { + // FCU to parent + let parent_fcu = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; + beacon_handle + .fork_choice_updated( + parent_fcu, + None, + EngineApiMessageVersion::V3, + ) + .await + .map_err(|e| { + eyre!("block {block_num}: FCU to parent failed on follower {follower_idx}: {e:?}") + })?; + + // newPayload + let execution_data = OpExecutionData { + payload: OpExecutionPayload::V4(payload.execution_payload.clone()), + sidecar: OpExecutionPayloadSidecar::v4( + CancunPayloadFields::new(payload.parent_beacon_block_root, vec![]), + alloy_rpc_types_engine::PraguePayloadFields { + requests: alloy_eips::eip7685::RequestsOrHash::Hash( + alloy_eips::eip7685::EMPTY_REQUESTS_HASH, + ), + }, + ), + }; -impl ReadOnlyAction for MineBlockWithState -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let client = &env.node_clients[self.node_idx]; - let engine = client.engine.http_client(); + let status = beacon_handle + .new_payload(execution_data) + .await + .map_err(|e| { + eyre!("block {block_num}: newPayload failed on follower {follower_idx}: {e:?}") + })?; + + if !matches!(status.status, PayloadStatusEnum::Valid) { + return Err(eyre!( + "block {block_num}: newPayload invalid on follower {follower_idx}: {:?}", + status + )); + } - let latest: Option = - EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - alloy_consensus::Receipt, - Header, - TransactionSigned, - >::block_by_number( - &client.rpc, alloy_eips::BlockNumberOrTag::Latest, false - ) - .await?; + // FCU to head + let head_fcu = ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash: block_hash, + finalized_block_hash: block_hash, + }; + beacon_handle + .fork_choice_updated( + head_fcu, + None, + EngineApiMessageVersion::V3, + ) + .await + .map_err(|e| { + eyre!("block {block_num}: FCU to head failed on follower {follower_idx}: {e:?}") + })?; - let parent_hash = latest - .ok_or_else(|| eyre!("No latest block"))? - .header - .hash_slow(); + info!( + target: "engine_driver", + block = block_num, + follower = follower_idx, + %block_hash, + "canonicalized on follower via beacon handle" + ); + } else { + return Err(eyre!( + "block {block_num}: follower {follower_idx} has no beacon_engine_handle" + )); + } + } - let fcu_state = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; + // 6. Invoke callback + if let Some(ref on_block) = self.on_block { + on_block(block_num, &payload).await?; + } - let fcu_result = if self.flashblocks { - FlashblocksEngineApiExtClient::::flashblocks_fork_choice_updated_v3( - &engine, - fcu_state, - Some(self.attributes.clone()), - Some((self.authorization_gen)(self.attributes.clone())), - ) - .await? - } else { - EngineApiClient::::fork_choice_updated_v3( - &engine, - fcu_state, - Some(self.attributes.clone()), - ) - .await? - }; + // 7. Advance head + parent_hash = block_hash; - if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "FCU status not valid: {:?}", - fcu_result.payload_status - )); + info!( + target: "engine_driver", + block = block_num, + %block_hash, + "block complete" + ); } - let payload_id = fcu_result - .payload_id - .ok_or_else(|| eyre!("No payload ID returned"))?; - - // Wait for block to be built - tokio::time::sleep(self.block_interval).await; - - let payload = - EngineApiClient::::get_payload_v3(&engine, payload_id).await?; - - let block_hash = payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - - info!( - target: "actions", - block_hash = ?block_hash, - tx_count = payload.execution_payload.payload_inner.payload_inner.transactions.len(), - "Mined block, storing in shared state" - ); - - // Store payload in shared state - self.state.set_payload(payload); - Ok(()) }) } } -/// Validate flashblocks and signal when complete, storing validated hashes in shared state -pub struct ValidateFlashblocksWithState { - pub flashblock_stream: Pin + Send>>, - pub beacon_handle: Arc>, - pub chain_spec: Arc, - pub state: BlockProductionState, -} - -impl ValidateFlashblocksWithState { - pub fn new( - flashblock_stream: Pin + Send>>, - beacon_handle: Arc>, - chain_spec: Arc, - state: BlockProductionState, - ) -> Self { - Self { - flashblock_stream, - beacon_handle, - chain_spec, - state, - } - } -} - -impl Action for ValidateFlashblocksWithState { +impl Action for EngineDriver +where + A: Fn(B256, OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, +{ fn execute<'a>( &'a mut self, - _env: &'a mut Environment, + env: &'a mut Environment, ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let mut flashblocks = Flashblocks::default(); - let stream = &mut self.flashblock_stream; - - // Wait for payload to be available - let target_hash = loop { - if let Some(payload) = self.state.get_payload() { - break payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - info!( - target: "actions", - target_hash = ?target_hash, - "Starting flashblock validation" - ); - - while let Some(fb_payload) = stream.next().await { - let index = fb_payload.index; - - let is_new = flashblocks - .push(Flashblock { - flashblock: fb_payload, - }) - .ok(); - - if is_new.is_some() { - info!( - target: "actions", - index = %index, - "New payload started, reset flashblock collection" - ); - } - - // Reduce to get current state - let Some(reduced) = Flashblock::reduce(flashblocks.clone()).ok() else { - continue; - }; - - let reduced_hash = reduced.diff().block_hash; - - // Store validated hash for GetBlockByHash - self.state.add_validated_hash(reduced_hash); - - // Construct execution data - let execution_data = - execution_data_from_from_reduced_flashblock(reduced, self.chain_spec.clone()); - - // Validate - let parent = execution_data.parent_hash(); - let forkchoice = ForkchoiceState { - head_block_hash: parent, - safe_block_hash: parent, - finalized_block_hash: parent, - }; - - self.beacon_handle - .fork_choice_updated(forkchoice, None, EngineApiMessageVersion::V3) - .await - .ok(); - - let status = self.beacon_handle.new_payload(execution_data.clone()).await; - - match &status { - Ok(s) => { - info!( - target: "actions", - index = %index, - ?reduced_hash, - status = ?s.status, - "Validated intermediate flashblock" - ); - } - Err(e) => { - error!( - target: "actions", - index = %index, - error = ?e, - "Flashblock validation failed" - ); - } - } - - // Check if final - if reduced_hash == target_hash { - info!( - target: "actions", - block_hash = ?reduced_hash, - index = %index, - "Final flashblock validated" - ); - self.state.signal_final(); - break; - } - } - - Ok(()) - }) - } -} - -// ============================================================================ -// Block Production Loop - Composable N-block production as a single Action -// ============================================================================ - -/// Configuration for block production loop -#[derive(Clone)] -pub struct BlockProductionConfig { - /// Builder node index - pub builder_node_idx: usize, - /// Follower node indexes for canonicalization - pub follower_node_idxs: Vec, - /// Authorization generator - pub authorization_gen: A, - /// Attributes builder function: (timestamp, eip1559_params) -> OpPayloadAttributes - pub attributes_builder: F, - /// Block interval - pub block_interval: Duration, - /// Number of blocks to produce - pub num_blocks: u64, - /// Starting timestamp - pub start_timestamp: u64, - /// Timestamp increment per block - pub timestamp_increment: u64, - /// Shared state for cross-action communication - pub state: BlockProductionState, - /// Chain spec for EIP-1559 params - pub chain_spec: Arc, -} - -/// A complete block production loop as a single composable Action. -/// -/// This action produces N blocks in sequence, with each block going through: -/// 1. Mine block (stores payload in shared state) -/// 2. Validate flashblocks (runs parallel, signals when done) -/// 3. Query blocks by hash (runs parallel, uses validated hashes) -/// 4. Query receipts (runs parallel, uses tx hashes from spammer) -/// 5. Canonicalize on follower nodes -/// 6. Reset state and advance to next block -pub struct BlockProductionLoop { - pub config: BlockProductionConfig, - /// Flashblocks handle for getting streams - pub flashblocks_handle: H, - /// Beacon handle for validation - pub beacon_handle: Arc>, - /// Hook called after each block with (block_num, block_hash, tx_count) - pub on_block_produced: Option>, - /// Hook for each validated flashblock hash - pub on_validated_hash: Option>, - /// Hook for each fetched receipt - pub on_receipt: Option>, -} - -// ============================================================================ -// Simple Action Primitives for Composition -// ============================================================================ - -/// Reset the shared state - use at the start of each block cycle -pub struct ResetState { - pub state: BlockProductionState, -} - -impl ResetState { - pub fn new(state: BlockProductionState) -> Self { - Self { state } - } -} - -impl Action for ResetState { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - self.state.reset(); - Ok(()) - }) - } -} - -/// Query all validated block hashes from shared state -pub struct QueryValidatedBlocks { - pub node_idxs: Vec, - pub state: BlockProductionState, - pub on_block: Option>, -} - -impl QueryValidatedBlocks { - pub fn new(node_idxs: Vec, state: BlockProductionState) -> Self { - Self { - node_idxs, - state, - on_block: None, - } - } - - pub fn on_block(mut self, f: F) -> Self - where - F: Fn(alloy_rpc_types_eth::Block) -> Result<()> + Send + Sync + 'static, - { - self.on_block = Some(hook(f)); - self - } -} - -impl ReadOnlyAction for QueryValidatedBlocks { - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let hashes = self.state.validated_block_hashes.read().clone(); - - for hash in &hashes { - for &node_idx in &self.node_idxs { - let block: Option = - EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - alloy_consensus::Receipt, - Header, - TransactionSigned, - >::block_by_hash( - &env.node_clients[node_idx].rpc, *hash, false - ) - .await?; - - if let Some(ref b) = block - && let Some(ref hook) = self.on_block - { - hook(b.clone())?; - } - } - } - Ok(()) - }) - } -} - -/// Query receipts for all transaction hashes in shared state -pub struct QueryTxReceipts { - pub node_idxs: Vec, - pub state: BlockProductionState, - pub on_receipt: Option>, -} - -impl QueryTxReceipts { - pub fn new(node_idxs: Vec, state: BlockProductionState) -> Self { - Self { - node_idxs, - state, - on_receipt: None, - } - } - - pub fn on_receipt(mut self, f: F) -> Self - where - F: Fn(OpTransactionReceipt) -> Result<()> + Send + Sync + 'static, - { - self.on_receipt = Some(hook(f)); - self - } -} - -impl ReadOnlyAction for QueryTxReceipts { - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let hashes = self.state.get_tx_hashes(); - - for hash in &hashes { - for &node_idx in &self.node_idxs { - let receipt: Option = EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - OpTransactionReceipt, - Header, - TransactionSigned, - >::transaction_receipt( - &env.node_clients[node_idx].rpc, - *hash, - ) - .await?; - - if let Some(ref r) = receipt - && let Some(ref hook) = self.on_receipt - { - hook(r.clone())?; - } - } - } - Ok(()) - }) - } -} - -/// A dynamic mining action that gets attributes from shared state -pub struct DynamicMineBlock { - pub node_idx: usize, - pub authorization_gen: A, - pub block_interval: Duration, - pub state: BlockProductionState, - /// Function to get current attributes (called at execution time) - pub get_attributes: Arc OpPayloadAttributes + Send + Sync>, -} - -impl DynamicMineBlock -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - pub fn new( - node_idx: usize, - authorization_gen: A, - state: BlockProductionState, - get_attributes: F, - ) -> Self - where - F: Fn() -> OpPayloadAttributes + Send + Sync + 'static, - { - Self { - node_idx, - authorization_gen, - block_interval: Duration::from_millis(2000), - state, - get_attributes: Arc::new(get_attributes), - } - } - - pub fn with_interval(mut self, interval: Duration) -> Self { - self.block_interval = interval; - self - } -} - -impl ReadOnlyAction for DynamicMineBlock -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let attributes = (self.get_attributes)(); - - let mine_action = MineBlockWithState::new( - self.node_idx, - attributes, - self.authorization_gen.clone(), - self.state.clone(), - ) - .with_interval(self.block_interval); - - mine_action.execute_readonly(env).await - }) - } -} - -/// A dynamic flashblock validator that gets streams from a flashblocks handle -pub struct DynamicValidateFlashblocks { - pub flashblocks_handle: flashblocks_p2p::protocol::handler::FlashblocksHandle, - pub beacon_handle: Arc>, - pub chain_spec: Arc, - pub state: BlockProductionState, -} - -impl DynamicValidateFlashblocks { - pub fn new( - flashblocks_handle: flashblocks_p2p::protocol::handler::FlashblocksHandle, - beacon_handle: Arc>, - chain_spec: Arc, - state: BlockProductionState, - ) -> Self { - Self { - flashblocks_handle, - beacon_handle, - chain_spec, - state, - } - } -} - -impl Action for DynamicValidateFlashblocks { - fn execute<'a>( - &'a mut self, - env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let stream = Box::pin(self.flashblocks_handle.live_flashblock_stream()); - - let mut validate_action = ValidateFlashblocksWithState::new( - stream, - self.beacon_handle.clone(), - self.chain_spec.clone(), - self.state.clone(), - ); - - validate_action.execute(env).await - }) - } -} - -impl ReadOnlyAction for DynamicValidateFlashblocks { - fn execute_readonly<'a>( - &'a self, - _env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let mut flashblocks = Flashblocks::default(); - let mut stream = Box::pin(self.flashblocks_handle.live_flashblock_stream()); - - // Wait for payload to be available - let target_hash = loop { - if let Some(payload) = self.state.get_payload() { - break payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - info!( - target: "actions", - target_hash = ?target_hash, - "Starting flashblock validation (parallel)" - ); - - while let Some(fb_payload) = stream.next().await { - let index = fb_payload.index; - - let is_new = flashblocks - .push(Flashblock { - flashblock: fb_payload, - }) - .ok(); - - if is_new.is_some() { - info!( - target: "actions", - index = %index, - "New flashblock received" - ); - } - - // Check if this is the final flashblock matching our target - if let Ok(reduced) = Flashblock::reduce(flashblocks.clone()) { - let block_hash = reduced.diff().block_hash; - if block_hash == target_hash { - info!( - target: "actions", - block_hash = ?block_hash, - "Final flashblock validated" - ); - self.state.add_validated_hash(block_hash); - self.state.signal_final(); - break; - } - } - } - - Ok(()) - }) - } -} - -/// Log current block production state -pub struct LogBlockComplete { - pub state: BlockProductionState, - pub on_complete: Option>, -} - -impl LogBlockComplete { - pub fn new(state: BlockProductionState) -> Self { - Self { - state, - on_complete: None, - } - } - - pub fn on_complete(mut self, f: F) -> Self - where - F: Fn((B256, usize)) -> Result<()> + Send + Sync + 'static, - { - self.on_complete = Some(hook(f)); - self - } -} - -impl Action for LogBlockComplete { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - if let Some(payload) = self.state.get_payload() { - let hash = payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - let tx_count = payload - .execution_payload - .payload_inner - .payload_inner - .transactions - .len(); - - info!( - target: "block_production", - ?hash, - tx_count, - validated_hashes = self.state.validated_block_hashes.read().len(), - tx_receipts = self.state.get_tx_hashes().len(), - "Block cycle complete" - ); - - if let Some(ref hook) = self.on_complete { - hook((hash, tx_count))?; - } - } - Ok(()) - }) - } -} - -/// Sleep for a duration - useful between block cycles -pub struct Sleep { - pub duration: Duration, -} - -impl Sleep { - pub fn new(duration: Duration) -> Self { - Self { duration } - } - - pub fn millis(ms: u64) -> Self { - Self::new(Duration::from_millis(ms)) - } -} - -impl Action for Sleep { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - tokio::time::sleep(self.duration).await; - Ok(()) - }) + EngineDriver::execute(self, env) } } diff --git a/crates/world/node/tests/e2e-testsuite/spammer.rs b/crates/world/node/tests/e2e-testsuite/spammer.rs index 020fbd14b..aa6cdac0a 100644 --- a/crates/world/node/tests/e2e-testsuite/spammer.rs +++ b/crates/world/node/tests/e2e-testsuite/spammer.rs @@ -12,11 +12,11 @@ use reth::{chainspec::EthChainSpec, rpc::api::EthApiClient}; use reth_e2e_test_utils::testsuite::NodeClient; use reth_optimism_node::OpEngineTypes; use reth_optimism_primitives::{OpReceipt, OpTransactionSigned}; -use revm_primitives::{Address, B256, Bytes}; +use revm_primitives::{Address, Bytes}; use tracing::{debug, error, info}; use world_chain_test::{node::tx, utils::signer}; -use crate::{actions::BlockProductionState, setup::CHAIN_SPEC}; +use crate::setup::CHAIN_SPEC; sol! { #[sol(rpc, bytecode = "6080604052348015600e575f5ffd5b506101338061001c5f395ff3fe608060405234801561000f575f5ffd5b506004361061004a575f3560e01c80632b68b9c61461004e578063703c2d1a14610056578063affed0e01461005e578063b8dda9c71461007a575b5f5ffd5b61005433ff5b005b6100546100ac565b61006760015481565b6040519081526020015b60405180910390f35b61009c6100883660046100f7565b5f6020819052908152604090205460ff1681565b6040519015158152602001610071565b5f5b60648110156100f4576001805f8282546100c8919061010e565b9091555050600180545f908152602081905260409020805460ff19811660ff90911615179055016100ae565b50565b5f60208284031215610107575f5ffd5b5035919050565b8082018082111561012d57634e487b7160e01b5f52601160045260245ffd5b9291505056")] @@ -137,46 +137,6 @@ impl TxSpammer { }); } - /// Spawns a background task that sends transactions and reports hashes to shared state. - /// - /// Same as `spawn` but also records transaction hashes in the provided `BlockProductionState` - /// for receipt querying by other actions. - pub fn spawn_with_state(self, tpf: u64, http_url: Url, state: BlockProductionState) { - tokio::spawn(async move { - // Deploy test contracts - let wallet = EthereumWallet::from(signer(0)); - let provider = Arc::new(ProviderBuilder::new().wallet(wallet).connect_http(http_url)); - - let contract = *TestContract::deploy(provider.clone()) - .await - .unwrap() - .address(); - - let factory = *TestContractFactory::deploy(provider) - .await - .unwrap() - .address(); - - info!("Deployed TestContract at {contract}, TestContractFactory at {factory}"); - - // Track nonce per signer (signers 1..=MAX_SIGNERS) - let mut nonces: Vec = vec![0; MAX_SIGNERS as usize]; - - loop { - let batch = self.build_batch(tpf, &mut nonces, contract, factory).await; - let tx_hashes = self.broadcast_batch_with_hashes(&batch).await; - - // Record submitted tx hashes in shared state - for hash in tx_hashes { - state.add_tx_hash(hash); - } - - info!("Submitted {} transactions", batch.len()); - tokio::time::sleep(Duration::from_millis(200)).await; - } - }); - } - /// Builds a batch of `tpf` raw transactions, distributing them across signers round-robin. async fn build_batch( &self, @@ -233,31 +193,4 @@ impl TxSpammer { futs.collect::<()>().await; } - - /// Broadcasts a batch of transactions and returns the successful tx hashes. - async fn broadcast_batch_with_hashes(&self, batch: &[Bytes]) -> Vec { - let mut tx_hashes = Vec::with_capacity(batch.len()); - - while let Some(client) = self.rpc.first() { - for tx in batch { - let result = EthApiClient::< - TransactionRequest, - OpTransactionSigned, - alloy_consensus::Block, - OpReceipt, - Header, - Bytes, - >::send_raw_transaction(&client.rpc, tx.clone()) - .await - .inspect_err(|e| error!("Error sending transaction: {:?}", e)); - - if let Ok(tx_hash) = result { - info!("Submitted tx: {:?}", tx_hash); - tx_hashes.push(tx_hash); - } - } - } - - tx_hashes - } } diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index d651fedbc..b731ded82 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -1,16 +1,12 @@ -use crate::{ - actions::{ - ActionSequence, BlockProductionState, DynamicMineBlock, DynamicValidateFlashblocks, - LogBlockComplete, QueryTxReceipts, QueryValidatedBlocks, ResetState, Sleep, - }, - setup::{TX_SET_L1_BLOCK, build_payload_attributes}, -}; +use crate::setup::{TX_SET_L1_BLOCK, build_payload_attributes}; use alloy_network::{Ethereum, EthereumWallet, TransactionBuilder, eip2718::Encodable2718}; use alloy_primitives::{Bytes, b64}; +use alloy_provider::ProviderBuilder; use alloy_rpc_types::TransactionRequest; use alloy_rpc_types_engine::PayloadStatusEnum; use eyre::eyre::eyre; -use futures::future::Either; +use flashblocks_p2p::protocol::event::{ChainEvent, WorldChainEvent}; +use op_alloy_consensus::OpTxEnvelope; use reth::{ chainspec::EthChainSpec, network::{NetworkSyncUpdater, SyncState}, @@ -63,7 +59,7 @@ async fn create_priority_transaction( Ok((signed.encoded_2718().into(), *signed.tx_hash())) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_can_build_pbh_payload() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let (signers, mut nodes, _tasks, _, _) = @@ -93,7 +89,7 @@ async fn test_can_build_pbh_payload() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_transaction_pool_ordering() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -139,7 +135,7 @@ async fn test_transaction_pool_ordering() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_enforces_block_uncompressed_size_limit() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -238,7 +234,7 @@ async fn test_enforces_block_uncompressed_size_limit() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_without_block_uncompressed_size_limit_includes_all_transactions() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -287,7 +283,7 @@ async fn test_without_block_uncompressed_size_limit_includes_all_transactions() Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_invalidate_dup_tx_and_nullifier() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let (_signers, mut nodes, _tasks, _, _) = @@ -301,7 +297,7 @@ async fn test_invalidate_dup_tx_and_nullifier() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_dup_pbh_nonce() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -410,7 +406,7 @@ async fn test_flashblocks() -> eyre::Result<()> { .as_ref() .unwrap() .flashblocks_handle - .live_flashblock_stream(), + .flashblock_stream(), ); let validation_stream = crate::actions::FlashblocksValidatonStream { @@ -488,7 +484,7 @@ async fn test_eth_api_receipt() -> eyre::Result<()> { .clone() .unwrap() .flashblocks_handle - .live_flashblock_stream(); + .flashblock_stream(); let mine_block = crate::actions::AssertMineBlock::new( 0, @@ -652,7 +648,7 @@ async fn test_eth_block_by_hash_pending() -> eyre::Result<()> { .clone() .unwrap() .flashblocks_handle - .live_flashblock_stream(); + .flashblock_stream(); let (sender, mut rx) = tokio::sync::mpsc::channel(1); let timestamp = crate::setup::current_timestamp(); @@ -704,7 +700,7 @@ async fn test_eth_block_by_hash_pending() -> eyre::Result<()> { /// /// Verifies that without tx_peers configuration, transactions propagate to ALL connected peers /// using Reth's default TransactionPropagationKind::All policy. -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_default_propagation_policy() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -778,8 +774,7 @@ async fn test_default_propagation_policy() -> eyre::Result<()> { /// Test Part 2: /// - Inject tx into Node 2 -> should propagate to both Node 0 and Node 1 /// - Verifies multi-peer whitelist works correctly -#[tokio::test] -#[ignore = "TODO: flaky - not sure what's causing this to fail"] +#[tokio::test(flavor = "multi_thread")] async fn test_selective_propagation_policy() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -877,9 +872,27 @@ async fn test_selective_propagation_policy() -> eyre::Result<()> { .node .inner .network - .add_peer(node_0_peer_id, node_0_addr); - - tokio::time::sleep(Duration::from_secs(3)).await; + .connect_peer(node_0_peer_id, node_0_addr); + + // Wait for reconnection to establish + let start = tokio::time::Instant::now(); + loop { + let peer = node_2_ctx + .node + .inner + .network + .get_peer_by_id(node_0_peer_id) + .await?; + if peer.is_some() { + break; + } + if start.elapsed() > Duration::from_secs(10) { + panic!("Timeout waiting for Node 0 <-> Node 2 reconnection"); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + // Extra time for sync state to stabilize + tokio::time::sleep(Duration::from_secs(1)).await; // Create a new transaction and inject into Node 2 // Node 2 has tx_peers = [Node 0, Node 1], so it should propagate to both @@ -924,7 +937,7 @@ async fn test_selective_propagation_policy() -> eyre::Result<()> { /// - Inject tx into Node 0 -> should NOT propagate to any node /// - Inject tx into Node 1 -> should NOT propagate to any node (even though Node 0 is whitelisted) /// - Verifies that disable_txpool_gossip takes precedence over tx_peers -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_gossip_disabled_no_propagation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -983,190 +996,607 @@ async fn test_gossip_disabled_no_propagation() -> eyre::Result<()> { Ok(()) } +/// End-to-end test: drives the builder's consensus engine through a block +/// building loop, using a hook on the `WorldChainEventsStream` to assert +/// stream invariants: +/// +/// 1. Canon events are always yielded +/// 2. Pending flashblocks are only yielded after their epoch parent is canonical +/// 3. Flashblock indices are monotonically increasing within an epoch +/// 4. The P2P state's flushed cursor tracks the latest yielded flashblock +/// 5. Stale flashblocks (from old epochs) are never yielded #[tokio::test(flavor = "multi_thread")] -#[ignore = "flaky test"] -async fn test_continuous_block_production_with_validation() -> eyre::Result<()> { +async fn test_event_stream_invariants() -> eyre::Result<()> { reth_tracing::init_test_tracing(); - const NUM_BLOCKS: u64 = 10; - const BLOCK_INTERVAL_MS: u64 = 2000; - const TXS_PER_FLASHBLOCK: u64 = 20; - - let (_, mut nodes, _tasks, mut flashblocks_env, tx_spammer) = - setup::(3, optimism_payload_attributes, true).await?; - - // Setup: 1 basic verifier node for flashblock validation - let (_, mut basic_validators, _tasks, _basic_env, _) = - setup_with_tx_peers::( - 1, - optimism_payload_attributes, - false, - false, - true, - ) - .await?; + const TRANSACTIONS_PER_FLASHBLOCK: u64 = 10; - let basic_validator = &mut basic_validators[0]; + tokio::time::sleep(Duration::from_millis(100)).await; - let [builder_node, follower_0, follower_1] = &mut nodes[..] else { - unreachable!("Expected exactly 2 nodes") - }; - - let builder_context = builder_node.ext_context.clone(); + let (_, mut nodes, _tasks, mut env, tx_spammer) = + setup::(1, optimism_payload_attributes, true).await?; - let basic_beacon_handle = - Arc::new(basic_validator.node.inner.consensus_engine_handle().clone()); + let builder_node = &mut nodes[0]; + let builder_context = builder_node.ext_context.clone().unwrap(); + let rpc_url = builder_node.node.rpc_url(); - let follower_context_0 = follower_0.ext_context.clone(); - let _follower_context_1 = follower_1.ext_context.clone(); + tx_spammer.spawn(TRANSACTIONS_PER_FLASHBLOCK, rpc_url); - // Create shared state for cross-action communication - let state = BlockProductionState::new(); + let block_hash = builder_node.node.block_hash(0); - // Create authorization generator - let genesis_hash = builder_node.node.block_hash(0); let authorization_generator = crate::setup::create_authorization_generator( - genesis_hash, + block_hash, builder_context - .unwrap() .flashblocks_handle .builder_sk() .unwrap() .verifying_key(), ); - // Spawn spammer with shared state to track tx hashes - let rpc_url = builder_node.node.rpc_url(); - tx_spammer.spawn_with_state(TXS_PER_FLASHBLOCK, rpc_url, state.clone()); + let timestamp = crate::setup::current_timestamp(); + let eip1559_params = + encode_eip1559_params(builder_node.node.inner.chain_spec().as_ref(), timestamp)?; - info!( - target: "test", - "Starting continuous block production test for {} blocks using ActionSequence", - NUM_BLOCKS + let attributes = build_payload_attributes( + timestamp, + eip1559_params, + Some(vec![TX_SET_L1_BLOCK.clone()]), ); - // Track statistics via hooks - let blocks_produced = Arc::new(AtomicU64::new(0)); - let validated_hashes = Arc::new(AtomicUsize::new(0)); - let receipts_fetched = Arc::new(AtomicUsize::new(0)); - - // Create timestamp state that advances with each iteration - let timestamp = Arc::new(AtomicU64::new(crate::setup::current_timestamp())); - let chain_spec = builder_node.node.inner.chain_spec().clone(); - - // Clone handles for the attribute builder closure - let timestamp_for_attrs = timestamp.clone(); - let chain_spec_for_attrs = chain_spec.clone(); - - let validated_hashes_counter = validated_hashes.clone(); - let receipts_counter = receipts_fetched.clone(); - - // Build the composable action sequence for ONE block cycle - let block_cycle = ActionSequence::new() - // 1. Reset state at start of each block - .then(ResetState::new(state.clone())) - // 2. Mine block + validate flashblocks in parallel - .with( - DynamicMineBlock::new( - 0, // builder node - authorization_generator.clone(), - state.clone(), - move || { - let ts = timestamp_for_attrs.load(Ordering::SeqCst); - let eip1559 = - crate::setup::encode_eip1559_params(chain_spec_for_attrs.as_ref(), ts) - .unwrap(); - crate::setup::build_payload_attributes( - ts, - eip1559, - Some(vec![crate::setup::TX_SET_L1_BLOCK.clone()]), - ) - }, - ) - .with_interval(Duration::from_millis(BLOCK_INTERVAL_MS)), - DynamicValidateFlashblocks::new( - follower_context_0.unwrap().flashblocks_handle.clone(), - basic_beacon_handle, - chain_spec.clone(), - state.clone(), - ), - ) - // 3. Query validated blocks and receipts in parallel (AFTER mining/validation) - .with( - QueryValidatedBlocks::new(vec![0, 1], state.clone()).on_block(move |_| { - validated_hashes_counter.fetch_add(1, Ordering::SeqCst); - Ok(()) - }), - QueryTxReceipts::new(vec![0, 1], state.clone()).on_receipt(move |_| { - receipts_counter.fetch_add(1, Ordering::SeqCst); - Ok(()) - }), - ) - // 6. Log completion and track stats - // Note: Skipping Canonicalize on follower nodes for now - // because Isthmus V4 payloads don't work with RPC new_payload_v3 - .then({ - let counter = blocks_produced.clone(); - LogBlockComplete::new(state.clone()).on_complete(move |_| { - counter.fetch_add(1, Ordering::SeqCst); - Ok(()) - }) - }) - // 8. Small delay between blocks - .then(Sleep::millis(100)); - - // Repeat the block cycle N times, advancing timestamp each iteration - let timestamp_for_repeat = timestamp.clone(); - let mut action = block_cycle.repeat(NUM_BLOCKS).on_each(move |i| { - // Advance timestamp for next block (after first iteration) - if i > 0 { - timestamp_for_repeat.fetch_add(2, Ordering::SeqCst); - } - info!( - target: "test", - iteration = i + 1, - total = NUM_BLOCKS, - "Starting block cycle" + // --- Assertion state shared with the hook --- + let canon_count = Arc::new(AtomicUsize::new(0)); + let pending_count = Arc::new(AtomicUsize::new(0)); + let last_index = Arc::new(AtomicU64::new(0)); + let saw_canon_before_pending = Arc::new(std::sync::atomic::AtomicBool::new(false)); + + let canon_count_hook = canon_count.clone(); + let pending_count_hook = pending_count.clone(); + let last_index_hook = last_index.clone(); + let saw_canon_hook = saw_canon_before_pending.clone(); + + // Create the event stream with a hook that asserts invariants + let p2p_state = builder_context.flashblocks_handle.state.clone(); + let mut stream = builder_context + .flashblocks_handle + .event_stream::<(), _, _, _>( + builder_node.node.inner.provider.clone(), + move |event: &WorldChainEvent<()>| { + match event { + WorldChainEvent::Chain(ChainEvent::Canon(_tip)) => { + canon_count_hook.fetch_add(1, Ordering::SeqCst); + } + WorldChainEvent::Chain(ChainEvent::Pending(fb)) => { + // Invariant: we must have seen at least one canon event + // before any pending flashblock is yielded. + if canon_count_hook.load(Ordering::SeqCst) > 0 { + saw_canon_hook.store(true, Ordering::SeqCst); + } + + // Invariant: indices are monotonically increasing + let prev = last_index_hook.swap(fb.index, Ordering::SeqCst); + if pending_count_hook.load(Ordering::SeqCst) > 0 { + assert!( + fb.index >= prev, + "flashblock index went backwards: {} -> {}", + prev, + fb.index + ); + } + + pending_count_hook.fetch_add(1, Ordering::SeqCst); + } + _ => {} + } + None + }, ); - Ok(()) + // Spawn the stream consumer + let _stream_handle = tokio::spawn(async move { + let mut count = 0usize; + while let Some(_event) = futures::StreamExt::next(&mut stream).await { + count += 1; + if count > 50 { + break; // safety valve + } + } + count }); - // Execute the entire repeated sequence as a single action - let fut = async { action.execute(&mut flashblocks_env).await }; - - futures::future::select( - Box::pin(fut), - Box::pin(if blocks_produced.load(Ordering::SeqCst) == NUM_BLOCKS { - Either::Left(futures::future::ready(())) - } else { - Either::Right(futures::future::pending::<()>()) - }), + // Mine a block + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let mine_block = crate::actions::AssertMineBlock::new( + 0, + None, + attributes, + authorization_generator, + Duration::from_millis(3000), + true, + tx, ) .await; - let final_blocks = blocks_produced.load(Ordering::SeqCst); - let final_hashes = validated_hashes.load(Ordering::SeqCst); - let final_receipts = receipts_fetched.load(Ordering::SeqCst); + tokio::spawn(async move { + let mut mine_action = mine_block; + mine_action.execute(&mut env).await + }); + + // Wait for mining to complete + rx.recv() + .await + .ok_or(eyre!("failed to receive mined block"))?; + + // Give the stream a moment to process remaining events + tokio::time::sleep(Duration::from_millis(500)).await; + + // --- Assert invariants --- + let canons = canon_count.load(Ordering::SeqCst); + let pendings = pending_count.load(Ordering::SeqCst); info!( target: "test", - blocks_produced = final_blocks, - validated_hashes = final_hashes, - receipts_fetched = final_receipts, - "Test completed successfully" + canon_events = canons, + pending_events = pendings, + "stream invariant results" + ); + + assert!(canons > 1, "expected at least one canon event"); + assert!(pendings > 1, "expected at least one pending flashblock"); + assert!( + saw_canon_before_pending.load(Ordering::SeqCst), + "expected canon event before first pending flashblock" + ); + + // Verify P2P state was updated by the hook + let state = p2p_state.lock(); + assert!( + state.canon_tip.is_some(), + "expected canon_tip to be set on P2P state" + ); + assert!( + state.flushed_payload_id.is_some(), + "expected flushed_payload_id to be set on P2P state" + ); + + Ok(()) +} + +/// End-to-end test: uses [`EngineDriver`] to build multiple blocks while +/// querying the pending block, logs, transactions, and receipts via the +/// Eth JSON-RPC API at each block boundary. +#[tokio::test(flavor = "multi_thread")] +async fn test_engine_driver_pending_block_queries() -> eyre::Result<()> { + use alloy_eips::BlockNumberOrTag; + use reth::rpc::api::EthApiClient; + + reth_tracing::init_test_tracing(); + + const NUM_BLOCKS: usize = 3; + const BLOCK_INTERVAL: Duration = Duration::from_millis(2000); + + tokio::time::sleep(Duration::from_millis(100)).await; + + // 2 nodes: builder + follower + let (_, nodes, _tasks, mut env, tx_spammer) = + setup::(2, optimism_payload_attributes, true).await?; + + let builder_context = nodes[0].ext_context.clone().unwrap(); + let block_hash = nodes[0].node.block_hash(0); + let chain_spec = nodes[0].node.inner.chain_spec().clone(); + let rpc_url = nodes[0].node.rpc_url(); + + // Initialize forkchoice on all nodes to genesis + for node in &nodes { + node.node.update_forkchoice(block_hash, block_hash).await?; + } + + // Spawn background transactions so blocks have content + tx_spammer.spawn(10, rpc_url); + + let builder_vk = builder_context + .flashblocks_handle + .builder_sk() + .unwrap() + .verifying_key(); + + let authorization_gen = + move |parent_hash: B256, attrs: reth_optimism_node::OpPayloadAttributes| { + let authorizer_sk = ed25519_dalek::SigningKey::from_bytes(&[0; 32]); + let payload_id = + reth_optimism_payload_builder::payload_id_optimism(&parent_hash, &attrs, 3); + flashblocks_primitives::p2p::Authorization::new( + payload_id, + reth_node_api::PayloadAttributes::timestamp(&attrs), + &authorizer_sk, + builder_vk, + ) + }; + + // --- Flashblock stream: capture latest pending flashblock --- + use flashblocks_primitives::primitives::FlashblocksPayloadV1; + use std::sync::RwLock; + + let latest_stream_fb: Arc>> = Arc::new(RwLock::new(None)); + let latest_stream_fb_writer = latest_stream_fb.clone(); + + // Use the raw flashblock_stream (no buffering) to capture flashblocks + // as they're broadcast, independent of canon state. + let mut fb_stream = builder_context.flashblocks_handle.flashblock_stream(); + + let _stream_task = tokio::spawn(async move { + while let Some(fb) = futures::StreamExt::next(&mut fb_stream).await { + *latest_stream_fb_writer.write().unwrap() = Some(fb); + } + }); + + // Track per-block results + let blocks_verified = Arc::new(AtomicUsize::new(0)); + let blocks_verified_cb = blocks_verified.clone(); + + let builder_rpc = env.node_clients[0].rpc.clone(); + + let mut driver = crate::actions::EngineDriver { + builder_idx: 0, + follower_idxs: vec![], + initial_parent_hash: Some(block_hash), + num_blocks: NUM_BLOCKS, + block_interval: BLOCK_INTERVAL, + flashblocks: true, + authorization_gen, + attributes_gen: Box::new({ + let chain_spec = chain_spec.clone(); + move |_block_number, timestamp| { + let eip1559 = encode_eip1559_params(chain_spec.as_ref(), timestamp)?; + Ok(build_payload_attributes( + timestamp, + eip1559, + Some(vec![TX_SET_L1_BLOCK.clone()]), + )) + } + }), + during_build: None, + on_block: Some(Box::new({ + let builder_rpc = builder_rpc.clone(); + let latest_stream_fb = latest_stream_fb.clone(); + move |block_num, payload| { + let builder_rpc = builder_rpc.clone(); + let blocks_verified = blocks_verified_cb.clone(); + let latest_stream_fb = latest_stream_fb.clone(); + + let payload_tx_count = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .transactions + .len(); + + Box::pin(async move { + assert!( + payload_tx_count > 0, + "block {block_num}: expected at least 1 transaction" + ); + + // Query the pending block from the Eth API + let pending: Option = + EthApiClient::< + TransactionRequest, + alloy_rpc_types::Transaction, + alloy_rpc_types_eth::Block, + alloy_consensus::Receipt, + alloy_consensus::Header, + reth_optimism_primitives::OpTransactionSigned, + >::block_by_number( + &builder_rpc, BlockNumberOrTag::Pending, false + ) + .await?; + + // Get the latest flashblock from the event stream + let stream_fb = latest_stream_fb.read().unwrap().clone(); + + if let (Some(pending_block), Some(stream_fb)) = (&pending, &stream_fb) { + info!( + target: "engine_driver_test", + block = block_num, + pending_number = pending_block.header.number, + pending_tx_count = pending_block.transactions.len(), + stream_payload_id = %stream_fb.payload_id, + stream_index = stream_fb.index, + stream_tx_count = stream_fb.diff.transactions.len(), + "comparing pending block vs event stream flashblock" + ); + + // The pending block from the Eth API should be for the + // same payload as the stream flashblock. + assert_eq!( + stream_fb.payload_id, + stream_fb.payload_id, // sanity + "block {block_num}: stream flashblock should have a valid payload_id" + ); + + // The pending block tx count should be >= the stream + // flashblock's cumulative tx count (pending block + // includes all transactions, stream fb has the diff). + assert!( + !pending_block.transactions.is_empty() + || stream_fb.diff.transactions.is_empty(), + "block {block_num}: pending block should have transactions if stream flashblock does" + ); + assert!( + pending_block.hash() == stream_fb.diff.block_hash, + "block {block_num}: pending block hash should match stream flashblock hash" + ); + } else { + info!( + target: "engine_driver_test", + block = block_num, + pending_available = pending.is_some(), + stream_available = stream_fb.is_some(), + "pending block or stream flashblock not yet available" + ); + } + + blocks_verified.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + } + })), + }; + + driver.execute(&mut env).await?; + + let verified = blocks_verified.load(Ordering::SeqCst); + + info!( + target: "engine_driver_test", + verified, + "engine driver test complete" ); assert_eq!( - final_blocks, NUM_BLOCKS, - "Should have produced {} blocks", - NUM_BLOCKS + verified, NUM_BLOCKS, + "expected to verify {NUM_BLOCKS} blocks" ); + // Verify the event stream captured flashblocks + let final_fb = latest_stream_fb.read().unwrap().clone(); assert!( - final_hashes > 0, - "Should have validated at least some block hashes" + final_fb.is_some(), + "expected the event stream to have captured at least one flashblock" + ); + + Ok(()) +} + +/// Large block production loop using [`EngineDriver`] that sanity-checks +/// all helper macros in the `on_block` hook: `provider!`, `fetch_block!`, +/// `fetch_tx!`, `fetch_receipt!`, `eth_call!`, `fetch_logs!`. +#[tokio::test(flavor = "multi_thread")] +async fn test_eth_api_assertions() -> eyre::Result<()> { + use crate::setup::encode_eip1559_params; + use alloy_provider::Provider; + use alloy_rpc_types::Filter; + + reth_tracing::init_test_tracing(); + tokio::time::sleep(Duration::from_millis(100)).await; + + const NUM_BLOCKS: usize = 5; + const BLOCK_INTERVAL: Duration = Duration::from_millis(2000); + + let (_, nodes, _tasks, mut env, tx_spammer) = + setup::(1, optimism_payload_attributes, true).await?; + + let builder_context = nodes[0].ext_context.clone().unwrap(); + let block_hash = nodes[0].node.block_hash(0); + let chain_spec = nodes[0].node.inner.chain_spec().clone(); + let rpc_url = nodes[0].node.rpc_url(); + + for node in &nodes { + node.node.update_forkchoice(block_hash, block_hash).await?; + } + + tx_spammer.spawn(10, rpc_url); + + let builder_vk = builder_context + .flashblocks_handle + .builder_sk() + .unwrap() + .verifying_key(); + + let authorization_gen = + move |parent_hash: B256, attrs: reth_optimism_node::OpPayloadAttributes| { + let authorizer_sk = ed25519_dalek::SigningKey::from_bytes(&[0; 32]); + let payload_id = + reth_optimism_payload_builder::payload_id_optimism(&parent_hash, &attrs, 3); + flashblocks_primitives::p2p::Authorization::new( + payload_id, + reth_node_api::PayloadAttributes::timestamp(&attrs), + &authorizer_sk, + builder_vk, + ) + }; + + let checks_passed = Arc::new(AtomicUsize::new(0)); + let checks_passed_cb = checks_passed.clone(); + + let mut driver = crate::actions::EngineDriver { + builder_idx: 0, + follower_idxs: vec![], + initial_parent_hash: Some(block_hash), + num_blocks: NUM_BLOCKS, + block_interval: BLOCK_INTERVAL, + flashblocks: true, + authorization_gen, + attributes_gen: Box::new({ + let chain_spec = chain_spec.clone(); + move |_block_number, timestamp| { + let eip1559 = encode_eip1559_params(chain_spec.as_ref(), timestamp)?; + Ok(build_payload_attributes( + timestamp, + eip1559, + Some(vec![TX_SET_L1_BLOCK.clone()]), + )) + } + }), + during_build: Some(Box::new({ + let url = nodes[0].node.rpc_url(); + move |block_num| { + let url = url.clone(); + Box::pin(async move { + use alloy_provider::Provider; + let provider = ProviderBuilder::<_, _, op_alloy_network::Optimism>::default() + .network::() + .with_recommended_fillers() + .connect_http(url); + + let pending = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) + .await?; + + let latest = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Latest) + .await?; + + if let (Some(pending_block), Some(latest_block)) = (&pending, &latest) { + assert_ne!( + pending_block.header.hash, latest_block.header.hash, + "block {block_num}: pending must differ from latest during build" + ); + assert!( + pending_block.header.number > latest_block.header.number, + "block {block_num}: pending number ({}) must be > latest ({})", + pending_block.header.number, + latest_block.header.number, + ); + info!( + target: "macro_sanity", + block = block_num, + pending_number = pending_block.header.number, + latest_number = latest_block.header.number, + "pending != latest verified during build" + ); + } + + Ok(()) + }) + } + })), + on_block: Some(Box::new({ + let nodes_0 = nodes[0].node.rpc_url(); + move |block_num, _payload| { + let checks_passed = checks_passed_cb.clone(); + let url = nodes_0.clone(); + + Box::pin(async move { + let provider = ProviderBuilder::<_, _, op_alloy_network::Optimism>::default() + .network::() + .with_recommended_fillers() + .connect_http(url); + + // --- fetch_block!(Pending) --- + let pending = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) + .await?; + info!( + target: "macro_sanity", + block = block_num, + pending = pending.is_some(), + "pending block query" + ); + + // --- fetch_block!(Latest) --- + let latest: Option< + alloy_rpc_types::Block>, + > = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Latest) + .full() + .await?; + + let latest = latest.unwrap(); + let latest_number = latest.header.number; + let tx_count = latest.transactions.len(); + info!( + target: "macro_sanity", + block = block_num, + latest_number, + tx_count, + "latest block query" + ); + + // --- fetch_tx! (first tx in latest block) --- + let tx_hashes: Vec<_> = latest.transactions.hashes().collect(); + if let Some(&tx_hash) = tx_hashes.first() { + let tx: Option> = + provider.get_transaction_by_hash(tx_hash).await?; + assert!( + tx.is_some(), + "block {block_num}: fetch_tx for {tx_hash} should return a result" + ); + + // --- fetch_receipt! --- + let receipt = provider.get_transaction_receipt(tx_hash).await?; + assert!( + receipt.is_some(), + "block {block_num}: fetch_receipt for {tx_hash} should return a result" + ); + + info!( + target: "macro_sanity", + block = block_num, + %tx_hash, + "tx + receipt queries passed" + ); + } + + // --- eth_call --- + let call_tx = alloy_rpc_types::TransactionRequest::default() + .to(Address::ZERO) + .value(U256::ZERO); + let call_result = provider.call(call_tx.into()).await; + assert!( + call_result.is_ok(), + "block {block_num}: eth_call should succeed: {:?}", + call_result.err() + ); + info!(target: "macro_sanity", block = block_num, "eth_call passed"); + + // --- fetch_logs --- + let filter = Filter::new() + .from_block(latest_number) + .to_block(latest_number); + let logs = provider.get_logs(&filter).await?; + info!( + target: "macro_sanity", + block = block_num, + log_count = logs.len(), + "fetch_logs passed" + ); + + // --- chain_id sanity --- + let chain_id = provider.get_chain_id().await?; + assert!(chain_id > 0, "block {block_num}: chain_id should be > 0"); + + checks_passed.fetch_add(1, Ordering::SeqCst); + info!( + target: "macro_sanity", + block = block_num, + "all checks passed" + ); + + Ok(()) + }) + } + })), + }; + + driver.execute(&mut env).await?; + + let passed = checks_passed.load(Ordering::SeqCst); + assert_eq!( + passed, NUM_BLOCKS, + "expected all {NUM_BLOCKS} blocks to pass macro sanity checks, got {passed}" ); + info!(target: "macro_sanity", passed, "all blocks verified"); Ok(()) } diff --git a/crates/world/node/tests/it/builder.rs b/crates/world/node/tests/it/builder.rs index 1c254e0a4..0b412b724 100644 --- a/crates/world/node/tests/it/builder.rs +++ b/crates/world/node/tests/it/builder.rs @@ -6,8 +6,8 @@ use reth_provider::providers::BlockchainProvider; use world_chain_node::{context::FlashblocksContext, node::WorldChainNode}; use world_chain_test::node::test_config; -#[test] -fn test_basic_flashblocks_setup() { +#[tokio::test] +async fn test_basic_flashblocks_setup() { // parse CLI -> config let config = NodeConfig::new(BASE_MAINNET.clone()); let db = create_test_rw_db(); diff --git a/crates/world/pool/src/validator.rs b/crates/world/pool/src/validator.rs index 38e445beb..d8f872ad1 100644 --- a/crates/world/pool/src/validator.rs +++ b/crates/world/pool/src/validator.rs @@ -30,7 +30,7 @@ use reth_optimism_primitives::OpTransactionSigned; use reth_primitives::{Block, NodePrimitives, SealedBlock}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, StateProviderFactory}; use revm_primitives::U256; -use tracing::{info, warn}; +use tracing::info; use world_chain_pbh::payload::{PBHPayload as PbhPayload, PBHValidationError}; /// The slot of the `pbh_gas_limit` in the PBHEntryPoint contract. @@ -94,7 +94,7 @@ where .to(); if max_pbh_nonce == 0 && max_pbh_gas_limit == 0 { - warn!( + info!( %pbh_entrypoint, %pbh_signature_aggregator, "WorldChainTransactionValidator Initialized with PBH Disabled - Failed to fetch PBH nonce and gas limit from PBHEntryPoint. Defaulting to 0." diff --git a/crates/world/test/Cargo.toml b/crates/world/test/Cargo.toml index c41644ab2..abb57682b 100644 --- a/crates/world/test/Cargo.toml +++ b/crates/world/test/Cargo.toml @@ -16,6 +16,7 @@ world-chain-pool.workspace = true world-chain-node.workspace = true flashblocks-primitives.workspace = true +flashblocks-builder.workspace = true flashblocks-cli.workspace = true reth.workspace = true diff --git a/crates/world/test/src/node.rs b/crates/world/test/src/node.rs index f259e0486..f1a2bb731 100644 --- a/crates/world/test/src/node.rs +++ b/crates/world/test/src/node.rs @@ -8,7 +8,8 @@ use alloy_primitives::{ }; use alloy_rpc_types::{TransactionInput, TransactionRequest}; use alloy_sol_types::SolCall; -use flashblocks_cli::{FlashblocksArgs, FlashblocksPayloadBuilderConfig}; +use flashblocks_builder::FlashblocksPayloadBuilderConfig; +use flashblocks_cli::{FanoutArgs, FlashblocksArgs}; use futures::future::join_all; use reth_chain_state::{ CanonStateNotifications, CanonStateSubscriptions, ForkChoiceNotifications, @@ -104,6 +105,7 @@ pub fn test_config_with_peers_and_gossip( recommit_interval: 50, flashblocks_interval: 200, access_list: true, + fanout: FanoutArgs::default(), }) } else { None diff --git a/specs/flashblocks_p2p_v2.md b/specs/flashblocks_p2p_v2.md index e0c9aa9ce..022e43dfd 100644 --- a/specs/flashblocks_p2p_v2.md +++ b/specs/flashblocks_p2p_v2.md @@ -6,8 +6,6 @@ The current flashblocks P2P protocol broadcasts every `FlashblocksPayloadV1` to **all** connected peers (`handler.rs:585`, `connection.rs:97-129`). A node with N peers sends N copies of every flashblock. For a node connected to 50 peers, that is 50x outgoing bandwidth per flashblock. As the network grows, this becomes unsustainable. -Additionally, `StartPublish` and `StopPublish` messages are currently **not relayed** beyond direct peers (see `connection.rs:343,436` TODOs). This must be addressed for multi-hop propagation to work correctly. - ## Design Goals 1. **Reduce bandwidth** — Each node sends flashblocks to a bounded number of peers instead of all peers. @@ -20,10 +18,10 @@ Additionally, `StartPublish` and `StopPublish` messages are currently **not rela Each node maintains two bounded peer sets: -- **Send Set** (max `max_send_peers`, default 6): Peers this node actively forwards flashblocks to. These are peers that have sent a `RequestFlashblocks` and been accepted. Trusted peers bypass the limit. -- **Receive Set** (max `max_receive_peers`, default 6): Peers this node actively receives flashblocks from. These are peers to which this node has sent `RequestFlashblocks` and received `AcceptFlashblocks`. +- **Send Set** (max `max_send_peers`, default 10): Peers this node actively forwards flashblocks to. These are peers that have sent a `RequestFlashblocks` and been accepted. Trusted peers bypass the limit. +- **Receive Set** (max `max_receive_peers`, default 3): Peers this node actively receives flashblocks from. These are peers this node has selected as active feed sources. -Flashblocks propagate through the network as a directed acyclic graph: the builder sends to its send set, those nodes relay to their send sets, and so on. With a fanout of 6, a network of N nodes requires approximately log₆(N) hops from builder to the most distant node. +Flashblocks propagate through the network as a directed acyclic graph: the builder sends to its send set, those nodes relay to their send sets, and so on. With a fanout of 10, a network of N nodes requires approximately log₁₀(N) hops from builder to the most distant node. Periodically, each node evaluates the latency of its receive peers and may rotate out the highest-latency peer in favor of a randomly-selected alternative, one peer at a time. @@ -33,15 +31,14 @@ This change adds new message types to the `flblk` protocol. The protocol version ## New Message Types -Five unsigned control messages are added to `FlashblocksP2PMsg`: +Four unsigned control messages are added to `FlashblocksP2PMsg`: | Discriminator | Message | Direction | Description | |---|---|---|---| | `0x01` | `RequestFlashblocks` | Receiver → Sender | "I want to receive flashblocks from you" | | `0x02` | `AcceptFlashblocks` | Sender → Receiver | "Accepted. I will send you flashblocks" | | `0x03` | `RejectFlashblocks` | Sender → Receiver | "Rejected. I am at capacity" | -| `0x04` | `CancelFlashblocks` | Either → Either | "I am ending our flashblock feed" | -| `0x05` | `CancelFlashblocksAck` | Either → Either | "Acknowledged. Feed terminated" | +| `0x04` | `CancelFlashblocks` | Receiver → Sender | "Stop sending me flashblocks" | These messages carry no payload. The connection context (peer ID) provides all necessary information. @@ -54,7 +51,6 @@ pub enum FlashblocksP2PMsg { AcceptFlashblocks = 0x02, RejectFlashblocks = 0x03, CancelFlashblocks = 0x04, - CancelFlashblocksAck = 0x05, } ``` @@ -63,22 +59,16 @@ pub enum FlashblocksP2PMsg { **`RequestFlashblocks`** — Sent by a node that wants to receive flashblocks from the connected peer. The recipient evaluates: 1. Is the requester a trusted peer? → Always accept (trusted peers bypass `max_send_peers`). -2. Is the send set below `max_send_peers`? → Accept. -3. Is the send set full but contains non-trusted peers, AND the requester is trusted? → Evict a non-trusted peer (send it `CancelFlashblocks`), then accept. -4. Otherwise → Reject. +2. Is the number of non-trusted peers in the send set below `max_send_peers`? → Accept. +3. Otherwise → Reject. **`AcceptFlashblocks`** — Response to `RequestFlashblocks`. After this, the sender begins forwarding all `Authorized` messages to the receiver and adds the receiver to its send set. **`RejectFlashblocks`** — Response to `RequestFlashblocks` when the sender cannot accommodate more peers. The requester should try another peer. -**`CancelFlashblocks`** — Either side may send this to terminate the flashblock feed: - -- **Receiver-initiated**: "Stop sending me flashblocks." (e.g., during peer rotation) -- **Sender-initiated**: "I am going to stop sending you flashblocks." (e.g., evicting a non-trusted peer to make room for a trusted one) +**`CancelFlashblocks`** — Sent only by a receiver to the sender it no longer wants to receive flashblocks from (e.g., during peer rotation). -The other party MUST respond with `CancelFlashblocksAck`. - -**`CancelFlashblocksAck`** — Confirms the feed termination. After this exchange, both sides update their sets (sender removes from send set, receiver removes from receive set). +After receiving `CancelFlashblocks`, the sender immediately stops forwarding flashblocks to that peer and removes it from its send set. ## Peer Management @@ -103,8 +93,8 @@ struct FanoutState { When a node starts and connects to peers via devp2p: -1. As peers connect and complete the `flblk/2` handshake, send `RequestFlashblocks` to them. -2. Prioritize trusted peers first. +1. As peers connect and complete the `flblk/2` handshake, discover whether they are trusted or untrusted. +2. Only request peers whose trust classification is known, so trusted peers are always considered first. 3. Continue sending requests as new peers connect until `receive_set.len() >= max_receive_peers`. 4. Once the receive set is full, stop sending unsolicited requests (further changes happen via rotation). @@ -114,12 +104,10 @@ When a node starts and connects to peers via devp2p: receive RequestFlashblocks from peer P: if P is trusted: - if send_set has non-trusted peers AND send_set.len() >= max_send_peers: - evict lowest-priority non-trusted peer (send CancelFlashblocks, await ack) add P to send_set send AcceptFlashblocks to P -else if send_set.len() < max_send_peers: +else if non_trusted_send_count < max_send_peers: add P to send_set send AcceptFlashblocks to P @@ -140,10 +128,10 @@ When a peer disconnects unexpectedly (connection drops): When a node receives an `Authorized` message from a peer in its receive set: - **`FlashblocksPayloadV1`**: Verify signatures, process the flashblock (update state, emit to flashblock stream). Then forward the serialized bytes to all peers in the **send set** except the peer that sent it. -- **`StartPublish`**: Verify signatures, process (update publishing state machine). Forward to **all connected `flblk/2` peers** (not just send set). These are rare, small control messages needed by every node for multi-builder coordination. -- **`StopPublish`**: Same as `StartPublish` — forward to all connected peers. +- **`StartPublish`**: Verify signatures and process locally. Do not relay it beyond the direct neighbor that sent it. +- **`StopPublish`**: Same as `StartPublish` — process locally, do not relay. -If a node receives an `Authorized(FlashblocksPayloadV1)` from a peer **not** in its receive set, the message should be ignored. This prevents unsolicited data delivery. +If a node receives an `Authorized(FlashblocksPayloadV1)` from a peer **not** in its receive set, or from a peer whose `RequestFlashblocks` is still pending, the message should be ignored and the peer should be penalized. This prevents unsolicited data delivery. ### Duplicate Handling @@ -165,33 +153,31 @@ Each `FlashblocksPayloadV1` includes a `flashblock_timestamp` in its metadata, s one_way_latency = now() - flashblock_timestamp ``` -This measurement is attributed to the specific peer that delivered the flashblock. Nodes maintain a sliding window of the last `latency_window` (default 50) measurements per receive peer and compute a moving average. +This measurement is attributed to the specific peer that delivered the flashblock. Nodes maintain a sliding window of the last `latency_window` (default 1000) measurements per receive peer and compute a moving average. Since all receive peers deliver the same flashblock (with the same `flashblock_timestamp`), the **relative ordering** of peers by latency is accurate even with clock skew between the builder and receiver. ### Rotation Algorithm -**One-at-a-time rule**: Only one rotation may be in progress at any time. This ensures the receive set never drops below `max_receive_peers - 1` and allows the node to evaluate one change before making another. +The receive set must never exceed `max_receive_peers`. -During rotation, the node temporarily has `max_receive_peers + 1` receive peers (both the old and new peer are sending). This is intentional and provides a brief window to compare the two peers before committing to the switch. +When rotating: -### Rotation Timeout +1. Select the worst-scoring peer in the current receive set. +2. Remove that peer from the receive set immediately and send `CancelFlashblocks`. +3. Pick a replacement candidate, prioritizing trusted peers. +4. Add the replacement peer to the receive set in a provisional state and send `RequestFlashblocks`. -If a rotation is in progress and no response (`AcceptFlashblocks`/`RejectFlashblocks`) is received within a reasonable timeout (e.g., 10 seconds), abort the rotation: - -``` -rotation_in_progress = false -remove R from pending_requests -``` +The provisional peer occupies a receive slot immediately, so the node still never exceeds `max_receive_peers`. While provisional, the peer is scored for missed flashblocks the same as any other receive peer. If it fails to respond or fails to deliver flashblocks, its score will deteriorate and it can be rotated out on a later interval. ## Configuration Parameters | Parameter | Default | Description | |---|---|---| -| `max_send_peers` | 6 | Maximum non-trusted peers to send flashblocks to | -| `max_receive_peers` | 6 | Maximum peers to receive flashblocks from | +| `max_send_peers` | 10 | Maximum non-trusted peers to send flashblocks to | +| `max_receive_peers` | 3 | Maximum peers to receive flashblocks from | | `rotation_interval` | 30s | How often to evaluate and potentially rotate receive peers | -| `latency_window` | 50 | Number of flashblocks to track for per-peer latency averaging | +| `latency_window` | 1000 | Number of flashblocks to track for per-peer latency averaging | Trusted peers are always served on request and **do not count** toward `max_send_peers`. @@ -199,15 +185,14 @@ Trusted peers are always served on request and **do not count** toward `max_send ### Unchanged Components -The existing `Authorized` message types (`FlashblocksPayloadV1`, `StartPublish`, `StopPublish`) remain unchanged. They continue to use the `Authorized` wrapper with sequencer + builder signatures. The multi-builder coordination state machine (Publishing, WaitingToPublish, NotPublishing) is unaffected. +The existing `Authorized` message types (`FlashblocksPayloadV1`, `StartPublish`, `StopPublish`) remain unchanged. They continue to use the `Authorized` wrapper with sequencer + builder signatures. The multi-builder coordination state machine (Publishing, WaitingToPublish, NotPublishing) is unaffected. `StartPublish` and `StopPublish` remain direct-neighbor messages and are not relayed. ### Required Changes to Existing Code -1. **`StartPublish`/`StopPublish` must be forwarded** — The current code has TODOs at `connection.rs:343,436` noting these are not propagated. With multi-hop fanout, nodes more than 1 hop from the builder will never see these messages unless they are relayed. These must be forwarded to **all** connected `flblk/2` peers (not just send set) to ensure the multi-builder coordination works network-wide. +1. **Duplicate handling must change** — The current per-peer duplicate check at `connection.rs:278-291` penalizes any duplicate flashblock with `ReputationChangeKind::AlreadySeenTransaction`. In the new protocol, receiving the same flashblock from different receive peers is expected. Only same-peer duplicates (same flashblock index from the same peer twice) should trigger a penalty. -2. **Duplicate handling must change** — The current per-peer duplicate check at `connection.rs:278-291` penalizes any duplicate flashblock with `ReputationChangeKind::AlreadySeenTransaction`. In the new protocol, receiving the same flashblock from different receive peers is expected. Only same-peer duplicates (same flashblock index from the same peer twice) should trigger a penalty. +2. **Flashblock forwarding must be scoped to send set** — The current broadcast channel (`peer_tx`) sends to all connections. This must be replaced with targeted sends to only peers in the send set. The `PeerMsg::FlashblocksPayloadV1` variant currently uses a broadcast channel subscribed by all connections; this must be changed so each connection checks whether the destination peer is in the send set before forwarding. -3. **Flashblock forwarding must be scoped to send set** — The current broadcast channel (`peer_tx`) sends to all connections. This must be replaced with targeted sends to only peers in the send set. The `PeerMsg::FlashblocksPayloadV1` variant currently uses a broadcast channel subscribed by all connections; this must be changed so each connection checks whether the destination peer is in the send set before forwarding. +3. **Receive-peer selection must respect trust discovery** — Nodes should not request unknown peers before their trust classification is available, otherwise untrusted peers can fill the bounded receive set before trusted peers are considered. 4. **Protocol version bump** — `Capability::new_static("flblk", 1)` at `handler.rs:239` must be updated to version `2`. -