From 87eed0e56c813305669c0ca1f62cdc4d2f0ef890 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 9 Mar 2026 17:00:09 -0700 Subject: [PATCH 01/43] feat: flashblocks p2p v2 --- Cargo.lock | 2 + crates/flashblocks/p2p/Cargo.toml | 2 + .../p2p/src/protocol/connection.rs | 201 ++++- .../flashblocks/p2p/src/protocol/handler.rs | 839 +++++++++++++++++- crates/flashblocks/primitives/src/p2p.rs | 40 + 5 files changed, 1060 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ae8afc12..49f8b38b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3888,12 +3888,14 @@ version = "1.10.1" dependencies = [ "alloy-primitives", "alloy-rlp", + "blake3", "chrono", "ed25519-dalek", "flashblocks-primitives", "futures", "metrics", "parking_lot", + "rand 0.9.2", "reth", "reth-eth-wire", "reth-ethereum", diff --git a/crates/flashblocks/p2p/Cargo.toml b/crates/flashblocks/p2p/Cargo.toml index 2bc32cc6a..d2f6da945 100644 --- a/crates/flashblocks/p2p/Cargo.toml +++ b/crates/flashblocks/p2p/Cargo.toml @@ -28,3 +28,5 @@ thiserror.workspace = true parking_lot.workspace = true chrono.workspace = true reth-tasks = { workspace = true } +rand.workspace = true +blake3.workspace = true diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index cafeb3ca8..5f2da545c 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -12,11 +12,13 @@ use flashblocks_primitives::{ }; use futures::{Stream, StreamExt}; use metrics::gauge; +use parking_lot::Mutex; use reth::payload::PayloadId; use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnection}; use reth_network::{cache::LruMap, types::ReputationChangeKind}; use std::{ pin::Pin, + sync::Arc, task::{Context, Poll, ready}, }; use tokio_stream::wrappers::BroadcastStream; @@ -31,6 +33,88 @@ const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; /// This should be large enough to retain entries across the grace window. const RECEIVED_CACHE_LEN: u32 = AUTHORIZATION_TIMESTAMP_GRACE_SEC as u32 * 20; +/// A lightweight rolling average with a configurable smoothing window. +#[derive(Clone, Debug)] +pub(crate) struct RollingAverage { + value: Option, + window: i64, +} + +impl RollingAverage { + pub(crate) fn new(window: i64) -> Self { + Self { + value: None, + window: window.max(1), + } + } + + pub(crate) fn record(&mut self, sample: i64) { + self.value = Some(match self.value { + Some(current) => (current * (self.window - 1) + sample) / self.window, + None => sample, + }); + } + + pub(crate) fn value(&self) -> Option { + self.value + } + + pub(crate) fn reset(&mut self) { + self.value = None; + } +} + +/// Shared fanout metadata for a single peer connection. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct FlashblocksConnectionFlags { + pub(crate) trusted: bool, + pub(crate) trusted_known: bool, + pub(crate) send_enabled: bool, + pub(crate) receive_enabled: bool, + pub(crate) request_in_flight: bool, + pub(crate) cancel_in_flight: bool, +} + +/// Shared fanout metadata for a single peer connection. +#[derive(Debug)] +pub(crate) struct FlashblocksConnectionState { + flags: Mutex, + latency_average: Mutex, +} + +impl FlashblocksConnectionState { + pub(crate) fn new(latency_window: i64) -> Self { + Self { + flags: Mutex::new(FlashblocksConnectionFlags::default()), + latency_average: Mutex::new(RollingAverage::new(latency_window)), + } + } + + pub(crate) fn flags(&self) -> FlashblocksConnectionFlags { + *self.flags.lock() + } + + pub(crate) fn update_flags(&self, update: F) + where + F: FnOnce(&mut FlashblocksConnectionFlags), + { + let mut flags = self.flags.lock(); + update(&mut flags); + } + + pub(crate) fn record_latency(&self, sample: i64) { + self.latency_average.lock().record(sample); + } + + pub(crate) fn average_latency(&self) -> Option { + self.latency_average.lock().value() + } + + pub(crate) fn reset_latency(&self) { + self.latency_average.lock().reset(); + } +} + /// Represents a single P2P connection for the flashblocks protocol. /// /// This struct manages the bidirectional communication with a single peer in the flashblocks @@ -49,6 +133,8 @@ pub struct FlashblocksConnection { /// Receiver for peer messages to be sent to all peers. /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. peer_rx: BroadcastStream, + /// Shared fanout state for this peer, also visible to the protocol handler. + fanout_state: Arc, /// Per-peer tracking of flashblocks this peer has already sent us. /// Uses `peek` for lookups to avoid LRU promotion, giving FIFO eviction semantics. received_cache: LruMap<(PayloadId, usize), ()>, @@ -62,12 +148,18 @@ impl FlashblocksConnection { /// * `conn` - The underlying protocol connection for sending and receiving messages. /// * `peer_id` - The unique identifier of the connected peer. /// * `peer_rx` - Receiver for peer messages to be sent to all peers. - pub fn new( + pub(crate) fn new( protocol: FlashblocksP2PProtocol, conn: ProtocolConnection, peer_id: PeerId, peer_rx: BroadcastStream, + fanout_state: Arc, ) -> Self { + protocol.handle.ensure_background_tasks(); + protocol + .handle + .on_peer_connected(protocol.network.clone(), peer_id, fanout_state.clone()); + gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); Self { @@ -75,6 +167,7 @@ impl FlashblocksConnection { conn, peer_id, peer_rx, + fanout_state, received_cache: LruMap::new(RECEIVED_CACHE_LEN), } } @@ -109,6 +202,7 @@ impl Drop for FlashblocksConnection { "dropping flashblocks connection" ); + self.protocol.handle.on_peer_disconnected(self.peer_id); gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).decrement(1); } } @@ -131,7 +225,11 @@ impl Stream for FlashblocksConnection { bytes, )) => { // Check if this flashblock actually originated from this peer. - if !this.received_cache_contains(&(payload_id, flashblock_index)) { + let send_enabled = this.fanout_state.flags().send_enabled; + if send_enabled + && !this + .received_cache_contains(&(payload_id, flashblock_index)) + { trace!( target: "flashblocks::p2p", peer_id = %this.peer_id, @@ -161,6 +259,16 @@ impl Stream for FlashblocksConnection { ); return Poll::Ready(Some(bytes_mut)); } + PeerMsg::Direct { peer_id, bytes } => { + if peer_id == this.peer_id { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "Sending direct flashblocks control message to peer" + ); + return Poll::Ready(Some(bytes)); + } + } } } Err(error) => { @@ -234,6 +342,21 @@ impl Stream for FlashblocksConnection { } } } + FlashblocksP2PMsg::RequestFlashblocks => { + this.protocol.handle.handle_request_message(this.peer_id); + } + FlashblocksP2PMsg::AcceptFlashblocks => { + this.protocol.handle.handle_accept_message(this.peer_id); + } + FlashblocksP2PMsg::RejectFlashblocks => { + this.protocol.handle.handle_reject_message(this.peer_id); + } + FlashblocksP2PMsg::CancelFlashblocks => { + this.protocol.handle.handle_cancel_message(this.peer_id); + } + FlashblocksP2PMsg::CancelFlashblocksAck => { + this.protocol.handle.handle_cancel_ack_message(this.peer_id); + } } } } @@ -298,6 +421,17 @@ impl FlashblocksConnection { return; } + if !self.fanout_state.flags().receive_enabled { + trace!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "ignoring flashblock from peer outside receive set", + ); + return; + } + // Check if this peer is spamming us with the same payload index if !self.received_cache_insert((msg.payload_id, msg.index as usize)) { // We've already seen this index from this peer. @@ -353,6 +487,7 @@ impl FlashblocksConnection { if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); + self.fanout_state.record_latency(latency); } self.protocol @@ -363,8 +498,6 @@ impl FlashblocksConnection { /// Handles incoming `StartPublish` messages from a peer. /// - /// TODO: handle propogating this if we care. For now we assume direct peering. - /// /// # Arguments /// * `authorized_payload` - The authorized `StartPublish` message received from the peer /// @@ -375,20 +508,20 @@ impl FlashblocksConnection { /// - If we are waiting to publish, updates the list of active publishers /// - If we are not publishing, adds the new publisher to the list of active publishers fn handle_start_publish(&mut self, authorized_payload: AuthorizedPayload) { - let state = self.protocol.handle.state.lock(); let Ok(builder_sk) = self.protocol.handle.builder_sk() else { return; }; let authorization = &authorized_payload.authorized.authorization; + let payload_timestamp = self.protocol.handle.state.lock().payload_timestamp; // Check if the request is expired for dos protection. // It's important to ensure that this `StartPublish` request // is very recent, or it could be used in a replay attack. - if state.payload_timestamp > authorization.timestamp { + if payload_timestamp > authorization.timestamp { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = state.payload_timestamp, + current_timestamp = payload_timestamp, timestamp = authorized_payload.authorized.authorization.timestamp, "received initiate build request with outdated timestamp", ); @@ -398,6 +531,20 @@ impl FlashblocksConnection { return; } + if !self + .protocol + .handle + .remember_control_message(&authorized_payload.authorized) + { + trace!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + "ignoring duplicate StartPublish message", + ); + return; + } + + let state = self.protocol.handle.state.lock(); state.publishing_status.send_modify(|status| { let active_publishers = match status { PublishingStatus::Publishing { @@ -452,12 +599,18 @@ impl FlashblocksConnection { active_publishers.push((authorization.builder_vk, authorization.timestamp)); } }); + + let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); + self.protocol + .handle + .ctx + .peer_tx + .send(PeerMsg::StartPublishing(p2p_msg.encode())) + .ok(); } /// Handles incoming `StopPublish` messages from a peer. - /// - /// TODO: handle propogating this if we care. For now we assume direct peering. - /// + /// # Arguments /// * `authorized_payload` - The authorized `StopPublish` message received from the peer /// @@ -468,17 +621,17 @@ impl FlashblocksConnection { /// - If we are waiting to publish, removes the publisher from the list of active publishers and checks if we can start publishing /// - If we are not publishing, removes the publisher from the list of active publishers fn handle_stop_publish(&mut self, authorized_payload: AuthorizedPayload) { - let state = self.protocol.handle.state.lock(); let authorization = &authorized_payload.authorized.authorization; + let payload_timestamp = self.protocol.handle.state.lock().payload_timestamp; // Check if the request is expired for dos protection. // It's important to ensure that this `StartPublish` request // is very recent, or it could be used in a replay attack. - if state.payload_timestamp > authorization.timestamp { + if payload_timestamp > authorization.timestamp { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = state.payload_timestamp, + current_timestamp = payload_timestamp, timestamp = authorized_payload.authorized.authorization.timestamp, "Received initiate build response with outdated timestamp", ); @@ -488,6 +641,20 @@ impl FlashblocksConnection { return; } + if !self + .protocol + .handle + .remember_control_message(&authorized_payload.authorized) + { + trace!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + "ignoring duplicate StopPublish message", + ); + return; + } + + let state = self.protocol.handle.state.lock(); state.publishing_status.send_modify(|status| { match status { PublishingStatus::Publishing { .. } => { @@ -556,5 +723,13 @@ impl FlashblocksConnection { } } }); + + let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); + self.protocol + .handle + .ctx + .peer_tx + .send(PeerMsg::StopPublishing(p2p_msg.encode())) + .ok(); } } diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 5651ca7c9..8cc92e99c 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,4 +1,7 @@ -use crate::protocol::{connection::FlashblocksConnection, error::FlashblocksP2PError}; +use crate::protocol::{ + connection::{FlashblocksConnection, FlashblocksConnectionState}, + error::FlashblocksP2PError, +}; use alloy_rlp::BytesMut; use chrono::Utc; use ed25519_dalek::{SigningKey, VerifyingKey}; @@ -12,13 +15,25 @@ use flashblocks_primitives::{ use futures::{Stream, StreamExt, stream}; use metrics::histogram; use parking_lot::Mutex; +use rand::seq::SliceRandom; use reth::payload::PayloadId; use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; -use reth_network::Peers; -use std::{net::SocketAddr, sync::Arc}; -use tokio::sync::{broadcast, watch}; -use tracing::{debug, info}; +use reth_network::{Peers, cache::LruCache}; +use std::{ + collections::{HashMap, HashSet}, + net::SocketAddr, + sync::{ + Arc, Weak, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; +use tokio::{ + sync::{broadcast, watch}, + time::{self, Instant}, +}; +use tracing::{debug, info, warn}; use reth_ethereum::network::{ api::Direction, @@ -43,6 +58,9 @@ const MAX_PUBLISH_WAIT_SEC: u64 = 2; /// before dropping them. In practice, we should rarely need to buffer any messages. const BROADCAST_BUFFER_CAPACITY: usize = 100; +/// Number of recently forwarded `StartPublish`/`StopPublish` messages we remember to avoid loops. +const CONTROL_MESSAGE_CACHE_LEN: u32 = 2048; + /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -63,6 +81,56 @@ pub enum PeerMsg { StartPublishing(BytesMut), /// Send a previously serialized StopPublish message to all peers. StopPublishing(BytesMut), + /// Send an already serialized control message to a single peer. + Direct { peer_id: PeerId, bytes: BytesMut }, +} + +/// Runtime configuration for bounded flashblocks fanout. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FanoutConfig { + /// Maximum number of non-trusted peers to send flashblocks to. + pub max_send_peers: usize, + /// Maximum number of peers to receive flashblocks from. + pub max_receive_peers: usize, + /// How often to evaluate latency-based peer rotation. + pub rotation_interval: Duration, + /// How long to wait for a rotation request to be answered. + pub rotation_timeout: Duration, + /// Number of latency measurements to retain per receive peer. + pub latency_window: i64, +} + +impl Default for FanoutConfig { + fn default() -> Self { + Self { + max_send_peers: 6, + max_receive_peers: 6, + rotation_interval: Duration::from_secs(30), + rotation_timeout: Duration::from_secs(10), + latency_window: 50, + } + } +} + +#[derive(Clone, Debug)] +enum RotationState { + WaitingForResponse { + candidate: PeerId, + evict: PeerId, + requested_at: Instant, + }, + WaitingForCancelAck { + candidate: PeerId, + evict: PeerId, + }, +} + +#[derive(Debug, Default)] +struct FanoutState { + send_set: HashSet, + receive_set: HashSet, + connections: HashMap>, + rotation: Option, } /// The current publishing status of this node in the flashblocks P2P network. @@ -106,7 +174,7 @@ impl Default for PublishingStatus { /// This struct maintains the current state of flashblock publishing, including coordination /// with other publishers, payload buffering, and ordering information. It serves as the /// central state management for the flashblocks P2P protocol handler. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct FlashblocksP2PState { /// Current publishing status indicating whether we're publishing, waiting, or not publishing. pub publishing_status: watch::Sender, @@ -123,6 +191,27 @@ pub struct FlashblocksP2PState { /// Contains `None` for flashblocks not yet received, enabling out-of-order receipt /// while maintaining in-order delivery. pub flashblocks: Vec>, + /// Fanout and peer-selection state for flashblock forwarding. + fanout: FanoutState, + /// Recently processed control messages to prevent rebroadcast loops. + seen_control_messages: LruCache<[u8; 32]>, +} + +impl Default for FlashblocksP2PState { + fn default() -> Self { + let (publishing_status, _) = watch::channel(PublishingStatus::default()); + + Self { + publishing_status, + payload_id: PayloadId::default(), + payload_timestamp: 0, + flashblock_timestamp: 0, + flashblock_index: 0, + flashblocks: Vec::new(), + fanout: FanoutState::default(), + seen_control_messages: LruCache::new(CONTROL_MESSAGE_CACHE_LEN), + } + } } impl FlashblocksP2PState { @@ -135,6 +224,383 @@ impl FlashblocksP2PState { } } +impl FanoutState { + fn connection_state(&self, peer_id: &PeerId) -> Option> { + self.connections.get(peer_id).and_then(Weak::upgrade) + } + + fn is_trusted(&self, peer_id: &PeerId) -> bool { + self.connection_state(peer_id) + .is_some_and(|peer| peer.flags().trusted) + } + + fn non_trusted_send_count(&self) -> usize { + self.send_set + .iter() + .filter(|peer_id| !self.is_trusted(peer_id)) + .count() + } + + fn request_in_flight_count(&self) -> usize { + self.connections + .values() + .filter_map(Weak::upgrade) + .filter(|peer| peer.flags().request_in_flight) + .count() + } + + fn available_receive_candidates(&self) -> Vec { + let mut trusted = Vec::new(); + let mut unknown = Vec::new(); + let mut untrusted = Vec::new(); + + for (peer_id, peer_state) in &self.connections { + let Some(peer_state) = peer_state.upgrade() else { + continue; + }; + + if self.receive_set.contains(peer_id) { + continue; + } + + let peer_state = peer_state.flags(); + if peer_state.request_in_flight || peer_state.cancel_in_flight { + continue; + } + + match (peer_state.trusted_known, peer_state.trusted) { + (true, true) => trusted.push(*peer_id), + (false, _) => unknown.push(*peer_id), + (true, false) => untrusted.push(*peer_id), + } + } + + let mut rng = rand::rng(); + trusted.shuffle(&mut rng); + unknown.shuffle(&mut rng); + untrusted.shuffle(&mut rng); + + trusted.extend(unknown); + trusted.extend(untrusted); + trusted + } + + fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { + if self.rotation.is_some() { + return; + } + + let target = ctx + .fanout_config + .max_receive_peers + .saturating_sub(self.receive_set.len() + self.request_in_flight_count()); + if target == 0 { + return; + } + + for peer_id in self.available_receive_candidates().into_iter().take(target) { + let Some(peer_state) = self.connection_state(&peer_id) else { + continue; + }; + + let flags = peer_state.flags(); + if flags.request_in_flight || flags.receive_enabled || flags.cancel_in_flight { + continue; + } + peer_state.update_flags(|flags| flags.request_in_flight = true); + + ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + } + } + + fn worst_receive_peer(&self) -> Option { + self.receive_set + .iter() + .filter_map(|peer_id| { + self.connection_state(peer_id).and_then(|peer_state| { + peer_state + .average_latency() + .map(|average| (*peer_id, average)) + }) + }) + .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) + .map(|(peer_id, _)| peer_id) + } + + fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { + if self.rotation.is_some() || self.receive_set.len() < ctx.fanout_config.max_receive_peers { + return; + } + + let Some(evict) = self.worst_receive_peer() else { + return; + }; + + let mut candidates = self.available_receive_candidates(); + if candidates.is_empty() { + return; + } + + let mut rng = rand::rng(); + candidates.shuffle(&mut rng); + let candidate = candidates[0]; + + let Some(candidate_state) = self.connection_state(&candidate) else { + return; + }; + + let flags = candidate_state.flags(); + if flags.request_in_flight || flags.receive_enabled || flags.cancel_in_flight { + return; + } + candidate_state.update_flags(|flags| flags.request_in_flight = true); + + self.rotation = Some(RotationState::WaitingForResponse { + candidate, + evict, + requested_at: Instant::now(), + }); + ctx.send_direct(candidate, FlashblocksP2PMsg::RequestFlashblocks); + } + + fn check_rotation_timeout(&mut self, ctx: &FlashblocksP2PCtx) { + let Some(RotationState::WaitingForResponse { + candidate, + requested_at, + .. + }) = self.rotation.as_ref() + else { + return; + }; + + if requested_at.elapsed() < ctx.fanout_config.rotation_timeout { + return; + } + + let candidate = *candidate; + if let Some(candidate_state) = self.connection_state(&candidate) { + candidate_state.update_flags(|flags| flags.request_in_flight = false); + } + self.rotation = None; + self.maybe_request_receive_peers(ctx); + } + + fn handle_disconnect(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + self.connections.remove(&peer_id); + self.send_set.remove(&peer_id); + self.receive_set.remove(&peer_id); + + if self + .rotation + .as_ref() + .is_some_and(|rotation| match rotation { + RotationState::WaitingForResponse { + candidate, evict, .. + } => *candidate == peer_id || *evict == peer_id, + RotationState::WaitingForCancelAck { candidate, evict } => { + *candidate == peer_id || *evict == peer_id + } + }) + { + self.rotation = None; + } + + self.maybe_request_receive_peers(ctx); + } + + fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; + + let flags = peer_state.flags(); + let peer_is_trusted = flags.trusted; + let send_enabled = flags.send_enabled; + let cancel_in_flight = flags.cancel_in_flight; + + if send_enabled { + ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + return; + } + + if cancel_in_flight { + ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); + return; + } + + if peer_is_trusted { + if self.non_trusted_send_count() >= ctx.fanout_config.max_send_peers { + if let Some(evicted_peer) = self.send_set.iter().copied().find(|candidate| { + !self.is_trusted(candidate) + && self + .connection_state(candidate) + .is_some_and(|state| !state.flags().cancel_in_flight) + }) { + if let Some(evicted_state) = self.connection_state(&evicted_peer) { + evicted_state.update_flags(|flags| { + flags.send_enabled = false; + flags.cancel_in_flight = true; + }); + } + self.send_set.remove(&evicted_peer); + ctx.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); + } + } + + peer_state.update_flags(|flags| { + flags.send_enabled = true; + flags.cancel_in_flight = false; + }); + self.send_set.insert(peer_id); + ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + return; + } + + if self.non_trusted_send_count() < ctx.fanout_config.max_send_peers { + peer_state.update_flags(|flags| { + flags.send_enabled = true; + flags.cancel_in_flight = false; + }); + self.send_set.insert(peer_id); + ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + } else { + ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); + } + } + + fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; + + if !peer_state.flags().request_in_flight { + return; + } + peer_state.update_flags(|flags| { + flags.request_in_flight = false; + flags.receive_enabled = true; + }); + + self.receive_set.insert(peer_id); + + if let Some(RotationState::WaitingForResponse { + candidate, evict, .. + }) = self.rotation.as_ref() + { + if *candidate == peer_id { + let evict = *evict; + if let Some(evict_state) = self.connection_state(&evict) { + evict_state.update_flags(|flags| flags.cancel_in_flight = true); + } else { + self.rotation = None; + self.maybe_request_receive_peers(ctx); + return; + } + self.rotation = Some(RotationState::WaitingForCancelAck { + candidate: peer_id, + evict, + }); + ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + return; + } + } + + self.maybe_request_receive_peers(ctx); + } + + fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; + + if !peer_state.flags().request_in_flight { + return; + } + peer_state.update_flags(|flags| flags.request_in_flight = false); + + if self.rotation.as_ref().is_some_and(|rotation| { + matches!( + rotation, + RotationState::WaitingForResponse { candidate, .. } if *candidate == peer_id + ) + }) { + self.rotation = None; + } + + self.maybe_request_receive_peers(ctx); + } + + fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + self.send_set.remove(&peer_id); + self.receive_set.remove(&peer_id); + + if let Some(peer_state) = self.connection_state(&peer_id) { + peer_state.update_flags(|flags| { + flags.send_enabled = false; + flags.receive_enabled = false; + flags.request_in_flight = false; + flags.cancel_in_flight = false; + }); + peer_state.reset_latency(); + } + + if self + .rotation + .as_ref() + .is_some_and(|rotation| match rotation { + RotationState::WaitingForResponse { + candidate, evict, .. + } => *candidate == peer_id || *evict == peer_id, + RotationState::WaitingForCancelAck { candidate, evict } => { + *candidate == peer_id || *evict == peer_id + } + }) + { + self.rotation = None; + } + + ctx.send_direct(peer_id, FlashblocksP2PMsg::CancelFlashblocksAck); + self.maybe_request_receive_peers(ctx); + } + + fn handle_cancel_ack(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; + + let clear_rotation = self.rotation.as_ref().is_some_and(|rotation| { + matches!( + rotation, + RotationState::WaitingForCancelAck { evict, .. } if *evict == peer_id + ) + }); + + if !peer_state.flags().cancel_in_flight { + return; + } + peer_state.update_flags(|flags| { + flags.cancel_in_flight = false; + flags.request_in_flight = false; + + if clear_rotation { + flags.receive_enabled = false; + } else { + flags.send_enabled = false; + } + }); + + if clear_rotation { + peer_state.reset_latency(); + self.rotation = None; + self.receive_set.remove(&peer_id); + } else { + self.send_set.remove(&peer_id); + } + + self.maybe_request_receive_peers(ctx); + } +} + /// Context struct containing shared resources for the flashblocks P2P protocol. /// /// This struct holds the network handle, cryptographic keys, and communication channels @@ -146,12 +612,16 @@ pub struct FlashblocksP2PCtx { pub authorizer_vk: VerifyingKey, /// Builder's signing key used to sign outgoing authorized P2P messages. pub builder_sk: Option, + /// Fanout configuration for peer selection and rotation. + pub fanout_config: FanoutConfig, /// Broadcast sender for peer messages that will be sent to all connected peers. /// Messages may not be strictly ordered due to network conditions. pub peer_tx: broadcast::Sender, /// Broadcast sender for verified and strictly ordered flashblock payloads. /// Used by RPC overlays and other consumers of flashblock data. pub flashblock_tx: broadcast::Sender, + /// Ensures rotation/background tasks are only started once per handle. + background_tasks_started: Arc, } /// Handle for the flashblocks P2P protocol. @@ -169,14 +639,24 @@ pub struct FlashblocksHandle { impl FlashblocksHandle { pub fn new(authorizer_vk: VerifyingKey, builder_sk: Option) -> Self { + Self::with_fanout_config(authorizer_vk, builder_sk, FanoutConfig::default()) + } + + pub fn with_fanout_config( + authorizer_vk: VerifyingKey, + builder_sk: Option, + fanout_config: FanoutConfig, + ) -> Self { let flashblock_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); let peer_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); let state = Arc::new(Mutex::new(FlashblocksP2PState::default())); let ctx = FlashblocksP2PCtx { authorizer_vk, builder_sk, + fanout_config, peer_tx, flashblock_tx, + background_tasks_started: Arc::new(AtomicBool::new(false)), }; Self { ctx, state } @@ -192,6 +672,133 @@ impl FlashblocksHandle { .as_ref() .ok_or(FlashblocksP2PError::MissingBuilderSk) } + + pub(crate) fn ensure_background_tasks(&self) { + if self + .ctx + .background_tasks_started + .swap(true, Ordering::AcqRel) + { + return; + } + + let handle = self.clone(); + tokio::spawn(async move { + let mut rotation_interval = time::interval(handle.ctx.fanout_config.rotation_interval); + let mut timeout_interval = time::interval(Duration::from_secs(1)); + + loop { + tokio::select! { + _ = rotation_interval.tick() => { + let mut state = handle.state.lock(); + state.fanout.maybe_start_rotation(&handle.ctx); + } + _ = timeout_interval.tick() => { + let mut state = handle.state.lock(); + state.fanout.check_rotation_timeout(&handle.ctx); + } + } + } + }); + } + + pub(crate) fn on_peer_connected( + &self, + network: N, + peer_id: PeerId, + fanout_state: Arc, + ) { + { + let mut state = self.state.lock(); + state + .fanout + .connections + .insert(peer_id, Arc::downgrade(&fanout_state)); + state.fanout.maybe_request_receive_peers(&self.ctx); + } + + let handle = self.clone(); + tokio::spawn(async move { + match network.get_peer_by_id(peer_id).await { + Ok(Some(peer_info)) => { + fanout_state.update_flags(|flags| { + flags.trusted = peer_info.kind.is_trusted(); + flags.trusted_known = true; + }); + + let mut state = handle.state.lock(); + if state + .fanout + .connections + .get(&peer_id) + .and_then(Weak::upgrade) + .is_some_and(|current| Arc::ptr_eq(¤t, &fanout_state)) + { + state.fanout.maybe_request_receive_peers(&handle.ctx); + } + } + Ok(None) => {} + Err(error) => { + warn!( + target: "flashblocks::p2p", + %peer_id, + %error, + "failed to load peer info for flashblocks fanout", + ); + } + } + }); + } + + pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.fanout.handle_disconnect(&self.ctx, peer_id); + } + + pub(crate) fn remember_control_message(&self, authorized: &Authorized) -> bool { + let encoded = FlashblocksP2PMsg::Authorized(authorized.clone()).encode(); + let hash = blake3::hash(&encoded); + self.state + .lock() + .seen_control_messages + .insert(*hash.as_bytes()) + } + + pub(crate) fn handle_request_message(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.fanout.handle_request(&self.ctx, peer_id); + } + + pub(crate) fn handle_accept_message(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.fanout.handle_accept(&self.ctx, peer_id); + } + + pub(crate) fn handle_reject_message(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.fanout.handle_reject(&self.ctx, peer_id); + } + + pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.fanout.handle_cancel(&self.ctx, peer_id); + } + + pub(crate) fn handle_cancel_ack_message(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.fanout.handle_cancel_ack(&self.ctx, peer_id); + } +} + +impl FlashblocksP2PCtx { + pub(crate) fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { + self.peer_tx + .send(PeerMsg::Direct { + peer_id, + bytes: msg.encode(), + }) + .ok(); + } } /// Main protocol handler for the flashblocks P2P protocol. @@ -231,12 +838,12 @@ impl FlashblocksP2PProtocol { } impl FlashblocksP2PProtocol { - /// Returns the P2P capability for the flashblocks v1 protocol. + /// Returns the P2P capability for the flashblocks v2 protocol. /// /// This capability is used during devp2p handshake to advertise support - /// for the flashblocks protocol with protocol name "flblk" and version 1. + /// for the flashblocks protocol with protocol name "flblk" and version 2. pub fn capability() -> Capability { - Capability::new_static("flblk", 1) + Capability::new_static("flblk", 2) } } @@ -633,7 +1240,7 @@ impl ConnectionHandler for FlashblocksP2PProtoco type Connection = FlashblocksConnection; fn protocol(&self) -> Protocol { - Protocol::new(Self::capability(), 1) + Protocol::new(Self::capability(), 6) } fn on_unsupported_by_peer( @@ -663,7 +1270,217 @@ impl ConnectionHandler for FlashblocksP2PProtoco ); let peer_rx = self.handle.ctx.peer_tx.subscribe(); + let fanout_state = Arc::new(FlashblocksConnectionState::new( + self.handle.ctx.fanout_config.latency_window, + )); + + FlashblocksConnection::new( + self, + conn, + peer_id, + BroadcastStream::new(peer_rx), + fanout_state, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::SigningKey; + + fn test_ctx(config: FanoutConfig) -> FlashblocksP2PCtx { + let authorizer = SigningKey::from_bytes(&[7; 32]); + + FlashblocksP2PCtx { + authorizer_vk: authorizer.verifying_key(), + builder_sk: Some(SigningKey::from_bytes(&[8; 32])), + fanout_config: config, + peer_tx: broadcast::Sender::new(16), + flashblock_tx: broadcast::Sender::new(16), + background_tasks_started: Arc::new(AtomicBool::new(false)), + } + } + + fn test_peer_state( + latency_window: i64, + trusted: bool, + trusted_known: bool, + ) -> Arc { + let state = Arc::new(FlashblocksConnectionState::new(latency_window)); + state.update_flags(|flags| { + flags.trusted = trusted; + flags.trusted_known = trusted_known; + }); + state + } + + #[test] + fn trusted_peers_are_requested_first() { + let config = FanoutConfig { + max_receive_peers: 1, + ..Default::default() + }; + let latency_window = config.latency_window; + let ctx = test_ctx(config); + let mut fanout = FanoutState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let trusted_peer = PeerId::random(); + let untrusted_peer = PeerId::random(); + let trusted_state = test_peer_state(latency_window, true, true); + let untrusted_state = test_peer_state(latency_window, false, true); + fanout + .connections + .insert(trusted_peer, Arc::downgrade(&trusted_state)); + fanout + .connections + .insert(untrusted_peer, Arc::downgrade(&untrusted_state)); + + fanout.maybe_request_receive_peers(&ctx); + + assert!(trusted_state.flags().request_in_flight); + assert!(!untrusted_state.flags().request_in_flight); + match rx.try_recv().expect("request sent") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, trusted_peer); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } + } + + #[test] + fn trusted_request_evicts_non_trusted_sender() { + let config = FanoutConfig { + max_send_peers: 1, + ..Default::default() + }; + let latency_window = config.latency_window; + let ctx = test_ctx(config); + let mut fanout = FanoutState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let victim = PeerId::random(); + let trusted_requester = PeerId::random(); + let victim_state = test_peer_state(latency_window, false, true); + let requester_state = test_peer_state(latency_window, true, true); + victim_state.update_flags(|flags| flags.send_enabled = true); + fanout + .connections + .insert(victim, Arc::downgrade(&victim_state)); + fanout + .connections + .insert(trusted_requester, Arc::downgrade(&requester_state)); + fanout.send_set.insert(victim); + + fanout.handle_request(&ctx, trusted_requester); + + assert!(!fanout.send_set.contains(&victim)); + assert!(fanout.send_set.contains(&trusted_requester)); + assert!(victim_state.flags().cancel_in_flight); + assert!(requester_state.flags().send_enabled); + + match rx.try_recv().expect("cancel sent") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, victim); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::CancelFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } + + match rx.try_recv().expect("accept sent") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, trusted_requester); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::AcceptFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } + } + + #[test] + fn rotation_accepts_candidate_then_waits_for_cancel_ack() { + let config = FanoutConfig { + max_receive_peers: 1, + latency_window: 4, + ..Default::default() + }; + let latency_window = config.latency_window; + let ctx = test_ctx(config); + let mut fanout = FanoutState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let current_peer = PeerId::random(); + let candidate_peer = PeerId::random(); + let current_state = test_peer_state(latency_window, false, true); + let candidate_state = test_peer_state(latency_window, false, true); + current_state.update_flags(|flags| flags.receive_enabled = true); + current_state.record_latency(42); + fanout + .connections + .insert(current_peer, Arc::downgrade(¤t_state)); + fanout + .connections + .insert(candidate_peer, Arc::downgrade(&candidate_state)); + fanout.receive_set.insert(current_peer); + + fanout.maybe_start_rotation(&ctx); + + assert!(candidate_state.flags().request_in_flight); + assert!(matches!( + fanout.rotation, + Some(RotationState::WaitingForResponse { candidate, evict, .. }) + if candidate == candidate_peer && evict == current_peer + )); + + match rx.try_recv().expect("rotation request sent") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, candidate_peer); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } + + fanout.handle_accept(&ctx, candidate_peer); + + assert!(fanout.receive_set.contains(¤t_peer)); + assert!(fanout.receive_set.contains(&candidate_peer)); + assert!(candidate_state.flags().receive_enabled); + assert!(current_state.flags().cancel_in_flight); + assert!(matches!( + fanout.rotation, + Some(RotationState::WaitingForCancelAck { candidate, evict }) + if candidate == candidate_peer && evict == current_peer + )); + + match rx.try_recv().expect("cancel sent to old peer") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, current_peer); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::CancelFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } + + fanout.handle_cancel_ack(&ctx, current_peer); - FlashblocksConnection::new(self, conn, peer_id, BroadcastStream::new(peer_rx)) + assert!(!fanout.receive_set.contains(¤t_peer)); + assert!(fanout.receive_set.contains(&candidate_peer)); + assert!(!current_state.flags().receive_enabled); + assert!(fanout.rotation.is_none()); } } diff --git a/crates/flashblocks/primitives/src/p2p.rs b/crates/flashblocks/primitives/src/p2p.rs index 3ef29e158..1470d7090 100644 --- a/crates/flashblocks/primitives/src/p2p.rs +++ b/crates/flashblocks/primitives/src/p2p.rs @@ -47,6 +47,16 @@ pub struct StopPublish; pub enum FlashblocksP2PMsg { /// An authorized message containing a signed and authorized payload Authorized(Authorized) = 0x00, + /// Requests that the remote peer begin forwarding flashblocks to us. + RequestFlashblocks = 0x01, + /// Accepts a previously sent [`Self::RequestFlashblocks`] request. + AcceptFlashblocks = 0x02, + /// Rejects a previously sent [`Self::RequestFlashblocks`] request. + RejectFlashblocks = 0x03, + /// Terminates an active flashblocks feed. + CancelFlashblocks = 0x04, + /// Acknowledges termination of an active flashblocks feed. + CancelFlashblocksAck = 0x05, } /// The different types of authorized messages that can be sent over the Flashblocks P2P network. @@ -443,6 +453,11 @@ impl FlashblocksP2PMsg { buf.put_u8(0x00); payload.encode(&mut buf); } + FlashblocksP2PMsg::RequestFlashblocks => buf.put_u8(0x01), + FlashblocksP2PMsg::AcceptFlashblocks => buf.put_u8(0x02), + FlashblocksP2PMsg::RejectFlashblocks => buf.put_u8(0x03), + FlashblocksP2PMsg::CancelFlashblocks => buf.put_u8(0x04), + FlashblocksP2PMsg::CancelFlashblocksAck => buf.put_u8(0x05), } buf } @@ -458,6 +473,11 @@ impl FlashblocksP2PMsg { let payload = Authorized::decode(buf)?; Ok(FlashblocksP2PMsg::Authorized(payload)) } + 0x01 => Ok(FlashblocksP2PMsg::RequestFlashblocks), + 0x02 => Ok(FlashblocksP2PMsg::AcceptFlashblocks), + 0x03 => Ok(FlashblocksP2PMsg::RejectFlashblocks), + 0x04 => Ok(FlashblocksP2PMsg::CancelFlashblocks), + 0x05 => Ok(FlashblocksP2PMsg::CancelFlashblocksAck), _ => Err(FlashblocksError::UnknownMessageType), } } @@ -815,6 +835,26 @@ mod tests { match decoded { FlashblocksP2PMsg::Authorized(inner) => assert_eq!(inner, authorized), + _ => panic!("decoded wrong message variant"), + } + } + + #[test] + fn p2p_control_msg_roundtrip() { + let variants = [ + FlashblocksP2PMsg::RequestFlashblocks, + FlashblocksP2PMsg::AcceptFlashblocks, + FlashblocksP2PMsg::RejectFlashblocks, + FlashblocksP2PMsg::CancelFlashblocks, + FlashblocksP2PMsg::CancelFlashblocksAck, + ]; + + for msg in variants { + let encoded = msg.encode(); + let mut view: &[u8] = &encoded; + let decoded = FlashblocksP2PMsg::decode(&mut view).expect("decoding succeeds"); + assert!(view.is_empty(), "all bytes consumed"); + assert_eq!(decoded, msg); } } From f611af4628e4795492e8ca82770484f6d34042ae Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 9 Mar 2026 20:56:43 -0700 Subject: [PATCH 02/43] refactor --- .../p2p/src/protocol/connection.rs | 39 +----- .../flashblocks/p2p/src/protocol/handler.rs | 127 ++++-------------- crates/flashblocks/primitives/src/p2p.rs | 5 - 3 files changed, 34 insertions(+), 137 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 5f2da545c..35e4d2a42 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -33,14 +33,14 @@ const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; /// This should be large enough to retain entries across the grace window. const RECEIVED_CACHE_LEN: u32 = AUTHORIZATION_TIMESTAMP_GRACE_SEC as u32 * 20; -/// A lightweight rolling average with a configurable smoothing window. +/// A lightweight moving average with a configurable smoothing window. #[derive(Clone, Debug)] -pub(crate) struct RollingAverage { +pub(crate) struct MovingAverage { value: Option, window: i64, } -impl RollingAverage { +impl MovingAverage { pub(crate) fn new(window: i64) -> Self { Self { value: None, @@ -79,14 +79,14 @@ pub(crate) struct FlashblocksConnectionFlags { #[derive(Debug)] pub(crate) struct FlashblocksConnectionState { flags: Mutex, - latency_average: Mutex, + latency_average: Mutex, } impl FlashblocksConnectionState { pub(crate) fn new(latency_window: i64) -> Self { Self { flags: Mutex::new(FlashblocksConnectionFlags::default()), - latency_average: Mutex::new(RollingAverage::new(latency_window)), + latency_average: Mutex::new(MovingAverage::new(latency_window)), } } @@ -354,9 +354,6 @@ impl Stream for FlashblocksConnection { FlashblocksP2PMsg::CancelFlashblocks => { this.protocol.handle.handle_cancel_message(this.peer_id); } - FlashblocksP2PMsg::CancelFlashblocksAck => { - this.protocol.handle.handle_cancel_ack_message(this.peer_id); - } } } } @@ -531,19 +528,6 @@ impl FlashblocksConnection { return; } - if !self - .protocol - .handle - .remember_control_message(&authorized_payload.authorized) - { - trace!( - target: "flashblocks::p2p", - peer_id = %self.peer_id, - "ignoring duplicate StartPublish message", - ); - return; - } - let state = self.protocol.handle.state.lock(); state.publishing_status.send_modify(|status| { let active_publishers = match status { @@ -641,19 +625,6 @@ impl FlashblocksConnection { return; } - if !self - .protocol - .handle - .remember_control_message(&authorized_payload.authorized) - { - trace!( - target: "flashblocks::p2p", - peer_id = %self.peer_id, - "ignoring duplicate StopPublish message", - ); - return; - } - let state = self.protocol.handle.state.lock(); state.publishing_status.send_modify(|status| { match status { diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 8cc92e99c..99eb75789 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -19,7 +19,7 @@ use rand::seq::SliceRandom; use reth::payload::PayloadId; use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; -use reth_network::{Peers, cache::LruCache}; +use reth_network::Peers; use std::{ collections::{HashMap, HashSet}, net::SocketAddr, @@ -58,9 +58,6 @@ const MAX_PUBLISH_WAIT_SEC: u64 = 2; /// before dropping them. In practice, we should rarely need to buffer any messages. const BROADCAST_BUFFER_CAPACITY: usize = 100; -/// Number of recently forwarded `StartPublish`/`StopPublish` messages we remember to avoid loops. -const CONTROL_MESSAGE_CACHE_LEN: u32 = 2048; - /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -94,8 +91,8 @@ pub struct FanoutConfig { pub max_receive_peers: usize, /// How often to evaluate latency-based peer rotation. pub rotation_interval: Duration, - /// How long to wait for a rotation request to be answered. - pub rotation_timeout: Duration, + /// How long to wait for request flashblocks to be answered. + pub request_flashblocks_timeout: Duration, /// Number of latency measurements to retain per receive peer. pub latency_window: i64, } @@ -103,34 +100,25 @@ pub struct FanoutConfig { impl Default for FanoutConfig { fn default() -> Self { Self { - max_send_peers: 6, - max_receive_peers: 6, + max_send_peers: 10, + max_receive_peers: 3, rotation_interval: Duration::from_secs(30), - rotation_timeout: Duration::from_secs(10), - latency_window: 50, + request_flashblocks_timeout: Duration::from_secs(2), + latency_window: 1000, } } } -#[derive(Clone, Debug)] -enum RotationState { - WaitingForResponse { - candidate: PeerId, - evict: PeerId, - requested_at: Instant, - }, - WaitingForCancelAck { - candidate: PeerId, - evict: PeerId, - }, -} - #[derive(Debug, Default)] struct FanoutState { - send_set: HashSet, - receive_set: HashSet, - connections: HashMap>, - rotation: Option, + /// Peers we are actively sending flashblocks to. + send_set: HashMap>, + /// Peers we are actively receiving flashblocks from. + receive_set: HashMap>, + /// Peers that are connected but idle, i.e. not currently sending or receiving flashblocks. + idle_set: HashMap>, + /// State for an ongoing rotation, if any. + awaiting_flashblocks_req: Option<(PeerId, Instant)>, } /// The current publishing status of this node in the flashblocks P2P network. @@ -192,9 +180,7 @@ pub struct FlashblocksP2PState { /// while maintaining in-order delivery. pub flashblocks: Vec>, /// Fanout and peer-selection state for flashblock forwarding. - fanout: FanoutState, - /// Recently processed control messages to prevent rebroadcast loops. - seen_control_messages: LruCache<[u8; 32]>, + pub fanout: FanoutState, } impl Default for FlashblocksP2PState { @@ -209,7 +195,6 @@ impl Default for FlashblocksP2PState { flashblock_index: 0, flashblocks: Vec::new(), fanout: FanoutState::default(), - seen_control_messages: LruCache::new(CONTROL_MESSAGE_CACHE_LEN), } } } @@ -226,7 +211,7 @@ impl FlashblocksP2PState { impl FanoutState { fn connection_state(&self, peer_id: &PeerId) -> Option> { - self.connections.get(peer_id).and_then(Weak::upgrade) + self.idle_set.get(peer_id).and_then(Weak::upgrade) } fn is_trusted(&self, peer_id: &PeerId) -> bool { @@ -242,7 +227,7 @@ impl FanoutState { } fn request_in_flight_count(&self) -> usize { - self.connections + self.idle_set .values() .filter_map(Weak::upgrade) .filter(|peer| peer.flags().request_in_flight) @@ -254,7 +239,7 @@ impl FanoutState { let mut unknown = Vec::new(); let mut untrusted = Vec::new(); - for (peer_id, peer_state) in &self.connections { + for (peer_id, peer_state) in &self.idle_set { let Some(peer_state) = peer_state.upgrade() else { continue; }; @@ -373,7 +358,7 @@ impl FanoutState { return; }; - if requested_at.elapsed() < ctx.fanout_config.rotation_timeout { + if requested_at.elapsed() < ctx.fanout_config.request_flashblocks_timeout { return; } @@ -386,7 +371,7 @@ impl FanoutState { } fn handle_disconnect(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - self.connections.remove(&peer_id); + self.idle_set.remove(&peer_id); self.send_set.remove(&peer_id); self.receive_set.remove(&peer_id); @@ -559,44 +544,6 @@ impl FanoutState { self.rotation = None; } - ctx.send_direct(peer_id, FlashblocksP2PMsg::CancelFlashblocksAck); - self.maybe_request_receive_peers(ctx); - } - - fn handle_cancel_ack(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - let Some(peer_state) = self.connection_state(&peer_id) else { - return; - }; - - let clear_rotation = self.rotation.as_ref().is_some_and(|rotation| { - matches!( - rotation, - RotationState::WaitingForCancelAck { evict, .. } if *evict == peer_id - ) - }); - - if !peer_state.flags().cancel_in_flight { - return; - } - peer_state.update_flags(|flags| { - flags.cancel_in_flight = false; - flags.request_in_flight = false; - - if clear_rotation { - flags.receive_enabled = false; - } else { - flags.send_enabled = false; - } - }); - - if clear_rotation { - peer_state.reset_latency(); - self.rotation = None; - self.receive_set.remove(&peer_id); - } else { - self.send_set.remove(&peer_id); - } - self.maybe_request_receive_peers(ctx); } } @@ -712,7 +659,7 @@ impl FlashblocksHandle { let mut state = self.state.lock(); state .fanout - .connections + .idle_set .insert(peer_id, Arc::downgrade(&fanout_state)); state.fanout.maybe_request_receive_peers(&self.ctx); } @@ -729,7 +676,7 @@ impl FlashblocksHandle { let mut state = handle.state.lock(); if state .fanout - .connections + .idle_set .get(&peer_id) .and_then(Weak::upgrade) .is_some_and(|current| Arc::ptr_eq(¤t, &fanout_state)) @@ -755,15 +702,6 @@ impl FlashblocksHandle { state.fanout.handle_disconnect(&self.ctx, peer_id); } - pub(crate) fn remember_control_message(&self, authorized: &Authorized) -> bool { - let encoded = FlashblocksP2PMsg::Authorized(authorized.clone()).encode(); - let hash = blake3::hash(&encoded); - self.state - .lock() - .seen_control_messages - .insert(*hash.as_bytes()) - } - pub(crate) fn handle_request_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); state.fanout.handle_request(&self.ctx, peer_id); @@ -783,11 +721,6 @@ impl FlashblocksHandle { let mut state = self.state.lock(); state.fanout.handle_cancel(&self.ctx, peer_id); } - - pub(crate) fn handle_cancel_ack_message(&self, peer_id: PeerId) { - let mut state = self.state.lock(); - state.fanout.handle_cancel_ack(&self.ctx, peer_id); - } } impl FlashblocksP2PCtx { @@ -1331,10 +1264,10 @@ mod tests { let trusted_state = test_peer_state(latency_window, true, true); let untrusted_state = test_peer_state(latency_window, false, true); fanout - .connections + .idle_set .insert(trusted_peer, Arc::downgrade(&trusted_state)); fanout - .connections + .idle_set .insert(untrusted_peer, Arc::downgrade(&untrusted_state)); fanout.maybe_request_receive_peers(&ctx); @@ -1370,10 +1303,10 @@ mod tests { let requester_state = test_peer_state(latency_window, true, true); victim_state.update_flags(|flags| flags.send_enabled = true); fanout - .connections + .idle_set .insert(victim, Arc::downgrade(&victim_state)); fanout - .connections + .idle_set .insert(trusted_requester, Arc::downgrade(&requester_state)); fanout.send_set.insert(victim); @@ -1426,10 +1359,10 @@ mod tests { current_state.update_flags(|flags| flags.receive_enabled = true); current_state.record_latency(42); fanout - .connections + .idle_set .insert(current_peer, Arc::downgrade(¤t_state)); fanout - .connections + .idle_set .insert(candidate_peer, Arc::downgrade(&candidate_state)); fanout.receive_set.insert(current_peer); @@ -1476,8 +1409,6 @@ mod tests { other => panic!("unexpected peer message: {other:?}"), } - fanout.handle_cancel_ack(&ctx, current_peer); - assert!(!fanout.receive_set.contains(¤t_peer)); assert!(fanout.receive_set.contains(&candidate_peer)); assert!(!current_state.flags().receive_enabled); diff --git a/crates/flashblocks/primitives/src/p2p.rs b/crates/flashblocks/primitives/src/p2p.rs index 1470d7090..d290525bf 100644 --- a/crates/flashblocks/primitives/src/p2p.rs +++ b/crates/flashblocks/primitives/src/p2p.rs @@ -55,8 +55,6 @@ pub enum FlashblocksP2PMsg { RejectFlashblocks = 0x03, /// Terminates an active flashblocks feed. CancelFlashblocks = 0x04, - /// Acknowledges termination of an active flashblocks feed. - CancelFlashblocksAck = 0x05, } /// The different types of authorized messages that can be sent over the Flashblocks P2P network. @@ -457,7 +455,6 @@ impl FlashblocksP2PMsg { FlashblocksP2PMsg::AcceptFlashblocks => buf.put_u8(0x02), FlashblocksP2PMsg::RejectFlashblocks => buf.put_u8(0x03), FlashblocksP2PMsg::CancelFlashblocks => buf.put_u8(0x04), - FlashblocksP2PMsg::CancelFlashblocksAck => buf.put_u8(0x05), } buf } @@ -477,7 +474,6 @@ impl FlashblocksP2PMsg { 0x02 => Ok(FlashblocksP2PMsg::AcceptFlashblocks), 0x03 => Ok(FlashblocksP2PMsg::RejectFlashblocks), 0x04 => Ok(FlashblocksP2PMsg::CancelFlashblocks), - 0x05 => Ok(FlashblocksP2PMsg::CancelFlashblocksAck), _ => Err(FlashblocksError::UnknownMessageType), } } @@ -846,7 +842,6 @@ mod tests { FlashblocksP2PMsg::AcceptFlashblocks, FlashblocksP2PMsg::RejectFlashblocks, FlashblocksP2PMsg::CancelFlashblocks, - FlashblocksP2PMsg::CancelFlashblocksAck, ]; for msg in variants { From 1ff0377bfa7d65d2408e5f723f940e735daa30e0 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 9 Mar 2026 21:30:18 -0700 Subject: [PATCH 03/43] wip --- .../p2p/src/protocol/connection.rs | 71 +++++-------------- .../flashblocks/p2p/src/protocol/handler.rs | 55 ++++---------- 2 files changed, 32 insertions(+), 94 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 35e4d2a42..07bb7d7c2 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -35,7 +35,7 @@ const RECEIVED_CACHE_LEN: u32 = AUTHORIZATION_TIMESTAMP_GRACE_SEC as u32 * 20; /// A lightweight moving average with a configurable smoothing window. #[derive(Clone, Debug)] -pub(crate) struct MovingAverage { +pub struct MovingAverage { value: Option, window: i64, } @@ -66,53 +66,18 @@ impl MovingAverage { /// Shared fanout metadata for a single peer connection. #[derive(Clone, Copy, Debug, Default)] -pub(crate) struct FlashblocksConnectionFlags { - pub(crate) trusted: bool, - pub(crate) trusted_known: bool, - pub(crate) send_enabled: bool, - pub(crate) receive_enabled: bool, - pub(crate) request_in_flight: bool, - pub(crate) cancel_in_flight: bool, -} +pub struct FlashblocksConnectionFlags {} /// Shared fanout metadata for a single peer connection. #[derive(Debug)] -pub(crate) struct FlashblocksConnectionState { - flags: Mutex, - latency_average: Mutex, -} - -impl FlashblocksConnectionState { - pub(crate) fn new(latency_window: i64) -> Self { - Self { - flags: Mutex::new(FlashblocksConnectionFlags::default()), - latency_average: Mutex::new(MovingAverage::new(latency_window)), - } - } - - pub(crate) fn flags(&self) -> FlashblocksConnectionFlags { - *self.flags.lock() - } - - pub(crate) fn update_flags(&self, update: F) - where - F: FnOnce(&mut FlashblocksConnectionFlags), - { - let mut flags = self.flags.lock(); - update(&mut flags); - } - - pub(crate) fn record_latency(&self, sample: i64) { - self.latency_average.lock().record(sample); - } - - pub(crate) fn average_latency(&self) -> Option { - self.latency_average.lock().value() - } - - pub(crate) fn reset_latency(&self) { - self.latency_average.lock().reset(); - } +pub struct FlashblocksConnectionState { + pub latency_average: MovingAverage, + pub trusted: bool, + pub trusted_known: bool, + pub send_enabled: bool, + pub receive_enabled: bool, + pub request_in_flight: bool, + pub cancel_in_flight: bool, } /// Represents a single P2P connection for the flashblocks protocol. @@ -133,8 +98,8 @@ pub struct FlashblocksConnection { /// Receiver for peer messages to be sent to all peers. /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. peer_rx: BroadcastStream, - /// Shared fanout state for this peer, also visible to the protocol handler. - fanout_state: Arc, + /// Shared connection state for this peer, also visible to the protocol handler. + state: Arc>, /// Per-peer tracking of flashblocks this peer has already sent us. /// Uses `peek` for lookups to avoid LRU promotion, giving FIFO eviction semantics. received_cache: LruMap<(PayloadId, usize), ()>, @@ -153,12 +118,12 @@ impl FlashblocksConnection { conn: ProtocolConnection, peer_id: PeerId, peer_rx: BroadcastStream, - fanout_state: Arc, + state: Arc>, ) -> Self { protocol.handle.ensure_background_tasks(); protocol .handle - .on_peer_connected(protocol.network.clone(), peer_id, fanout_state.clone()); + .on_peer_connected(protocol.network.clone(), peer_id, state.clone()); gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); @@ -167,7 +132,7 @@ impl FlashblocksConnection { conn, peer_id, peer_rx, - fanout_state, + state, received_cache: LruMap::new(RECEIVED_CACHE_LEN), } } @@ -225,7 +190,7 @@ impl Stream for FlashblocksConnection { bytes, )) => { // Check if this flashblock actually originated from this peer. - let send_enabled = this.fanout_state.flags().send_enabled; + let send_enabled = this.state.flags().send_enabled; if send_enabled && !this .received_cache_contains(&(payload_id, flashblock_index)) @@ -418,7 +383,7 @@ impl FlashblocksConnection { return; } - if !self.fanout_state.flags().receive_enabled { + if !self.state.flags().receive_enabled { trace!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -484,7 +449,7 @@ impl FlashblocksConnection { if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); - self.fanout_state.record_latency(latency); + self.state.record_latency(latency); } self.protocol diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 99eb75789..e34365273 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -109,18 +109,6 @@ impl Default for FanoutConfig { } } -#[derive(Debug, Default)] -struct FanoutState { - /// Peers we are actively sending flashblocks to. - send_set: HashMap>, - /// Peers we are actively receiving flashblocks from. - receive_set: HashMap>, - /// Peers that are connected but idle, i.e. not currently sending or receiving flashblocks. - idle_set: HashMap>, - /// State for an ongoing rotation, if any. - awaiting_flashblocks_req: Option<(PeerId, Instant)>, -} - /// The current publishing status of this node in the flashblocks P2P network. /// /// This enum tracks whether we are actively publishing flashblocks, waiting to publish, @@ -179,8 +167,14 @@ pub struct FlashblocksP2PState { /// Contains `None` for flashblocks not yet received, enabling out-of-order receipt /// while maintaining in-order delivery. pub flashblocks: Vec>, - /// Fanout and peer-selection state for flashblock forwarding. - pub fanout: FanoutState, + /// Peers we are actively sending flashblocks to. + pub send_set: HashMap>, + /// Peers we are actively receiving flashblocks from. + pub receive_set: HashMap>, + /// Peers that are connected but idle, i.e. not currently sending or receiving flashblocks. + pub idle_set: HashMap>, + /// State for an ongoing rotation, if any. + pub awaiting_flashblocks_req: Option<(PeerId, Instant)>, } impl Default for FlashblocksP2PState { @@ -194,7 +188,10 @@ impl Default for FlashblocksP2PState { flashblock_timestamp: 0, flashblock_index: 0, flashblocks: Vec::new(), - fanout: FanoutState::default(), + send_set: HashMap::new(), + receive_set: HashMap::new(), + idle_set: HashMap::new(), + awaiting_flashblocks_req: None, } } } @@ -209,31 +206,7 @@ impl FlashblocksP2PState { } } -impl FanoutState { - fn connection_state(&self, peer_id: &PeerId) -> Option> { - self.idle_set.get(peer_id).and_then(Weak::upgrade) - } - - fn is_trusted(&self, peer_id: &PeerId) -> bool { - self.connection_state(peer_id) - .is_some_and(|peer| peer.flags().trusted) - } - - fn non_trusted_send_count(&self) -> usize { - self.send_set - .iter() - .filter(|peer_id| !self.is_trusted(peer_id)) - .count() - } - - fn request_in_flight_count(&self) -> usize { - self.idle_set - .values() - .filter_map(Weak::upgrade) - .filter(|peer| peer.flags().request_in_flight) - .count() - } - +impl FlashblocksP2PState { fn available_receive_candidates(&self) -> Vec { let mut trusted = Vec::new(); let mut unknown = Vec::new(); @@ -653,7 +626,7 @@ impl FlashblocksHandle { &self, network: N, peer_id: PeerId, - fanout_state: Arc, + fanout_state: Arc>, ) { { let mut state = self.state.lock(); From 94fdb5ba45f04f32d47b1ea77c7f9a6e01b9af49 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Mon, 9 Mar 2026 22:30:20 -0700 Subject: [PATCH 04/43] :wip --- Cargo.lock | 1 - crates/flashblocks/p2p/Cargo.toml | 1 - .../p2p/src/protocol/connection.rs | 118 ++-- .../flashblocks/p2p/src/protocol/handler.rs | 598 +++++++++++------- 4 files changed, 441 insertions(+), 277 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49f8b38b4..3d271d579 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3888,7 +3888,6 @@ version = "1.10.1" dependencies = [ "alloy-primitives", "alloy-rlp", - "blake3", "chrono", "ed25519-dalek", "flashblocks-primitives", diff --git a/crates/flashblocks/p2p/Cargo.toml b/crates/flashblocks/p2p/Cargo.toml index d2f6da945..b6721ec6d 100644 --- a/crates/flashblocks/p2p/Cargo.toml +++ b/crates/flashblocks/p2p/Cargo.toml @@ -29,4 +29,3 @@ parking_lot.workspace = true chrono.workspace = true reth-tasks = { workspace = true } rand.workspace = true -blake3.workspace = true diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 07bb7d7c2..2d3caddce 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -17,6 +17,7 @@ use reth::payload::PayloadId; use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnection}; use reth_network::{cache::LruMap, types::ReputationChangeKind}; use std::{ + fmt, pin::Pin, sync::Arc, task::{Context, Poll, ready}, @@ -64,20 +65,70 @@ impl MovingAverage { } } -/// Shared fanout metadata for a single peer connection. -#[derive(Clone, Copy, Debug, Default)] -pub struct FlashblocksConnectionFlags {} - -/// Shared fanout metadata for a single peer connection. -#[derive(Debug)] +/// Shared connection metadata for a single peer connection. pub struct FlashblocksConnectionState { - pub latency_average: MovingAverage, pub trusted: bool, pub trusted_known: bool, pub send_enabled: bool, pub receive_enabled: bool, pub request_in_flight: bool, - pub cancel_in_flight: bool, + score_average: MovingAverage, + received_cache: LruMap<(PayloadId, usize), ()>, +} + +impl fmt::Debug for FlashblocksConnectionState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FlashblocksConnectionState") + .field("trusted", &self.trusted) + .field("trusted_known", &self.trusted_known) + .field("send_enabled", &self.send_enabled) + .field("receive_enabled", &self.receive_enabled) + .field("request_in_flight", &self.request_in_flight) + .field("score_average", &self.score_average) + .finish() + } +} + +impl FlashblocksConnectionState { + pub(crate) fn new(latency_window: i64) -> Self { + Self { + trusted: false, + trusted_known: false, + send_enabled: false, + receive_enabled: false, + request_in_flight: false, + score_average: MovingAverage::new(latency_window), + received_cache: LruMap::new(RECEIVED_CACHE_LEN), + } + } + + pub(crate) fn record_latency(&mut self, sample: i64) { + self.score_average.record(sample); + } + + pub(crate) fn record_missed_flashblock(&mut self, penalty: i64) { + self.score_average.record(penalty); + } + + pub(crate) fn score(&self) -> Option { + self.score_average.value() + } + + pub(crate) fn note_received_flashblock(&mut self, key: (PayloadId, usize)) -> bool { + if self.received_cache.peek(&key).is_some() { + return false; + } + self.received_cache.insert(key, ()) + } + + pub(crate) fn has_received_flashblock(&self, key: &(PayloadId, usize)) -> bool { + self.received_cache.peek(key).is_some() + } + + pub(crate) fn reset_receive_tracking(&mut self) { + self.score_average.reset(); + self.received_cache = LruMap::new(RECEIVED_CACHE_LEN); + } } /// Represents a single P2P connection for the flashblocks protocol. @@ -100,9 +151,6 @@ pub struct FlashblocksConnection { peer_rx: BroadcastStream, /// Shared connection state for this peer, also visible to the protocol handler. state: Arc>, - /// Per-peer tracking of flashblocks this peer has already sent us. - /// Uses `peek` for lookups to avoid LRU promotion, giving FIFO eviction semantics. - received_cache: LruMap<(PayloadId, usize), ()>, } impl FlashblocksConnection { @@ -133,32 +181,10 @@ impl FlashblocksConnection { peer_id, peer_rx, state, - received_cache: LruMap::new(RECEIVED_CACHE_LEN), } } } -impl FlashblocksConnection { - /// Insert a `(payload_id, flashblock_index)` into the received cache. - /// - /// Uses [`LruMap::peek`] before insert to avoid promoting duplicates, - /// giving FIFO eviction semantics instead of LRU. - /// - /// Returns `true` if the key was newly inserted, `false` if it already existed. - fn received_cache_insert(&mut self, key: (PayloadId, usize)) -> bool { - if self.received_cache.peek(&key).is_some() { - return false; - } - self.received_cache.insert(key, ()) - } - - /// Check if a `(payload_id, flashblock_index)` exists in the received cache - /// without promoting it (preserves FIFO eviction order). - fn received_cache_contains(&self, key: &(PayloadId, usize)) -> bool { - self.received_cache.peek(key).is_some() - } -} - impl Drop for FlashblocksConnection { fn drop(&mut self) { info!( @@ -190,11 +216,15 @@ impl Stream for FlashblocksConnection { bytes, )) => { // Check if this flashblock actually originated from this peer. - let send_enabled = this.state.flags().send_enabled; - if send_enabled - && !this - .received_cache_contains(&(payload_id, flashblock_index)) - { + let should_send = { + let state = this.state.lock(); + state.send_enabled + && !state.has_received_flashblock(&( + payload_id, + flashblock_index, + )) + }; + if should_send { trace!( target: "flashblocks::p2p", peer_id = %this.peer_id, @@ -383,7 +413,7 @@ impl FlashblocksConnection { return; } - if !self.state.flags().receive_enabled { + if !self.state.lock().receive_enabled { trace!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -395,7 +425,11 @@ impl FlashblocksConnection { } // Check if this peer is spamming us with the same payload index - if !self.received_cache_insert((msg.payload_id, msg.index as usize)) { + if !self + .state + .lock() + .note_received_flashblock((msg.payload_id, msg.index as usize)) + { // We've already seen this index from this peer. // They could be trying to DOS us. tracing::warn!( @@ -449,13 +483,13 @@ impl FlashblocksConnection { if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); - self.state.record_latency(latency); + self.state.lock().record_latency(latency); } self.protocol .handle .ctx - .publish(&mut state, authorized_payload); + .publish(&mut state, authorized_payload, Some(self.peer_id)); } /// Handles incoming `StartPublish` messages from a peer. diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index e34365273..76614e6d0 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -24,7 +24,7 @@ use std::{ collections::{HashMap, HashSet}, net::SocketAddr, sync::{ - Arc, Weak, + Arc, atomic::{AtomicBool, Ordering}, }, time::Duration, @@ -58,6 +58,11 @@ const MAX_PUBLISH_WAIT_SEC: u64 = 2; /// before dropping them. In practice, we should rarely need to buffer any messages. const BROADCAST_BUFFER_CAPACITY: usize = 100; +/// A missed flashblock should dominate modest latency differences when rotating receive peers. +const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; +/// Grace window for peers in the receive set to deliver a flashblock after we first observe it. +const RECEIVE_FLASHBLOCK_GRACE_WINDOW: Duration = Duration::from_secs(10); + /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -82,6 +87,18 @@ pub enum PeerMsg { Direct { peer_id: PeerId, bytes: BytesMut }, } +#[derive(Clone, Debug)] +struct ObservedFlashblock { + observed_at: Instant, + expected_peers: HashSet, + scored: bool, +} + +#[derive(Debug, Default)] +struct ObservedPayload { + flashblocks: Vec>, +} + /// Runtime configuration for bounded flashblocks fanout. #[derive(Clone, Debug, PartialEq, Eq)] pub struct FanoutConfig { @@ -167,12 +184,14 @@ pub struct FlashblocksP2PState { /// Contains `None` for flashblocks not yet received, enabling out-of-order receipt /// while maintaining in-order delivery. pub flashblocks: Vec>, + /// Flashblocks observed from network peers, tracked until their receive grace windows expire. + observed_payloads: HashMap, + /// All currently connected peers and their shared connection state. + pub connections: HashMap>>, /// Peers we are actively sending flashblocks to. - pub send_set: HashMap>, + pub send_set: HashSet, /// Peers we are actively receiving flashblocks from. - pub receive_set: HashMap>, - /// Peers that are connected but idle, i.e. not currently sending or receiving flashblocks. - pub idle_set: HashMap>, + pub receive_set: HashSet, /// State for an ongoing rotation, if any. pub awaiting_flashblocks_req: Option<(PeerId, Instant)>, } @@ -188,9 +207,10 @@ impl Default for FlashblocksP2PState { flashblock_timestamp: 0, flashblock_index: 0, flashblocks: Vec::new(), - send_set: HashMap::new(), - receive_set: HashMap::new(), - idle_set: HashMap::new(), + observed_payloads: HashMap::new(), + connections: HashMap::new(), + send_set: HashSet::new(), + receive_set: HashSet::new(), awaiting_flashblocks_req: None, } } @@ -204,25 +224,69 @@ impl FlashblocksP2PState { pub fn publishing_status(&self) -> PublishingStatus { self.publishing_status.borrow().clone() } -} -impl FlashblocksP2PState { + fn connection_state(&self, peer_id: &PeerId) -> Option>> { + self.connections.get(peer_id).cloned() + } + + fn insert_connection( + &mut self, + peer_id: PeerId, + peer_state: &Arc>, + ) { + self.connections.insert(peer_id, Arc::clone(peer_state)); + } + + fn remove_connection(&mut self, peer_id: &PeerId) { + self.connections.remove(peer_id); + self.send_set.remove(peer_id); + self.receive_set.remove(peer_id); + } + + fn set_send_enabled(&mut self, peer_id: PeerId, enabled: bool) { + if enabled { + self.send_set.insert(peer_id); + } else { + self.send_set.remove(&peer_id); + } + } + + fn set_receive_enabled(&mut self, peer_id: PeerId, enabled: bool) { + if enabled { + self.receive_set.insert(peer_id); + } else { + self.receive_set.remove(&peer_id); + } + } + + fn is_trusted(&self, peer_id: &PeerId) -> bool { + self.connection_state(peer_id) + .is_some_and(|peer| peer.lock().trusted) + } + + fn non_trusted_send_count(&self) -> usize { + self.send_set + .iter() + .filter(|peer_id| !self.is_trusted(peer_id)) + .count() + } + fn available_receive_candidates(&self) -> Vec { let mut trusted = Vec::new(); let mut unknown = Vec::new(); let mut untrusted = Vec::new(); - for (peer_id, peer_state) in &self.idle_set { - let Some(peer_state) = peer_state.upgrade() else { - continue; - }; - + for peer_id in self.connections.keys() { if self.receive_set.contains(peer_id) { continue; } - let peer_state = peer_state.flags(); - if peer_state.request_in_flight || peer_state.cancel_in_flight { + let Some(peer_state) = self.connection_state(peer_id) else { + continue; + }; + + let peer_state = peer_state.lock(); + if peer_state.request_in_flight { continue; } @@ -244,55 +308,52 @@ impl FlashblocksP2PState { } fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { - if self.rotation.is_some() { + if self.awaiting_flashblocks_req.is_some() + || self.receive_set.len() >= ctx.fanout_config.max_receive_peers + { return; } - let target = ctx - .fanout_config - .max_receive_peers - .saturating_sub(self.receive_set.len() + self.request_in_flight_count()); - if target == 0 { + let Some(peer_id) = self.available_receive_candidates().into_iter().next() else { return; - } - - for peer_id in self.available_receive_candidates().into_iter().take(target) { - let Some(peer_state) = self.connection_state(&peer_id) else { - continue; - }; + }; + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; - let flags = peer_state.flags(); - if flags.request_in_flight || flags.receive_enabled || flags.cancel_in_flight { - continue; + { + let mut peer_state = peer_state.lock(); + if peer_state.request_in_flight || peer_state.receive_enabled { + return; } - peer_state.update_flags(|flags| flags.request_in_flight = true); - - ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + peer_state.request_in_flight = true; } + + self.awaiting_flashblocks_req = Some((peer_id, Instant::now())); + ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } fn worst_receive_peer(&self) -> Option { self.receive_set .iter() .filter_map(|peer_id| { - self.connection_state(peer_id).and_then(|peer_state| { - peer_state - .average_latency() - .map(|average| (*peer_id, average)) - }) + self.connection_state(peer_id) + .and_then(|peer_state| peer_state.lock().score().map(|score| (*peer_id, score))) }) .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) .map(|(peer_id, _)| peer_id) } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { - if self.rotation.is_some() || self.receive_set.len() < ctx.fanout_config.max_receive_peers { + if self.awaiting_flashblocks_req.is_some() + || self.receive_set.len() < ctx.fanout_config.max_receive_peers + { return; } - let Some(evict) = self.worst_receive_peer() else { + if self.worst_receive_peer().is_none() { return; - }; + } let mut candidates = self.available_receive_candidates(); if candidates.is_empty() { @@ -307,27 +368,20 @@ impl FlashblocksP2PState { return; }; - let flags = candidate_state.flags(); - if flags.request_in_flight || flags.receive_enabled || flags.cancel_in_flight { - return; + { + let mut candidate_state = candidate_state.lock(); + if candidate_state.request_in_flight || candidate_state.receive_enabled { + return; + } + candidate_state.request_in_flight = true; } - candidate_state.update_flags(|flags| flags.request_in_flight = true); - self.rotation = Some(RotationState::WaitingForResponse { - candidate, - evict, - requested_at: Instant::now(), - }); + self.awaiting_flashblocks_req = Some((candidate, Instant::now())); ctx.send_direct(candidate, FlashblocksP2PMsg::RequestFlashblocks); } fn check_rotation_timeout(&mut self, ctx: &FlashblocksP2PCtx) { - let Some(RotationState::WaitingForResponse { - candidate, - requested_at, - .. - }) = self.rotation.as_ref() - else { + let Some((candidate, requested_at)) = self.awaiting_flashblocks_req else { return; }; @@ -335,32 +389,21 @@ impl FlashblocksP2PState { return; } - let candidate = *candidate; if let Some(candidate_state) = self.connection_state(&candidate) { - candidate_state.update_flags(|flags| flags.request_in_flight = false); + candidate_state.lock().request_in_flight = false; } - self.rotation = None; + self.awaiting_flashblocks_req = None; self.maybe_request_receive_peers(ctx); } fn handle_disconnect(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - self.idle_set.remove(&peer_id); - self.send_set.remove(&peer_id); - self.receive_set.remove(&peer_id); + self.remove_connection(&peer_id); if self - .rotation - .as_ref() - .is_some_and(|rotation| match rotation { - RotationState::WaitingForResponse { - candidate, evict, .. - } => *candidate == peer_id || *evict == peer_id, - RotationState::WaitingForCancelAck { candidate, evict } => { - *candidate == peer_id || *evict == peer_id - } - }) + .awaiting_flashblocks_req + .is_some_and(|(candidate, _)| candidate == peer_id) { - self.rotation = None; + self.awaiting_flashblocks_req = None; } self.maybe_request_receive_peers(ctx); @@ -371,55 +414,47 @@ impl FlashblocksP2PState { return; }; - let flags = peer_state.flags(); - let peer_is_trusted = flags.trusted; - let send_enabled = flags.send_enabled; - let cancel_in_flight = flags.cancel_in_flight; + let (peer_is_trusted, send_enabled) = { + let peer_state = peer_state.lock(); + (peer_state.trusted, peer_state.send_enabled) + }; if send_enabled { ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } - if cancel_in_flight { - ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); - return; - } - if peer_is_trusted { if self.non_trusted_send_count() >= ctx.fanout_config.max_send_peers { - if let Some(evicted_peer) = self.send_set.iter().copied().find(|candidate| { - !self.is_trusted(candidate) - && self - .connection_state(candidate) - .is_some_and(|state| !state.flags().cancel_in_flight) - }) { + if let Some(evicted_peer) = self + .send_set + .iter() + .copied() + .find(|candidate| !self.is_trusted(candidate)) + { if let Some(evicted_state) = self.connection_state(&evicted_peer) { - evicted_state.update_flags(|flags| { - flags.send_enabled = false; - flags.cancel_in_flight = true; - }); + evicted_state.lock().send_enabled = false; + self.set_send_enabled(evicted_peer, false); } - self.send_set.remove(&evicted_peer); ctx.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); } } - peer_state.update_flags(|flags| { - flags.send_enabled = true; - flags.cancel_in_flight = false; - }); - self.send_set.insert(peer_id); + { + let mut peer_state = peer_state.lock(); + peer_state.send_enabled = true; + } + self.set_send_enabled(peer_id, true); ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } if self.non_trusted_send_count() < ctx.fanout_config.max_send_peers { - peer_state.update_flags(|flags| { - flags.send_enabled = true; - flags.cancel_in_flight = false; - }); - self.send_set.insert(peer_id); + { + let mut peer_state = peer_state.lock(); + peer_state.send_enabled = true; + } + self.set_send_enabled(peer_id, true); ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); } else { ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); @@ -431,35 +466,40 @@ impl FlashblocksP2PState { return; }; - if !peer_state.flags().request_in_flight { + if !peer_state.lock().request_in_flight { return; } - peer_state.update_flags(|flags| { - flags.request_in_flight = false; - flags.receive_enabled = true; - }); + let evict = if self.receive_set.len() >= ctx.fanout_config.max_receive_peers + && !self.receive_set.contains(&peer_id) + { + self.worst_receive_peer() + } else { + None + }; - self.receive_set.insert(peer_id); + { + let mut peer_state = peer_state.lock(); + peer_state.request_in_flight = false; + peer_state.receive_enabled = true; + } + self.set_receive_enabled(peer_id, true); - if let Some(RotationState::WaitingForResponse { - candidate, evict, .. - }) = self.rotation.as_ref() + if self + .awaiting_flashblocks_req + .is_some_and(|(candidate, _)| candidate == peer_id) { - if *candidate == peer_id { - let evict = *evict; - if let Some(evict_state) = self.connection_state(&evict) { - evict_state.update_flags(|flags| flags.cancel_in_flight = true); - } else { - self.rotation = None; - self.maybe_request_receive_peers(ctx); - return; + self.awaiting_flashblocks_req = None; + } + + if let Some(evict) = evict.filter(|evict| *evict != peer_id) { + if let Some(evict_state) = self.connection_state(&evict) { + { + let mut evict_state = evict_state.lock(); + evict_state.receive_enabled = false; + evict_state.reset_receive_tracking(); } - self.rotation = Some(RotationState::WaitingForCancelAck { - candidate: peer_id, - evict, - }); + self.set_receive_enabled(evict, false); ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); - return; } } @@ -471,54 +511,110 @@ impl FlashblocksP2PState { return; }; - if !peer_state.flags().request_in_flight { + if !peer_state.lock().request_in_flight { return; } - peer_state.update_flags(|flags| flags.request_in_flight = false); - - if self.rotation.as_ref().is_some_and(|rotation| { - matches!( - rotation, - RotationState::WaitingForResponse { candidate, .. } if *candidate == peer_id - ) - }) { - self.rotation = None; + peer_state.lock().request_in_flight = false; + + if self + .awaiting_flashblocks_req + .is_some_and(|(candidate, _)| candidate == peer_id) + { + self.awaiting_flashblocks_req = None; } self.maybe_request_receive_peers(ctx); } fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - self.send_set.remove(&peer_id); - self.receive_set.remove(&peer_id); - if let Some(peer_state) = self.connection_state(&peer_id) { - peer_state.update_flags(|flags| { - flags.send_enabled = false; - flags.receive_enabled = false; - flags.request_in_flight = false; - flags.cancel_in_flight = false; - }); - peer_state.reset_latency(); + { + let mut peer_state = peer_state.lock(); + peer_state.send_enabled = false; + peer_state.receive_enabled = false; + peer_state.request_in_flight = false; + peer_state.reset_receive_tracking(); + } + self.set_send_enabled(peer_id, false); + self.set_receive_enabled(peer_id, false); + } else { + self.remove_connection(&peer_id); } if self - .rotation - .as_ref() - .is_some_and(|rotation| match rotation { - RotationState::WaitingForResponse { - candidate, evict, .. - } => *candidate == peer_id || *evict == peer_id, - RotationState::WaitingForCancelAck { candidate, evict } => { - *candidate == peer_id || *evict == peer_id - } - }) + .awaiting_flashblocks_req + .is_some_and(|(candidate, _)| candidate == peer_id) { - self.rotation = None; + self.awaiting_flashblocks_req = None; } self.maybe_request_receive_peers(ctx); } + + fn record_receive_observation(&mut self, payload_id: PayloadId, flashblock_index: usize) { + let expected_peers = self.receive_set.clone(); + + let payload = self.observed_payloads.entry(payload_id).or_default(); + if flashblock_index >= payload.flashblocks.len() { + payload + .flashblocks + .resize_with(flashblock_index + 1, || None); + } + payload.flashblocks[flashblock_index].get_or_insert_with(|| ObservedFlashblock { + observed_at: Instant::now(), + expected_peers, + scored: false, + }); + } + + fn score_expired_receive_observations(&mut self) { + let mut expired = Vec::new(); + + for (payload_id, observed_payload) in &mut self.observed_payloads { + for (index, observed_flashblock) in observed_payload.flashblocks.iter_mut().enumerate() + { + let Some(observed_flashblock) = observed_flashblock.as_mut() else { + continue; + }; + + if observed_flashblock.scored + || observed_flashblock.observed_at.elapsed() < RECEIVE_FLASHBLOCK_GRACE_WINDOW + { + continue; + } + + observed_flashblock.scored = true; + expired.push(( + *payload_id, + index, + observed_flashblock.expected_peers.clone(), + )); + } + } + + for (payload_id, index, expected_peers) in expired { + for peer_id in expected_peers { + let Some(peer_state) = self.connection_state(&peer_id) else { + continue; + }; + + let mut peer_state = peer_state.lock(); + if !peer_state.has_received_flashblock(&(payload_id, index)) { + peer_state.record_missed_flashblock(MISSED_FLASHBLOCK_PENALTY_NS); + } + } + } + + self.observed_payloads + .retain(|payload_id, observed_payload| { + *payload_id == self.payload_id + || observed_payload + .flashblocks + .iter() + .flatten() + .any(|observed_flashblock| !observed_flashblock.scored) + }); + } } /// Context struct containing shared resources for the flashblocks P2P protocol. @@ -611,11 +707,12 @@ impl FlashblocksHandle { tokio::select! { _ = rotation_interval.tick() => { let mut state = handle.state.lock(); - state.fanout.maybe_start_rotation(&handle.ctx); + state.maybe_start_rotation(&handle.ctx); } _ = timeout_interval.tick() => { let mut state = handle.state.lock(); - state.fanout.check_rotation_timeout(&handle.ctx); + state.check_rotation_timeout(&handle.ctx); + state.score_expired_receive_observations(); } } } @@ -630,31 +727,25 @@ impl FlashblocksHandle { ) { { let mut state = self.state.lock(); - state - .fanout - .idle_set - .insert(peer_id, Arc::downgrade(&fanout_state)); - state.fanout.maybe_request_receive_peers(&self.ctx); + state.insert_connection(peer_id, &fanout_state); + state.maybe_request_receive_peers(&self.ctx); } let handle = self.clone(); tokio::spawn(async move { match network.get_peer_by_id(peer_id).await { Ok(Some(peer_info)) => { - fanout_state.update_flags(|flags| { - flags.trusted = peer_info.kind.is_trusted(); - flags.trusted_known = true; - }); + let mut fanout_state_guard = fanout_state.lock(); + fanout_state_guard.trusted = peer_info.kind.is_trusted(); + fanout_state_guard.trusted_known = true; + drop(fanout_state_guard); let mut state = handle.state.lock(); if state - .fanout - .idle_set - .get(&peer_id) - .and_then(Weak::upgrade) + .connection_state(&peer_id) .is_some_and(|current| Arc::ptr_eq(¤t, &fanout_state)) { - state.fanout.maybe_request_receive_peers(&handle.ctx); + state.maybe_request_receive_peers(&handle.ctx); } } Ok(None) => {} @@ -672,27 +763,27 @@ impl FlashblocksHandle { pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.fanout.handle_disconnect(&self.ctx, peer_id); + state.handle_disconnect(&self.ctx, peer_id); } pub(crate) fn handle_request_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.fanout.handle_request(&self.ctx, peer_id); + state.handle_request(&self.ctx, peer_id); } pub(crate) fn handle_accept_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.fanout.handle_accept(&self.ctx, peer_id); + state.handle_accept(&self.ctx, peer_id); } pub(crate) fn handle_reject_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.fanout.handle_reject(&self.ctx, peer_id); + state.handle_reject(&self.ctx, peer_id); } pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.fanout.handle_cancel(&self.ctx, peer_id); + state.handle_cancel(&self.ctx, peer_id); } } @@ -783,7 +874,7 @@ impl FlashblocksHandle { if authorization != authorized_payload.authorized.authorization { return Err(FlashblocksP2PError::ExpiredAuthorization); } - self.ctx.publish(&mut state, authorized_payload); + self.ctx.publish(&mut state, authorized_payload, None); Ok(()) } @@ -1011,6 +1102,7 @@ impl FlashblocksP2PCtx { &self, state: &mut FlashblocksP2PState, authorized_payload: AuthorizedPayload, + source_peer_id: Option, ) { let payload = authorized_payload.msg(); let authorization = authorized_payload.authorized.authorization; @@ -1097,6 +1189,10 @@ impl FlashblocksP2PCtx { self.peer_tx.send(peer_msg).ok(); + if source_peer_id.is_some() { + state.record_receive_observation(payload.payload_id, payload.index as usize); + } + let now = Utc::now() .timestamp_nanos_opt() .expect("time went backwards"); @@ -1176,9 +1272,9 @@ impl ConnectionHandler for FlashblocksP2PProtoco ); let peer_rx = self.handle.ctx.peer_tx.subscribe(); - let fanout_state = Arc::new(FlashblocksConnectionState::new( + let fanout_state = Arc::new(Mutex::new(FlashblocksConnectionState::new( self.handle.ctx.fanout_config.latency_window, - )); + ))); FlashblocksConnection::new( self, @@ -1212,12 +1308,13 @@ mod tests { latency_window: i64, trusted: bool, trusted_known: bool, - ) -> Arc { - let state = Arc::new(FlashblocksConnectionState::new(latency_window)); - state.update_flags(|flags| { - flags.trusted = trusted; - flags.trusted_known = trusted_known; - }); + ) -> Arc> { + let state = Arc::new(Mutex::new(FlashblocksConnectionState::new(latency_window))); + { + let mut state_guard = state.lock(); + state_guard.trusted = trusted; + state_guard.trusted_known = trusted_known; + } state } @@ -1229,24 +1326,20 @@ mod tests { }; let latency_window = config.latency_window; let ctx = test_ctx(config); - let mut fanout = FanoutState::default(); + let mut fanout = FlashblocksP2PState::default(); let mut rx = ctx.peer_tx.subscribe(); let trusted_peer = PeerId::random(); let untrusted_peer = PeerId::random(); let trusted_state = test_peer_state(latency_window, true, true); let untrusted_state = test_peer_state(latency_window, false, true); - fanout - .idle_set - .insert(trusted_peer, Arc::downgrade(&trusted_state)); - fanout - .idle_set - .insert(untrusted_peer, Arc::downgrade(&untrusted_state)); + fanout.insert_connection(trusted_peer, &trusted_state); + fanout.insert_connection(untrusted_peer, &untrusted_state); fanout.maybe_request_receive_peers(&ctx); - assert!(trusted_state.flags().request_in_flight); - assert!(!untrusted_state.flags().request_in_flight); + assert!(trusted_state.lock().request_in_flight); + assert!(!untrusted_state.lock().request_in_flight); match rx.try_recv().expect("request sent") { PeerMsg::Direct { peer_id, bytes } => { assert_eq!(peer_id, trusted_peer); @@ -1267,28 +1360,24 @@ mod tests { }; let latency_window = config.latency_window; let ctx = test_ctx(config); - let mut fanout = FanoutState::default(); + let mut fanout = FlashblocksP2PState::default(); let mut rx = ctx.peer_tx.subscribe(); let victim = PeerId::random(); let trusted_requester = PeerId::random(); let victim_state = test_peer_state(latency_window, false, true); let requester_state = test_peer_state(latency_window, true, true); - victim_state.update_flags(|flags| flags.send_enabled = true); - fanout - .idle_set - .insert(victim, Arc::downgrade(&victim_state)); - fanout - .idle_set - .insert(trusted_requester, Arc::downgrade(&requester_state)); - fanout.send_set.insert(victim); + victim_state.lock().send_enabled = true; + fanout.insert_connection(victim, &victim_state); + fanout.insert_connection(trusted_requester, &requester_state); + fanout.set_send_enabled(victim, true); fanout.handle_request(&ctx, trusted_requester); assert!(!fanout.send_set.contains(&victim)); assert!(fanout.send_set.contains(&trusted_requester)); - assert!(victim_state.flags().cancel_in_flight); - assert!(requester_state.flags().send_enabled); + assert!(!victim_state.lock().send_enabled); + assert!(requester_state.lock().send_enabled); match rx.try_recv().expect("cancel sent") { PeerMsg::Direct { peer_id, bytes } => { @@ -1314,7 +1403,7 @@ mod tests { } #[test] - fn rotation_accepts_candidate_then_waits_for_cancel_ack() { + fn rotation_accepts_candidate_and_cancels_current_peer() { let config = FanoutConfig { max_receive_peers: 1, latency_window: 4, @@ -1322,30 +1411,25 @@ mod tests { }; let latency_window = config.latency_window; let ctx = test_ctx(config); - let mut fanout = FanoutState::default(); + let mut fanout = FlashblocksP2PState::default(); let mut rx = ctx.peer_tx.subscribe(); let current_peer = PeerId::random(); let candidate_peer = PeerId::random(); let current_state = test_peer_state(latency_window, false, true); let candidate_state = test_peer_state(latency_window, false, true); - current_state.update_flags(|flags| flags.receive_enabled = true); - current_state.record_latency(42); - fanout - .idle_set - .insert(current_peer, Arc::downgrade(¤t_state)); - fanout - .idle_set - .insert(candidate_peer, Arc::downgrade(&candidate_state)); - fanout.receive_set.insert(current_peer); + current_state.lock().receive_enabled = true; + current_state.lock().record_latency(42); + fanout.insert_connection(current_peer, ¤t_state); + fanout.insert_connection(candidate_peer, &candidate_state); + fanout.set_receive_enabled(current_peer, true); fanout.maybe_start_rotation(&ctx); - assert!(candidate_state.flags().request_in_flight); + assert!(candidate_state.lock().request_in_flight); assert!(matches!( - fanout.rotation, - Some(RotationState::WaitingForResponse { candidate, evict, .. }) - if candidate == candidate_peer && evict == current_peer + fanout.awaiting_flashblocks_req, + Some((candidate, _)) if candidate == candidate_peer )); match rx.try_recv().expect("rotation request sent") { @@ -1361,15 +1445,11 @@ mod tests { fanout.handle_accept(&ctx, candidate_peer); - assert!(fanout.receive_set.contains(¤t_peer)); + assert!(!fanout.receive_set.contains(¤t_peer)); assert!(fanout.receive_set.contains(&candidate_peer)); - assert!(candidate_state.flags().receive_enabled); - assert!(current_state.flags().cancel_in_flight); - assert!(matches!( - fanout.rotation, - Some(RotationState::WaitingForCancelAck { candidate, evict }) - if candidate == candidate_peer && evict == current_peer - )); + assert!(candidate_state.lock().receive_enabled); + assert!(!current_state.lock().receive_enabled); + assert!(fanout.awaiting_flashblocks_req.is_none()); match rx.try_recv().expect("cancel sent to old peer") { PeerMsg::Direct { peer_id, bytes } => { @@ -1384,7 +1464,59 @@ mod tests { assert!(!fanout.receive_set.contains(¤t_peer)); assert!(fanout.receive_set.contains(&candidate_peer)); - assert!(!current_state.flags().receive_enabled); - assert!(fanout.rotation.is_none()); + assert!(!current_state.lock().receive_enabled); + assert!(fanout.awaiting_flashblocks_req.is_none()); + } + + #[test] + fn peer_score_penalizes_missed_flashblocks() { + let config = FanoutConfig { + max_receive_peers: 2, + latency_window: 4, + ..Default::default() + }; + let latency_window = config.latency_window; + let mut fanout = FlashblocksP2PState::default(); + + let steady_peer = PeerId::random(); + let lagging_peer = PeerId::random(); + let steady_state = test_peer_state(latency_window, false, true); + let lagging_state = test_peer_state(latency_window, false, true); + + { + let mut steady = steady_state.lock(); + steady.receive_enabled = true; + steady.record_latency(10); + steady.note_received_flashblock((PayloadId::default(), 0)); + } + { + let mut lagging = lagging_state.lock(); + lagging.receive_enabled = true; + lagging.record_latency(100); + } + + fanout.insert_connection(steady_peer, &steady_state); + fanout.insert_connection(lagging_peer, &lagging_state); + fanout.set_receive_enabled(steady_peer, true); + fanout.set_receive_enabled(lagging_peer, true); + + fanout.record_receive_observation(PayloadId::default(), 0); + fanout + .observed_payloads + .get_mut(&PayloadId::default()) + .unwrap() + .flashblocks[0] + .as_mut() + .unwrap() + .observed_at = + Instant::now() - RECEIVE_FLASHBLOCK_GRACE_WINDOW - Duration::from_secs(1); + fanout.score_expired_receive_observations(); + + assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); + assert_eq!(steady_state.lock().score(), Some(10)); + assert_eq!( + lagging_state.lock().score(), + Some((100 * (latency_window - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / latency_window) + ); } } From 487855d8f6bf7bc75a8fb4af2d832b775fc40286 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 20:31:12 -0700 Subject: [PATCH 05/43] wip --- .../p2p/src/protocol/connection.rs | 145 +++--- .../flashblocks/p2p/src/protocol/handler.rs | 470 +++++++----------- 2 files changed, 240 insertions(+), 375 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 2d3caddce..c539ad131 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -13,11 +13,9 @@ use flashblocks_primitives::{ use futures::{Stream, StreamExt}; use metrics::gauge; use parking_lot::Mutex; -use reth::payload::PayloadId; use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnection}; -use reth_network::{cache::LruMap, types::ReputationChangeKind}; +use reth_network::types::ReputationChangeKind; use std::{ - fmt, pin::Pin, sync::Arc, task::{Context, Poll, ready}, @@ -29,11 +27,6 @@ use tracing::{info, trace}; /// minor skew/races between peers. const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; -/// Number of payload receive-sets cached per peer. -/// -/// This should be large enough to retain entries across the grace window. -const RECEIVED_CACHE_LEN: u32 = AUTHORIZATION_TIMESTAMP_GRACE_SEC as u32 * 20; - /// A lightweight moving average with a configurable smoothing window. #[derive(Clone, Debug)] pub struct MovingAverage { @@ -66,69 +59,36 @@ impl MovingAverage { } /// Shared connection metadata for a single peer connection. +#[derive(Clone, Debug)] pub struct FlashblocksConnectionState { + /// Whether this peer is marked as trusted or not. pub trusted: bool, - pub trusted_known: bool, + /// Whether we currently have an outstanding flashblocks request to this peer. + pub request_in_flight: bool, + /// Whether we are currently sending flashblocks to this peer. pub send_enabled: bool, + /// Whether we are currently requesting flashblocks from this peer. pub receive_enabled: bool, - pub request_in_flight: bool, - score_average: MovingAverage, - received_cache: LruMap<(PayloadId, usize), ()>, -} - -impl fmt::Debug for FlashblocksConnectionState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("FlashblocksConnectionState") - .field("trusted", &self.trusted) - .field("trusted_known", &self.trusted_known) - .field("send_enabled", &self.send_enabled) - .field("receive_enabled", &self.receive_enabled) - .field("request_in_flight", &self.request_in_flight) - .field("score_average", &self.score_average) - .finish() - } + /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. + pub receive_enabled_timestamp: u64, + /// Score for this peer connection, used for adaptive timeouts and peer selection. + /// + /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks + /// counting as 10s + pub score: MovingAverage, } impl FlashblocksConnectionState { pub(crate) fn new(latency_window: i64) -> Self { Self { trusted: false, - trusted_known: false, + request_in_flight: false, send_enabled: false, receive_enabled: false, - request_in_flight: false, - score_average: MovingAverage::new(latency_window), - received_cache: LruMap::new(RECEIVED_CACHE_LEN), + receive_enabled_timestamp: 0, + score: MovingAverage::new(latency_window), } } - - pub(crate) fn record_latency(&mut self, sample: i64) { - self.score_average.record(sample); - } - - pub(crate) fn record_missed_flashblock(&mut self, penalty: i64) { - self.score_average.record(penalty); - } - - pub(crate) fn score(&self) -> Option { - self.score_average.value() - } - - pub(crate) fn note_received_flashblock(&mut self, key: (PayloadId, usize)) -> bool { - if self.received_cache.peek(&key).is_some() { - return false; - } - self.received_cache.insert(key, ()) - } - - pub(crate) fn has_received_flashblock(&self, key: &(PayloadId, usize)) -> bool { - self.received_cache.peek(key).is_some() - } - - pub(crate) fn reset_receive_tracking(&mut self) { - self.score_average.reset(); - self.received_cache = LruMap::new(RECEIVED_CACHE_LEN); - } } /// Represents a single P2P connection for the flashblocks protocol. @@ -217,12 +177,14 @@ impl Stream for FlashblocksConnection { )) => { // Check if this flashblock actually originated from this peer. let should_send = { - let state = this.state.lock(); - state.send_enabled - && !state.has_received_flashblock(&( + let is_send_enabled = this.state.lock().send_enabled; + let already_received = + this.protocol.handle.state.lock().peer_received_flashblock( + this.peer_id, payload_id, - flashblock_index, - )) + flashblock_index as u64, + ); + is_send_enabled && !already_received }; if should_send { trace!( @@ -375,21 +337,23 @@ impl FlashblocksConnection { authorized_payload: AuthorizedPayload, ) { let state_handle = self.protocol.handle.state.clone(); - let mut state = state_handle.lock(); + let conn_state = self.state.lock().clone(); + let mut p2p_state = state_handle.lock(); + let authorization = &authorized_payload.authorized.authorization; let msg = authorized_payload.msg(); // Check if this payload is older than our current view by more than the allowed // grace window. if authorization.timestamp - < state + < p2p_state .payload_timestamp .saturating_sub(AUTHORIZATION_TIMESTAMP_GRACE_SEC) { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = state.payload_timestamp, + current_timestamp = p2p_state.payload_timestamp, timestamp = authorization.timestamp, grace_sec = AUTHORIZATION_TIMESTAMP_GRACE_SEC, "received flashblock with outdated timestamp", @@ -413,7 +377,7 @@ impl FlashblocksConnection { return; } - if !self.state.lock().receive_enabled { + if !conn_state.receive_enabled { trace!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -424,12 +388,45 @@ impl FlashblocksConnection { return; } - // Check if this peer is spamming us with the same payload index - if !self - .state - .lock() - .note_received_flashblock((msg.payload_id, msg.index as usize)) + if msg.payload_id == p2p_state.payload_id + && (msg.index as usize) + .saturating_add(crate::protocol::handler::RECEIVE_FLASHBLOCK_GRACE_WINDOW) + < p2p_state.flashblock_index { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + current_index = p2p_state.flashblock_index, + grace_window = crate::protocol::handler::RECEIVE_FLASHBLOCK_GRACE_WINDOW, + "received flashblock outside receive grace window", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + return; + } + + // Check if we're expecting to see flashblocks from this peer + if !conn_state.receive_enabled + && conn_state.receive_enabled_timestamp + 2 < authorization.timestamp + { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock from peer outside receive window", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + return; + } + + // Check if this peer is spamming us with the same payload index + if !p2p_state.note_peer_received_flashblock(&authorization, &msg, self.peer_id) { // We've already seen this index from this peer. // They could be trying to DOS us. tracing::warn!( @@ -445,7 +442,7 @@ impl FlashblocksConnection { return; } - state.publishing_status.send_modify(|status| { + p2p_state.publishing_status.send_modify(|status| { let active_publishers = match status { PublishingStatus::Publishing { .. } => { // We are currently building, so we should not be seeing any new flashblocks @@ -483,13 +480,13 @@ impl FlashblocksConnection { if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); - self.state.lock().record_latency(latency); + self.state.lock().score.record(latency); } self.protocol .handle .ctx - .publish(&mut state, authorized_payload, Some(self.peer_id)); + .publish(&mut p2p_state, authorized_payload, Some(self.peer_id)); } /// Handles incoming `StartPublish` messages from a peer. diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 76614e6d0..0707ccb6e 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -15,13 +15,13 @@ use flashblocks_primitives::{ use futures::{Stream, StreamExt, stream}; use metrics::histogram; use parking_lot::Mutex; -use rand::seq::SliceRandom; +use rand::{Rng, seq::SliceRandom}; use reth::payload::PayloadId; use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; use reth_network::Peers; use std::{ - collections::{HashMap, HashSet}, + collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, sync::{ Arc, @@ -60,8 +60,11 @@ const BROADCAST_BUFFER_CAPACITY: usize = 100; /// A missed flashblock should dominate modest latency differences when rotating receive peers. const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; -/// Grace window for peers in the receive set to deliver a flashblock after we first observe it. -const RECEIVE_FLASHBLOCK_GRACE_WINDOW: Duration = Duration::from_secs(10); +/// Grace window in number of flashblocks to receive late flashblocks from peers before scoring them for missing flashblocks. +/// +/// This must be at least long enough to cover the max authorization age to prevent a griefing +/// attack. +pub(crate) const RECEIVE_FLASHBLOCK_GRACE_WINDOW: usize = 50; /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// @@ -88,15 +91,11 @@ pub enum PeerMsg { } #[derive(Clone, Debug)] -struct ObservedFlashblock { - observed_at: Instant, - expected_peers: HashSet, - scored: bool, -} - -#[derive(Debug, Default)] -struct ObservedPayload { - flashblocks: Vec>, +pub struct ObservedPayload { + payload_id: PayloadId, + timestamp: u64, + flashblock_index: u64, + received_peers: HashSet, } /// Runtime configuration for bounded flashblocks fanout. @@ -185,13 +184,9 @@ pub struct FlashblocksP2PState { /// while maintaining in-order delivery. pub flashblocks: Vec>, /// Flashblocks observed from network peers, tracked until their receive grace windows expire. - observed_payloads: HashMap, + pub observed_payloads: VecDeque, /// All currently connected peers and their shared connection state. pub connections: HashMap>>, - /// Peers we are actively sending flashblocks to. - pub send_set: HashSet, - /// Peers we are actively receiving flashblocks from. - pub receive_set: HashSet, /// State for an ongoing rotation, if any. pub awaiting_flashblocks_req: Option<(PeerId, Instant)>, } @@ -207,10 +202,8 @@ impl Default for FlashblocksP2PState { flashblock_timestamp: 0, flashblock_index: 0, flashblocks: Vec::new(), - observed_payloads: HashMap::new(), + observed_payloads: VecDeque::new(), connections: HashMap::new(), - send_set: HashSet::new(), - receive_set: HashSet::new(), awaiting_flashblocks_req: None, } } @@ -225,120 +218,130 @@ impl FlashblocksP2PState { self.publishing_status.borrow().clone() } - fn connection_state(&self, peer_id: &PeerId) -> Option>> { - self.connections.get(peer_id).cloned() + /// Returns the connection state of a peer. + fn connection_state( + &self, + peer_id: &PeerId, + ) -> Option<&Arc>> { + self.connections.get(peer_id) } - fn insert_connection( + /// Marks receiving a flashblock from a peer and returns whether this is the first time we've observed this peer receive this flashblock. + /// + /// Called when a flashblock is received from any peer. + pub(crate) fn note_peer_received_flashblock( &mut self, + authorization: &Authorization, + flashblock: &FlashblocksPayloadV1, peer_id: PeerId, - peer_state: &Arc>, - ) { - self.connections.insert(peer_id, Arc::clone(peer_state)); - } - - fn remove_connection(&mut self, peer_id: &PeerId) { - self.connections.remove(peer_id); - self.send_set.remove(peer_id); - self.receive_set.remove(peer_id); - } - - fn set_send_enabled(&mut self, peer_id: PeerId, enabled: bool) { - if enabled { - self.send_set.insert(peer_id); - } else { - self.send_set.remove(&peer_id); + ) -> bool { + if let Some(observed_payload) = self.observed_payloads.iter_mut().find(|observed_payload| { + observed_payload.payload_id == flashblock.payload_id + && observed_payload.flashblock_index == flashblock.index + }) { + return observed_payload.received_peers.insert(peer_id); } - } - fn set_receive_enabled(&mut self, peer_id: PeerId, enabled: bool) { - if enabled { - self.receive_set.insert(peer_id); - } else { - self.receive_set.remove(&peer_id); + if self.observed_payloads.len() >= RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let evicted = self.observed_payloads.pop_front().unwrap(); + for (peer_id, connection) in self.connections.iter() { + let mut connection = connection.lock(); + if connection.receive_enabled + && connection.receive_enabled_timestamp < evicted.timestamp + 2 + && !evicted.received_peers.contains(peer_id) + { + debug!( + target: "flashblocks::p2p", + %peer_id, + payload_id = %evicted.payload_id, + flashblock_index = evicted.flashblock_index, + "scoring peer for missed flashblock", + ); + connection.score.record(MISSED_FLASHBLOCK_PENALTY_NS); + } + } } + + self.observed_payloads.push_back(ObservedPayload { + payload_id: flashblock.payload_id, + timestamp: authorization.timestamp, + flashblock_index: flashblock.index, + received_peers: HashSet::from([peer_id]), + }); + + true } - fn is_trusted(&self, peer_id: &PeerId) -> bool { - self.connection_state(peer_id) - .is_some_and(|peer| peer.lock().trusted) + /// Returns whether we've seen a given flashblock from a given peer. + pub(crate) fn peer_received_flashblock( + &self, + peer_id: PeerId, + payload_id: PayloadId, + index: u64, + ) -> bool { + self.observed_payloads + .iter() + .find(|observed_payload| { + observed_payload.payload_id == payload_id + && observed_payload.flashblock_index == index + }) + .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) } - fn non_trusted_send_count(&self) -> usize { - self.send_set + fn receive_peers(&self) -> HashSet { + self.connections .iter() - .filter(|peer_id| !self.is_trusted(peer_id)) - .count() + .filter_map(|(peer_id, peer_state)| { + peer_state.lock().receive_enabled.then_some(*peer_id) + }) + .collect() } fn available_receive_candidates(&self) -> Vec { - let mut trusted = Vec::new(); - let mut unknown = Vec::new(); - let mut untrusted = Vec::new(); - - for peer_id in self.connections.keys() { - if self.receive_set.contains(peer_id) { - continue; - } - - let Some(peer_state) = self.connection_state(peer_id) else { - continue; - }; - - let peer_state = peer_state.lock(); - if peer_state.request_in_flight { - continue; - } - - match (peer_state.trusted_known, peer_state.trusted) { - (true, true) => trusted.push(*peer_id), - (false, _) => unknown.push(*peer_id), - (true, false) => untrusted.push(*peer_id), - } - } - - let mut rng = rand::rng(); - trusted.shuffle(&mut rng); - unknown.shuffle(&mut rng); - untrusted.shuffle(&mut rng); - - trusted.extend(unknown); - trusted.extend(untrusted); - trusted + self.connections + .iter() + .filter_map(|(peer_id, peer_state)| { + let peer_state = peer_state.lock(); + if !peer_state.receive_enabled && !peer_state.request_in_flight { + Some(*peer_id) + } else { + None + } + }) + .collect() } fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { if self.awaiting_flashblocks_req.is_some() - || self.receive_set.len() >= ctx.fanout_config.max_receive_peers + || self.receive_peer_count() >= ctx.fanout_config.max_receive_peers { return; } - let Some(peer_id) = self.available_receive_candidates().into_iter().next() else { - return; - }; - let Some(peer_state) = self.connection_state(&peer_id) else { + let candidates = self.available_receive_candidates(); + if candidates.is_empty() { return; - }; - - { - let mut peer_state = peer_state.lock(); - if peer_state.request_in_flight || peer_state.receive_enabled { - return; - } - peer_state.request_in_flight = true; } + let rand = rand::rng().random_range(0..candidates.len()); + let peer_id = candidates[rand]; + + // get seconds since unix epoch for request timestamp + let requested_at = Utc::now().timestamp() as u64; - self.awaiting_flashblocks_req = Some((peer_id, Instant::now())); + self.awaiting_flashblocks_req = Some((peer_id, requested_at)); ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } fn worst_receive_peer(&self) -> Option { - self.receive_set + self.connections .iter() - .filter_map(|peer_id| { - self.connection_state(peer_id) - .and_then(|peer_state| peer_state.lock().score().map(|score| (*peer_id, score))) + .filter_map(|(peer_id, peer_state)| { + let peer_state = peer_state.lock(); + if peer_state.receive_enabled { + peer_state.score().map(|score| (*peer_id, score)) + } else { + None + } }) .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) .map(|(peer_id, _)| peer_id) @@ -346,7 +349,7 @@ impl FlashblocksP2PState { fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { if self.awaiting_flashblocks_req.is_some() - || self.receive_set.len() < ctx.fanout_config.max_receive_peers + || self.receive_peer_count() < ctx.fanout_config.max_receive_peers { return; } @@ -367,16 +370,11 @@ impl FlashblocksP2PState { let Some(candidate_state) = self.connection_state(&candidate) else { return; }; + drop(candidate_state); - { - let mut candidate_state = candidate_state.lock(); - if candidate_state.request_in_flight || candidate_state.receive_enabled { - return; - } - candidate_state.request_in_flight = true; - } - - self.awaiting_flashblocks_req = Some((candidate, Instant::now())); + let requested_at = Instant::now(); + self.start_requesting_peer(candidate); + self.awaiting_flashblocks_req = Some((candidate, requested_at)); ctx.send_direct(candidate, FlashblocksP2PMsg::RequestFlashblocks); } @@ -389,15 +387,13 @@ impl FlashblocksP2PState { return; } - if let Some(candidate_state) = self.connection_state(&candidate) { - candidate_state.lock().request_in_flight = false; - } + self.set_request_in_flight(candidate, false); self.awaiting_flashblocks_req = None; self.maybe_request_receive_peers(ctx); } fn handle_disconnect(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - self.remove_connection(&peer_id); + self.connections.remove(&peer_id); if self .awaiting_flashblocks_req @@ -414,46 +410,31 @@ impl FlashblocksP2PState { return; }; - let (peer_is_trusted, send_enabled) = { - let peer_state = peer_state.lock(); - (peer_state.trusted, peer_state.send_enabled) - }; + let peer_is_trusted = peer_state.lock().trusted; - if send_enabled { + if self.send_enabled(&peer_id) { ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } if peer_is_trusted { if self.non_trusted_send_count() >= ctx.fanout_config.max_send_peers { - if let Some(evicted_peer) = self - .send_set - .iter() - .copied() - .find(|candidate| !self.is_trusted(candidate)) + if let Some(evicted_peer) = + self.connections.keys().copied().find(|candidate| { + self.send_enabled(candidate) && !self.is_trusted(candidate) + }) { - if let Some(evicted_state) = self.connection_state(&evicted_peer) { - evicted_state.lock().send_enabled = false; - self.set_send_enabled(evicted_peer, false); - } + self.set_send_enabled(evicted_peer, false); ctx.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); } } - { - let mut peer_state = peer_state.lock(); - peer_state.send_enabled = true; - } self.set_send_enabled(peer_id, true); ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } if self.non_trusted_send_count() < ctx.fanout_config.max_send_peers { - { - let mut peer_state = peer_state.lock(); - peer_state.send_enabled = true; - } self.set_send_enabled(peer_id, true); ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); } else { @@ -466,11 +447,14 @@ impl FlashblocksP2PState { return; }; - if !peer_state.lock().request_in_flight { + if self + .awaiting_flashblocks_req + .is_none_or(|(candidate, _)| candidate != peer_id) + { return; } - let evict = if self.receive_set.len() >= ctx.fanout_config.max_receive_peers - && !self.receive_set.contains(&peer_id) + let evict = if self.receive_peer_count() >= ctx.fanout_config.max_receive_peers + && !self.receive_enabled(&peer_id) { self.worst_receive_peer() } else { @@ -479,10 +463,9 @@ impl FlashblocksP2PState { { let mut peer_state = peer_state.lock(); - peer_state.request_in_flight = false; peer_state.receive_enabled = true; + peer_state.request_in_flight = false; } - self.set_receive_enabled(peer_id, true); if self .awaiting_flashblocks_req @@ -493,128 +476,16 @@ impl FlashblocksP2PState { if let Some(evict) = evict.filter(|evict| *evict != peer_id) { if let Some(evict_state) = self.connection_state(&evict) { - { - let mut evict_state = evict_state.lock(); - evict_state.receive_enabled = false; - evict_state.reset_receive_tracking(); - } - self.set_receive_enabled(evict, false); + let mut evict_state = evict_state.lock(); + evict_state.receive_enabled = false; + evict_state.request_in_flight = false; + evict_state.reset_receive_tracking(); ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); } } self.maybe_request_receive_peers(ctx); } - - fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - let Some(peer_state) = self.connection_state(&peer_id) else { - return; - }; - - if !peer_state.lock().request_in_flight { - return; - } - peer_state.lock().request_in_flight = false; - - if self - .awaiting_flashblocks_req - .is_some_and(|(candidate, _)| candidate == peer_id) - { - self.awaiting_flashblocks_req = None; - } - - self.maybe_request_receive_peers(ctx); - } - - fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - if let Some(peer_state) = self.connection_state(&peer_id) { - { - let mut peer_state = peer_state.lock(); - peer_state.send_enabled = false; - peer_state.receive_enabled = false; - peer_state.request_in_flight = false; - peer_state.reset_receive_tracking(); - } - self.set_send_enabled(peer_id, false); - self.set_receive_enabled(peer_id, false); - } else { - self.remove_connection(&peer_id); - } - - if self - .awaiting_flashblocks_req - .is_some_and(|(candidate, _)| candidate == peer_id) - { - self.awaiting_flashblocks_req = None; - } - - self.maybe_request_receive_peers(ctx); - } - - fn record_receive_observation(&mut self, payload_id: PayloadId, flashblock_index: usize) { - let expected_peers = self.receive_set.clone(); - - let payload = self.observed_payloads.entry(payload_id).or_default(); - if flashblock_index >= payload.flashblocks.len() { - payload - .flashblocks - .resize_with(flashblock_index + 1, || None); - } - payload.flashblocks[flashblock_index].get_or_insert_with(|| ObservedFlashblock { - observed_at: Instant::now(), - expected_peers, - scored: false, - }); - } - - fn score_expired_receive_observations(&mut self) { - let mut expired = Vec::new(); - - for (payload_id, observed_payload) in &mut self.observed_payloads { - for (index, observed_flashblock) in observed_payload.flashblocks.iter_mut().enumerate() - { - let Some(observed_flashblock) = observed_flashblock.as_mut() else { - continue; - }; - - if observed_flashblock.scored - || observed_flashblock.observed_at.elapsed() < RECEIVE_FLASHBLOCK_GRACE_WINDOW - { - continue; - } - - observed_flashblock.scored = true; - expired.push(( - *payload_id, - index, - observed_flashblock.expected_peers.clone(), - )); - } - } - - for (payload_id, index, expected_peers) in expired { - for peer_id in expected_peers { - let Some(peer_state) = self.connection_state(&peer_id) else { - continue; - }; - - let mut peer_state = peer_state.lock(); - if !peer_state.has_received_flashblock(&(payload_id, index)) { - peer_state.record_missed_flashblock(MISSED_FLASHBLOCK_PENALTY_NS); - } - } - } - - self.observed_payloads - .retain(|payload_id, observed_payload| { - *payload_id == self.payload_id - || observed_payload - .flashblocks - .iter() - .flatten() - .any(|observed_flashblock| !observed_flashblock.scored) - }); - } } /// Context struct containing shared resources for the flashblocks P2P protocol. @@ -778,12 +649,31 @@ impl FlashblocksHandle { pub(crate) fn handle_reject_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.handle_reject(&self.ctx, peer_id); + let this = &mut state; + let ctx: &FlashblocksP2PCtx = &self.ctx; + let Some(peer_state) = this.connection_state(&peer_id) else { + return; + }; + + if this + .awaiting_flashblocks_req + .is_none_or(|(candidate, _)| candidate != peer_id) + { + return; + } + peer_state.lock().request_in_flight = false; + this.awaiting_flashblocks_req = None; + + this.maybe_request_receive_peers(ctx); } pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.handle_cancel(&self.ctx, peer_id); + let this = &mut state; + if let Some(peer_state) = this.connection_state(&peer_id) { + let mut peer_state = peer_state.lock(); + peer_state.send_enabled = false; + } } } @@ -1102,7 +992,7 @@ impl FlashblocksP2PCtx { &self, state: &mut FlashblocksP2PState, authorized_payload: AuthorizedPayload, - source_peer_id: Option, + _source_peer_id: Option, ) { let payload = authorized_payload.msg(); let authorization = authorized_payload.authorized.authorization; @@ -1189,10 +1079,6 @@ impl FlashblocksP2PCtx { self.peer_tx.send(peer_msg).ok(); - if source_peer_id.is_some() { - state.record_receive_observation(payload.payload_id, payload.index as usize); - } - let now = Utc::now() .timestamp_nanos_opt() .expect("time went backwards"); @@ -1338,6 +1224,10 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); + assert!(matches!( + fanout.awaiting_flashblocks_req, + Some((peer_id, _)) if peer_id == trusted_peer + )); assert!(trusted_state.lock().request_in_flight); assert!(!untrusted_state.lock().request_in_flight); match rx.try_recv().expect("request sent") { @@ -1370,12 +1260,9 @@ mod tests { victim_state.lock().send_enabled = true; fanout.insert_connection(victim, &victim_state); fanout.insert_connection(trusted_requester, &requester_state); - fanout.set_send_enabled(victim, true); fanout.handle_request(&ctx, trusted_requester); - assert!(!fanout.send_set.contains(&victim)); - assert!(fanout.send_set.contains(&trusted_requester)); assert!(!victim_state.lock().send_enabled); assert!(requester_state.lock().send_enabled); @@ -1422,7 +1309,6 @@ mod tests { current_state.lock().record_latency(42); fanout.insert_connection(current_peer, ¤t_state); fanout.insert_connection(candidate_peer, &candidate_state); - fanout.set_receive_enabled(current_peer, true); fanout.maybe_start_rotation(&ctx); @@ -1445,10 +1331,8 @@ mod tests { fanout.handle_accept(&ctx, candidate_peer); - assert!(!fanout.receive_set.contains(¤t_peer)); - assert!(fanout.receive_set.contains(&candidate_peer)); - assert!(candidate_state.lock().receive_enabled); assert!(!current_state.lock().receive_enabled); + assert!(candidate_state.lock().receive_enabled); assert!(fanout.awaiting_flashblocks_req.is_none()); match rx.try_recv().expect("cancel sent to old peer") { @@ -1462,9 +1346,8 @@ mod tests { other => panic!("unexpected peer message: {other:?}"), } - assert!(!fanout.receive_set.contains(¤t_peer)); - assert!(fanout.receive_set.contains(&candidate_peer)); assert!(!current_state.lock().receive_enabled); + assert!(candidate_state.lock().receive_enabled); assert!(fanout.awaiting_flashblocks_req.is_none()); } @@ -1483,33 +1366,18 @@ mod tests { let steady_state = test_peer_state(latency_window, false, true); let lagging_state = test_peer_state(latency_window, false, true); - { - let mut steady = steady_state.lock(); - steady.receive_enabled = true; - steady.record_latency(10); - steady.note_received_flashblock((PayloadId::default(), 0)); - } - { - let mut lagging = lagging_state.lock(); - lagging.receive_enabled = true; - lagging.record_latency(100); - } + steady_state.lock().receive_enabled = true; + lagging_state.lock().receive_enabled = true; + steady_state.lock().record_latency(10); + lagging_state.lock().record_latency(100); fanout.insert_connection(steady_peer, &steady_state); fanout.insert_connection(lagging_peer, &lagging_state); - fanout.set_receive_enabled(steady_peer, true); - fanout.set_receive_enabled(lagging_peer, true); - - fanout.record_receive_observation(PayloadId::default(), 0); - fanout - .observed_payloads - .get_mut(&PayloadId::default()) - .unwrap() - .flashblocks[0] - .as_mut() - .unwrap() - .observed_at = - Instant::now() - RECEIVE_FLASHBLOCK_GRACE_WINDOW - Duration::from_secs(1); + fanout.note_peer_received_flashblock(steady_peer, (PayloadId::default(), 0)); + + for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { + fanout.note_peer_received_flashblock(steady_peer, (PayloadId::default(), index)); + } fanout.score_expired_receive_observations(); assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); From 7d0f608caf5fc6e216ae147d0793df18a9a2f82f Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 21:14:47 -0700 Subject: [PATCH 06/43] wip --- .../p2p/src/protocol/connection.rs | 123 ++++---- .../flashblocks/p2p/src/protocol/handler.rs | 273 ++++++++++-------- 2 files changed, 216 insertions(+), 180 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index c539ad131..65bf9b1fb 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -27,66 +27,36 @@ use tracing::{info, trace}; /// minor skew/races between peers. const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; -/// A lightweight moving average with a configurable smoothing window. -#[derive(Clone, Debug)] -pub struct MovingAverage { - value: Option, - window: i64, -} - -impl MovingAverage { - pub(crate) fn new(window: i64) -> Self { - Self { - value: None, - window: window.max(1), - } - } - - pub(crate) fn record(&mut self, sample: i64) { - self.value = Some(match self.value { - Some(current) => (current * (self.window - 1) + sample) / self.window, - None => sample, - }); - } - - pub(crate) fn value(&self) -> Option { - self.value - } - - pub(crate) fn reset(&mut self) { - self.value = None; - } -} - /// Shared connection metadata for a single peer connection. #[derive(Clone, Debug)] pub struct FlashblocksConnectionState { /// Whether this peer is marked as trusted or not. pub trusted: bool, + /// Whether we have loaded the peer's trust classification from the network yet. + pub trusted_known: bool, /// Whether we currently have an outstanding flashblocks request to this peer. pub request_in_flight: bool, /// Whether we are currently sending flashblocks to this peer. pub send_enabled: bool, /// Whether we are currently requesting flashblocks from this peer. - pub receive_enabled: bool, - /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. - pub receive_enabled_timestamp: u64, - /// Score for this peer connection, used for adaptive timeouts and peer selection. /// + /// Optional score for this peer connection, used for adaptive timeouts and peer selection. /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks /// counting as 10s - pub score: MovingAverage, + pub receive_enabled: Option, + /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. + pub receive_enabled_timestamp: u64, } impl FlashblocksConnectionState { - pub(crate) fn new(latency_window: i64) -> Self { + pub(crate) fn new() -> Self { Self { trusted: false, + trusted_known: false, request_in_flight: false, send_enabled: false, - receive_enabled: false, + receive_enabled: None, receive_enabled_timestamp: 0, - score: MovingAverage::new(latency_window), } } } @@ -128,7 +98,6 @@ impl FlashblocksConnection { peer_rx: BroadcastStream, state: Arc>, ) -> Self { - protocol.handle.ensure_background_tasks(); protocol .handle .on_peer_connected(protocol.network.clone(), peer_id, state.clone()); @@ -337,7 +306,7 @@ impl FlashblocksConnection { authorized_payload: AuthorizedPayload, ) { let state_handle = self.protocol.handle.state.clone(); - let conn_state = self.state.lock().clone(); + let mut conn_state = self.state.lock(); let mut p2p_state = state_handle.lock(); let authorization = &authorized_payload.authorized.authorization; @@ -377,17 +346,6 @@ impl FlashblocksConnection { return; } - if !conn_state.receive_enabled { - trace!( - target: "flashblocks::p2p", - peer_id = %self.peer_id, - payload_id = %msg.payload_id, - index = msg.index, - "ignoring flashblock from peer outside receive set", - ); - return; - } - if msg.payload_id == p2p_state.payload_id && (msg.index as usize) .saturating_add(crate::protocol::handler::RECEIVE_FLASHBLOCK_GRACE_WINDOW) @@ -409,21 +367,21 @@ impl FlashblocksConnection { } // Check if we're expecting to see flashblocks from this peer - if !conn_state.receive_enabled - && conn_state.receive_enabled_timestamp + 2 < authorization.timestamp - { - tracing::warn!( - target: "flashblocks::p2p", - peer_id = %self.peer_id, - payload_id = %msg.payload_id, - index = msg.index, - "received flashblock from peer outside receive window", - ); - self.protocol - .network - .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + let Some(score) = conn_state.receive_enabled.as_mut() else { + if conn_state.receive_enabled_timestamp + 2 < authorization.timestamp { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock from peer outside receive window", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + } return; - } + }; // Check if this peer is spamming us with the same payload index if !p2p_state.note_peer_received_flashblock(&authorization, &msg, self.peer_id) { @@ -480,7 +438,7 @@ impl FlashblocksConnection { if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); - self.state.lock().score.record(latency); + score.record(latency); } self.protocol @@ -700,3 +658,34 @@ impl FlashblocksConnection { .ok(); } } + +/// A lightweight moving average with a configurable smoothing window. +#[derive(Clone, Debug)] +pub struct MovingAverage { + value: Option, + window: i64, +} + +impl MovingAverage { + pub(crate) fn new(window: i64) -> Self { + Self { + value: None, + window: window.max(1), + } + } + + pub(crate) fn record(&mut self, sample: i64) { + self.value = Some(match self.value { + Some(current) => (current * (self.window - 1) + sample) / self.window, + None => sample, + }); + } + + pub(crate) fn value(&self) -> Option { + self.value + } + + pub(crate) fn reset(&mut self) { + self.value = None; + } +} diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 0707ccb6e..7d666e3a9 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -62,7 +62,7 @@ const BROADCAST_BUFFER_CAPACITY: usize = 100; const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; /// Grace window in number of flashblocks to receive late flashblocks from peers before scoring them for missing flashblocks. /// -/// This must be at least long enough to cover the max authorization age to prevent a griefing +/// This must be at least long enough to cover the max authorization age to prevent a spam /// attack. pub(crate) const RECEIVE_FLASHBLOCK_GRACE_WINDOW: usize = 50; @@ -288,22 +288,20 @@ impl FlashblocksP2PState { .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) } - fn receive_peers(&self) -> HashSet { + fn num_receive_peers(&self) -> usize { self.connections .iter() - .filter_map(|(peer_id, peer_state)| { - peer_state.lock().receive_enabled.then_some(*peer_id) - }) - .collect() + .filter(|(_, peer_state)| peer_state.lock().receive_enabled) + .count() } - fn available_receive_candidates(&self) -> Vec { + fn available_receive_candidates(&self) -> Vec<(PeerId, bool, bool)> { self.connections .iter() .filter_map(|(peer_id, peer_state)| { let peer_state = peer_state.lock(); if !peer_state.receive_enabled && !peer_state.request_in_flight { - Some(*peer_id) + Some((*peer_id, peer_state.trusted, peer_state.trusted_known)) } else { None } @@ -313,7 +311,7 @@ impl FlashblocksP2PState { fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { if self.awaiting_flashblocks_req.is_some() - || self.receive_peer_count() >= ctx.fanout_config.max_receive_peers + || self.num_receive_peers() >= ctx.fanout_config.max_receive_peers { return; } @@ -322,34 +320,50 @@ impl FlashblocksP2PState { if candidates.is_empty() { return; } - let rand = rand::rng().random_range(0..candidates.len()); - let peer_id = candidates[rand]; - - // get seconds since unix epoch for request timestamp - let requested_at = Utc::now().timestamp() as u64; + let trusted_candidates: Vec<_> = candidates + .iter() + .filter_map(|(peer_id, trusted, trusted_known)| { + (*trusted && *trusted_known).then_some(*peer_id) + }) + .collect(); + let candidate_pool = if trusted_candidates.is_empty() { + candidates + .iter() + .map(|(peer_id, _, _)| *peer_id) + .collect::>() + } else { + trusted_candidates + }; + let rand = rand::rng().random_range(0..candidate_pool.len()); + let peer_id = candidate_pool[rand]; - self.awaiting_flashblocks_req = Some((peer_id, requested_at)); + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; + peer_state.lock().request_in_flight = true; + self.awaiting_flashblocks_req = Some((peer_id, Instant::now())); ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } fn worst_receive_peer(&self) -> Option { self.connections .iter() - .filter_map(|(peer_id, peer_state)| { - let peer_state = peer_state.lock(); - if peer_state.receive_enabled { - peer_state.score().map(|score| (*peer_id, score)) - } else { - None - } - }) - .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) - .map(|(peer_id, _)| peer_id) + .find(|(_, peer_state)| peer_state.lock().receive_enabled) + // .filter_map(|(peer_id, peer_state)| { + // let peer_state = peer_state.lock(); + // if peer_state.receive_enabled { + // peer_state.score.value().map(|score| (*peer_id, score)) + // } else { + // None + // } + // }) + // .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) + // .map(|(peer_id, _)| peer_id) } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { if self.awaiting_flashblocks_req.is_some() - || self.receive_peer_count() < ctx.fanout_config.max_receive_peers + || self.num_receive_peers() < ctx.fanout_config.max_receive_peers { return; } @@ -365,16 +379,18 @@ impl FlashblocksP2PState { let mut rng = rand::rng(); candidates.shuffle(&mut rng); - let candidate = candidates[0]; + let candidate = candidates + .iter() + .find_map(|(peer_id, trusted, trusted_known)| { + (*trusted && *trusted_known).then_some(*peer_id) + }) + .unwrap_or(candidates[0].0); let Some(candidate_state) = self.connection_state(&candidate) else { return; }; - drop(candidate_state); - - let requested_at = Instant::now(); - self.start_requesting_peer(candidate); - self.awaiting_flashblocks_req = Some((candidate, requested_at)); + candidate_state.lock().request_in_flight = true; + self.awaiting_flashblocks_req = Some((candidate, Instant::now())); ctx.send_direct(candidate, FlashblocksP2PMsg::RequestFlashblocks); } @@ -387,7 +403,9 @@ impl FlashblocksP2PState { return; } - self.set_request_in_flight(candidate, false); + if let Some(candidate_state) = self.connection_state(&candidate) { + candidate_state.lock().request_in_flight = false; + } self.awaiting_flashblocks_req = None; self.maybe_request_receive_peers(ctx); } @@ -410,32 +428,53 @@ impl FlashblocksP2PState { return; }; - let peer_is_trusted = peer_state.lock().trusted; - - if self.send_enabled(&peer_id) { + if peer_state.lock().send_enabled { ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } + let peer_is_trusted = peer_state.lock().trusted; if peer_is_trusted { - if self.non_trusted_send_count() >= ctx.fanout_config.max_send_peers { + let non_trusted_send_count = self + .connections + .values() + .filter(|candidate_state| { + let candidate_state = candidate_state.lock(); + candidate_state.send_enabled && !candidate_state.trusted + }) + .count(); + if non_trusted_send_count >= ctx.fanout_config.max_send_peers { if let Some(evicted_peer) = - self.connections.keys().copied().find(|candidate| { - self.send_enabled(candidate) && !self.is_trusted(candidate) - }) + self.connections + .iter() + .find_map(|(candidate, candidate_state)| { + let candidate_state = candidate_state.lock(); + (candidate_state.send_enabled && !candidate_state.trusted) + .then_some(*candidate) + }) { - self.set_send_enabled(evicted_peer, false); + if let Some(evicted_state) = self.connection_state(&evicted_peer) { + evicted_state.lock().send_enabled = false; + } ctx.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); } } - self.set_send_enabled(peer_id, true); + peer_state.lock().send_enabled = true; ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } - if self.non_trusted_send_count() < ctx.fanout_config.max_send_peers { - self.set_send_enabled(peer_id, true); + let non_trusted_send_count = self + .connections + .values() + .filter(|candidate_state| { + let candidate_state = candidate_state.lock(); + candidate_state.send_enabled && !candidate_state.trusted + }) + .count(); + if non_trusted_send_count < ctx.fanout_config.max_send_peers { + peer_state.lock().send_enabled = true; ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); } else { ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); @@ -453,18 +492,20 @@ impl FlashblocksP2PState { { return; } - let evict = if self.receive_peer_count() >= ctx.fanout_config.max_receive_peers - && !self.receive_enabled(&peer_id) + let evict = if self.num_receive_peers() >= ctx.fanout_config.max_receive_peers + && !peer_state.lock().receive_enabled { self.worst_receive_peer() } else { None }; + let timestamp = Utc::now().timestamp() as u64; { let mut peer_state = peer_state.lock(); peer_state.receive_enabled = true; peer_state.request_in_flight = false; + peer_state.receive_enabled_timestamp = timestamp; } if self @@ -479,9 +520,10 @@ impl FlashblocksP2PState { let mut evict_state = evict_state.lock(); evict_state.receive_enabled = false; evict_state.request_in_flight = false; - evict_state.reset_receive_tracking(); - ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + evict_state.receive_enabled_timestamp = timestamp; + evict_state.score.reset(); } + ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); } self.maybe_request_receive_peers(ctx); @@ -507,8 +549,6 @@ pub struct FlashblocksP2PCtx { /// Broadcast sender for verified and strictly ordered flashblock payloads. /// Used by RPC overlays and other consumers of flashblock data. pub flashblock_tx: broadcast::Sender, - /// Ensures rotation/background tasks are only started once per handle. - background_tasks_started: Arc, } /// Handle for the flashblocks P2P protocol. @@ -543,10 +583,26 @@ impl FlashblocksHandle { fanout_config, peer_tx, flashblock_tx, - background_tasks_started: Arc::new(AtomicBool::new(false)), }; + let handle = Self { ctx, state }; + let moved_handle = handle.clone(); - Self { ctx, state } + tokio::spawn(async move { + let mut rotation_interval = + time::interval(moved_handle.ctx.fanout_config.rotation_interval); + rotation_interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); + rotation_interval.tick().await; + + loop { + rotation_interval.tick().await; + moved_handle + .state + .lock() + .maybe_start_rotation(&moved_handle.ctx) + } + }); + + handle } pub fn flashblocks_tx(&self) -> broadcast::Sender { @@ -560,36 +616,6 @@ impl FlashblocksHandle { .ok_or(FlashblocksP2PError::MissingBuilderSk) } - pub(crate) fn ensure_background_tasks(&self) { - if self - .ctx - .background_tasks_started - .swap(true, Ordering::AcqRel) - { - return; - } - - let handle = self.clone(); - tokio::spawn(async move { - let mut rotation_interval = time::interval(handle.ctx.fanout_config.rotation_interval); - let mut timeout_interval = time::interval(Duration::from_secs(1)); - - loop { - tokio::select! { - _ = rotation_interval.tick() => { - let mut state = handle.state.lock(); - state.maybe_start_rotation(&handle.ctx); - } - _ = timeout_interval.tick() => { - let mut state = handle.state.lock(); - state.check_rotation_timeout(&handle.ctx); - state.score_expired_receive_observations(); - } - } - } - }); - } - pub(crate) fn on_peer_connected( &self, network: N, @@ -598,7 +624,7 @@ impl FlashblocksHandle { ) { { let mut state = self.state.lock(); - state.insert_connection(peer_id, &fanout_state); + state.connections.insert(peer_id, fanout_state.clone()); state.maybe_request_receive_peers(&self.ctx); } @@ -1158,9 +1184,7 @@ impl ConnectionHandler for FlashblocksP2PProtoco ); let peer_rx = self.handle.ctx.peer_tx.subscribe(); - let fanout_state = Arc::new(Mutex::new(FlashblocksConnectionState::new( - self.handle.ctx.fanout_config.latency_window, - ))); + let fanout_state = Arc::new(Mutex::new(FlashblocksConnectionState::new())); FlashblocksConnection::new( self, @@ -1174,6 +1198,8 @@ impl ConnectionHandler for FlashblocksP2PProtoco #[cfg(test)] mod tests { + use crate::protocol::connection::Score; + use super::*; use ed25519_dalek::SigningKey; @@ -1186,16 +1212,14 @@ mod tests { fanout_config: config, peer_tx: broadcast::Sender::new(16), flashblock_tx: broadcast::Sender::new(16), - background_tasks_started: Arc::new(AtomicBool::new(false)), } } fn test_peer_state( - latency_window: i64, trusted: bool, trusted_known: bool, ) -> Arc> { - let state = Arc::new(Mutex::new(FlashblocksConnectionState::new(latency_window))); + let state = Arc::new(Mutex::new(FlashblocksConnectionState::new())); { let mut state_guard = state.lock(); state_guard.trusted = trusted; @@ -1210,17 +1234,20 @@ mod tests { max_receive_peers: 1, ..Default::default() }; - let latency_window = config.latency_window; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); let mut rx = ctx.peer_tx.subscribe(); let trusted_peer = PeerId::random(); let untrusted_peer = PeerId::random(); - let trusted_state = test_peer_state(latency_window, true, true); - let untrusted_state = test_peer_state(latency_window, false, true); - fanout.insert_connection(trusted_peer, &trusted_state); - fanout.insert_connection(untrusted_peer, &untrusted_state); + let trusted_state = test_peer_state(true, true); + let untrusted_state = test_peer_state(false, true); + fanout + .connections + .insert(trusted_peer, trusted_state.clone()); + fanout + .connections + .insert(untrusted_peer, untrusted_state.clone()); fanout.maybe_request_receive_peers(&ctx); @@ -1255,11 +1282,13 @@ mod tests { let victim = PeerId::random(); let trusted_requester = PeerId::random(); - let victim_state = test_peer_state(latency_window, false, true); - let requester_state = test_peer_state(latency_window, true, true); + let victim_state = test_peer_state(false, true); + let requester_state = test_peer_state(true, true); victim_state.lock().send_enabled = true; - fanout.insert_connection(victim, &victim_state); - fanout.insert_connection(trusted_requester, &requester_state); + fanout.connections.insert(victim, victim_state.clone()); + fanout + .connections + .insert(trusted_requester, requester_state.clone()); fanout.handle_request(&ctx, trusted_requester); @@ -1303,12 +1332,17 @@ mod tests { let current_peer = PeerId::random(); let candidate_peer = PeerId::random(); - let current_state = test_peer_state(latency_window, false, true); - let candidate_state = test_peer_state(latency_window, false, true); - current_state.lock().receive_enabled = true; - current_state.lock().record_latency(42); - fanout.insert_connection(current_peer, ¤t_state); - fanout.insert_connection(candidate_peer, &candidate_state); + let current_state = test_peer_state(false, true); + let candidate_state = test_peer_state(false, true); + let mut score = Score::new(latency_window); + score.record(42); + current_state.lock().receive_enabled = Some(score); + fanout + .connections + .insert(current_peer, current_state.clone()); + fanout + .connections + .insert(candidate_peer, candidate_state.clone()); fanout.maybe_start_rotation(&ctx); @@ -1365,25 +1399,38 @@ mod tests { let lagging_peer = PeerId::random(); let steady_state = test_peer_state(latency_window, false, true); let lagging_state = test_peer_state(latency_window, false, true); + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); steady_state.lock().receive_enabled = true; lagging_state.lock().receive_enabled = true; - steady_state.lock().record_latency(10); - lagging_state.lock().record_latency(100); + steady_state.lock().score.record(10); + lagging_state.lock().score.record(100); - fanout.insert_connection(steady_peer, &steady_state); - fanout.insert_connection(lagging_peer, &lagging_state); - fanout.note_peer_received_flashblock(steady_peer, (PayloadId::default(), 0)); + fanout.connections.insert(steady_peer, steady_state.clone()); + fanout + .connections + .insert(lagging_peer, lagging_state.clone()); for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { - fanout.note_peer_received_flashblock(steady_peer, (PayloadId::default(), index)); + let authorization = Authorization::new( + PayloadId::default(), + index as u64, + &authorizer, + builder.verifying_key(), + ); + let flashblock = FlashblocksPayloadV1 { + payload_id: PayloadId::default(), + index: index as u64, + ..Default::default() + }; + fanout.note_peer_received_flashblock(&authorization, &flashblock, steady_peer); } - fanout.score_expired_receive_observations(); assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); - assert_eq!(steady_state.lock().score(), Some(10)); + assert_eq!(steady_state.lock().score.value(), Some(10)); assert_eq!( - lagging_state.lock().score(), + lagging_state.lock().score.value(), Some((100 * (latency_window - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / latency_window) ); } From 78be1cd891fbe224530b7f84ec1ca70dd839e0c4 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 21:37:51 -0700 Subject: [PATCH 07/43] wip --- .../p2p/src/protocol/connection.rs | 17 +- .../flashblocks/p2p/src/protocol/handler.rs | 164 ++++++++---------- 2 files changed, 80 insertions(+), 101 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 65bf9b1fb..97b557f1c 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -43,7 +43,7 @@ pub struct FlashblocksConnectionState { /// Optional score for this peer connection, used for adaptive timeouts and peer selection. /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks /// counting as 10s - pub receive_enabled: Option, + pub receive_enabled: Option, /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. pub receive_enabled_timestamp: u64, } @@ -122,7 +122,12 @@ impl Drop for FlashblocksConnection { "dropping flashblocks connection" ); - self.protocol.handle.on_peer_disconnected(self.peer_id); + let handle = &self.protocol.handle; + let peer_id = self.peer_id; + let mut state = handle.state.lock(); + state.connections.remove(&peer_id); + state.maybe_request_receive_peers(&handle.ctx); + gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).decrement(1); } } @@ -661,12 +666,12 @@ impl FlashblocksConnection { /// A lightweight moving average with a configurable smoothing window. #[derive(Clone, Debug)] -pub struct MovingAverage { +pub struct Score { value: Option, window: i64, } -impl MovingAverage { +impl Score { pub(crate) fn new(window: i64) -> Self { Self { value: None, @@ -684,8 +689,4 @@ impl MovingAverage { pub(crate) fn value(&self) -> Option { self.value } - - pub(crate) fn reset(&mut self) { - self.value = None; - } } diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 7d666e3a9..1f7e1d858 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,5 +1,5 @@ use crate::protocol::{ - connection::{FlashblocksConnection, FlashblocksConnectionState}, + connection::{FlashblocksConnection, FlashblocksConnectionState, Score}, error::FlashblocksP2PError, }; use alloy_rlp::BytesMut; @@ -23,10 +23,7 @@ use reth_network::Peers; use std::{ collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, + sync::Arc, time::Duration, }; use tokio::{ @@ -246,18 +243,19 @@ impl FlashblocksP2PState { let evicted = self.observed_payloads.pop_front().unwrap(); for (peer_id, connection) in self.connections.iter() { let mut connection = connection.lock(); - if connection.receive_enabled - && connection.receive_enabled_timestamp < evicted.timestamp + 2 + if connection.receive_enabled_timestamp < evicted.timestamp + 2 && !evicted.received_peers.contains(peer_id) { - debug!( - target: "flashblocks::p2p", - %peer_id, - payload_id = %evicted.payload_id, - flashblock_index = evicted.flashblock_index, - "scoring peer for missed flashblock", - ); - connection.score.record(MISSED_FLASHBLOCK_PENALTY_NS); + if let Some(score) = connection.receive_enabled.as_mut() { + debug!( + target: "flashblocks::p2p", + %peer_id, + payload_id = %evicted.payload_id, + flashblock_index = evicted.flashblock_index, + "scoring peer for missed flashblock", + ); + score.record(MISSED_FLASHBLOCK_PENALTY_NS); + } } } } @@ -291,7 +289,7 @@ impl FlashblocksP2PState { fn num_receive_peers(&self) -> usize { self.connections .iter() - .filter(|(_, peer_state)| peer_state.lock().receive_enabled) + .filter(|(_, peer_state)| peer_state.lock().receive_enabled.is_some()) .count() } @@ -300,7 +298,7 @@ impl FlashblocksP2PState { .iter() .filter_map(|(peer_id, peer_state)| { let peer_state = peer_state.lock(); - if !peer_state.receive_enabled && !peer_state.request_in_flight { + if peer_state.receive_enabled.is_none() && !peer_state.request_in_flight { Some((*peer_id, peer_state.trusted, peer_state.trusted_known)) } else { None @@ -309,7 +307,7 @@ impl FlashblocksP2PState { .collect() } - fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { + pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { if self.awaiting_flashblocks_req.is_some() || self.num_receive_peers() >= ctx.fanout_config.max_receive_peers { @@ -348,17 +346,15 @@ impl FlashblocksP2PState { fn worst_receive_peer(&self) -> Option { self.connections .iter() - .find(|(_, peer_state)| peer_state.lock().receive_enabled) - // .filter_map(|(peer_id, peer_state)| { - // let peer_state = peer_state.lock(); - // if peer_state.receive_enabled { - // peer_state.score.value().map(|score| (*peer_id, score)) - // } else { - // None - // } - // }) - // .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) - // .map(|(peer_id, _)| peer_id) + .filter_map(|(peer_id, peer_state)| { + let peer_state = peer_state.lock(); + peer_state + .receive_enabled + .as_ref() + .and_then(|score| score.value().map(|score| (*peer_id, score))) + }) + .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) + .map(|(peer_id, _)| peer_id) } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { @@ -394,35 +390,6 @@ impl FlashblocksP2PState { ctx.send_direct(candidate, FlashblocksP2PMsg::RequestFlashblocks); } - fn check_rotation_timeout(&mut self, ctx: &FlashblocksP2PCtx) { - let Some((candidate, requested_at)) = self.awaiting_flashblocks_req else { - return; - }; - - if requested_at.elapsed() < ctx.fanout_config.request_flashblocks_timeout { - return; - } - - if let Some(candidate_state) = self.connection_state(&candidate) { - candidate_state.lock().request_in_flight = false; - } - self.awaiting_flashblocks_req = None; - self.maybe_request_receive_peers(ctx); - } - - fn handle_disconnect(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - self.connections.remove(&peer_id); - - if self - .awaiting_flashblocks_req - .is_some_and(|(candidate, _)| candidate == peer_id) - { - self.awaiting_flashblocks_req = None; - } - - self.maybe_request_receive_peers(ctx); - } - fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { let Some(peer_state) = self.connection_state(&peer_id) else { return; @@ -493,7 +460,7 @@ impl FlashblocksP2PState { return; } let evict = if self.num_receive_peers() >= ctx.fanout_config.max_receive_peers - && !peer_state.lock().receive_enabled + && peer_state.lock().receive_enabled.is_none() { self.worst_receive_peer() } else { @@ -503,7 +470,9 @@ impl FlashblocksP2PState { { let mut peer_state = peer_state.lock(); - peer_state.receive_enabled = true; + peer_state + .receive_enabled + .get_or_insert_with(|| Score::new(ctx.fanout_config.latency_window)); peer_state.request_in_flight = false; peer_state.receive_enabled_timestamp = timestamp; } @@ -518,10 +487,9 @@ impl FlashblocksP2PState { if let Some(evict) = evict.filter(|evict| *evict != peer_id) { if let Some(evict_state) = self.connection_state(&evict) { let mut evict_state = evict_state.lock(); - evict_state.receive_enabled = false; + evict_state.receive_enabled = None; evict_state.request_in_flight = false; evict_state.receive_enabled_timestamp = timestamp; - evict_state.score.reset(); } ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); } @@ -605,17 +573,6 @@ impl FlashblocksHandle { handle } - pub fn flashblocks_tx(&self) -> broadcast::Sender { - self.ctx.flashblock_tx.clone() - } - - pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { - self.ctx - .builder_sk - .as_ref() - .ok_or(FlashblocksP2PError::MissingBuilderSk) - } - pub(crate) fn on_peer_connected( &self, network: N, @@ -658,11 +615,6 @@ impl FlashblocksHandle { }); } - pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { - let mut state = self.state.lock(); - state.handle_disconnect(&self.ctx, peer_id); - } - pub(crate) fn handle_request_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); state.handle_request(&self.ctx, peer_id); @@ -761,6 +713,14 @@ impl FlashblocksP2PProtocol { } impl FlashblocksHandle { + /// Returns the builder signing key if configured. + pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { + self.ctx + .builder_sk + .as_ref() + .ok_or(FlashblocksP2PError::MissingBuilderSk) + } + /// Publishes a newly created flashblock from the payload builder to the P2P network. /// /// This method validates that the builder has authorization to publish and that @@ -1198,8 +1158,6 @@ impl ConnectionHandler for FlashblocksP2PProtoco #[cfg(test)] mod tests { - use crate::protocol::connection::Score; - use super::*; use ed25519_dalek::SigningKey; @@ -1275,7 +1233,6 @@ mod tests { max_send_peers: 1, ..Default::default() }; - let latency_window = config.latency_window; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); let mut rx = ctx.peer_tx.subscribe(); @@ -1365,8 +1322,8 @@ mod tests { fanout.handle_accept(&ctx, candidate_peer); - assert!(!current_state.lock().receive_enabled); - assert!(candidate_state.lock().receive_enabled); + assert!(current_state.lock().receive_enabled.is_none()); + assert!(candidate_state.lock().receive_enabled.is_some()); assert!(fanout.awaiting_flashblocks_req.is_none()); match rx.try_recv().expect("cancel sent to old peer") { @@ -1380,8 +1337,8 @@ mod tests { other => panic!("unexpected peer message: {other:?}"), } - assert!(!current_state.lock().receive_enabled); - assert!(candidate_state.lock().receive_enabled); + assert!(current_state.lock().receive_enabled.is_none()); + assert!(candidate_state.lock().receive_enabled.is_some()); assert!(fanout.awaiting_flashblocks_req.is_none()); } @@ -1397,15 +1354,25 @@ mod tests { let steady_peer = PeerId::random(); let lagging_peer = PeerId::random(); - let steady_state = test_peer_state(latency_window, false, true); - let lagging_state = test_peer_state(latency_window, false, true); + let steady_state = test_peer_state(false, true); + let lagging_state = test_peer_state(false, true); let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); - steady_state.lock().receive_enabled = true; - lagging_state.lock().receive_enabled = true; - steady_state.lock().score.record(10); - lagging_state.lock().score.record(100); + steady_state.lock().receive_enabled = Some(Score::new(latency_window)); + lagging_state.lock().receive_enabled = Some(Score::new(latency_window)); + steady_state + .lock() + .receive_enabled + .as_mut() + .expect("steady peer score") + .record(10); + lagging_state + .lock() + .receive_enabled + .as_mut() + .expect("lagging peer score") + .record(100); fanout.connections.insert(steady_peer, steady_state.clone()); fanout @@ -1428,9 +1395,20 @@ mod tests { } assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); - assert_eq!(steady_state.lock().score.value(), Some(10)); assert_eq!( - lagging_state.lock().score.value(), + steady_state + .lock() + .receive_enabled + .as_ref() + .and_then(Score::value), + Some(10) + ); + assert_eq!( + lagging_state + .lock() + .receive_enabled + .as_ref() + .and_then(Score::value), Some((100 * (latency_window - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / latency_window) ); } From 31c1013ab896561a8f92b2b5465cf29633a8e901 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 21:59:57 -0700 Subject: [PATCH 08/43] wip --- .../p2p/src/protocol/connection.rs | 20 +- .../flashblocks/p2p/src/protocol/handler.rs | 300 ++++++++++++------ specs/flashblocks_p2p_v2.md | 57 ++-- 3 files changed, 225 insertions(+), 152 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 97b557f1c..25785aca6 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -122,11 +122,7 @@ impl Drop for FlashblocksConnection { "dropping flashblocks connection" ); - let handle = &self.protocol.handle; - let peer_id = self.peer_id; - let mut state = handle.state.lock(); - state.connections.remove(&peer_id); - state.maybe_request_receive_peers(&handle.ctx); + self.protocol.handle.on_peer_disconnected(self.peer_id); gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).decrement(1); } @@ -543,13 +539,6 @@ impl FlashblocksConnection { } }); - let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); - self.protocol - .handle - .ctx - .peer_tx - .send(PeerMsg::StartPublishing(p2p_msg.encode())) - .ok(); } /// Handles incoming `StopPublish` messages from a peer. @@ -654,13 +643,6 @@ impl FlashblocksConnection { } }); - let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); - self.protocol - .handle - .ctx - .peer_tx - .send(PeerMsg::StopPublishing(p2p_msg.encode())) - .ok(); } } diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 1f7e1d858..92071393f 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -28,7 +28,7 @@ use std::{ }; use tokio::{ sync::{broadcast, watch}, - time::{self, Instant}, + time, }; use tracing::{debug, info, warn}; @@ -184,8 +184,8 @@ pub struct FlashblocksP2PState { pub observed_payloads: VecDeque, /// All currently connected peers and their shared connection state. pub connections: HashMap>>, - /// State for an ongoing rotation, if any. - pub awaiting_flashblocks_req: Option<(PeerId, Instant)>, + /// The peer currently occupying the outstanding request slot, if any. + pub awaiting_flashblocks_req: Option, } impl Default for FlashblocksP2PState { @@ -293,13 +293,16 @@ impl FlashblocksP2PState { .count() } - fn available_receive_candidates(&self) -> Vec<(PeerId, bool, bool)> { + fn available_receive_candidates(&self) -> Vec<(PeerId, bool)> { self.connections .iter() .filter_map(|(peer_id, peer_state)| { let peer_state = peer_state.lock(); - if peer_state.receive_enabled.is_none() && !peer_state.request_in_flight { - Some((*peer_id, peer_state.trusted, peer_state.trusted_known)) + if peer_state.trusted_known + && peer_state.receive_enabled.is_none() + && !peer_state.request_in_flight + { + Some((*peer_id, peer_state.trusted)) } else { None } @@ -307,10 +310,22 @@ impl FlashblocksP2PState { .collect() } + fn begin_requesting_peer(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state(&peer_id) else { + return; + }; + let timestamp = Utc::now().timestamp() as u64; + let mut peer_state = peer_state.lock(); + peer_state.request_in_flight = true; + peer_state.receive_enabled = Some(Score::new(ctx.fanout_config.latency_window)); + peer_state.receive_enabled_timestamp = timestamp; + drop(peer_state); + self.awaiting_flashblocks_req = Some(peer_id); + ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + } + pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { - if self.awaiting_flashblocks_req.is_some() - || self.num_receive_peers() >= ctx.fanout_config.max_receive_peers - { + if self.num_receive_peers() >= ctx.fanout_config.max_receive_peers { return; } @@ -320,30 +335,21 @@ impl FlashblocksP2PState { } let trusted_candidates: Vec<_> = candidates .iter() - .filter_map(|(peer_id, trusted, trusted_known)| { - (*trusted && *trusted_known).then_some(*peer_id) - }) + .filter_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) .collect(); let candidate_pool = if trusted_candidates.is_empty() { candidates .iter() - .map(|(peer_id, _, _)| *peer_id) + .map(|(peer_id, _)| *peer_id) .collect::>() } else { trusted_candidates }; let rand = rand::rng().random_range(0..candidate_pool.len()); - let peer_id = candidate_pool[rand]; - - let Some(peer_state) = self.connection_state(&peer_id) else { - return; - }; - peer_state.lock().request_in_flight = true; - self.awaiting_flashblocks_req = Some((peer_id, Instant::now())); - ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + self.begin_requesting_peer(ctx, candidate_pool[rand]); } - fn worst_receive_peer(&self) -> Option { + fn worst_receive_peer(&self) -> Option<(PeerId, i64)> { self.connections .iter() .filter_map(|(peer_id, peer_state)| { @@ -354,19 +360,16 @@ impl FlashblocksP2PState { .and_then(|score| score.value().map(|score| (*peer_id, score))) }) .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) - .map(|(peer_id, _)| peer_id) } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { - if self.awaiting_flashblocks_req.is_some() - || self.num_receive_peers() < ctx.fanout_config.max_receive_peers - { + if self.num_receive_peers() < ctx.fanout_config.max_receive_peers { return; } - if self.worst_receive_peer().is_none() { + let Some((evict, _)) = self.worst_receive_peer() else { return; - } + }; let mut candidates = self.available_receive_candidates(); if candidates.is_empty() { @@ -377,17 +380,22 @@ impl FlashblocksP2PState { candidates.shuffle(&mut rng); let candidate = candidates .iter() - .find_map(|(peer_id, trusted, trusted_known)| { - (*trusted && *trusted_known).then_some(*peer_id) - }) + .find_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) .unwrap_or(candidates[0].0); - let Some(candidate_state) = self.connection_state(&candidate) else { - return; - }; - candidate_state.lock().request_in_flight = true; - self.awaiting_flashblocks_req = Some((candidate, Instant::now())); - ctx.send_direct(candidate, FlashblocksP2PMsg::RequestFlashblocks); + if let Some(evict_state) = self.connection_state(&evict) { + let timestamp = Utc::now().timestamp() as u64; + let mut evict_state = evict_state.lock(); + evict_state.receive_enabled = None; + evict_state.request_in_flight = false; + evict_state.receive_enabled_timestamp = timestamp; + } + if self.awaiting_flashblocks_req == Some(evict) { + self.awaiting_flashblocks_req = None; + } + ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + + self.begin_requesting_peer(ctx, candidate); } fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { @@ -448,53 +456,17 @@ impl FlashblocksP2PState { } } - fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + fn handle_accept(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) { let Some(peer_state) = self.connection_state(&peer_id) else { return; }; - if self - .awaiting_flashblocks_req - .is_none_or(|(candidate, _)| candidate != peer_id) - { + if self.awaiting_flashblocks_req != Some(peer_id) { return; } - let evict = if self.num_receive_peers() >= ctx.fanout_config.max_receive_peers - && peer_state.lock().receive_enabled.is_none() - { - self.worst_receive_peer() - } else { - None - }; - let timestamp = Utc::now().timestamp() as u64; - { - let mut peer_state = peer_state.lock(); - peer_state - .receive_enabled - .get_or_insert_with(|| Score::new(ctx.fanout_config.latency_window)); - peer_state.request_in_flight = false; - peer_state.receive_enabled_timestamp = timestamp; - } - - if self - .awaiting_flashblocks_req - .is_some_and(|(candidate, _)| candidate == peer_id) - { - self.awaiting_flashblocks_req = None; - } - - if let Some(evict) = evict.filter(|evict| *evict != peer_id) { - if let Some(evict_state) = self.connection_state(&evict) { - let mut evict_state = evict_state.lock(); - evict_state.receive_enabled = None; - evict_state.request_in_flight = false; - evict_state.receive_enabled_timestamp = timestamp; - } - ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); - } - - self.maybe_request_receive_peers(ctx); + peer_state.lock().request_in_flight = false; + self.awaiting_flashblocks_req = None; } } @@ -615,6 +587,15 @@ impl FlashblocksHandle { }); } + pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.connections.remove(&peer_id); + if state.awaiting_flashblocks_req == Some(peer_id) { + state.awaiting_flashblocks_req = None; + } + state.maybe_request_receive_peers(&self.ctx); + } + pub(crate) fn handle_request_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); state.handle_request(&self.ctx, peer_id); @@ -633,13 +614,14 @@ impl FlashblocksHandle { return; }; - if this - .awaiting_flashblocks_req - .is_none_or(|(candidate, _)| candidate != peer_id) - { + if this.awaiting_flashblocks_req != Some(peer_id) { return; } - peer_state.lock().request_in_flight = false; + { + let mut peer_state = peer_state.lock(); + peer_state.request_in_flight = false; + peer_state.receive_enabled = None; + } this.awaiting_flashblocks_req = None; this.maybe_request_receive_peers(ctx); @@ -648,9 +630,21 @@ impl FlashblocksHandle { pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); let this = &mut state; - if let Some(peer_state) = this.connection_state(&peer_id) { + let mut should_refill = false; + if let Some(peer_state) = this.connection_state(&peer_id).cloned() { let mut peer_state = peer_state.lock(); peer_state.send_enabled = false; + if peer_state.receive_enabled.is_some() || peer_state.request_in_flight { + peer_state.receive_enabled = None; + peer_state.request_in_flight = false; + if this.awaiting_flashblocks_req == Some(peer_id) { + this.awaiting_flashblocks_req = None; + } + should_refill = true; + } + } + if should_refill { + this.maybe_request_receive_peers(&self.ctx); } } } @@ -1211,9 +1205,10 @@ mod tests { assert!(matches!( fanout.awaiting_flashblocks_req, - Some((peer_id, _)) if peer_id == trusted_peer + Some(peer_id) if peer_id == trusted_peer )); assert!(trusted_state.lock().request_in_flight); + assert!(trusted_state.lock().receive_enabled.is_some()); assert!(!untrusted_state.lock().request_in_flight); match rx.try_recv().expect("request sent") { PeerMsg::Direct { peer_id, bytes } => { @@ -1227,6 +1222,35 @@ mod tests { } } + #[test] + fn unknown_peers_are_not_requested_until_trust_is_known() { + let config = FanoutConfig { + max_receive_peers: 1, + ..Default::default() + }; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let trusted_peer = PeerId::random(); + let untrusted_peer = PeerId::random(); + let trusted_state = test_peer_state(true, false); + let untrusted_state = test_peer_state(false, false); + fanout + .connections + .insert(trusted_peer, trusted_state.clone()); + fanout + .connections + .insert(untrusted_peer, untrusted_state.clone()); + + fanout.maybe_request_receive_peers(&ctx); + + assert!(fanout.awaiting_flashblocks_req.is_none()); + assert!(!trusted_state.lock().request_in_flight); + assert!(!untrusted_state.lock().request_in_flight); + assert!(rx.try_recv().is_err()); + } + #[test] fn trusted_request_evicts_non_trusted_sender() { let config = FanoutConfig { @@ -1276,7 +1300,7 @@ mod tests { } #[test] - fn rotation_accepts_candidate_and_cancels_current_peer() { + fn rotation_replaces_peer_before_requesting_candidate() { let config = FanoutConfig { max_receive_peers: 1, latency_window: 4, @@ -1303,41 +1327,39 @@ mod tests { fanout.maybe_start_rotation(&ctx); + assert!(current_state.lock().receive_enabled.is_none()); assert!(candidate_state.lock().request_in_flight); + assert!(candidate_state.lock().receive_enabled.is_some()); assert!(matches!( fanout.awaiting_flashblocks_req, - Some((candidate, _)) if candidate == candidate_peer + Some(candidate) if candidate == candidate_peer )); - match rx.try_recv().expect("rotation request sent") { + match rx.try_recv().expect("cancel sent to old peer") { PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, candidate_peer); + assert_eq!(peer_id, current_peer); assert_eq!( FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::RequestFlashblocks + FlashblocksP2PMsg::CancelFlashblocks ); } other => panic!("unexpected peer message: {other:?}"), } - fanout.handle_accept(&ctx, candidate_peer); - - assert!(current_state.lock().receive_enabled.is_none()); - assert!(candidate_state.lock().receive_enabled.is_some()); - assert!(fanout.awaiting_flashblocks_req.is_none()); - - match rx.try_recv().expect("cancel sent to old peer") { + match rx.try_recv().expect("rotation request sent") { PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, current_peer); + assert_eq!(peer_id, candidate_peer); assert_eq!( FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::CancelFlashblocks + FlashblocksP2PMsg::RequestFlashblocks ); } other => panic!("unexpected peer message: {other:?}"), } - assert!(current_state.lock().receive_enabled.is_none()); + fanout.handle_accept(&ctx, candidate_peer); + + assert!(!candidate_state.lock().request_in_flight); assert!(candidate_state.lock().receive_enabled.is_some()); assert!(fanout.awaiting_flashblocks_req.is_none()); } @@ -1394,7 +1416,7 @@ mod tests { fanout.note_peer_received_flashblock(&authorization, &flashblock, steady_peer); } - assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); + assert_eq!(fanout.worst_receive_peer().map(|(peer_id, _)| peer_id), Some(lagging_peer)); assert_eq!( steady_state .lock() @@ -1412,4 +1434,82 @@ mod tests { Some((100 * (latency_window - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / latency_window) ); } + + #[test] + fn pending_candidate_is_rotated_out_after_missing_blocks() { + let config = FanoutConfig { + max_receive_peers: 2, + latency_window: 4, + ..Default::default() + }; + let latency_window = config.latency_window; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let steady_peer = PeerId::random(); + let rotating_peer = PeerId::random(); + let candidate_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + + let steady_state = test_peer_state(false, true); + let rotating_state = test_peer_state(false, true); + let candidate_state = test_peer_state(true, true); + let replacement_state = test_peer_state(true, true); + + steady_state.lock().receive_enabled = Some(Score::new(latency_window)); + rotating_state.lock().receive_enabled = Some(Score::new(latency_window)); + steady_state + .lock() + .receive_enabled + .as_mut() + .expect("steady peer score") + .record(10); + rotating_state + .lock() + .receive_enabled + .as_mut() + .expect("rotating peer score") + .record(100); + + fanout.connections.insert(steady_peer, steady_state.clone()); + fanout + .connections + .insert(rotating_peer, rotating_state.clone()); + fanout + .connections + .insert(candidate_peer, candidate_state.clone()); + + fanout.maybe_start_rotation(&ctx); + + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let authorization = Authorization::new( + PayloadId::default(), + index as u64, + &authorizer, + builder.verifying_key(), + ); + let flashblock = FlashblocksPayloadV1 { + payload_id: PayloadId::default(), + index: index as u64, + ..Default::default() + }; + fanout.note_peer_received_flashblock(&authorization, &flashblock, steady_peer); + } + + assert_eq!( + fanout.worst_receive_peer().map(|(peer_id, _)| peer_id), + Some(candidate_peer) + ); + + fanout + .connections + .insert(replacement_peer, replacement_state.clone()); + fanout.maybe_start_rotation(&ctx); + + assert!(candidate_state.lock().receive_enabled.is_none()); + assert!(replacement_state.lock().request_in_flight); + assert_eq!(fanout.awaiting_flashblocks_req, Some(replacement_peer)); + } } diff --git a/specs/flashblocks_p2p_v2.md b/specs/flashblocks_p2p_v2.md index e0c9aa9ce..cecff7d41 100644 --- a/specs/flashblocks_p2p_v2.md +++ b/specs/flashblocks_p2p_v2.md @@ -6,8 +6,6 @@ The current flashblocks P2P protocol broadcasts every `FlashblocksPayloadV1` to **all** connected peers (`handler.rs:585`, `connection.rs:97-129`). A node with N peers sends N copies of every flashblock. For a node connected to 50 peers, that is 50x outgoing bandwidth per flashblock. As the network grows, this becomes unsustainable. -Additionally, `StartPublish` and `StopPublish` messages are currently **not relayed** beyond direct peers (see `connection.rs:343,436` TODOs). This must be addressed for multi-hop propagation to work correctly. - ## Design Goals 1. **Reduce bandwidth** — Each node sends flashblocks to a bounded number of peers instead of all peers. @@ -20,10 +18,10 @@ Additionally, `StartPublish` and `StopPublish` messages are currently **not rela Each node maintains two bounded peer sets: -- **Send Set** (max `max_send_peers`, default 6): Peers this node actively forwards flashblocks to. These are peers that have sent a `RequestFlashblocks` and been accepted. Trusted peers bypass the limit. -- **Receive Set** (max `max_receive_peers`, default 6): Peers this node actively receives flashblocks from. These are peers to which this node has sent `RequestFlashblocks` and received `AcceptFlashblocks`. +- **Send Set** (max `max_send_peers`, default 10): Peers this node actively forwards flashblocks to. These are peers that have sent a `RequestFlashblocks` and been accepted. Trusted peers bypass the limit. +- **Receive Set** (max `max_receive_peers`, default 3): Peers this node actively receives flashblocks from. These are peers this node has selected as active feed sources. -Flashblocks propagate through the network as a directed acyclic graph: the builder sends to its send set, those nodes relay to their send sets, and so on. With a fanout of 6, a network of N nodes requires approximately log₆(N) hops from builder to the most distant node. +Flashblocks propagate through the network as a directed acyclic graph: the builder sends to its send set, those nodes relay to their send sets, and so on. With a fanout of 10, a network of N nodes requires approximately log₁₀(N) hops from builder to the most distant node. Periodically, each node evaluates the latency of its receive peers and may rotate out the highest-latency peer in favor of a randomly-selected alternative, one peer at a time. @@ -33,7 +31,7 @@ This change adds new message types to the `flblk` protocol. The protocol version ## New Message Types -Five unsigned control messages are added to `FlashblocksP2PMsg`: +Four unsigned control messages are added to `FlashblocksP2PMsg`: | Discriminator | Message | Direction | Description | |---|---|---|---| @@ -41,7 +39,6 @@ Five unsigned control messages are added to `FlashblocksP2PMsg`: | `0x02` | `AcceptFlashblocks` | Sender → Receiver | "Accepted. I will send you flashblocks" | | `0x03` | `RejectFlashblocks` | Sender → Receiver | "Rejected. I am at capacity" | | `0x04` | `CancelFlashblocks` | Either → Either | "I am ending our flashblock feed" | -| `0x05` | `CancelFlashblocksAck` | Either → Either | "Acknowledged. Feed terminated" | These messages carry no payload. The connection context (peer ID) provides all necessary information. @@ -54,7 +51,6 @@ pub enum FlashblocksP2PMsg { AcceptFlashblocks = 0x02, RejectFlashblocks = 0x03, CancelFlashblocks = 0x04, - CancelFlashblocksAck = 0x05, } ``` @@ -76,9 +72,7 @@ pub enum FlashblocksP2PMsg { - **Receiver-initiated**: "Stop sending me flashblocks." (e.g., during peer rotation) - **Sender-initiated**: "I am going to stop sending you flashblocks." (e.g., evicting a non-trusted peer to make room for a trusted one) -The other party MUST respond with `CancelFlashblocksAck`. - -**`CancelFlashblocksAck`** — Confirms the feed termination. After this exchange, both sides update their sets (sender removes from send set, receiver removes from receive set). +After receiving `CancelFlashblocks`, the other side immediately updates its local send/receive state for that feed. ## Peer Management @@ -103,8 +97,8 @@ struct FanoutState { When a node starts and connects to peers via devp2p: -1. As peers connect and complete the `flblk/2` handshake, send `RequestFlashblocks` to them. -2. Prioritize trusted peers first. +1. As peers connect and complete the `flblk/2` handshake, discover whether they are trusted or untrusted. +2. Only request peers whose trust classification is known, so trusted peers are always considered first. 3. Continue sending requests as new peers connect until `receive_set.len() >= max_receive_peers`. 4. Once the receive set is full, stop sending unsolicited requests (further changes happen via rotation). @@ -140,8 +134,8 @@ When a peer disconnects unexpectedly (connection drops): When a node receives an `Authorized` message from a peer in its receive set: - **`FlashblocksPayloadV1`**: Verify signatures, process the flashblock (update state, emit to flashblock stream). Then forward the serialized bytes to all peers in the **send set** except the peer that sent it. -- **`StartPublish`**: Verify signatures, process (update publishing state machine). Forward to **all connected `flblk/2` peers** (not just send set). These are rare, small control messages needed by every node for multi-builder coordination. -- **`StopPublish`**: Same as `StartPublish` — forward to all connected peers. +- **`StartPublish`**: Verify signatures and process locally. Do not relay it beyond the direct neighbor that sent it. +- **`StopPublish`**: Same as `StartPublish` — process locally, do not relay. If a node receives an `Authorized(FlashblocksPayloadV1)` from a peer **not** in its receive set, the message should be ignored. This prevents unsolicited data delivery. @@ -165,33 +159,31 @@ Each `FlashblocksPayloadV1` includes a `flashblock_timestamp` in its metadata, s one_way_latency = now() - flashblock_timestamp ``` -This measurement is attributed to the specific peer that delivered the flashblock. Nodes maintain a sliding window of the last `latency_window` (default 50) measurements per receive peer and compute a moving average. +This measurement is attributed to the specific peer that delivered the flashblock. Nodes maintain a sliding window of the last `latency_window` (default 1000) measurements per receive peer and compute a moving average. Since all receive peers deliver the same flashblock (with the same `flashblock_timestamp`), the **relative ordering** of peers by latency is accurate even with clock skew between the builder and receiver. ### Rotation Algorithm -**One-at-a-time rule**: Only one rotation may be in progress at any time. This ensures the receive set never drops below `max_receive_peers - 1` and allows the node to evaluate one change before making another. - -During rotation, the node temporarily has `max_receive_peers + 1` receive peers (both the old and new peer are sending). This is intentional and provides a brief window to compare the two peers before committing to the switch. +The receive set must never exceed `max_receive_peers`. -### Rotation Timeout +When rotating: -If a rotation is in progress and no response (`AcceptFlashblocks`/`RejectFlashblocks`) is received within a reasonable timeout (e.g., 10 seconds), abort the rotation: +1. Select the worst-scoring peer in the current receive set. +2. Remove that peer from the receive set immediately and send `CancelFlashblocks`. +3. Pick a replacement candidate, prioritizing trusted peers. +4. Add the replacement peer to the receive set in a provisional state and send `RequestFlashblocks`. -``` -rotation_in_progress = false -remove R from pending_requests -``` +The provisional peer occupies a receive slot immediately, so the node still never exceeds `max_receive_peers`. While provisional, the peer is scored for missed flashblocks the same as any other receive peer. If it fails to respond or fails to deliver flashblocks, its score will deteriorate and it can be rotated out on a later interval. ## Configuration Parameters | Parameter | Default | Description | |---|---|---| -| `max_send_peers` | 6 | Maximum non-trusted peers to send flashblocks to | -| `max_receive_peers` | 6 | Maximum peers to receive flashblocks from | +| `max_send_peers` | 10 | Maximum non-trusted peers to send flashblocks to | +| `max_receive_peers` | 3 | Maximum peers to receive flashblocks from | | `rotation_interval` | 30s | How often to evaluate and potentially rotate receive peers | -| `latency_window` | 50 | Number of flashblocks to track for per-peer latency averaging | +| `latency_window` | 1000 | Number of flashblocks to track for per-peer latency averaging | Trusted peers are always served on request and **do not count** toward `max_send_peers`. @@ -199,15 +191,14 @@ Trusted peers are always served on request and **do not count** toward `max_send ### Unchanged Components -The existing `Authorized` message types (`FlashblocksPayloadV1`, `StartPublish`, `StopPublish`) remain unchanged. They continue to use the `Authorized` wrapper with sequencer + builder signatures. The multi-builder coordination state machine (Publishing, WaitingToPublish, NotPublishing) is unaffected. +The existing `Authorized` message types (`FlashblocksPayloadV1`, `StartPublish`, `StopPublish`) remain unchanged. They continue to use the `Authorized` wrapper with sequencer + builder signatures. The multi-builder coordination state machine (Publishing, WaitingToPublish, NotPublishing) is unaffected. `StartPublish` and `StopPublish` remain direct-neighbor messages and are not relayed. ### Required Changes to Existing Code -1. **`StartPublish`/`StopPublish` must be forwarded** — The current code has TODOs at `connection.rs:343,436` noting these are not propagated. With multi-hop fanout, nodes more than 1 hop from the builder will never see these messages unless they are relayed. These must be forwarded to **all** connected `flblk/2` peers (not just send set) to ensure the multi-builder coordination works network-wide. +1. **Duplicate handling must change** — The current per-peer duplicate check at `connection.rs:278-291` penalizes any duplicate flashblock with `ReputationChangeKind::AlreadySeenTransaction`. In the new protocol, receiving the same flashblock from different receive peers is expected. Only same-peer duplicates (same flashblock index from the same peer twice) should trigger a penalty. -2. **Duplicate handling must change** — The current per-peer duplicate check at `connection.rs:278-291` penalizes any duplicate flashblock with `ReputationChangeKind::AlreadySeenTransaction`. In the new protocol, receiving the same flashblock from different receive peers is expected. Only same-peer duplicates (same flashblock index from the same peer twice) should trigger a penalty. +2. **Flashblock forwarding must be scoped to send set** — The current broadcast channel (`peer_tx`) sends to all connections. This must be replaced with targeted sends to only peers in the send set. The `PeerMsg::FlashblocksPayloadV1` variant currently uses a broadcast channel subscribed by all connections; this must be changed so each connection checks whether the destination peer is in the send set before forwarding. -3. **Flashblock forwarding must be scoped to send set** — The current broadcast channel (`peer_tx`) sends to all connections. This must be replaced with targeted sends to only peers in the send set. The `PeerMsg::FlashblocksPayloadV1` variant currently uses a broadcast channel subscribed by all connections; this must be changed so each connection checks whether the destination peer is in the send set before forwarding. +3. **Receive-peer selection must respect trust discovery** — Nodes should not request unknown peers before their trust classification is available, otherwise untrusted peers can fill the bounded receive set before trusted peers are considered. 4. **Protocol version bump** — `Capability::new_static("flblk", 1)` at `handler.rs:239` must be updated to version `2`. - From b9eebab81d27e2cdbc90051a0612c828e96d6f32 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 22:17:11 -0700 Subject: [PATCH 09/43] wip --- .../p2p/src/protocol/connection.rs | 72 ++- .../flashblocks/p2p/src/protocol/handler.rs | 542 +++++++++++------- 2 files changed, 372 insertions(+), 242 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 25785aca6..cb59386d7 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -12,13 +12,12 @@ use flashblocks_primitives::{ }; use futures::{Stream, StreamExt}; use metrics::gauge; -use parking_lot::Mutex; use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnection}; use reth_network::types::ReputationChangeKind; use std::{ pin::Pin, - sync::Arc, task::{Context, Poll, ready}, + time::Instant, }; use tokio_stream::wrappers::BroadcastStream; use tracing::{info, trace}; @@ -46,6 +45,8 @@ pub struct FlashblocksConnectionState { pub receive_enabled: Option, /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. pub receive_enabled_timestamp: u64, + /// Earliest time at which this peer is eligible for another control-plane retry. + pub request_backoff_until: Option, } impl FlashblocksConnectionState { @@ -57,6 +58,7 @@ impl FlashblocksConnectionState { send_enabled: false, receive_enabled: None, receive_enabled_timestamp: 0, + request_backoff_until: None, } } } @@ -79,8 +81,6 @@ pub struct FlashblocksConnection { /// Receiver for peer messages to be sent to all peers. /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. peer_rx: BroadcastStream, - /// Shared connection state for this peer, also visible to the protocol handler. - state: Arc>, } impl FlashblocksConnection { @@ -96,11 +96,10 @@ impl FlashblocksConnection { conn: ProtocolConnection, peer_id: PeerId, peer_rx: BroadcastStream, - state: Arc>, ) -> Self { protocol .handle - .on_peer_connected(protocol.network.clone(), peer_id, state.clone()); + .on_peer_connected(protocol.network.clone(), peer_id); gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); @@ -109,7 +108,6 @@ impl FlashblocksConnection { conn, peer_id, peer_rx, - state, } } } @@ -147,13 +145,15 @@ impl Stream for FlashblocksConnection { )) => { // Check if this flashblock actually originated from this peer. let should_send = { - let is_send_enabled = this.state.lock().send_enabled; - let already_received = - this.protocol.handle.state.lock().peer_received_flashblock( - this.peer_id, - payload_id, - flashblock_index as u64, - ); + let state = this.protocol.handle.state.lock(); + let already_received = state.peer_received_flashblock( + this.peer_id, + payload_id, + flashblock_index as u64, + ); + let is_send_enabled = state + .connection_state(&this.peer_id) + .is_some_and(|peer_state| peer_state.send_enabled); is_send_enabled && !already_received }; if should_send { @@ -306,12 +306,10 @@ impl FlashblocksConnection { &mut self, authorized_payload: AuthorizedPayload, ) { - let state_handle = self.protocol.handle.state.clone(); - let mut conn_state = self.state.lock(); - let mut p2p_state = state_handle.lock(); - let authorization = &authorized_payload.authorized.authorization; let msg = authorized_payload.msg(); + let flashblock_timestamp = msg.metadata.flashblock_timestamp; + let mut p2p_state = self.protocol.handle.state.lock(); // Check if this payload is older than our current view by more than the allowed // grace window. @@ -367,8 +365,10 @@ impl FlashblocksConnection { return; } - // Check if we're expecting to see flashblocks from this peer - let Some(score) = conn_state.receive_enabled.as_mut() else { + let Some(conn_state) = p2p_state.connection_state(&self.peer_id) else { + return; + }; + if conn_state.receive_enabled.is_none() { if conn_state.receive_enabled_timestamp + 2 < authorization.timestamp { tracing::warn!( target: "flashblocks::p2p", @@ -382,12 +382,10 @@ impl FlashblocksConnection { .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); } return; - }; + } - // Check if this peer is spamming us with the same payload index + // Check if this peer is spamming us with the same payload index. if !p2p_state.note_peer_received_flashblock(&authorization, &msg, self.peer_id) { - // We've already seen this index from this peer. - // They could be trying to DOS us. tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -404,8 +402,6 @@ impl FlashblocksConnection { p2p_state.publishing_status.send_modify(|status| { let active_publishers = match status { PublishingStatus::Publishing { .. } => { - // We are currently building, so we should not be seeing any new flashblocks - // over the p2p network. tracing::error!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -419,33 +415,31 @@ impl FlashblocksConnection { PublishingStatus::NotPublishing { active_publishers } => active_publishers, }; - // Update the list of active publishers if let Some((_, timestamp)) = active_publishers .iter_mut() .find(|(publisher, _)| *publisher == authorization.builder_vk) { - // This is an existing publisher, we should update their block number *timestamp = authorization.timestamp; } else { - // This is a new publisher, we should add them to the list of active publishers active_publishers.push((authorization.builder_vk, authorization.timestamp)); } }); - let now = Utc::now() - .timestamp_nanos_opt() - .expect("time went backwards"); - - if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { + if let Some(flashblock_timestamp) = flashblock_timestamp { + let now = Utc::now() + .timestamp_nanos_opt() + .expect("time went backwards"); let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); - score.record(latency); + if let Some(score) = p2p_state + .connection_state_mut(&self.peer_id) + .and_then(|peer_state| peer_state.receive_enabled.as_mut()) + { + score.record(latency); + } } - self.protocol - .handle - .ctx - .publish(&mut p2p_state, authorized_payload, Some(self.peer_id)); + self.protocol.handle.ctx.publish(&mut p2p_state, authorized_payload); } /// Handles incoming `StartPublish` messages from a peer. diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 92071393f..9ba7b0b93 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -24,7 +24,7 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, net::SocketAddr, sync::Arc, - time::Duration, + time::{Duration, Instant}, }; use tokio::{ sync::{broadcast, watch}, @@ -104,8 +104,6 @@ pub struct FanoutConfig { pub max_receive_peers: usize, /// How often to evaluate latency-based peer rotation. pub rotation_interval: Duration, - /// How long to wait for request flashblocks to be answered. - pub request_flashblocks_timeout: Duration, /// Number of latency measurements to retain per receive peer. pub latency_window: i64, } @@ -116,7 +114,6 @@ impl Default for FanoutConfig { max_send_peers: 10, max_receive_peers: 3, rotation_interval: Duration::from_secs(30), - request_flashblocks_timeout: Duration::from_secs(2), latency_window: 1000, } } @@ -182,10 +179,8 @@ pub struct FlashblocksP2PState { pub flashblocks: Vec>, /// Flashblocks observed from network peers, tracked until their receive grace windows expire. pub observed_payloads: VecDeque, - /// All currently connected peers and their shared connection state. - pub connections: HashMap>>, - /// The peer currently occupying the outstanding request slot, if any. - pub awaiting_flashblocks_req: Option, + /// All currently connected peers and their connection state. + pub connections: HashMap, } impl Default for FlashblocksP2PState { @@ -201,7 +196,6 @@ impl Default for FlashblocksP2PState { flashblocks: Vec::new(), observed_payloads: VecDeque::new(), connections: HashMap::new(), - awaiting_flashblocks_req: None, } } } @@ -216,13 +210,17 @@ impl FlashblocksP2PState { } /// Returns the connection state of a peer. - fn connection_state( - &self, - peer_id: &PeerId, - ) -> Option<&Arc>> { + pub(crate) fn connection_state(&self, peer_id: &PeerId) -> Option<&FlashblocksConnectionState> { self.connections.get(peer_id) } + pub(crate) fn connection_state_mut( + &mut self, + peer_id: &PeerId, + ) -> Option<&mut FlashblocksConnectionState> { + self.connections.get_mut(peer_id) + } + /// Marks receiving a flashblock from a peer and returns whether this is the first time we've observed this peer receive this flashblock. /// /// Called when a flashblock is received from any peer. @@ -241,8 +239,7 @@ impl FlashblocksP2PState { if self.observed_payloads.len() >= RECEIVE_FLASHBLOCK_GRACE_WINDOW { let evicted = self.observed_payloads.pop_front().unwrap(); - for (peer_id, connection) in self.connections.iter() { - let mut connection = connection.lock(); + for (peer_id, connection) in &mut self.connections { if connection.receive_enabled_timestamp < evicted.timestamp + 2 && !evicted.received_peers.contains(peer_id) { @@ -288,19 +285,39 @@ impl FlashblocksP2PState { fn num_receive_peers(&self) -> usize { self.connections - .iter() - .filter(|(_, peer_state)| peer_state.lock().receive_enabled.is_some()) + .values() + .filter(|peer_state| peer_state.receive_enabled.is_some()) .count() } + fn request_backoff_deadline(ctx: &FlashblocksP2PCtx) -> Instant { + Instant::now() + ctx.fanout_config.rotation_interval + } + + fn clear_receive_state( + peer_state: &mut FlashblocksConnectionState, + receive_enabled_timestamp: u64, + request_backoff_until: Option, + ) -> bool { + let had_receive_state = peer_state.receive_enabled.is_some() || peer_state.request_in_flight; + peer_state.receive_enabled = None; + peer_state.request_in_flight = false; + peer_state.receive_enabled_timestamp = receive_enabled_timestamp; + peer_state.request_backoff_until = request_backoff_until; + had_receive_state + } + fn available_receive_candidates(&self) -> Vec<(PeerId, bool)> { + let now = Instant::now(); self.connections .iter() .filter_map(|(peer_id, peer_state)| { - let peer_state = peer_state.lock(); if peer_state.trusted_known && peer_state.receive_enabled.is_none() && !peer_state.request_in_flight + && peer_state + .request_backoff_until + .is_none_or(|until| until <= now) { Some((*peer_id, peer_state.trusted)) } else { @@ -311,55 +328,62 @@ impl FlashblocksP2PState { } fn begin_requesting_peer(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - let Some(peer_state) = self.connection_state(&peer_id) else { + let Some(peer_state) = self.connection_state_mut(&peer_id) else { return; }; let timestamp = Utc::now().timestamp() as u64; - let mut peer_state = peer_state.lock(); peer_state.request_in_flight = true; peer_state.receive_enabled = Some(Score::new(ctx.fanout_config.latency_window)); peer_state.receive_enabled_timestamp = timestamp; - drop(peer_state); - self.awaiting_flashblocks_req = Some(peer_id); + peer_state.request_backoff_until = None; ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { - if self.num_receive_peers() >= ctx.fanout_config.max_receive_peers { - return; - } - - let candidates = self.available_receive_candidates(); - if candidates.is_empty() { - return; - } - let trusted_candidates: Vec<_> = candidates - .iter() - .filter_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) - .collect(); - let candidate_pool = if trusted_candidates.is_empty() { - candidates + while self.num_receive_peers() < ctx.fanout_config.max_receive_peers { + let candidates = self.available_receive_candidates(); + if candidates.is_empty() { + return; + } + let trusted_candidates: Vec<_> = candidates .iter() - .map(|(peer_id, _)| *peer_id) - .collect::>() - } else { - trusted_candidates - }; - let rand = rand::rng().random_range(0..candidate_pool.len()); - self.begin_requesting_peer(ctx, candidate_pool[rand]); + .filter_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) + .collect(); + let candidate_pool = if trusted_candidates.is_empty() { + candidates + .iter() + .map(|(peer_id, _)| *peer_id) + .collect::>() + } else { + trusted_candidates + }; + let rand = rand::rng().random_range(0..candidate_pool.len()); + self.begin_requesting_peer(ctx, candidate_pool[rand]); + } } - fn worst_receive_peer(&self) -> Option<(PeerId, i64)> { + fn worst_receive_peer(&self) -> Option { self.connections .iter() .filter_map(|(peer_id, peer_state)| { - let peer_state = peer_state.lock(); - peer_state - .receive_enabled - .as_ref() - .and_then(|score| score.value().map(|score| (*peer_id, score))) + let score = peer_state.receive_enabled.as_ref()?; + Some(( + *peer_id, + score.value(), + peer_state.receive_enabled_timestamp, + )) }) - .max_by(|(_, lhs), (_, rhs)| lhs.cmp(rhs)) + .max_by(|(_, lhs_score, lhs_timestamp), (_, rhs_score, rhs_timestamp)| { + match (lhs_score, rhs_score) { + (None, None) => rhs_timestamp.cmp(lhs_timestamp), + (None, Some(_)) => std::cmp::Ordering::Greater, + (Some(_), None) => std::cmp::Ordering::Less, + (Some(lhs_score), Some(rhs_score)) => lhs_score + .cmp(rhs_score) + .then_with(|| rhs_timestamp.cmp(lhs_timestamp)), + } + }) + .map(|(peer_id, _, _)| peer_id) } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { @@ -367,7 +391,7 @@ impl FlashblocksP2PState { return; } - let Some((evict, _)) = self.worst_receive_peer() else { + let Some(evict) = self.worst_receive_peer() else { return; }; @@ -383,15 +407,12 @@ impl FlashblocksP2PState { .find_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) .unwrap_or(candidates[0].0); - if let Some(evict_state) = self.connection_state(&evict) { - let timestamp = Utc::now().timestamp() as u64; - let mut evict_state = evict_state.lock(); - evict_state.receive_enabled = None; - evict_state.request_in_flight = false; - evict_state.receive_enabled_timestamp = timestamp; - } - if self.awaiting_flashblocks_req == Some(evict) { - self.awaiting_flashblocks_req = None; + if let Some(evict_state) = self.connection_state_mut(&evict) { + Self::clear_receive_state( + evict_state, + Utc::now().timestamp() as u64, + Some(Self::request_backoff_deadline(ctx)), + ); } ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); @@ -403,39 +424,43 @@ impl FlashblocksP2PState { return; }; - if peer_state.lock().send_enabled { - ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + let now = Instant::now(); + if peer_state.send_enabled { return; } - - let peer_is_trusted = peer_state.lock().trusted; + if !peer_state.trusted + && peer_state + .request_backoff_until + .is_some_and(|until| until > now) + { + return; + } + let peer_is_trusted = peer_state.trusted; if peer_is_trusted { let non_trusted_send_count = self .connections .values() - .filter(|candidate_state| { - let candidate_state = candidate_state.lock(); - candidate_state.send_enabled && !candidate_state.trusted - }) + .filter(|candidate_state| candidate_state.send_enabled && !candidate_state.trusted) .count(); if non_trusted_send_count >= ctx.fanout_config.max_send_peers { if let Some(evicted_peer) = self.connections .iter() .find_map(|(candidate, candidate_state)| { - let candidate_state = candidate_state.lock(); (candidate_state.send_enabled && !candidate_state.trusted) .then_some(*candidate) }) { - if let Some(evicted_state) = self.connection_state(&evicted_peer) { - evicted_state.lock().send_enabled = false; + if let Some(evicted_state) = self.connection_state_mut(&evicted_peer) { + evicted_state.send_enabled = false; } ctx.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); } } - peer_state.lock().send_enabled = true; + let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); + peer_state.send_enabled = true; + peer_state.request_backoff_until = None; ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); return; } @@ -443,30 +468,69 @@ impl FlashblocksP2PState { let non_trusted_send_count = self .connections .values() - .filter(|candidate_state| { - let candidate_state = candidate_state.lock(); - candidate_state.send_enabled && !candidate_state.trusted - }) + .filter(|candidate_state| candidate_state.send_enabled && !candidate_state.trusted) .count(); if non_trusted_send_count < ctx.fanout_config.max_send_peers { - peer_state.lock().send_enabled = true; + let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); + peer_state.send_enabled = true; + peer_state.request_backoff_until = None; ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); } else { + self.connection_state_mut(&peer_id) + .expect("peer exists") + .request_backoff_until = Some(Self::request_backoff_deadline(ctx)); ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); } } fn handle_accept(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) { - let Some(peer_state) = self.connection_state(&peer_id) else { + let Some(peer_state) = self.connection_state_mut(&peer_id) else { return; }; - if self.awaiting_flashblocks_req != Some(peer_id) { + if !peer_state.request_in_flight { return; } - peer_state.lock().request_in_flight = false; - self.awaiting_flashblocks_req = None; + peer_state.request_in_flight = false; + peer_state.request_backoff_until = None; + } + + fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return; + }; + + if !peer_state.request_in_flight { + return; + } + + Self::clear_receive_state( + peer_state, + Utc::now().timestamp() as u64, + Some(Self::request_backoff_deadline(ctx)), + ); + self.maybe_request_receive_peers(ctx); + } + + fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return; + }; + + let mut should_refill = false; + peer_state.send_enabled = false; + if peer_state.receive_enabled.is_some() || peer_state.request_in_flight { + peer_state.receive_enabled = None; + peer_state.request_in_flight = false; + peer_state.receive_enabled_timestamp = Utc::now().timestamp() as u64; + peer_state.request_backoff_until = Some(Self::request_backoff_deadline(ctx)); + should_refill = true; + } + + if should_refill { + self.maybe_request_receive_peers(ctx); + } } } @@ -535,10 +599,9 @@ impl FlashblocksHandle { loop { rotation_interval.tick().await; - moved_handle - .state - .lock() - .maybe_start_rotation(&moved_handle.ctx) + let mut state = moved_handle.state.lock(); + state.maybe_request_receive_peers(&moved_handle.ctx); + state.maybe_start_rotation(&moved_handle.ctx); } }); @@ -549,11 +612,10 @@ impl FlashblocksHandle { &self, network: N, peer_id: PeerId, - fanout_state: Arc>, ) { { let mut state = self.state.lock(); - state.connections.insert(peer_id, fanout_state.clone()); + state.connections.insert(peer_id, FlashblocksConnectionState::new()); state.maybe_request_receive_peers(&self.ctx); } @@ -561,16 +623,10 @@ impl FlashblocksHandle { tokio::spawn(async move { match network.get_peer_by_id(peer_id).await { Ok(Some(peer_info)) => { - let mut fanout_state_guard = fanout_state.lock(); - fanout_state_guard.trusted = peer_info.kind.is_trusted(); - fanout_state_guard.trusted_known = true; - drop(fanout_state_guard); - let mut state = handle.state.lock(); - if state - .connection_state(&peer_id) - .is_some_and(|current| Arc::ptr_eq(¤t, &fanout_state)) - { + if let Some(peer_state) = state.connection_state_mut(&peer_id) { + peer_state.trusted = peer_info.kind.is_trusted(); + peer_state.trusted_known = true; state.maybe_request_receive_peers(&handle.ctx); } } @@ -590,9 +646,6 @@ impl FlashblocksHandle { pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { let mut state = self.state.lock(); state.connections.remove(&peer_id); - if state.awaiting_flashblocks_req == Some(peer_id) { - state.awaiting_flashblocks_req = None; - } state.maybe_request_receive_peers(&self.ctx); } @@ -608,44 +661,12 @@ impl FlashblocksHandle { pub(crate) fn handle_reject_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - let this = &mut state; - let ctx: &FlashblocksP2PCtx = &self.ctx; - let Some(peer_state) = this.connection_state(&peer_id) else { - return; - }; - - if this.awaiting_flashblocks_req != Some(peer_id) { - return; - } - { - let mut peer_state = peer_state.lock(); - peer_state.request_in_flight = false; - peer_state.receive_enabled = None; - } - this.awaiting_flashblocks_req = None; - - this.maybe_request_receive_peers(ctx); + state.handle_reject(&self.ctx, peer_id); } pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) { let mut state = self.state.lock(); - let this = &mut state; - let mut should_refill = false; - if let Some(peer_state) = this.connection_state(&peer_id).cloned() { - let mut peer_state = peer_state.lock(); - peer_state.send_enabled = false; - if peer_state.receive_enabled.is_some() || peer_state.request_in_flight { - peer_state.receive_enabled = None; - peer_state.request_in_flight = false; - if this.awaiting_flashblocks_req == Some(peer_id) { - this.awaiting_flashblocks_req = None; - } - should_refill = true; - } - } - if should_refill { - this.maybe_request_receive_peers(&self.ctx); - } + state.handle_cancel(&self.ctx, peer_id); } } @@ -744,7 +765,7 @@ impl FlashblocksHandle { if authorization != authorized_payload.authorized.authorization { return Err(FlashblocksP2PError::ExpiredAuthorization); } - self.ctx.publish(&mut state, authorized_payload, None); + self.ctx.publish(&mut state, authorized_payload); Ok(()) } @@ -972,7 +993,6 @@ impl FlashblocksP2PCtx { &self, state: &mut FlashblocksP2PState, authorized_payload: AuthorizedPayload, - _source_peer_id: Option, ) { let payload = authorized_payload.msg(); let authorization = authorized_payload.authorized.authorization; @@ -1138,15 +1158,8 @@ impl ConnectionHandler for FlashblocksP2PProtoco ); let peer_rx = self.handle.ctx.peer_tx.subscribe(); - let fanout_state = Arc::new(Mutex::new(FlashblocksConnectionState::new())); - - FlashblocksConnection::new( - self, - conn, - peer_id, - BroadcastStream::new(peer_rx), - fanout_state, - ) + + FlashblocksConnection::new(self, conn, peer_id, BroadcastStream::new(peer_rx)) } } @@ -1170,16 +1183,29 @@ mod tests { fn test_peer_state( trusted: bool, trusted_known: bool, - ) -> Arc> { - let state = Arc::new(Mutex::new(FlashblocksConnectionState::new())); - { - let mut state_guard = state.lock(); - state_guard.trusted = trusted; - state_guard.trusted_known = trusted_known; - } + ) -> FlashblocksConnectionState { + let mut state = FlashblocksConnectionState::new(); + state.trusted = trusted; + state.trusted_known = trusted_known; state } + fn peer_state( + fanout: &FlashblocksP2PState, + peer_id: PeerId, + ) -> &FlashblocksConnectionState { + fanout.connection_state(&peer_id).expect("peer exists") + } + + fn apply_observation( + fanout: &mut FlashblocksP2PState, + authorization: &Authorization, + flashblock: &FlashblocksPayloadV1, + peer_id: PeerId, + ) { + fanout.note_peer_received_flashblock(authorization, flashblock, peer_id); + } + #[test] fn trusted_peers_are_requested_first() { let config = FanoutConfig { @@ -1203,13 +1229,9 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); - assert!(matches!( - fanout.awaiting_flashblocks_req, - Some(peer_id) if peer_id == trusted_peer - )); - assert!(trusted_state.lock().request_in_flight); - assert!(trusted_state.lock().receive_enabled.is_some()); - assert!(!untrusted_state.lock().request_in_flight); + assert!(peer_state(&fanout, trusted_peer).request_in_flight); + assert!(peer_state(&fanout, trusted_peer).receive_enabled.is_some()); + assert!(!peer_state(&fanout, untrusted_peer).request_in_flight); match rx.try_recv().expect("request sent") { PeerMsg::Direct { peer_id, bytes } => { assert_eq!(peer_id, trusted_peer); @@ -1245,9 +1267,8 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); - assert!(fanout.awaiting_flashblocks_req.is_none()); - assert!(!trusted_state.lock().request_in_flight); - assert!(!untrusted_state.lock().request_in_flight); + assert!(!peer_state(&fanout, trusted_peer).request_in_flight); + assert!(!peer_state(&fanout, untrusted_peer).request_in_flight); assert!(rx.try_recv().is_err()); } @@ -1263,9 +1284,9 @@ mod tests { let victim = PeerId::random(); let trusted_requester = PeerId::random(); - let victim_state = test_peer_state(false, true); + let mut victim_state = test_peer_state(false, true); let requester_state = test_peer_state(true, true); - victim_state.lock().send_enabled = true; + victim_state.send_enabled = true; fanout.connections.insert(victim, victim_state.clone()); fanout .connections @@ -1273,8 +1294,8 @@ mod tests { fanout.handle_request(&ctx, trusted_requester); - assert!(!victim_state.lock().send_enabled); - assert!(requester_state.lock().send_enabled); + assert!(!peer_state(&fanout, victim).send_enabled); + assert!(peer_state(&fanout, trusted_requester).send_enabled); match rx.try_recv().expect("cancel sent") { PeerMsg::Direct { peer_id, bytes } => { @@ -1313,11 +1334,11 @@ mod tests { let current_peer = PeerId::random(); let candidate_peer = PeerId::random(); - let current_state = test_peer_state(false, true); + let mut current_state = test_peer_state(false, true); let candidate_state = test_peer_state(false, true); let mut score = Score::new(latency_window); score.record(42); - current_state.lock().receive_enabled = Some(score); + current_state.receive_enabled = Some(score); fanout .connections .insert(current_peer, current_state.clone()); @@ -1327,13 +1348,9 @@ mod tests { fanout.maybe_start_rotation(&ctx); - assert!(current_state.lock().receive_enabled.is_none()); - assert!(candidate_state.lock().request_in_flight); - assert!(candidate_state.lock().receive_enabled.is_some()); - assert!(matches!( - fanout.awaiting_flashblocks_req, - Some(candidate) if candidate == candidate_peer - )); + assert!(peer_state(&fanout, current_peer).receive_enabled.is_none()); + assert!(peer_state(&fanout, candidate_peer).request_in_flight); + assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_some()); match rx.try_recv().expect("cancel sent to old peer") { PeerMsg::Direct { peer_id, bytes } => { @@ -1359,9 +1376,138 @@ mod tests { fanout.handle_accept(&ctx, candidate_peer); - assert!(!candidate_state.lock().request_in_flight); - assert!(candidate_state.lock().receive_enabled.is_some()); - assert!(fanout.awaiting_flashblocks_req.is_none()); + assert!(!peer_state(&fanout, candidate_peer).request_in_flight); + assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_some()); + } + + #[test] + fn multiple_pending_requests_clear_independently() { + let config = FanoutConfig { + max_receive_peers: 2, + ..Default::default() + }; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let first_peer = PeerId::random(); + let second_peer = PeerId::random(); + let first_state = test_peer_state(false, true); + let second_state = test_peer_state(false, true); + fanout.connections.insert(first_peer, first_state.clone()); + fanout.connections.insert(second_peer, second_state.clone()); + + fanout.maybe_request_receive_peers(&ctx); + + assert!(peer_state(&fanout, first_peer).request_in_flight); + assert!(peer_state(&fanout, second_peer).request_in_flight); + + let mut requested_peers = HashSet::new(); + for _ in 0..2 { + match rx.try_recv().expect("request sent") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::RequestFlashblocks + ); + requested_peers.insert(peer_id); + } + other => panic!("unexpected peer message: {other:?}"), + } + } + assert_eq!(requested_peers, HashSet::from([first_peer, second_peer])); + + fanout.handle_accept(&ctx, first_peer); + fanout.handle_accept(&ctx, second_peer); + + assert!(!peer_state(&fanout, first_peer).request_in_flight); + assert!(!peer_state(&fanout, second_peer).request_in_flight); + assert!(peer_state(&fanout, first_peer).receive_enabled.is_some()); + assert!(peer_state(&fanout, second_peer).receive_enabled.is_some()); + } + + #[test] + fn rejected_peer_is_not_immediately_retried() { + let config = FanoutConfig { + max_receive_peers: 1, + ..Default::default() + }; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let peer = PeerId::random(); + let candidate_state = test_peer_state(false, true); + fanout.connections.insert(peer, candidate_state.clone()); + + fanout.maybe_request_receive_peers(&ctx); + let _ = rx.try_recv().expect("initial request sent"); + + fanout.handle_reject(&ctx, peer); + + assert!(!peer_state(&fanout, peer).request_in_flight); + assert!(peer_state(&fanout, peer).receive_enabled.is_none()); + assert!(peer_state(&fanout, peer).request_backoff_until.is_some()); + assert!(rx.try_recv().is_err()); + } + + #[test] + fn silent_receive_peer_can_be_rotated_out_without_samples() { + let config = FanoutConfig { + max_receive_peers: 2, + ..Default::default() + }; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + let mut rx = ctx.peer_tx.subscribe(); + + let oldest_peer = PeerId::random(); + let newer_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + + let mut oldest_state = test_peer_state(false, true); + let mut newer_state = test_peer_state(false, true); + let replacement_state = test_peer_state(true, true); + + oldest_state.receive_enabled = Some(Score::new(4)); + oldest_state.request_in_flight = true; + oldest_state.receive_enabled_timestamp = 1; + newer_state.receive_enabled = Some(Score::new(4)); + newer_state.request_in_flight = true; + newer_state.receive_enabled_timestamp = 2; + + fanout.connections.insert(oldest_peer, oldest_state.clone()); + fanout.connections.insert(newer_peer, newer_state.clone()); + fanout + .connections + .insert(replacement_peer, replacement_state.clone()); + + fanout.maybe_start_rotation(&ctx); + + assert!(peer_state(&fanout, oldest_peer).receive_enabled.is_none()); + assert!(peer_state(&fanout, replacement_peer).request_in_flight); + + match rx.try_recv().expect("cancel sent to oldest peer") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, oldest_peer); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::CancelFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } + + match rx.try_recv().expect("replacement request sent") { + PeerMsg::Direct { peer_id, bytes } => { + assert_eq!(peer_id, replacement_peer); + assert_eq!( + FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + other => panic!("unexpected peer message: {other:?}"), + } } #[test] @@ -1376,21 +1522,19 @@ mod tests { let steady_peer = PeerId::random(); let lagging_peer = PeerId::random(); - let steady_state = test_peer_state(false, true); - let lagging_state = test_peer_state(false, true); + let mut steady_state = test_peer_state(false, true); + let mut lagging_state = test_peer_state(false, true); let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); - steady_state.lock().receive_enabled = Some(Score::new(latency_window)); - lagging_state.lock().receive_enabled = Some(Score::new(latency_window)); + steady_state.receive_enabled = Some(Score::new(latency_window)); + lagging_state.receive_enabled = Some(Score::new(latency_window)); steady_state - .lock() .receive_enabled .as_mut() .expect("steady peer score") .record(10); lagging_state - .lock() .receive_enabled .as_mut() .expect("lagging peer score") @@ -1413,21 +1557,19 @@ mod tests { index: index as u64, ..Default::default() }; - fanout.note_peer_received_flashblock(&authorization, &flashblock, steady_peer); + apply_observation(&mut fanout, &authorization, &flashblock, steady_peer); } - assert_eq!(fanout.worst_receive_peer().map(|(peer_id, _)| peer_id), Some(lagging_peer)); + assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); assert_eq!( - steady_state - .lock() + peer_state(&fanout, steady_peer) .receive_enabled .as_ref() .and_then(Score::value), Some(10) ); assert_eq!( - lagging_state - .lock() + peer_state(&fanout, lagging_peer) .receive_enabled .as_ref() .and_then(Score::value), @@ -1451,21 +1593,19 @@ mod tests { let candidate_peer = PeerId::random(); let replacement_peer = PeerId::random(); - let steady_state = test_peer_state(false, true); - let rotating_state = test_peer_state(false, true); + let mut steady_state = test_peer_state(false, true); + let mut rotating_state = test_peer_state(false, true); let candidate_state = test_peer_state(true, true); let replacement_state = test_peer_state(true, true); - steady_state.lock().receive_enabled = Some(Score::new(latency_window)); - rotating_state.lock().receive_enabled = Some(Score::new(latency_window)); + steady_state.receive_enabled = Some(Score::new(latency_window)); + rotating_state.receive_enabled = Some(Score::new(latency_window)); steady_state - .lock() .receive_enabled .as_mut() .expect("steady peer score") .record(10); rotating_state - .lock() .receive_enabled .as_mut() .expect("rotating peer score") @@ -1495,21 +1635,17 @@ mod tests { index: index as u64, ..Default::default() }; - fanout.note_peer_received_flashblock(&authorization, &flashblock, steady_peer); + apply_observation(&mut fanout, &authorization, &flashblock, steady_peer); } - assert_eq!( - fanout.worst_receive_peer().map(|(peer_id, _)| peer_id), - Some(candidate_peer) - ); + assert_eq!(fanout.worst_receive_peer(), Some(candidate_peer)); fanout .connections .insert(replacement_peer, replacement_state.clone()); fanout.maybe_start_rotation(&ctx); - assert!(candidate_state.lock().receive_enabled.is_none()); - assert!(replacement_state.lock().request_in_flight); - assert_eq!(fanout.awaiting_flashblocks_req, Some(replacement_peer)); + assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_none()); + assert!(peer_state(&fanout, replacement_peer).request_in_flight); } } From f0e94c5594c67d722b28584ef3d904cc6e7294f3 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 23:11:37 -0700 Subject: [PATCH 10/43] wip --- .../p2p/src/protocol/connection.rs | 79 ++- .../flashblocks/p2p/src/protocol/handler.rs | 525 +++++++++++------- 2 files changed, 378 insertions(+), 226 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index cb59386d7..5bcd3d0d6 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -19,6 +19,7 @@ use std::{ task::{Context, Poll, ready}, time::Instant, }; +use tokio::sync::mpsc; use tokio_stream::wrappers::BroadcastStream; use tracing::{info, trace}; @@ -47,6 +48,12 @@ pub struct FlashblocksConnectionState { pub receive_enabled_timestamp: u64, /// Earliest time at which this peer is eligible for another control-plane retry. pub request_backoff_until: Option, + /// Per-peer channel for sending direct (control) messages without broadcasting. + pub direct_tx: Option>, + /// Number of control messages received in the current rate-limit window. + pub control_msg_count: u32, + /// Start of the current rate-limit window. + pub control_msg_window_start: Instant, } impl FlashblocksConnectionState { @@ -59,6 +66,9 @@ impl FlashblocksConnectionState { receive_enabled: None, receive_enabled_timestamp: 0, request_backoff_until: None, + direct_tx: None, + control_msg_count: 0, + control_msg_window_start: Instant::now(), } } } @@ -81,6 +91,8 @@ pub struct FlashblocksConnection { /// Receiver for peer messages to be sent to all peers. /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. peer_rx: BroadcastStream, + /// Receiver for direct (control) messages targeted at this specific peer. + direct_rx: mpsc::UnboundedReceiver, } impl FlashblocksConnection { @@ -96,10 +108,12 @@ impl FlashblocksConnection { conn: ProtocolConnection, peer_id: PeerId, peer_rx: BroadcastStream, + direct_tx: mpsc::UnboundedSender, + direct_rx: mpsc::UnboundedReceiver, ) -> Self { protocol .handle - .on_peer_connected(protocol.network.clone(), peer_id); + .on_peer_connected(protocol.network.clone(), peer_id, direct_tx); gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); @@ -108,6 +122,7 @@ impl FlashblocksConnection { conn, peer_id, peer_rx, + direct_rx, } } } @@ -133,6 +148,16 @@ impl Stream for FlashblocksConnection { let this = self.get_mut(); loop { + // Check per-peer direct channel first (control messages). + if let Poll::Ready(Some(bytes)) = this.direct_rx.poll_recv(cx) { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "Sending direct flashblocks control message to peer" + ); + return Poll::Ready(Some(bytes)); + } + // Check if there are any flashblocks ready to broadcast to our peers. if let Poll::Ready(Some(res)) = this.peer_rx.poll_next_unpin(cx) { match res { @@ -186,16 +211,6 @@ impl Stream for FlashblocksConnection { ); return Poll::Ready(Some(bytes_mut)); } - PeerMsg::Direct { peer_id, bytes } => { - if peer_id == this.peer_id { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Sending direct flashblocks control message to peer" - ); - return Poll::Ready(Some(bytes)); - } - } } } Err(error) => { @@ -270,16 +285,32 @@ impl Stream for FlashblocksConnection { } } FlashblocksP2PMsg::RequestFlashblocks => { - this.protocol.handle.handle_request_message(this.peer_id); + if this.protocol.handle.handle_request_message(this.peer_id) { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } } FlashblocksP2PMsg::AcceptFlashblocks => { - this.protocol.handle.handle_accept_message(this.peer_id); + if this.protocol.handle.handle_accept_message(this.peer_id) { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } } FlashblocksP2PMsg::RejectFlashblocks => { - this.protocol.handle.handle_reject_message(this.peer_id); + if this.protocol.handle.handle_reject_message(this.peer_id) { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } } FlashblocksP2PMsg::CancelFlashblocks => { - this.protocol.handle.handle_cancel_message(this.peer_id); + if this.protocol.handle.handle_cancel_message(this.peer_id) { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } } } } @@ -458,26 +489,26 @@ impl FlashblocksConnection { return; }; let authorization = &authorized_payload.authorized.authorization; - let payload_timestamp = self.protocol.handle.state.lock().payload_timestamp; + let state = self.protocol.handle.state.lock(); // Check if the request is expired for dos protection. // It's important to ensure that this `StartPublish` request // is very recent, or it could be used in a replay attack. - if payload_timestamp > authorization.timestamp { + if state.payload_timestamp > authorization.timestamp { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = payload_timestamp, + current_timestamp = state.payload_timestamp, timestamp = authorized_payload.authorized.authorization.timestamp, "received initiate build request with outdated timestamp", ); + drop(state); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); return; } - let state = self.protocol.handle.state.lock(); state.publishing_status.send_modify(|status| { let active_publishers = match status { PublishingStatus::Publishing { @@ -548,26 +579,26 @@ impl FlashblocksConnection { /// - If we are not publishing, removes the publisher from the list of active publishers fn handle_stop_publish(&mut self, authorized_payload: AuthorizedPayload) { let authorization = &authorized_payload.authorized.authorization; - let payload_timestamp = self.protocol.handle.state.lock().payload_timestamp; + let state = self.protocol.handle.state.lock(); // Check if the request is expired for dos protection. - // It's important to ensure that this `StartPublish` request + // It's important to ensure that this `StopPublish` request // is very recent, or it could be used in a replay attack. - if payload_timestamp > authorization.timestamp { + if state.payload_timestamp > authorization.timestamp { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = payload_timestamp, + current_timestamp = state.payload_timestamp, timestamp = authorized_payload.authorized.authorization.timestamp, "Received initiate build response with outdated timestamp", ); + drop(state); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); return; } - let state = self.protocol.handle.state.lock(); state.publishing_status.send_modify(|status| { match status { PublishingStatus::Publishing { .. } => { diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 9ba7b0b93..004eaaeec 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -27,7 +27,7 @@ use std::{ time::{Duration, Instant}, }; use tokio::{ - sync::{broadcast, watch}, + sync::{broadcast, mpsc, watch}, time, }; use tracing::{debug, info, warn}; @@ -63,6 +63,13 @@ const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; /// attack. pub(crate) const RECEIVE_FLASHBLOCK_GRACE_WINDOW: usize = 50; +/// Maximum number of control messages (Request/Accept/Reject/Cancel) a peer may send +/// within a sliding window before being penalized. +const MAX_CONTROL_MSGS_PER_WINDOW: u32 = 10; + +/// Duration of the per-peer control-message rate-limit window. +const CONTROL_MSG_WINDOW: Duration = Duration::from_secs(30); + /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -83,8 +90,6 @@ pub enum PeerMsg { StartPublishing(BytesMut), /// Send a previously serialized StopPublish message to all peers. StopPublishing(BytesMut), - /// Send an already serialized control message to a single peer. - Direct { peer_id: PeerId, bytes: BytesMut }, } #[derive(Clone, Debug)] @@ -283,6 +288,29 @@ impl FlashblocksP2PState { .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) } + /// Sends a control message directly to a specific peer via its per-peer channel. + fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { + if let Some(conn) = self.connections.get(&peer_id) { + if let Some(tx) = &conn.direct_tx { + tx.send(msg.encode()).ok(); + } + } + } + + /// Returns `true` if the peer has exceeded the control-message rate limit. + fn check_control_rate_limit(&mut self, peer_id: &PeerId) -> bool { + let Some(peer_state) = self.connections.get_mut(peer_id) else { + return true; + }; + let now = Instant::now(); + if now.duration_since(peer_state.control_msg_window_start) > CONTROL_MSG_WINDOW { + peer_state.control_msg_count = 0; + peer_state.control_msg_window_start = now; + } + peer_state.control_msg_count += 1; + peer_state.control_msg_count > MAX_CONTROL_MSGS_PER_WINDOW + } + fn num_receive_peers(&self) -> usize { self.connections .values() @@ -336,7 +364,7 @@ impl FlashblocksP2PState { peer_state.receive_enabled = Some(Score::new(ctx.fanout_config.latency_window)); peer_state.receive_enabled_timestamp = timestamp; peer_state.request_backoff_until = None; - ctx.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { @@ -414,26 +442,33 @@ impl FlashblocksP2PState { Some(Self::request_backoff_deadline(ctx)), ); } - ctx.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); self.begin_requesting_peer(ctx, candidate); } - fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + if self.check_control_rate_limit(&peer_id) { + return true; + } + let Some(peer_state) = self.connection_state(&peer_id) else { - return; + return false; }; - let now = Instant::now(); if peer_state.send_enabled { - return; + // Already sending to this peer — repeated request is spam. + return true; } + let now = Instant::now(); if !peer_state.trusted && peer_state .request_backoff_until .is_some_and(|until| until > now) { - return; + // Non-trusted peer requesting during backoff is spam. + return true; } let peer_is_trusted = peer_state.trusted; if peer_is_trusted { @@ -454,15 +489,15 @@ impl FlashblocksP2PState { if let Some(evicted_state) = self.connection_state_mut(&evicted_peer) { evicted_state.send_enabled = false; } - ctx.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); + self.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); } } let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); peer_state.send_enabled = true; peer_state.request_backoff_until = None; - ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); - return; + self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + return false; } let non_trusted_send_count = self @@ -474,35 +509,49 @@ impl FlashblocksP2PState { let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); peer_state.send_enabled = true; peer_state.request_backoff_until = None; - ctx.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); } else { self.connection_state_mut(&peer_id) .expect("peer exists") .request_backoff_until = Some(Self::request_backoff_deadline(ctx)); - ctx.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); + self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); } + false } - fn handle_accept(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + fn handle_accept(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + if self.check_control_rate_limit(&peer_id) { + return true; + } + let Some(peer_state) = self.connection_state_mut(&peer_id) else { - return; + return false; }; if !peer_state.request_in_flight { - return; + // Unsolicited accept — we never asked this peer. + return true; } peer_state.request_in_flight = false; peer_state.request_backoff_until = None; + false } - fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + if self.check_control_rate_limit(&peer_id) { + return true; + } + let Some(peer_state) = self.connection_state_mut(&peer_id) else { - return; + return false; }; if !peer_state.request_in_flight { - return; + // Unsolicited reject — we never asked this peer. + return true; } Self::clear_receive_state( @@ -511,26 +560,42 @@ impl FlashblocksP2PState { Some(Self::request_backoff_deadline(ctx)), ); self.maybe_request_receive_peers(ctx); + false } - fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + if self.check_control_rate_limit(&peer_id) { + return true; + } + let Some(peer_state) = self.connection_state_mut(&peer_id) else { - return; + return false; }; - let mut should_refill = false; - peer_state.send_enabled = false; - if peer_state.receive_enabled.is_some() || peer_state.request_in_flight { + let has_send = peer_state.send_enabled; + let has_receive = peer_state.receive_enabled.is_some() || peer_state.request_in_flight; + + if !has_send && !has_receive { + // No active relationship — unsolicited cancel is spam. + return true; + } + + // Only clear the directions that actually have a relationship. + if has_send { + peer_state.send_enabled = false; + } + if has_receive { peer_state.receive_enabled = None; peer_state.request_in_flight = false; peer_state.receive_enabled_timestamp = Utc::now().timestamp() as u64; peer_state.request_backoff_until = Some(Self::request_backoff_deadline(ctx)); - should_refill = true; } - if should_refill { + if has_receive { self.maybe_request_receive_peers(ctx); } + false } } @@ -612,10 +677,13 @@ impl FlashblocksHandle { &self, network: N, peer_id: PeerId, + direct_tx: mpsc::UnboundedSender, ) { { let mut state = self.state.lock(); - state.connections.insert(peer_id, FlashblocksConnectionState::new()); + let mut conn_state = FlashblocksConnectionState::new(); + conn_state.direct_tx = Some(direct_tx); + state.connections.insert(peer_id, conn_state); state.maybe_request_receive_peers(&self.ctx); } @@ -649,35 +717,28 @@ impl FlashblocksHandle { state.maybe_request_receive_peers(&self.ctx); } - pub(crate) fn handle_request_message(&self, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + pub(crate) fn handle_request_message(&self, peer_id: PeerId) -> bool { let mut state = self.state.lock(); - state.handle_request(&self.ctx, peer_id); + state.handle_request(&self.ctx, peer_id) } - pub(crate) fn handle_accept_message(&self, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + pub(crate) fn handle_accept_message(&self, peer_id: PeerId) -> bool { let mut state = self.state.lock(); - state.handle_accept(&self.ctx, peer_id); + state.handle_accept(&self.ctx, peer_id) } - pub(crate) fn handle_reject_message(&self, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + pub(crate) fn handle_reject_message(&self, peer_id: PeerId) -> bool { let mut state = self.state.lock(); - state.handle_reject(&self.ctx, peer_id); + state.handle_reject(&self.ctx, peer_id) } - pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) { + /// Returns `true` if the peer should receive a reputation penalty. + pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) -> bool { let mut state = self.state.lock(); - state.handle_cancel(&self.ctx, peer_id); - } -} - -impl FlashblocksP2PCtx { - pub(crate) fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { - self.peer_tx - .send(PeerMsg::Direct { - peer_id, - bytes: msg.encode(), - }) - .ok(); + state.handle_cancel(&self.ctx, peer_id) } } @@ -1158,8 +1219,16 @@ impl ConnectionHandler for FlashblocksP2PProtoco ); let peer_rx = self.handle.ctx.peer_tx.subscribe(); - - FlashblocksConnection::new(self, conn, peer_id, BroadcastStream::new(peer_rx)) + let (direct_tx, direct_rx) = mpsc::unbounded_channel(); + + FlashblocksConnection::new( + self, + conn, + peer_id, + BroadcastStream::new(peer_rx), + direct_tx, + direct_rx, + ) } } @@ -1190,6 +1259,25 @@ mod tests { state } + /// Creates a peer state with a per-peer direct channel for message assertions. + fn test_peer_state_with_channel( + trusted: bool, + trusted_known: bool, + ) -> (FlashblocksConnectionState, mpsc::UnboundedReceiver) { + let (tx, rx) = mpsc::unbounded_channel(); + let mut state = FlashblocksConnectionState::new(); + state.trusted = trusted; + state.trusted_known = trusted_known; + state.direct_tx = Some(tx); + (state, rx) + } + + /// Receives and decodes a direct control message from a per-peer channel. + fn recv_direct(rx: &mut mpsc::UnboundedReceiver) -> FlashblocksP2PMsg { + let bytes = rx.try_recv().expect("expected a direct message"); + FlashblocksP2PMsg::decode(&mut &bytes[..]).expect("valid message") + } + fn peer_state( fanout: &FlashblocksP2PState, peer_id: PeerId, @@ -1214,34 +1302,23 @@ mod tests { }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let trusted_peer = PeerId::random(); let untrusted_peer = PeerId::random(); - let trusted_state = test_peer_state(true, true); + let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true, true); let untrusted_state = test_peer_state(false, true); - fanout - .connections - .insert(trusted_peer, trusted_state.clone()); - fanout - .connections - .insert(untrusted_peer, untrusted_state.clone()); + fanout.connections.insert(trusted_peer, trusted_state); + fanout.connections.insert(untrusted_peer, untrusted_state); fanout.maybe_request_receive_peers(&ctx); assert!(peer_state(&fanout, trusted_peer).request_in_flight); assert!(peer_state(&fanout, trusted_peer).receive_enabled.is_some()); assert!(!peer_state(&fanout, untrusted_peer).request_in_flight); - match rx.try_recv().expect("request sent") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, trusted_peer); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::RequestFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } + assert_eq!( + recv_direct(&mut trusted_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); } #[test] @@ -1252,24 +1329,20 @@ mod tests { }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let trusted_peer = PeerId::random(); let untrusted_peer = PeerId::random(); - let trusted_state = test_peer_state(true, false); - let untrusted_state = test_peer_state(false, false); - fanout - .connections - .insert(trusted_peer, trusted_state.clone()); - fanout - .connections - .insert(untrusted_peer, untrusted_state.clone()); + let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true, false); + let (untrusted_state, mut untrusted_rx) = test_peer_state_with_channel(false, false); + fanout.connections.insert(trusted_peer, trusted_state); + fanout.connections.insert(untrusted_peer, untrusted_state); fanout.maybe_request_receive_peers(&ctx); assert!(!peer_state(&fanout, trusted_peer).request_in_flight); assert!(!peer_state(&fanout, untrusted_peer).request_in_flight); - assert!(rx.try_recv().is_err()); + assert!(trusted_rx.try_recv().is_err()); + assert!(untrusted_rx.try_recv().is_err()); } #[test] @@ -1280,44 +1353,28 @@ mod tests { }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let victim = PeerId::random(); let trusted_requester = PeerId::random(); - let mut victim_state = test_peer_state(false, true); - let requester_state = test_peer_state(true, true); + let (mut victim_state, mut victim_rx) = test_peer_state_with_channel(false, true); + let (requester_state, mut requester_rx) = test_peer_state_with_channel(true, true); victim_state.send_enabled = true; - fanout.connections.insert(victim, victim_state.clone()); - fanout - .connections - .insert(trusted_requester, requester_state.clone()); + fanout.connections.insert(victim, victim_state); + fanout.connections.insert(trusted_requester, requester_state); - fanout.handle_request(&ctx, trusted_requester); + assert!(!fanout.handle_request(&ctx, trusted_requester)); assert!(!peer_state(&fanout, victim).send_enabled); assert!(peer_state(&fanout, trusted_requester).send_enabled); - match rx.try_recv().expect("cancel sent") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, victim); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::CancelFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } - - match rx.try_recv().expect("accept sent") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, trusted_requester); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::AcceptFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } + assert_eq!( + recv_direct(&mut victim_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut requester_rx), + FlashblocksP2PMsg::AcceptFlashblocks + ); } #[test] @@ -1330,21 +1387,16 @@ mod tests { let latency_window = config.latency_window; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let current_peer = PeerId::random(); let candidate_peer = PeerId::random(); - let mut current_state = test_peer_state(false, true); - let candidate_state = test_peer_state(false, true); + let (mut current_state, mut current_rx) = test_peer_state_with_channel(false, true); + let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false, true); let mut score = Score::new(latency_window); score.record(42); current_state.receive_enabled = Some(score); - fanout - .connections - .insert(current_peer, current_state.clone()); - fanout - .connections - .insert(candidate_peer, candidate_state.clone()); + fanout.connections.insert(current_peer, current_state); + fanout.connections.insert(candidate_peer, candidate_state); fanout.maybe_start_rotation(&ctx); @@ -1352,29 +1404,16 @@ mod tests { assert!(peer_state(&fanout, candidate_peer).request_in_flight); assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_some()); - match rx.try_recv().expect("cancel sent to old peer") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, current_peer); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::CancelFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } - - match rx.try_recv().expect("rotation request sent") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, candidate_peer); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::RequestFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } + assert_eq!( + recv_direct(&mut current_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut candidate_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); - fanout.handle_accept(&ctx, candidate_peer); + assert!(!fanout.handle_accept(&ctx, candidate_peer)); assert!(!peer_state(&fanout, candidate_peer).request_in_flight); assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_some()); @@ -1388,37 +1427,30 @@ mod tests { }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let first_peer = PeerId::random(); let second_peer = PeerId::random(); - let first_state = test_peer_state(false, true); - let second_state = test_peer_state(false, true); - fanout.connections.insert(first_peer, first_state.clone()); - fanout.connections.insert(second_peer, second_state.clone()); + let (first_state, mut first_rx) = test_peer_state_with_channel(false, true); + let (second_state, mut second_rx) = test_peer_state_with_channel(false, true); + fanout.connections.insert(first_peer, first_state); + fanout.connections.insert(second_peer, second_state); fanout.maybe_request_receive_peers(&ctx); assert!(peer_state(&fanout, first_peer).request_in_flight); assert!(peer_state(&fanout, second_peer).request_in_flight); - let mut requested_peers = HashSet::new(); - for _ in 0..2 { - match rx.try_recv().expect("request sent") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::RequestFlashblocks - ); - requested_peers.insert(peer_id); - } - other => panic!("unexpected peer message: {other:?}"), - } - } - assert_eq!(requested_peers, HashSet::from([first_peer, second_peer])); + assert_eq!( + recv_direct(&mut first_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + assert_eq!( + recv_direct(&mut second_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); - fanout.handle_accept(&ctx, first_peer); - fanout.handle_accept(&ctx, second_peer); + assert!(!fanout.handle_accept(&ctx, first_peer)); + assert!(!fanout.handle_accept(&ctx, second_peer)); assert!(!peer_state(&fanout, first_peer).request_in_flight); assert!(!peer_state(&fanout, second_peer).request_in_flight); @@ -1434,21 +1466,23 @@ mod tests { }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let peer = PeerId::random(); - let candidate_state = test_peer_state(false, true); - fanout.connections.insert(peer, candidate_state.clone()); + let (candidate_state, mut peer_rx) = test_peer_state_with_channel(false, true); + fanout.connections.insert(peer, candidate_state); fanout.maybe_request_receive_peers(&ctx); - let _ = rx.try_recv().expect("initial request sent"); + assert_eq!( + recv_direct(&mut peer_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); - fanout.handle_reject(&ctx, peer); + assert!(!fanout.handle_reject(&ctx, peer)); assert!(!peer_state(&fanout, peer).request_in_flight); assert!(peer_state(&fanout, peer).receive_enabled.is_none()); assert!(peer_state(&fanout, peer).request_backoff_until.is_some()); - assert!(rx.try_recv().is_err()); + assert!(peer_rx.try_recv().is_err()); } #[test] @@ -1459,15 +1493,14 @@ mod tests { }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - let mut rx = ctx.peer_tx.subscribe(); let oldest_peer = PeerId::random(); let newer_peer = PeerId::random(); let replacement_peer = PeerId::random(); - let mut oldest_state = test_peer_state(false, true); + let (mut oldest_state, mut oldest_rx) = test_peer_state_with_channel(false, true); let mut newer_state = test_peer_state(false, true); - let replacement_state = test_peer_state(true, true); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true, true); oldest_state.receive_enabled = Some(Score::new(4)); oldest_state.request_in_flight = true; @@ -1476,38 +1509,23 @@ mod tests { newer_state.request_in_flight = true; newer_state.receive_enabled_timestamp = 2; - fanout.connections.insert(oldest_peer, oldest_state.clone()); - fanout.connections.insert(newer_peer, newer_state.clone()); - fanout - .connections - .insert(replacement_peer, replacement_state.clone()); + fanout.connections.insert(oldest_peer, oldest_state); + fanout.connections.insert(newer_peer, newer_state); + fanout.connections.insert(replacement_peer, replacement_state); fanout.maybe_start_rotation(&ctx); assert!(peer_state(&fanout, oldest_peer).receive_enabled.is_none()); assert!(peer_state(&fanout, replacement_peer).request_in_flight); - match rx.try_recv().expect("cancel sent to oldest peer") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, oldest_peer); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::CancelFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } - - match rx.try_recv().expect("replacement request sent") { - PeerMsg::Direct { peer_id, bytes } => { - assert_eq!(peer_id, replacement_peer); - assert_eq!( - FlashblocksP2PMsg::decode(&mut &bytes[..]).unwrap(), - FlashblocksP2PMsg::RequestFlashblocks - ); - } - other => panic!("unexpected peer message: {other:?}"), - } + assert_eq!( + recv_direct(&mut oldest_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); } #[test] @@ -1540,10 +1558,8 @@ mod tests { .expect("lagging peer score") .record(100); - fanout.connections.insert(steady_peer, steady_state.clone()); - fanout - .connections - .insert(lagging_peer, lagging_state.clone()); + fanout.connections.insert(steady_peer, steady_state); + fanout.connections.insert(lagging_peer, lagging_state); for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { let authorization = Authorization::new( @@ -1611,13 +1627,13 @@ mod tests { .expect("rotating peer score") .record(100); - fanout.connections.insert(steady_peer, steady_state.clone()); + fanout.connections.insert(steady_peer, steady_state); fanout .connections - .insert(rotating_peer, rotating_state.clone()); + .insert(rotating_peer, rotating_state); fanout .connections - .insert(candidate_peer, candidate_state.clone()); + .insert(candidate_peer, candidate_state); fanout.maybe_start_rotation(&ctx); @@ -1642,10 +1658,115 @@ mod tests { fanout .connections - .insert(replacement_peer, replacement_state.clone()); + .insert(replacement_peer, replacement_state); fanout.maybe_start_rotation(&ctx); assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_none()); assert!(peer_state(&fanout, replacement_peer).request_in_flight); } + + #[test] + fn unsolicited_accept_is_penalized() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false, true); + fanout.connections.insert(peer, state); + + // Accept without a prior request should be penalized. + assert!(fanout.handle_accept(&ctx, peer)); + } + + #[test] + fn unsolicited_reject_is_penalized() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false, true); + fanout.connections.insert(peer, state); + + // Reject without a prior request should be penalized. + assert!(fanout.handle_reject(&ctx, peer)); + } + + #[test] + fn cancel_without_relationship_is_penalized() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false, true); + fanout.connections.insert(peer, state); + + // Cancel with no send/receive relationship should be penalized. + assert!(fanout.handle_cancel(&ctx, peer)); + } + + #[test] + fn cancel_only_clears_relevant_direction() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + // Peer we are only sending to — cancel should clear send but not touch receive. + let send_peer = PeerId::random(); + let mut send_state = test_peer_state(false, true); + send_state.send_enabled = true; + fanout.connections.insert(send_peer, send_state); + + assert!(!fanout.handle_cancel(&ctx, send_peer)); + assert!(!peer_state(&fanout, send_peer).send_enabled); + // Should NOT set a request_backoff (no receive state was cleared). + assert!(peer_state(&fanout, send_peer).request_backoff_until.is_none()); + + // Peer we are only receiving from — cancel should clear receive but not touch send. + let recv_peer = PeerId::random(); + let mut recv_state = test_peer_state(false, true); + recv_state.receive_enabled = Some(Score::new(4)); + fanout.connections.insert(recv_peer, recv_state); + + assert!(!fanout.handle_cancel(&ctx, recv_peer)); + assert!(peer_state(&fanout, recv_peer).receive_enabled.is_none()); + assert!(peer_state(&fanout, recv_peer).request_backoff_until.is_some()); + } + + #[test] + fn duplicate_request_when_already_sending_is_penalized() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false, true); + state.send_enabled = true; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_request(&ctx, peer)); + } + + #[test] + fn control_message_rate_limit_triggers_penalty() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false, true); + state.send_enabled = true; + fanout.connections.insert(peer, state); + + // Spam requests to exceed the rate limit. + for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { + // These return true because send_enabled is already set (duplicate request), + // but the rate limit hasn't been hit yet. + assert!(fanout.handle_request(&ctx, peer)); + } + // The next one should hit the rate limit. + assert!(fanout.handle_request(&ctx, peer)); + } } From 30cafa3d62df9166fd05fd4cfa8fcc4b9a523af3 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Tue, 10 Mar 2026 23:16:40 -0700 Subject: [PATCH 11/43] cleanup --- .../flashblocks/p2p/src/protocol/handler.rs | 47 +++++++------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 004eaaeec..34a5f66ea 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -326,13 +326,11 @@ impl FlashblocksP2PState { peer_state: &mut FlashblocksConnectionState, receive_enabled_timestamp: u64, request_backoff_until: Option, - ) -> bool { - let had_receive_state = peer_state.receive_enabled.is_some() || peer_state.request_in_flight; + ) { peer_state.receive_enabled = None; peer_state.request_in_flight = false; peer_state.receive_enabled_timestamp = receive_enabled_timestamp; peer_state.request_backoff_until = request_backoff_until; - had_receive_state } fn available_receive_candidates(&self) -> Vec<(PeerId, bool)> { @@ -471,20 +469,19 @@ impl FlashblocksP2PState { return true; } let peer_is_trusted = peer_state.trusted; + let non_trusted_send_count = self + .connections + .values() + .filter(|s| s.send_enabled && !s.trusted) + .count(); + if peer_is_trusted { - let non_trusted_send_count = self - .connections - .values() - .filter(|candidate_state| candidate_state.send_enabled && !candidate_state.trusted) - .count(); + // Trusted peers always get accepted; evict an untrusted sender if at capacity. if non_trusted_send_count >= ctx.fanout_config.max_send_peers { if let Some(evicted_peer) = self.connections .iter() - .find_map(|(candidate, candidate_state)| { - (candidate_state.send_enabled && !candidate_state.trusted) - .then_some(*candidate) - }) + .find_map(|(id, s)| (s.send_enabled && !s.trusted).then_some(*id)) { if let Some(evicted_state) = self.connection_state_mut(&evicted_peer) { evicted_state.send_enabled = false; @@ -492,30 +489,18 @@ impl FlashblocksP2PState { self.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); } } - - let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); - peer_state.send_enabled = true; - peer_state.request_backoff_until = None; - self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); - return false; - } - - let non_trusted_send_count = self - .connections - .values() - .filter(|candidate_state| candidate_state.send_enabled && !candidate_state.trusted) - .count(); - if non_trusted_send_count < ctx.fanout_config.max_send_peers { - let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); - peer_state.send_enabled = true; - peer_state.request_backoff_until = None; - self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); - } else { + } else if non_trusted_send_count >= ctx.fanout_config.max_send_peers { self.connection_state_mut(&peer_id) .expect("peer exists") .request_backoff_until = Some(Self::request_backoff_deadline(ctx)); self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); + return false; } + + let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); + peer_state.send_enabled = true; + peer_state.request_backoff_until = None; + self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); false } From b2b1dea3f14df06092181ad7a3a1a60382cf7f80 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 00:02:15 -0700 Subject: [PATCH 12/43] wip --- crates/flashblocks/node/tests/p2p.rs | 94 +++++--- .../p2p/src/protocol/connection.rs | 23 +- .../flashblocks/p2p/src/protocol/handler.rs | 227 +++++++++--------- crates/flashblocks/primitives/src/p2p.rs | 2 +- .../node/tests/e2e-testsuite/testsuite.rs | 18 +- crates/world/node/tests/it/builder.rs | 4 +- specs/flashblocks_p2p_v2.md | 20 +- 7 files changed, 211 insertions(+), 177 deletions(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index a438537b7..697dfdfce 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -38,10 +38,11 @@ use serde::{Deserialize, Serialize}; use std::{ any::Any, collections::HashMap, + fmt, io::Write, net::{IpAddr, SocketAddr}, path::PathBuf, - sync::Arc, + sync::{Arc, Mutex}, }; use tempfile::NamedTempFile; use tokio::time::{Duration, Instant, sleep}; @@ -58,6 +59,42 @@ use world_chain_test::{ utils::{account, eip1559, raw_tx, signer}, }; +/// Thread-safe log buffer for capturing tracing output across threads. +#[derive(Clone, Default)] +struct SharedLogBuffer(Arc>>); + +impl SharedLogBuffer { + fn logs(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +impl tracing_subscriber::Layer for SharedLogBuffer { + fn on_event( + &self, + event: &tracing::Event<'_>, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut visitor = LogVisitor(String::new()); + visitor.0.push_str(&format!("{} ", event.metadata().level())); + visitor.0.push_str(&format!("{}: ", event.metadata().target())); + event.record(&mut visitor); + self.0.lock().unwrap().push(visitor.0); + } +} + +struct LogVisitor(String); + +impl tracing::field::Visit for LogVisitor { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) { + if field.name() == "message" { + self.0.push_str(&format!("{:?}", value)); + } else { + self.0.push_str(&format!(" {}={:?}", field.name(), value)); + } + } +} + #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct Metadata { pub receipts: HashMap, @@ -365,7 +402,7 @@ async fn setup_nodes(n: u8) -> eyre::Result { }) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] #[ignore] async fn test_double_failover() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -457,7 +494,7 @@ async fn test_double_failover() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_force_race_condition() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -591,7 +628,7 @@ async fn test_force_race_condition() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_get_block_by_number_pending() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -670,7 +707,7 @@ async fn test_get_block_by_number_pending() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_peer_reputation() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -723,9 +760,15 @@ async fn test_peer_reputation() -> eyre::Result<()> { Ok(()) } -#[tokio::test] -#[tracing_test::traced_test] +#[tokio::test(flavor = "multi_thread")] async fn test_peer_monitoring() -> eyre::Result<()> { + use tracing_subscriber::layer::SubscriberExt; + + let log_buffer = SharedLogBuffer::default(); + let subscriber = tracing_subscriber::registry().with(log_buffer.clone()); + tracing::subscriber::set_global_default(subscriber) + .expect("failed to set global subscriber"); + let authorizer = SigningKey::from_bytes(&[0; 32]); // Create a temporary P2P secret key file for node1 to ensure consistent peer ID across restarts @@ -804,18 +847,17 @@ async fn test_peer_monitoring() -> eyre::Result<()> { sleep(Duration::from_millis(500)).await; // Check that disconnection was logged by the event listener (immediate detection) - logs_assert(|logs: &[&str]| { + { + let logs = log_buffer.logs(); let disconnect_log_exists = logs.iter().any(|log| { log.contains("trusted peer disconnected") && log.contains(&peer1_id.to_string()) }); - assert!( disconnect_log_exists, "Should have logged 'trusted peer disconnected' for peer {} from event listener", peer1_id ); - Ok(()) - }); + } // Wait for PeerMonitor periodic checks to detect the disconnection and emit multiple warning logs // Wait for at least 2 periodic ticks to ensure we get multiple log outputs (1s * 3 + 1s buffer for safety) @@ -892,75 +934,63 @@ async fn test_peer_monitoring() -> eyre::Result<()> { } // Assert that the "connection to trusted peer established" log appears for node1 - logs_assert(|logs: &[&str]| { + { + let logs = log_buffer.logs(); let reconnection_log_exists = logs.iter().any(|log| { log.contains("connection to trusted peer established") && log.contains(&peer1_id.to_string()) }); - assert!( reconnection_log_exists, "Should have logged 'connection to trusted peer established' for peer {} after restart", peer1_id ); - Ok(()) - }); + } // Wait for at least one more monitor tick to verify warnings stopped (1s interval + 1s buffer) sleep(monitor::PEER_MONITOR_INTERVAL + Duration::from_secs(1)).await; // Count the number of warning logs before and after reconnection to ensure they stopped - logs_assert(|logs: &[&str]| { - // Find the index where reconnection happened (use rposition to find the LAST occurrence) + { + let logs = log_buffer.logs(); let reconnection_log_idx = logs .iter() .rposition(|log| log.contains("connection to trusted peer established")) - .ok_or_else(|| { - "Could not find 'connection to trusted peer established' log".to_string() - })?; + .expect("Could not find 'connection to trusted peer established' log"); - // Split logs at the reconnection point let (logs_before_reconnect, logs_after_reconnect) = logs.split_at(reconnection_log_idx); - // Filter for disconnect warnings in logs before reconnection - let warnings_before_reconnect: Vec<&str> = logs_before_reconnect + let warnings_before_reconnect: Vec<&String> = logs_before_reconnect .iter() .filter(|log| { log.contains(&peer1_id.to_string()) && log.contains("WARN") && log.contains("trusted peer disconnected") }) - .copied() .collect(); - // We should have seen at least 2 warnings before reconnection assert!( warnings_before_reconnect.len() >= 2, "Should have had at least 2 warnings before reconnection, found {}", warnings_before_reconnect.len() ); - // Filter for disconnect warnings in logs after reconnection - let warnings_after_reconnect: Vec<&str> = logs_after_reconnect + let warnings_after_reconnect: Vec<&String> = logs_after_reconnect .iter() .filter(|log| { log.contains(&peer1_id.to_string()) && log.contains("WARN") && log.contains("trusted peer disconnected") }) - .copied() .collect(); - // There should be no warnings after reconnection assert!( warnings_after_reconnect.is_empty(), "Should have no warnings after reconnection, found {}: {:?}", warnings_after_reconnect.len(), warnings_after_reconnect ); - - Ok(()) - }); + } Ok(()) } diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 5bcd3d0d6..a5ea0276a 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -42,7 +42,8 @@ pub struct FlashblocksConnectionState { /// /// Optional score for this peer connection, used for adaptive timeouts and peer selection. /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks - /// counting as 10s + /// counting as 10s. While `request_in_flight` is true, the peer is only a provisional + /// candidate and must not deliver flashblocks yet. pub receive_enabled: Option, /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. pub receive_enabled_timestamp: u64, @@ -399,6 +400,19 @@ impl FlashblocksConnection { let Some(conn_state) = p2p_state.connection_state(&self.peer_id) else { return; }; + if conn_state.request_in_flight { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock before request was accepted", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + return; + } if conn_state.receive_enabled.is_none() { if conn_state.receive_enabled_timestamp + 2 < authorization.timestamp { tracing::warn!( @@ -470,7 +484,10 @@ impl FlashblocksConnection { } } - self.protocol.handle.ctx.publish(&mut p2p_state, authorized_payload); + self.protocol + .handle + .ctx + .publish(&mut p2p_state, authorized_payload); } /// Handles incoming `StartPublish` messages from a peer. @@ -563,7 +580,6 @@ impl FlashblocksConnection { active_publishers.push((authorization.builder_vk, authorization.timestamp)); } }); - } /// Handles incoming `StopPublish` messages from a peer. @@ -667,7 +683,6 @@ impl FlashblocksConnection { } } }); - } } diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 34a5f66ea..892df3e91 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -399,16 +399,18 @@ impl FlashblocksP2PState { peer_state.receive_enabled_timestamp, )) }) - .max_by(|(_, lhs_score, lhs_timestamp), (_, rhs_score, rhs_timestamp)| { - match (lhs_score, rhs_score) { + .max_by( + |(_, lhs_score, lhs_timestamp), (_, rhs_score, rhs_timestamp)| match ( + lhs_score, rhs_score, + ) { (None, None) => rhs_timestamp.cmp(lhs_timestamp), (None, Some(_)) => std::cmp::Ordering::Greater, (Some(_), None) => std::cmp::Ordering::Less, (Some(lhs_score), Some(rhs_score)) => lhs_score .cmp(rhs_score) .then_with(|| rhs_timestamp.cmp(lhs_timestamp)), - } - }) + }, + ) .map(|(peer_id, _, _)| peer_id) } @@ -455,6 +457,11 @@ impl FlashblocksP2PState { return false; }; + if !peer_state.trusted_known { + // Wait until trust classification is known before deciding whether this peer should + // bypass the non-trusted limit. + return false; + } if peer_state.send_enabled { // Already sending to this peer — repeated request is spam. return true; @@ -475,21 +482,7 @@ impl FlashblocksP2PState { .filter(|s| s.send_enabled && !s.trusted) .count(); - if peer_is_trusted { - // Trusted peers always get accepted; evict an untrusted sender if at capacity. - if non_trusted_send_count >= ctx.fanout_config.max_send_peers { - if let Some(evicted_peer) = - self.connections - .iter() - .find_map(|(id, s)| (s.send_enabled && !s.trusted).then_some(*id)) - { - if let Some(evicted_state) = self.connection_state_mut(&evicted_peer) { - evicted_state.send_enabled = false; - } - self.send_direct(evicted_peer, FlashblocksP2PMsg::CancelFlashblocks); - } - } - } else if non_trusted_send_count >= ctx.fanout_config.max_send_peers { + if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_config.max_send_peers { self.connection_state_mut(&peer_id) .expect("peer exists") .request_backoff_until = Some(Self::request_backoff_deadline(ctx)); @@ -549,7 +542,7 @@ impl FlashblocksP2PState { } /// Returns `true` if the peer should receive a reputation penalty. - fn handle_cancel(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + fn handle_cancel(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { if self.check_control_rate_limit(&peer_id) { return true; } @@ -558,28 +551,12 @@ impl FlashblocksP2PState { return false; }; - let has_send = peer_state.send_enabled; - let has_receive = peer_state.receive_enabled.is_some() || peer_state.request_in_flight; - - if !has_send && !has_receive { - // No active relationship — unsolicited cancel is spam. + if !peer_state.send_enabled { + // Cancel is only valid from a receiver to its sender. return true; } - // Only clear the directions that actually have a relationship. - if has_send { - peer_state.send_enabled = false; - } - if has_receive { - peer_state.receive_enabled = None; - peer_state.request_in_flight = false; - peer_state.receive_enabled_timestamp = Utc::now().timestamp() as u64; - peer_state.request_backoff_until = Some(Self::request_backoff_deadline(ctx)); - } - - if has_receive { - self.maybe_request_receive_peers(ctx); - } + peer_state.send_enabled = false; false } } @@ -664,36 +641,30 @@ impl FlashblocksHandle { peer_id: PeerId, direct_tx: mpsc::UnboundedSender, ) { - { - let mut state = self.state.lock(); - let mut conn_state = FlashblocksConnectionState::new(); - conn_state.direct_tx = Some(direct_tx); - state.connections.insert(peer_id, conn_state); - state.maybe_request_receive_peers(&self.ctx); - } + let trusted = tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(network.get_peer_by_id(peer_id)) + }); - let handle = self.clone(); - tokio::spawn(async move { - match network.get_peer_by_id(peer_id).await { - Ok(Some(peer_info)) => { - let mut state = handle.state.lock(); - if let Some(peer_state) = state.connection_state_mut(&peer_id) { - peer_state.trusted = peer_info.kind.is_trusted(); - peer_state.trusted_known = true; - state.maybe_request_receive_peers(&handle.ctx); - } - } - Ok(None) => {} - Err(error) => { - warn!( - target: "flashblocks::p2p", - %peer_id, - %error, - "failed to load peer info for flashblocks fanout", - ); - } + let mut state = self.state.lock(); + let mut conn_state = FlashblocksConnectionState::new(); + conn_state.direct_tx = Some(direct_tx); + conn_state.trusted_known = true; + match trusted { + Ok(Some(peer_info)) => { + conn_state.trusted = peer_info.kind.is_trusted(); } - }); + Ok(None) => {} + Err(error) => { + warn!( + target: "flashblocks::p2p", + %peer_id, + %error, + "failed to load peer info for flashblocks fanout", + ); + } + } + state.connections.insert(peer_id, conn_state); + state.maybe_request_receive_peers(&self.ctx); } pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { @@ -1174,7 +1145,7 @@ impl ConnectionHandler for FlashblocksP2PProtoco type Connection = FlashblocksConnection; fn protocol(&self) -> Protocol { - Protocol::new(Self::capability(), 6) + Protocol::new(Self::capability(), 5) } fn on_unsupported_by_peer( @@ -1234,10 +1205,7 @@ mod tests { } } - fn test_peer_state( - trusted: bool, - trusted_known: bool, - ) -> FlashblocksConnectionState { + fn test_peer_state(trusted: bool, trusted_known: bool) -> FlashblocksConnectionState { let mut state = FlashblocksConnectionState::new(); state.trusted = trusted; state.trusted_known = trusted_known; @@ -1248,7 +1216,10 @@ mod tests { fn test_peer_state_with_channel( trusted: bool, trusted_known: bool, - ) -> (FlashblocksConnectionState, mpsc::UnboundedReceiver) { + ) -> ( + FlashblocksConnectionState, + mpsc::UnboundedReceiver, + ) { let (tx, rx) = mpsc::unbounded_channel(); let mut state = FlashblocksConnectionState::new(); state.trusted = trusted; @@ -1263,10 +1234,7 @@ mod tests { FlashblocksP2PMsg::decode(&mut &bytes[..]).expect("valid message") } - fn peer_state( - fanout: &FlashblocksP2PState, - peer_id: PeerId, - ) -> &FlashblocksConnectionState { + fn peer_state(fanout: &FlashblocksP2PState, peer_id: PeerId) -> &FlashblocksConnectionState { fanout.connection_state(&peer_id).expect("peer exists") } @@ -1331,7 +1299,7 @@ mod tests { } #[test] - fn trusted_request_evicts_non_trusted_sender() { + fn trusted_request_bypasses_non_trusted_limit() { let config = FanoutConfig { max_send_peers: 1, ..Default::default() @@ -1345,17 +1313,15 @@ mod tests { let (requester_state, mut requester_rx) = test_peer_state_with_channel(true, true); victim_state.send_enabled = true; fanout.connections.insert(victim, victim_state); - fanout.connections.insert(trusted_requester, requester_state); + fanout + .connections + .insert(trusted_requester, requester_state); assert!(!fanout.handle_request(&ctx, trusted_requester)); - assert!(!peer_state(&fanout, victim).send_enabled); + assert!(peer_state(&fanout, victim).send_enabled); assert!(peer_state(&fanout, trusted_requester).send_enabled); - - assert_eq!( - recv_direct(&mut victim_rx), - FlashblocksP2PMsg::CancelFlashblocks - ); + assert!(victim_rx.try_recv().is_err()); assert_eq!( recv_direct(&mut requester_rx), FlashblocksP2PMsg::AcceptFlashblocks @@ -1387,7 +1353,11 @@ mod tests { assert!(peer_state(&fanout, current_peer).receive_enabled.is_none()); assert!(peer_state(&fanout, candidate_peer).request_in_flight); - assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_some()); + assert!( + peer_state(&fanout, candidate_peer) + .receive_enabled + .is_some() + ); assert_eq!( recv_direct(&mut current_rx), @@ -1401,7 +1371,11 @@ mod tests { assert!(!fanout.handle_accept(&ctx, candidate_peer)); assert!(!peer_state(&fanout, candidate_peer).request_in_flight); - assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_some()); + assert!( + peer_state(&fanout, candidate_peer) + .receive_enabled + .is_some() + ); } #[test] @@ -1496,7 +1470,9 @@ mod tests { fanout.connections.insert(oldest_peer, oldest_state); fanout.connections.insert(newer_peer, newer_state); - fanout.connections.insert(replacement_peer, replacement_state); + fanout + .connections + .insert(replacement_peer, replacement_state); fanout.maybe_start_rotation(&ctx); @@ -1613,12 +1589,8 @@ mod tests { .record(100); fanout.connections.insert(steady_peer, steady_state); - fanout - .connections - .insert(rotating_peer, rotating_state); - fanout - .connections - .insert(candidate_peer, candidate_state); + fanout.connections.insert(rotating_peer, rotating_state); + fanout.connections.insert(candidate_peer, candidate_state); fanout.maybe_start_rotation(&ctx); @@ -1646,7 +1618,11 @@ mod tests { .insert(replacement_peer, replacement_state); fanout.maybe_start_rotation(&ctx); - assert!(peer_state(&fanout, candidate_peer).receive_enabled.is_none()); + assert!( + peer_state(&fanout, candidate_peer) + .receive_enabled + .is_none() + ); assert!(peer_state(&fanout, replacement_peer).request_in_flight); } @@ -1693,31 +1669,50 @@ mod tests { } #[test] - fn cancel_only_clears_relevant_direction() { + fn cancel_only_clears_send_direction() { let config = FanoutConfig::default(); let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); - // Peer we are only sending to — cancel should clear send but not touch receive. - let send_peer = PeerId::random(); - let mut send_state = test_peer_state(false, true); - send_state.send_enabled = true; - fanout.connections.insert(send_peer, send_state); - - assert!(!fanout.handle_cancel(&ctx, send_peer)); - assert!(!peer_state(&fanout, send_peer).send_enabled); - // Should NOT set a request_backoff (no receive state was cleared). - assert!(peer_state(&fanout, send_peer).request_backoff_until.is_none()); - - // Peer we are only receiving from — cancel should clear receive but not touch send. - let recv_peer = PeerId::random(); - let mut recv_state = test_peer_state(false, true); - recv_state.receive_enabled = Some(Score::new(4)); - fanout.connections.insert(recv_peer, recv_state); - - assert!(!fanout.handle_cancel(&ctx, recv_peer)); - assert!(peer_state(&fanout, recv_peer).receive_enabled.is_none()); - assert!(peer_state(&fanout, recv_peer).request_backoff_until.is_some()); + let peer = PeerId::random(); + let mut state = test_peer_state(false, true); + state.send_enabled = true; + state.receive_enabled = Some(Score::new(4)); + fanout.connections.insert(peer, state); + + assert!(!fanout.handle_cancel(&ctx, peer)); + assert!(!peer_state(&fanout, peer).send_enabled); + assert!(peer_state(&fanout, peer).receive_enabled.is_some()); + assert!(!peer_state(&fanout, peer).request_in_flight); + } + + #[test] + fn cancel_from_sender_is_penalized() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false, true); + state.receive_enabled = Some(Score::new(4)); + fanout.connections.insert(peer, state); + + assert!(fanout.handle_cancel(&ctx, peer)); + } + + #[test] + fn request_from_unknown_trust_peer_is_deferred() { + let config = FanoutConfig::default(); + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (state, mut rx) = test_peer_state_with_channel(false, false); + fanout.connections.insert(peer, state); + + assert!(!fanout.handle_request(&ctx, peer)); + assert!(!peer_state(&fanout, peer).send_enabled); + assert!(rx.try_recv().is_err()); } #[test] diff --git a/crates/flashblocks/primitives/src/p2p.rs b/crates/flashblocks/primitives/src/p2p.rs index d290525bf..cd43b507c 100644 --- a/crates/flashblocks/primitives/src/p2p.rs +++ b/crates/flashblocks/primitives/src/p2p.rs @@ -53,7 +53,7 @@ pub enum FlashblocksP2PMsg { AcceptFlashblocks = 0x02, /// Rejects a previously sent [`Self::RequestFlashblocks`] request. RejectFlashblocks = 0x03, - /// Terminates an active flashblocks feed. + /// Sent by a receiver to terminate an active flashblocks feed from a sender. CancelFlashblocks = 0x04, } diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index a8eb3ce5f..b95f7fbc2 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -63,7 +63,7 @@ async fn create_priority_transaction( Ok((signed.encoded_2718().into(), *signed.tx_hash())) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_can_build_pbh_payload() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let (signers, mut nodes, _tasks, _, _) = @@ -93,7 +93,7 @@ async fn test_can_build_pbh_payload() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_transaction_pool_ordering() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -139,7 +139,7 @@ async fn test_transaction_pool_ordering() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_enforces_block_uncompressed_size_limit() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -238,7 +238,7 @@ async fn test_enforces_block_uncompressed_size_limit() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_without_block_uncompressed_size_limit_includes_all_transactions() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -287,7 +287,7 @@ async fn test_without_block_uncompressed_size_limit_includes_all_transactions() Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_invalidate_dup_tx_and_nullifier() -> eyre::Result<()> { reth_tracing::init_test_tracing(); let (_signers, mut nodes, _tasks, _, _) = @@ -301,7 +301,7 @@ async fn test_invalidate_dup_tx_and_nullifier() -> eyre::Result<()> { Ok(()) } -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_dup_pbh_nonce() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -704,7 +704,7 @@ async fn test_eth_block_by_hash_pending() -> eyre::Result<()> { /// /// Verifies that without tx_peers configuration, transactions propagate to ALL connected peers /// using Reth's default TransactionPropagationKind::All policy. -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_default_propagation_policy() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -778,7 +778,7 @@ async fn test_default_propagation_policy() -> eyre::Result<()> { /// Test Part 2: /// - Inject tx into Node 2 -> should propagate to both Node 0 and Node 1 /// - Verifies multi-peer whitelist works correctly -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] #[ignore = "TODO: flaky - not sure what's causing this to fail"] async fn test_selective_propagation_policy() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -924,7 +924,7 @@ async fn test_selective_propagation_policy() -> eyre::Result<()> { /// - Inject tx into Node 0 -> should NOT propagate to any node /// - Inject tx into Node 1 -> should NOT propagate to any node (even though Node 0 is whitelisted) /// - Verifies that disable_txpool_gossip takes precedence over tx_peers -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_gossip_disabled_no_propagation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/world/node/tests/it/builder.rs b/crates/world/node/tests/it/builder.rs index 1c254e0a4..0b412b724 100644 --- a/crates/world/node/tests/it/builder.rs +++ b/crates/world/node/tests/it/builder.rs @@ -6,8 +6,8 @@ use reth_provider::providers::BlockchainProvider; use world_chain_node::{context::FlashblocksContext, node::WorldChainNode}; use world_chain_test::node::test_config; -#[test] -fn test_basic_flashblocks_setup() { +#[tokio::test] +async fn test_basic_flashblocks_setup() { // parse CLI -> config let config = NodeConfig::new(BASE_MAINNET.clone()); let db = create_test_rw_db(); diff --git a/specs/flashblocks_p2p_v2.md b/specs/flashblocks_p2p_v2.md index cecff7d41..022e43dfd 100644 --- a/specs/flashblocks_p2p_v2.md +++ b/specs/flashblocks_p2p_v2.md @@ -38,7 +38,7 @@ Four unsigned control messages are added to `FlashblocksP2PMsg`: | `0x01` | `RequestFlashblocks` | Receiver → Sender | "I want to receive flashblocks from you" | | `0x02` | `AcceptFlashblocks` | Sender → Receiver | "Accepted. I will send you flashblocks" | | `0x03` | `RejectFlashblocks` | Sender → Receiver | "Rejected. I am at capacity" | -| `0x04` | `CancelFlashblocks` | Either → Either | "I am ending our flashblock feed" | +| `0x04` | `CancelFlashblocks` | Receiver → Sender | "Stop sending me flashblocks" | These messages carry no payload. The connection context (peer ID) provides all necessary information. @@ -59,20 +59,16 @@ pub enum FlashblocksP2PMsg { **`RequestFlashblocks`** — Sent by a node that wants to receive flashblocks from the connected peer. The recipient evaluates: 1. Is the requester a trusted peer? → Always accept (trusted peers bypass `max_send_peers`). -2. Is the send set below `max_send_peers`? → Accept. -3. Is the send set full but contains non-trusted peers, AND the requester is trusted? → Evict a non-trusted peer (send it `CancelFlashblocks`), then accept. -4. Otherwise → Reject. +2. Is the number of non-trusted peers in the send set below `max_send_peers`? → Accept. +3. Otherwise → Reject. **`AcceptFlashblocks`** — Response to `RequestFlashblocks`. After this, the sender begins forwarding all `Authorized` messages to the receiver and adds the receiver to its send set. **`RejectFlashblocks`** — Response to `RequestFlashblocks` when the sender cannot accommodate more peers. The requester should try another peer. -**`CancelFlashblocks`** — Either side may send this to terminate the flashblock feed: +**`CancelFlashblocks`** — Sent only by a receiver to the sender it no longer wants to receive flashblocks from (e.g., during peer rotation). -- **Receiver-initiated**: "Stop sending me flashblocks." (e.g., during peer rotation) -- **Sender-initiated**: "I am going to stop sending you flashblocks." (e.g., evicting a non-trusted peer to make room for a trusted one) - -After receiving `CancelFlashblocks`, the other side immediately updates its local send/receive state for that feed. +After receiving `CancelFlashblocks`, the sender immediately stops forwarding flashblocks to that peer and removes it from its send set. ## Peer Management @@ -108,12 +104,10 @@ When a node starts and connects to peers via devp2p: receive RequestFlashblocks from peer P: if P is trusted: - if send_set has non-trusted peers AND send_set.len() >= max_send_peers: - evict lowest-priority non-trusted peer (send CancelFlashblocks, await ack) add P to send_set send AcceptFlashblocks to P -else if send_set.len() < max_send_peers: +else if non_trusted_send_count < max_send_peers: add P to send_set send AcceptFlashblocks to P @@ -137,7 +131,7 @@ When a node receives an `Authorized` message from a peer in its receive set: - **`StartPublish`**: Verify signatures and process locally. Do not relay it beyond the direct neighbor that sent it. - **`StopPublish`**: Same as `StartPublish` — process locally, do not relay. -If a node receives an `Authorized(FlashblocksPayloadV1)` from a peer **not** in its receive set, the message should be ignored. This prevents unsolicited data delivery. +If a node receives an `Authorized(FlashblocksPayloadV1)` from a peer **not** in its receive set, or from a peer whose `RequestFlashblocks` is still pending, the message should be ignored and the peer should be penalized. This prevents unsolicited data delivery. ### Duplicate Handling From 0d02eb085d9ecf4ebf8bb67e2f4dd70148f2cea5 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 00:45:30 -0700 Subject: [PATCH 13/43] wip --- Cargo.lock | 3 + crates/flashblocks/node/tests/p2p.rs | 12 +- crates/flashblocks/p2p/Cargo.toml | 5 + .../p2p/src/protocol/connection.rs | 16 +- .../flashblocks/p2p/src/protocol/handler.rs | 539 ++++++++++++++---- .../node/tests/e2e-testsuite/testsuite.rs | 26 +- 6 files changed, 473 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d271d579..2cfdf5ef5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3890,6 +3890,7 @@ dependencies = [ "alloy-rlp", "chrono", "ed25519-dalek", + "enr", "flashblocks-primitives", "futures", "metrics", @@ -3899,6 +3900,8 @@ dependencies = [ "reth-eth-wire", "reth-ethereum", "reth-network", + "reth-network-api", + "reth-network-peers", "reth-tasks", "thiserror 2.0.18", "tokio", diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 697dfdfce..03db65bb5 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -76,8 +76,12 @@ impl tracing_subscriber::Layer for SharedLogBuffer { _ctx: tracing_subscriber::layer::Context<'_, S>, ) { let mut visitor = LogVisitor(String::new()); - visitor.0.push_str(&format!("{} ", event.metadata().level())); - visitor.0.push_str(&format!("{}: ", event.metadata().target())); + visitor + .0 + .push_str(&format!("{} ", event.metadata().level())); + visitor + .0 + .push_str(&format!("{}: ", event.metadata().target())); event.record(&mut visitor); self.0.lock().unwrap().push(visitor.0); } @@ -403,7 +407,6 @@ async fn setup_nodes(n: u8) -> eyre::Result { } #[tokio::test(flavor = "multi_thread")] -#[ignore] async fn test_double_failover() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -766,8 +769,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { let log_buffer = SharedLogBuffer::default(); let subscriber = tracing_subscriber::registry().with(log_buffer.clone()); - tracing::subscriber::set_global_default(subscriber) - .expect("failed to set global subscriber"); + tracing::subscriber::set_global_default(subscriber).expect("failed to set global subscriber"); let authorizer = SigningKey::from_bytes(&[0; 32]); diff --git a/crates/flashblocks/p2p/Cargo.toml b/crates/flashblocks/p2p/Cargo.toml index b6721ec6d..078c241a0 100644 --- a/crates/flashblocks/p2p/Cargo.toml +++ b/crates/flashblocks/p2p/Cargo.toml @@ -29,3 +29,8 @@ parking_lot.workspace = true chrono.workspace = true reth-tasks = { workspace = true } rand.workspace = true + +[dev-dependencies] +reth-network-api.workspace = true +reth-network-peers.workspace = true +enr = { version = "0.13.0", default-features = false, features = ["rust-secp256k1"] } diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index a5ea0276a..d6ed28764 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -32,10 +32,11 @@ const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; pub struct FlashblocksConnectionState { /// Whether this peer is marked as trusted or not. pub trusted: bool, - /// Whether we have loaded the peer's trust classification from the network yet. - pub trusted_known: bool, /// Whether we currently have an outstanding flashblocks request to this peer. pub request_in_flight: bool, + /// Whether we intentionally abandoned an in-flight request and should treat a late + /// Accept/Reject as stale instead of malicious. + pub abandoned_request_in_flight: bool, /// Whether we are currently sending flashblocks to this peer. pub send_enabled: bool, /// Whether we are currently requesting flashblocks from this peer. @@ -47,8 +48,10 @@ pub struct FlashblocksConnectionState { pub receive_enabled: Option, /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. pub receive_enabled_timestamp: u64, - /// Earliest time at which this peer is eligible for another control-plane retry. - pub request_backoff_until: Option, + /// Earliest time at which this peer is eligible for another receive-side request. + pub receive_request_backoff_until: Option, + /// Earliest time at which this peer may retry an inbound send-set request after rejection. + pub send_request_backoff_until: Option, /// Per-peer channel for sending direct (control) messages without broadcasting. pub direct_tx: Option>, /// Number of control messages received in the current rate-limit window. @@ -61,12 +64,13 @@ impl FlashblocksConnectionState { pub(crate) fn new() -> Self { Self { trusted: false, - trusted_known: false, request_in_flight: false, + abandoned_request_in_flight: false, send_enabled: false, receive_enabled: None, receive_enabled_timestamp: 0, - request_backoff_until: None, + receive_request_backoff_until: None, + send_request_backoff_until: None, direct_tx: None, control_msg_count: 0, control_msg_window_start: Instant::now(), diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 892df3e91..dbcbfee2b 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -70,6 +70,12 @@ const MAX_CONTROL_MSGS_PER_WINDOW: u32 = 10; /// Duration of the per-peer control-message rate-limit window. const CONTROL_MSG_WINDOW: Duration = Duration::from_secs(30); +/// Maximum time to wait for the network manager to expose the newly connected peer's trust info. +const PEER_INFO_LOOKUP_TIMEOUT: Duration = Duration::from_secs(1); + +/// Poll interval while waiting for connected peer metadata to become available. +const PEER_INFO_LOOKUP_RETRY_INTERVAL: Duration = Duration::from_millis(10); + /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -325,12 +331,25 @@ impl FlashblocksP2PState { fn clear_receive_state( peer_state: &mut FlashblocksConnectionState, receive_enabled_timestamp: u64, - request_backoff_until: Option, + receive_request_backoff_until: Option, + ) { + peer_state.receive_enabled = None; + peer_state.request_in_flight = false; + peer_state.abandoned_request_in_flight = false; + peer_state.receive_enabled_timestamp = receive_enabled_timestamp; + peer_state.receive_request_backoff_until = receive_request_backoff_until; + } + + fn abandon_receive_request( + peer_state: &mut FlashblocksConnectionState, + receive_enabled_timestamp: u64, + receive_request_backoff_until: Option, ) { peer_state.receive_enabled = None; peer_state.request_in_flight = false; + peer_state.abandoned_request_in_flight = true; peer_state.receive_enabled_timestamp = receive_enabled_timestamp; - peer_state.request_backoff_until = request_backoff_until; + peer_state.receive_request_backoff_until = receive_request_backoff_until; } fn available_receive_candidates(&self) -> Vec<(PeerId, bool)> { @@ -338,11 +357,11 @@ impl FlashblocksP2PState { self.connections .iter() .filter_map(|(peer_id, peer_state)| { - if peer_state.trusted_known - && peer_state.receive_enabled.is_none() + if peer_state.receive_enabled.is_none() && !peer_state.request_in_flight + && !peer_state.abandoned_request_in_flight && peer_state - .request_backoff_until + .receive_request_backoff_until .is_none_or(|until| until <= now) { Some((*peer_id, peer_state.trusted)) @@ -359,9 +378,10 @@ impl FlashblocksP2PState { }; let timestamp = Utc::now().timestamp() as u64; peer_state.request_in_flight = true; + peer_state.abandoned_request_in_flight = false; peer_state.receive_enabled = Some(Score::new(ctx.fanout_config.latency_window)); peer_state.receive_enabled_timestamp = timestamp; - peer_state.request_backoff_until = None; + peer_state.receive_request_backoff_until = None; self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } @@ -435,14 +455,27 @@ impl FlashblocksP2PState { .find_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) .unwrap_or(candidates[0].0); + let evict_timestamp = Utc::now().timestamp() as u64; + let mut should_cancel = false; if let Some(evict_state) = self.connection_state_mut(&evict) { - Self::clear_receive_state( - evict_state, - Utc::now().timestamp() as u64, - Some(Self::request_backoff_deadline(ctx)), - ); + if evict_state.request_in_flight { + Self::abandon_receive_request( + evict_state, + evict_timestamp, + Some(Self::request_backoff_deadline(ctx)), + ); + } else { + Self::clear_receive_state( + evict_state, + evict_timestamp, + Some(Self::request_backoff_deadline(ctx)), + ); + should_cancel = true; + } + } + if should_cancel { + self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); } - self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); self.begin_requesting_peer(ctx, candidate); } @@ -457,11 +490,6 @@ impl FlashblocksP2PState { return false; }; - if !peer_state.trusted_known { - // Wait until trust classification is known before deciding whether this peer should - // bypass the non-trusted limit. - return false; - } if peer_state.send_enabled { // Already sending to this peer — repeated request is spam. return true; @@ -469,7 +497,7 @@ impl FlashblocksP2PState { let now = Instant::now(); if !peer_state.trusted && peer_state - .request_backoff_until + .send_request_backoff_until .is_some_and(|until| until > now) { // Non-trusted peer requesting during backoff is spam. @@ -485,14 +513,14 @@ impl FlashblocksP2PState { if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_config.max_send_peers { self.connection_state_mut(&peer_id) .expect("peer exists") - .request_backoff_until = Some(Self::request_backoff_deadline(ctx)); + .send_request_backoff_until = Some(Self::request_backoff_deadline(ctx)); self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); return false; } let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); peer_state.send_enabled = true; - peer_state.request_backoff_until = None; + peer_state.send_request_backoff_until = None; self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); false } @@ -507,14 +535,20 @@ impl FlashblocksP2PState { return false; }; - if !peer_state.request_in_flight { - // Unsolicited accept — we never asked this peer. - return true; + if peer_state.request_in_flight { + peer_state.request_in_flight = false; + peer_state.receive_request_backoff_until = None; + return false; } - peer_state.request_in_flight = false; - peer_state.request_backoff_until = None; - false + if peer_state.abandoned_request_in_flight { + peer_state.abandoned_request_in_flight = false; + self.send_direct(peer_id, FlashblocksP2PMsg::CancelFlashblocks); + return false; + } + + // Unsolicited accept — we never asked this peer. + true } /// Returns `true` if the peer should receive a reputation penalty. @@ -527,18 +561,23 @@ impl FlashblocksP2PState { return false; }; - if !peer_state.request_in_flight { - // Unsolicited reject — we never asked this peer. - return true; + if peer_state.request_in_flight { + Self::clear_receive_state( + peer_state, + Utc::now().timestamp() as u64, + Some(Self::request_backoff_deadline(ctx)), + ); + self.maybe_request_receive_peers(ctx); + return false; } - Self::clear_receive_state( - peer_state, - Utc::now().timestamp() as u64, - Some(Self::request_backoff_deadline(ctx)), - ); - self.maybe_request_receive_peers(ctx); - false + if peer_state.abandoned_request_in_flight { + peer_state.abandoned_request_in_flight = false; + return false; + } + + // Unsolicited reject — we never asked this peer. + true } /// Returns `true` if the peer should receive a reputation penalty. @@ -642,27 +681,58 @@ impl FlashblocksHandle { direct_tx: mpsc::UnboundedSender, ) { let trusted = tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on(network.get_peer_by_id(peer_id)) + let network = network.clone(); + tokio::runtime::Handle::current().block_on(async move { + let deadline = Instant::now() + PEER_INFO_LOOKUP_TIMEOUT; + + loop { + match network.get_peer_by_id(peer_id).await { + Ok(Some(peer_info)) => return Ok(peer_info.kind.is_trusted()), + Ok(None) if Instant::now() < deadline => { + time::sleep(PEER_INFO_LOOKUP_RETRY_INTERVAL).await; + } + Ok(None) => { + return Err( + "timed out waiting for peer info after connection".to_owned() + ); + } + Err(error) if Instant::now() < deadline => { + time::sleep(PEER_INFO_LOOKUP_RETRY_INTERVAL).await; + tracing::debug!( + target: "flashblocks::p2p", + %peer_id, + %error, + "retrying peer info lookup for flashblocks fanout" + ); + } + Err(error) => { + return Err(format!( + "failed to load peer info for flashblocks fanout: {error}" + )); + } + } + } + }) }); - let mut state = self.state.lock(); - let mut conn_state = FlashblocksConnectionState::new(); - conn_state.direct_tx = Some(direct_tx); - conn_state.trusted_known = true; - match trusted { - Ok(Some(peer_info)) => { - conn_state.trusted = peer_info.kind.is_trusted(); - } - Ok(None) => {} + let trusted = match trusted { + Ok(trusted) => trusted, Err(error) => { warn!( target: "flashblocks::p2p", %peer_id, %error, - "failed to load peer info for flashblocks fanout", + "failed to classify peer for flashblocks fanout; disconnecting" ); + network.disconnect_peer(peer_id); + return; } - } + }; + + let mut state = self.state.lock(); + let mut conn_state = FlashblocksConnectionState::new(); + conn_state.direct_tx = Some(direct_tx); + conn_state.trusted = trusted; state.connections.insert(peer_id, conn_state); state.maybe_request_receive_peers(&self.ctx); } @@ -1192,6 +1262,126 @@ impl ConnectionHandler for FlashblocksP2PProtoco mod tests { use super::*; use ed25519_dalek::SigningKey; + use enr::{Enr, secp256k1::SecretKey}; + use reth_eth_wire::{Capabilities, EthVersion, Status, StatusMessage, UnifiedStatus}; + use reth_network::{ + PeerInfo, PeersInfo, + types::{PeerKind, Reputation, ReputationChangeKind}, + }; + use reth_network_api::{NetworkError, noop::NoopNetwork}; + use reth_network_peers::NodeRecord; + use std::{ + collections::VecDeque, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; + + #[derive(Clone, Debug, Default)] + struct MockNetwork { + noop: NoopNetwork, + peer_lookup_responses: Arc>>>, + lookup_calls: Arc, + disconnected_peers: Arc>>, + } + + impl MockNetwork { + fn with_peer_lookup_responses(peer_lookup_responses: Vec>) -> Self { + Self { + peer_lookup_responses: Arc::new(Mutex::new(peer_lookup_responses.into())), + ..Default::default() + } + } + + fn lookup_calls(&self) -> usize { + self.lookup_calls.load(Ordering::SeqCst) + } + + fn disconnected_peers(&self) -> Vec { + self.disconnected_peers.lock().clone() + } + } + + impl PeersInfo for MockNetwork { + fn num_connected_peers(&self) -> usize { + self.noop.num_connected_peers() + } + + fn local_node_record(&self) -> NodeRecord { + self.noop.local_node_record() + } + + fn local_enr(&self) -> Enr { + self.noop.local_enr() + } + } + + impl Peers for MockNetwork { + fn add_trusted_peer_id(&self, _peer: PeerId) {} + + fn add_peer_kind( + &self, + _peer: PeerId, + _kind: PeerKind, + _tcp_addr: SocketAddr, + _udp_addr: Option, + ) { + } + + async fn get_peers_by_kind(&self, _kind: PeerKind) -> Result, NetworkError> { + Ok(vec![]) + } + + async fn get_all_peers(&self) -> Result, NetworkError> { + Ok(vec![]) + } + + async fn get_peer_by_id(&self, _peer_id: PeerId) -> Result, NetworkError> { + self.lookup_calls.fetch_add(1, Ordering::SeqCst); + Ok(self.peer_lookup_responses.lock().pop_front().flatten()) + } + + async fn get_peers_by_id( + &self, + _peer_ids: Vec, + ) -> Result, NetworkError> { + Ok(vec![]) + } + + fn remove_peer(&self, _peer: PeerId, _kind: PeerKind) {} + + fn disconnect_peer(&self, peer: PeerId) { + self.disconnected_peers.lock().push(peer); + } + + fn disconnect_peer_with_reason( + &self, + peer: PeerId, + _reason: reth_eth_wire::DisconnectReason, + ) { + self.disconnect_peer(peer); + } + + fn connect_peer_kind( + &self, + _peer: PeerId, + _kind: PeerKind, + _tcp_addr: SocketAddr, + _udp_addr: Option, + ) { + } + + fn reputation_change(&self, _peer_id: PeerId, _kind: ReputationChangeKind) {} + + async fn reputation_by_id( + &self, + _peer_id: PeerId, + ) -> Result, NetworkError> { + Ok(None) + } + } fn test_ctx(config: FanoutConfig) -> FlashblocksP2PCtx { let authorizer = SigningKey::from_bytes(&[7; 32]); @@ -1205,17 +1395,15 @@ mod tests { } } - fn test_peer_state(trusted: bool, trusted_known: bool) -> FlashblocksConnectionState { + fn test_peer_state(trusted: bool) -> FlashblocksConnectionState { let mut state = FlashblocksConnectionState::new(); state.trusted = trusted; - state.trusted_known = trusted_known; state } /// Creates a peer state with a per-peer direct channel for message assertions. fn test_peer_state_with_channel( trusted: bool, - trusted_known: bool, ) -> ( FlashblocksConnectionState, mpsc::UnboundedReceiver, @@ -1223,7 +1411,6 @@ mod tests { let (tx, rx) = mpsc::unbounded_channel(); let mut state = FlashblocksConnectionState::new(); state.trusted = trusted; - state.trusted_known = trusted_known; state.direct_tx = Some(tx); (state, rx) } @@ -1247,6 +1434,89 @@ mod tests { fanout.note_peer_received_flashblock(authorization, flashblock, peer_id); } + fn test_peer_info(peer_id: PeerId, trusted: bool) -> PeerInfo { + PeerInfo { + capabilities: Arc::new(Capabilities::new(vec![])), + remote_id: peer_id, + client_version: Arc::::from("mock"), + enode: "enode://mock".to_owned(), + enr: None, + remote_addr: SocketAddr::from((IpAddr::V4(Ipv4Addr::LOCALHOST), 30303)), + local_addr: None, + direction: Direction::Incoming, + eth_version: EthVersion::Eth67, + status: Arc::new(UnifiedStatus::from_message(StatusMessage::Legacy(Status { + version: EthVersion::Eth67, + ..Status::default() + }))), + session_established: Instant::now(), + kind: if trusted { + PeerKind::Trusted + } else { + PeerKind::Basic + }, + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn on_peer_connected_retries_until_peer_info_is_available() { + let authorizer = SigningKey::from_bytes(&[7; 32]); + let handle = FlashblocksHandle::with_fanout_config( + authorizer.verifying_key(), + Some(SigningKey::from_bytes(&[8; 32])), + FanoutConfig { + max_receive_peers: 1, + ..Default::default() + }, + ); + let peer_id = PeerId::random(); + let network = MockNetwork::with_peer_lookup_responses(vec![ + None, + Some(test_peer_info(peer_id, true)), + ]); + let (direct_tx, mut direct_rx) = mpsc::unbounded_channel(); + + handle.on_peer_connected(network.clone(), peer_id, direct_tx); + + assert!(network.lookup_calls() >= 2); + assert!(network.disconnected_peers().is_empty()); + let state = handle.state.lock(); + assert!( + state + .connection_state(&peer_id) + .expect("peer exists") + .trusted + ); + drop(state); + assert_eq!( + recv_direct(&mut direct_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn on_peer_connected_disconnects_when_peer_info_never_arrives() { + let authorizer = SigningKey::from_bytes(&[7; 32]); + let handle = FlashblocksHandle::with_fanout_config( + authorizer.verifying_key(), + Some(SigningKey::from_bytes(&[8; 32])), + FanoutConfig { + max_receive_peers: 1, + ..Default::default() + }, + ); + let peer_id = PeerId::random(); + let network = MockNetwork::default(); + let (direct_tx, mut direct_rx) = mpsc::unbounded_channel(); + + handle.on_peer_connected(network.clone(), peer_id, direct_tx); + + assert!(network.lookup_calls() > 1); + assert_eq!(network.disconnected_peers(), vec![peer_id]); + assert!(handle.state.lock().connection_state(&peer_id).is_none()); + assert!(direct_rx.try_recv().is_err()); + } + #[test] fn trusted_peers_are_requested_first() { let config = FanoutConfig { @@ -1258,8 +1528,8 @@ mod tests { let trusted_peer = PeerId::random(); let untrusted_peer = PeerId::random(); - let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true, true); - let untrusted_state = test_peer_state(false, true); + let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true); + let untrusted_state = test_peer_state(false); fanout.connections.insert(trusted_peer, trusted_state); fanout.connections.insert(untrusted_peer, untrusted_state); @@ -1274,30 +1544,6 @@ mod tests { ); } - #[test] - fn unknown_peers_are_not_requested_until_trust_is_known() { - let config = FanoutConfig { - max_receive_peers: 1, - ..Default::default() - }; - let ctx = test_ctx(config); - let mut fanout = FlashblocksP2PState::default(); - - let trusted_peer = PeerId::random(); - let untrusted_peer = PeerId::random(); - let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true, false); - let (untrusted_state, mut untrusted_rx) = test_peer_state_with_channel(false, false); - fanout.connections.insert(trusted_peer, trusted_state); - fanout.connections.insert(untrusted_peer, untrusted_state); - - fanout.maybe_request_receive_peers(&ctx); - - assert!(!peer_state(&fanout, trusted_peer).request_in_flight); - assert!(!peer_state(&fanout, untrusted_peer).request_in_flight); - assert!(trusted_rx.try_recv().is_err()); - assert!(untrusted_rx.try_recv().is_err()); - } - #[test] fn trusted_request_bypasses_non_trusted_limit() { let config = FanoutConfig { @@ -1309,8 +1555,8 @@ mod tests { let victim = PeerId::random(); let trusted_requester = PeerId::random(); - let (mut victim_state, mut victim_rx) = test_peer_state_with_channel(false, true); - let (requester_state, mut requester_rx) = test_peer_state_with_channel(true, true); + let (mut victim_state, mut victim_rx) = test_peer_state_with_channel(false); + let (requester_state, mut requester_rx) = test_peer_state_with_channel(true); victim_state.send_enabled = true; fanout.connections.insert(victim, victim_state); fanout @@ -1341,8 +1587,8 @@ mod tests { let current_peer = PeerId::random(); let candidate_peer = PeerId::random(); - let (mut current_state, mut current_rx) = test_peer_state_with_channel(false, true); - let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false, true); + let (mut current_state, mut current_rx) = test_peer_state_with_channel(false); + let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false); let mut score = Score::new(latency_window); score.record(42); current_state.receive_enabled = Some(score); @@ -1389,8 +1635,8 @@ mod tests { let first_peer = PeerId::random(); let second_peer = PeerId::random(); - let (first_state, mut first_rx) = test_peer_state_with_channel(false, true); - let (second_state, mut second_rx) = test_peer_state_with_channel(false, true); + let (first_state, mut first_rx) = test_peer_state_with_channel(false); + let (second_state, mut second_rx) = test_peer_state_with_channel(false); fanout.connections.insert(first_peer, first_state); fanout.connections.insert(second_peer, second_state); @@ -1427,7 +1673,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let (candidate_state, mut peer_rx) = test_peer_state_with_channel(false, true); + let (candidate_state, mut peer_rx) = test_peer_state_with_channel(false); fanout.connections.insert(peer, candidate_state); fanout.maybe_request_receive_peers(&ctx); @@ -1440,10 +1686,60 @@ mod tests { assert!(!peer_state(&fanout, peer).request_in_flight); assert!(peer_state(&fanout, peer).receive_enabled.is_none()); - assert!(peer_state(&fanout, peer).request_backoff_until.is_some()); + assert!( + peer_state(&fanout, peer) + .receive_request_backoff_until + .is_some() + ); assert!(peer_rx.try_recv().is_err()); } + #[test] + fn stale_accept_after_abandoned_request_is_canceled_without_penalty() { + let config = FanoutConfig { + max_receive_peers: 1, + latency_window: 4, + ..Default::default() + }; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let abandoned_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + let (mut abandoned_state, mut abandoned_rx) = test_peer_state_with_channel(false); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); + + abandoned_state.receive_enabled = Some(Score::new(4)); + abandoned_state.request_in_flight = true; + abandoned_state.receive_enabled_timestamp = 1; + + fanout.connections.insert(abandoned_peer, abandoned_state); + fanout + .connections + .insert(replacement_peer, replacement_state); + + fanout.maybe_start_rotation(&ctx); + + assert!( + peer_state(&fanout, abandoned_peer) + .receive_enabled + .is_none() + ); + assert!(peer_state(&fanout, abandoned_peer).abandoned_request_in_flight); + assert!(abandoned_rx.try_recv().is_err()); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(!fanout.handle_accept(&ctx, abandoned_peer)); + assert!(!peer_state(&fanout, abandoned_peer).abandoned_request_in_flight); + assert_eq!( + recv_direct(&mut abandoned_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + } + #[test] fn silent_receive_peer_can_be_rotated_out_without_samples() { let config = FanoutConfig { @@ -1457,15 +1753,13 @@ mod tests { let newer_peer = PeerId::random(); let replacement_peer = PeerId::random(); - let (mut oldest_state, mut oldest_rx) = test_peer_state_with_channel(false, true); - let mut newer_state = test_peer_state(false, true); - let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true, true); + let (mut oldest_state, mut oldest_rx) = test_peer_state_with_channel(false); + let mut newer_state = test_peer_state(false); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); oldest_state.receive_enabled = Some(Score::new(4)); - oldest_state.request_in_flight = true; oldest_state.receive_enabled_timestamp = 1; newer_state.receive_enabled = Some(Score::new(4)); - newer_state.request_in_flight = true; newer_state.receive_enabled_timestamp = 2; fanout.connections.insert(oldest_peer, oldest_state); @@ -1501,8 +1795,8 @@ mod tests { let steady_peer = PeerId::random(); let lagging_peer = PeerId::random(); - let mut steady_state = test_peer_state(false, true); - let mut lagging_state = test_peer_state(false, true); + let mut steady_state = test_peer_state(false); + let mut lagging_state = test_peer_state(false); let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); @@ -1570,10 +1864,10 @@ mod tests { let candidate_peer = PeerId::random(); let replacement_peer = PeerId::random(); - let mut steady_state = test_peer_state(false, true); - let mut rotating_state = test_peer_state(false, true); - let candidate_state = test_peer_state(true, true); - let replacement_state = test_peer_state(true, true); + let mut steady_state = test_peer_state(false); + let mut rotating_state = test_peer_state(false); + let candidate_state = test_peer_state(true); + let replacement_state = test_peer_state(true); steady_state.receive_enabled = Some(Score::new(latency_window)); rotating_state.receive_enabled = Some(Score::new(latency_window)); @@ -1633,7 +1927,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let state = test_peer_state(false, true); + let state = test_peer_state(false); fanout.connections.insert(peer, state); // Accept without a prior request should be penalized. @@ -1647,7 +1941,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let state = test_peer_state(false, true); + let state = test_peer_state(false); fanout.connections.insert(peer, state); // Reject without a prior request should be penalized. @@ -1661,7 +1955,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let state = test_peer_state(false, true); + let state = test_peer_state(false); fanout.connections.insert(peer, state); // Cancel with no send/receive relationship should be penalized. @@ -1675,7 +1969,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let mut state = test_peer_state(false, true); + let mut state = test_peer_state(false); state.send_enabled = true; state.receive_enabled = Some(Score::new(4)); fanout.connections.insert(peer, state); @@ -1693,7 +1987,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let mut state = test_peer_state(false, true); + let mut state = test_peer_state(false); state.receive_enabled = Some(Score::new(4)); fanout.connections.insert(peer, state); @@ -1701,32 +1995,53 @@ mod tests { } #[test] - fn request_from_unknown_trust_peer_is_deferred() { + fn duplicate_request_when_already_sending_is_penalized() { let config = FanoutConfig::default(); let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let (state, mut rx) = test_peer_state_with_channel(false, false); + let mut state = test_peer_state(false); + state.send_enabled = true; fanout.connections.insert(peer, state); - assert!(!fanout.handle_request(&ctx, peer)); - assert!(!peer_state(&fanout, peer).send_enabled); - assert!(rx.try_recv().is_err()); + assert!(fanout.handle_request(&ctx, peer)); } #[test] - fn duplicate_request_when_already_sending_is_penalized() { + fn receive_backoff_does_not_penalize_inbound_request() { let config = FanoutConfig::default(); let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let mut state = test_peer_state(false, true); - state.send_enabled = true; + let (mut state, mut rx) = test_peer_state_with_channel(false); + state.receive_request_backoff_until = Some(Instant::now() + Duration::from_secs(60)); fanout.connections.insert(peer, state); - assert!(fanout.handle_request(&ctx, peer)); + assert!(!fanout.handle_request(&ctx, peer)); + assert!(peer_state(&fanout, peer).send_enabled); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::AcceptFlashblocks); + } + + #[test] + fn send_backoff_does_not_block_receive_selection() { + let config = FanoutConfig { + max_receive_peers: 1, + ..Default::default() + }; + let ctx = test_ctx(config); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (mut state, mut rx) = test_peer_state_with_channel(false); + state.send_request_backoff_until = Some(Instant::now() + Duration::from_secs(60)); + fanout.connections.insert(peer, state); + + fanout.maybe_request_receive_peers(&ctx); + + assert!(peer_state(&fanout, peer).request_in_flight); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::RequestFlashblocks); } #[test] @@ -1736,7 +2051,7 @@ mod tests { let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let mut state = test_peer_state(false, true); + let mut state = test_peer_state(false); state.send_enabled = true; fanout.connections.insert(peer, state); diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index b95f7fbc2..9dcf88f37 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -779,7 +779,6 @@ async fn test_default_propagation_policy() -> eyre::Result<()> { /// - Inject tx into Node 2 -> should propagate to both Node 0 and Node 1 /// - Verifies multi-peer whitelist works correctly #[tokio::test(flavor = "multi_thread")] -#[ignore = "TODO: flaky - not sure what's causing this to fail"] async fn test_selective_propagation_policy() -> eyre::Result<()> { reth_tracing::init_test_tracing(); @@ -877,9 +876,27 @@ async fn test_selective_propagation_policy() -> eyre::Result<()> { .node .inner .network - .add_peer(node_0_peer_id, node_0_addr); - - tokio::time::sleep(Duration::from_secs(3)).await; + .connect_peer(node_0_peer_id, node_0_addr); + + // Wait for reconnection to establish + let start = tokio::time::Instant::now(); + loop { + let peer = node_2_ctx + .node + .inner + .network + .get_peer_by_id(node_0_peer_id) + .await?; + if peer.is_some() { + break; + } + if start.elapsed() > Duration::from_secs(10) { + panic!("Timeout waiting for Node 0 <-> Node 2 reconnection"); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + // Extra time for sync state to stabilize + tokio::time::sleep(Duration::from_secs(1)).await; // Create a new transaction and inject into Node 2 // Node 2 has tx_peers = [Node 0, Node 1], so it should propagate to both @@ -984,7 +1001,6 @@ async fn test_gossip_disabled_no_propagation() -> eyre::Result<()> { } #[tokio::test(flavor = "multi_thread")] -#[ignore = "flaky test"] async fn test_continuous_block_production_with_validation() -> eyre::Result<()> { reth_tracing::init_test_tracing(); From 16675939d389b33490d74bc96083908eccbb7126 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 00:49:47 -0700 Subject: [PATCH 14/43] wip --- .../p2p/src/protocol/connection.rs | 9 +- .../flashblocks/p2p/src/protocol/handler.rs | 95 ++++++++----------- 2 files changed, 39 insertions(+), 65 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index d6ed28764..0d89a40ec 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -46,12 +46,9 @@ pub struct FlashblocksConnectionState { /// counting as 10s. While `request_in_flight` is true, the peer is only a provisional /// candidate and must not deliver flashblocks yet. pub receive_enabled: Option, - /// Timestamp of when we enabled/disabled receiving flashblocks from this peer. + /// Timestamp of the last receive-side state transition for this peer. + /// Used for late-message grace checks and receive retry cooldown. pub receive_enabled_timestamp: u64, - /// Earliest time at which this peer is eligible for another receive-side request. - pub receive_request_backoff_until: Option, - /// Earliest time at which this peer may retry an inbound send-set request after rejection. - pub send_request_backoff_until: Option, /// Per-peer channel for sending direct (control) messages without broadcasting. pub direct_tx: Option>, /// Number of control messages received in the current rate-limit window. @@ -69,8 +66,6 @@ impl FlashblocksConnectionState { send_enabled: false, receive_enabled: None, receive_enabled_timestamp: 0, - receive_request_backoff_until: None, - send_request_backoff_until: None, direct_tx: None, control_msg_count: 0, control_msg_window_start: Instant::now(), diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index dbcbfee2b..ae40a8ac7 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -324,45 +324,41 @@ impl FlashblocksP2PState { .count() } - fn request_backoff_deadline(ctx: &FlashblocksP2PCtx) -> Instant { - Instant::now() + ctx.fanout_config.rotation_interval + fn receive_retry_cooldown_secs(ctx: &FlashblocksP2PCtx) -> u64 { + ctx.fanout_config.rotation_interval.as_secs().max(1) } fn clear_receive_state( peer_state: &mut FlashblocksConnectionState, receive_enabled_timestamp: u64, - receive_request_backoff_until: Option, ) { peer_state.receive_enabled = None; peer_state.request_in_flight = false; peer_state.abandoned_request_in_flight = false; peer_state.receive_enabled_timestamp = receive_enabled_timestamp; - peer_state.receive_request_backoff_until = receive_request_backoff_until; } fn abandon_receive_request( peer_state: &mut FlashblocksConnectionState, receive_enabled_timestamp: u64, - receive_request_backoff_until: Option, ) { peer_state.receive_enabled = None; peer_state.request_in_flight = false; peer_state.abandoned_request_in_flight = true; peer_state.receive_enabled_timestamp = receive_enabled_timestamp; - peer_state.receive_request_backoff_until = receive_request_backoff_until; } - fn available_receive_candidates(&self) -> Vec<(PeerId, bool)> { - let now = Instant::now(); + fn available_receive_candidates(&self, ctx: &FlashblocksP2PCtx) -> Vec<(PeerId, bool)> { + let now = Utc::now().timestamp() as u64; + let retry_cooldown = Self::receive_retry_cooldown_secs(ctx); self.connections .iter() .filter_map(|(peer_id, peer_state)| { if peer_state.receive_enabled.is_none() && !peer_state.request_in_flight && !peer_state.abandoned_request_in_flight - && peer_state - .receive_request_backoff_until - .is_none_or(|until| until <= now) + && (peer_state.receive_enabled_timestamp == 0 + || peer_state.receive_enabled_timestamp + retry_cooldown <= now) { Some((*peer_id, peer_state.trusted)) } else { @@ -381,13 +377,12 @@ impl FlashblocksP2PState { peer_state.abandoned_request_in_flight = false; peer_state.receive_enabled = Some(Score::new(ctx.fanout_config.latency_window)); peer_state.receive_enabled_timestamp = timestamp; - peer_state.receive_request_backoff_until = None; self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { while self.num_receive_peers() < ctx.fanout_config.max_receive_peers { - let candidates = self.available_receive_candidates(); + let candidates = self.available_receive_candidates(ctx); if candidates.is_empty() { return; } @@ -443,7 +438,7 @@ impl FlashblocksP2PState { return; }; - let mut candidates = self.available_receive_candidates(); + let mut candidates = self.available_receive_candidates(ctx); if candidates.is_empty() { return; } @@ -459,17 +454,9 @@ impl FlashblocksP2PState { let mut should_cancel = false; if let Some(evict_state) = self.connection_state_mut(&evict) { if evict_state.request_in_flight { - Self::abandon_receive_request( - evict_state, - evict_timestamp, - Some(Self::request_backoff_deadline(ctx)), - ); + Self::abandon_receive_request(evict_state, evict_timestamp); } else { - Self::clear_receive_state( - evict_state, - evict_timestamp, - Some(Self::request_backoff_deadline(ctx)), - ); + Self::clear_receive_state(evict_state, evict_timestamp); should_cancel = true; } } @@ -494,15 +481,6 @@ impl FlashblocksP2PState { // Already sending to this peer — repeated request is spam. return true; } - let now = Instant::now(); - if !peer_state.trusted - && peer_state - .send_request_backoff_until - .is_some_and(|until| until > now) - { - // Non-trusted peer requesting during backoff is spam. - return true; - } let peer_is_trusted = peer_state.trusted; let non_trusted_send_count = self .connections @@ -511,16 +489,12 @@ impl FlashblocksP2PState { .count(); if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_config.max_send_peers { - self.connection_state_mut(&peer_id) - .expect("peer exists") - .send_request_backoff_until = Some(Self::request_backoff_deadline(ctx)); self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); return false; } let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); peer_state.send_enabled = true; - peer_state.send_request_backoff_until = None; self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); false } @@ -537,7 +511,6 @@ impl FlashblocksP2PState { if peer_state.request_in_flight { peer_state.request_in_flight = false; - peer_state.receive_request_backoff_until = None; return false; } @@ -562,11 +535,7 @@ impl FlashblocksP2PState { }; if peer_state.request_in_flight { - Self::clear_receive_state( - peer_state, - Utc::now().timestamp() as u64, - Some(Self::request_backoff_deadline(ctx)), - ); + Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); self.maybe_request_receive_peers(ctx); return false; } @@ -1686,12 +1655,22 @@ mod tests { assert!(!peer_state(&fanout, peer).request_in_flight); assert!(peer_state(&fanout, peer).receive_enabled.is_none()); - assert!( - peer_state(&fanout, peer) - .receive_request_backoff_until - .is_some() - ); assert!(peer_rx.try_recv().is_err()); + + fanout.maybe_request_receive_peers(&ctx); + assert!(peer_rx.try_recv().is_err()); + + fanout + .connection_state_mut(&peer) + .expect("peer exists") + .receive_enabled_timestamp = + Utc::now().timestamp() as u64 - ctx.fanout_config.rotation_interval.as_secs().max(1); + + fanout.maybe_request_receive_peers(&ctx); + assert_eq!( + recv_direct(&mut peer_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); } #[test] @@ -2009,14 +1988,14 @@ mod tests { } #[test] - fn receive_backoff_does_not_penalize_inbound_request() { + fn receive_retry_cooldown_does_not_penalize_inbound_request() { let config = FanoutConfig::default(); let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); let (mut state, mut rx) = test_peer_state_with_channel(false); - state.receive_request_backoff_until = Some(Instant::now() + Duration::from_secs(60)); + state.receive_enabled_timestamp = Utc::now().timestamp() as u64; fanout.connections.insert(peer, state); assert!(!fanout.handle_request(&ctx, peer)); @@ -2025,23 +2004,23 @@ mod tests { } #[test] - fn send_backoff_does_not_block_receive_selection() { + fn repeated_rejected_requests_are_rate_limited() { let config = FanoutConfig { - max_receive_peers: 1, + max_send_peers: 0, ..Default::default() }; let ctx = test_ctx(config); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); - let (mut state, mut rx) = test_peer_state_with_channel(false); - state.send_request_backoff_until = Some(Instant::now() + Duration::from_secs(60)); + let (state, mut rx) = test_peer_state_with_channel(false); fanout.connections.insert(peer, state); - fanout.maybe_request_receive_peers(&ctx); - - assert!(peer_state(&fanout, peer).request_in_flight); - assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::RequestFlashblocks); + for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { + assert!(!fanout.handle_request(&ctx, peer)); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::RejectFlashblocks); + } + assert!(fanout.handle_request(&ctx, peer)); } #[test] From cca37d0aa03c04d40ecec12e7e035f964974960b Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 01:20:11 -0700 Subject: [PATCH 15/43] fix: tests --- crates/flashblocks/node/tests/p2p.rs | 88 +++++++++++++++++++++------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 03db65bb5..17dbe4e4d 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -155,6 +155,67 @@ impl NodeContext { } } +async fn wait_for_pending_block( + node: &NodeContext, + expected_number: u64, + expected_txs: usize, +) -> eyre::Result<()> { + let provider = node.provider().await?; + let timeout = Duration::from_secs(10); + let poll_interval = Duration::from_millis(50); + let start = Instant::now(); + let mut last_observed = "no pending block".to_string(); + + loop { + let pending_block = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) + .await?; + + if let Some(pending_block) = pending_block { + let observed_number = pending_block.number(); + let observed_txs = pending_block.transactions.hashes().len(); + if observed_number == expected_number && observed_txs == expected_txs { + return Ok(()); + } + + last_observed = format!("number {observed_number}, txs {observed_txs}"); + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for pending block state: expected number {expected_number}, txs {expected_txs}; last observed {last_observed}" + )); + } + + sleep(poll_interval).await; + } +} + +async fn wait_for_trusted_peers( + node: &NodeContext, + expected_connections: usize, +) -> eyre::Result<()> { + let timeout = Duration::from_secs(10); + let poll_interval = Duration::from_millis(100); + let start = Instant::now(); + + loop { + let trusted_peers = node.network_handle.get_trusted_peers().await?; + if trusted_peers.len() == expected_connections { + return Ok(()); + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for trusted peers: expected {expected_connections}, last observed {}", + trusted_peers.len() + )); + } + + sleep(poll_interval).await; + } +} + fn init_tracing(filter: &str) -> tracing::subscriber::DefaultGuard { let sub = tracing_subscriber::fmt() .with_env_filter(filter) @@ -392,13 +453,14 @@ async fn setup_nodes(n: u8) -> eyre::Result { for i in 0..n { let builder = SigningKey::from_bytes(&[(i + 1) % n; 32]); let node = setup_node(exec.clone(), authorizer.clone(), builder, peers.clone()).await?; + if !peers.is_empty() { + wait_for_trusted_peers(&node, peers.len()).await?; + } let enr = node.local_node_record; peers.push((enr.id, enr.tcp_addr())); nodes.push(node); } - sleep(Duration::from_millis(6000)).await; - Ok(NodeTestFixture { nodes, authorizer, @@ -546,18 +608,9 @@ async fn test_force_race_condition() -> eyre::Result<()> { let authorized = AuthorizedPayload::new(nodes[0].p2p_handle.builder_sk()?, authorization, msg); nodes[0].p2p_handle.start_publishing(authorization)?; nodes[0].p2p_handle.publish_new(authorized).unwrap(); - sleep(Duration::from_millis(100)).await; // Query pending block after sending the base payload with an empty delta - let pending_block = nodes[1] - .provider() - .await? - .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) - .await? - .expect("pending block expected"); - - assert_eq!(pending_block.number(), expected_pending_number); - assert_eq!(pending_block.transactions.hashes().len(), 0); + wait_for_pending_block(&nodes[0], expected_pending_number, 0).await?; info!("Sending payload 0, index 1"); let payload_1 = next_payload(payload_0.payload_id, 1).await; @@ -573,18 +626,9 @@ async fn test_force_race_condition() -> eyre::Result<()> { payload_1.clone(), ); nodes[0].p2p_handle.publish_new(authorized).unwrap(); - sleep(Duration::from_millis(100)).await; // Query pending block after sending the second payload with two transactions - let block = nodes[1] - .provider() - .await? - .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) - .await? - .expect("pending block expected"); - - assert_eq!(block.number(), expected_pending_number); - assert_eq!(block.transactions.hashes().len(), 0); + wait_for_pending_block(&nodes[0], expected_pending_number, 0).await?; // Send a new block, this time from node 1 let payload_2 = base_payload(1, test_payload_id(21), 0, latest_block.hash(), AUTH_TS_NEXT); From 1517afbf2f5855225b5b690735abb17d76a1f02c Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 01:47:55 -0700 Subject: [PATCH 16/43] fix: more tests --- .../flashblocks/p2p/src/protocol/handler.rs | 23 +++++++++---- .../world/node/tests/e2e-testsuite/spammer.rs | 34 ++++++++++--------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index ae40a8ac7..4c9700725 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -691,10 +691,9 @@ impl FlashblocksHandle { target: "flashblocks::p2p", %peer_id, %error, - "failed to classify peer for flashblocks fanout; disconnecting" + "failed to classify peer for flashblocks fanout; defaulting to untrusted" ); - network.disconnect_peer(peer_id); - return; + false } }; @@ -1464,7 +1463,7 @@ mod tests { } #[tokio::test(flavor = "multi_thread")] - async fn on_peer_connected_disconnects_when_peer_info_never_arrives() { + async fn on_peer_connected_defaults_to_untrusted_when_peer_info_never_arrives() { let authorizer = SigningKey::from_bytes(&[7; 32]); let handle = FlashblocksHandle::with_fanout_config( authorizer.verifying_key(), @@ -1481,9 +1480,19 @@ mod tests { handle.on_peer_connected(network.clone(), peer_id, direct_tx); assert!(network.lookup_calls() > 1); - assert_eq!(network.disconnected_peers(), vec![peer_id]); - assert!(handle.state.lock().connection_state(&peer_id).is_none()); - assert!(direct_rx.try_recv().is_err()); + assert!(network.disconnected_peers().is_empty()); + let state = handle.state.lock(); + assert!( + !state + .connection_state(&peer_id) + .expect("peer exists") + .trusted + ); + drop(state); + assert_eq!( + recv_direct(&mut direct_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); } #[test] diff --git a/crates/world/node/tests/e2e-testsuite/spammer.rs b/crates/world/node/tests/e2e-testsuite/spammer.rs index 020fbd14b..227971372 100644 --- a/crates/world/node/tests/e2e-testsuite/spammer.rs +++ b/crates/world/node/tests/e2e-testsuite/spammer.rs @@ -238,23 +238,25 @@ impl TxSpammer { async fn broadcast_batch_with_hashes(&self, batch: &[Bytes]) -> Vec { let mut tx_hashes = Vec::with_capacity(batch.len()); - while let Some(client) = self.rpc.first() { - for tx in batch { - let result = EthApiClient::< - TransactionRequest, - OpTransactionSigned, - alloy_consensus::Block, - OpReceipt, - Header, - Bytes, - >::send_raw_transaction(&client.rpc, tx.clone()) - .await - .inspect_err(|e| error!("Error sending transaction: {:?}", e)); + let Some(client) = self.rpc.first() else { + return tx_hashes; + }; - if let Ok(tx_hash) = result { - info!("Submitted tx: {:?}", tx_hash); - tx_hashes.push(tx_hash); - } + for tx in batch { + let result = EthApiClient::< + TransactionRequest, + OpTransactionSigned, + alloy_consensus::Block, + OpReceipt, + Header, + Bytes, + >::send_raw_transaction(&client.rpc, tx.clone()) + .await + .inspect_err(|e| error!("Error sending transaction: {:?}", e)); + + if let Ok(tx_hash) = result { + info!("Submitted tx: {:?}", tx_hash); + tx_hashes.push(tx_hash); } } From 4f83221526c338faf880fce3afd87d5538ba4c3f Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 02:02:46 -0700 Subject: [PATCH 17/43] chore: clippy --- .../p2p/src/protocol/connection.rs | 4 +-- .../flashblocks/p2p/src/protocol/handler.rs | 27 +++++++++---------- crates/flashblocks/primitives/src/p2p.rs | 1 + 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 0d89a40ec..d9f765247 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -429,7 +429,7 @@ impl FlashblocksConnection { } // Check if this peer is spamming us with the same payload index. - if !p2p_state.note_peer_received_flashblock(&authorization, &msg, self.peer_id) { + if !p2p_state.note_peer_received_flashblock(authorization, msg, self.peer_id) { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -582,7 +582,7 @@ impl FlashblocksConnection { } /// Handles incoming `StopPublish` messages from a peer. - + /// /// # Arguments /// * `authorized_payload` - The authorized `StopPublish` message received from the peer /// diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 4c9700725..a71b8adc6 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -253,17 +253,16 @@ impl FlashblocksP2PState { for (peer_id, connection) in &mut self.connections { if connection.receive_enabled_timestamp < evicted.timestamp + 2 && !evicted.received_peers.contains(peer_id) + && let Some(score) = connection.receive_enabled.as_mut() { - if let Some(score) = connection.receive_enabled.as_mut() { - debug!( - target: "flashblocks::p2p", - %peer_id, - payload_id = %evicted.payload_id, - flashblock_index = evicted.flashblock_index, - "scoring peer for missed flashblock", - ); - score.record(MISSED_FLASHBLOCK_PENALTY_NS); - } + debug!( + target: "flashblocks::p2p", + %peer_id, + payload_id = %evicted.payload_id, + flashblock_index = evicted.flashblock_index, + "scoring peer for missed flashblock", + ); + score.record(MISSED_FLASHBLOCK_PENALTY_NS); } } } @@ -296,10 +295,10 @@ impl FlashblocksP2PState { /// Sends a control message directly to a specific peer via its per-peer channel. fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { - if let Some(conn) = self.connections.get(&peer_id) { - if let Some(tx) = &conn.direct_tx { - tx.send(msg.encode()).ok(); - } + if let Some(conn) = self.connections.get(&peer_id) + && let Some(tx) = &conn.direct_tx + { + tx.send(msg.encode()).ok(); } } diff --git a/crates/flashblocks/primitives/src/p2p.rs b/crates/flashblocks/primitives/src/p2p.rs index cd43b507c..ae3a06977 100644 --- a/crates/flashblocks/primitives/src/p2p.rs +++ b/crates/flashblocks/primitives/src/p2p.rs @@ -42,6 +42,7 @@ pub struct StopPublish; /// This enum represents the top-level message types that can be transmitted /// over the P2P network. Currently all messages are wrapped in authorization to ensure /// only authorized builders can create new messages. +#[allow(clippy::large_enum_variant)] #[repr(u8)] #[derive(Clone, Debug, PartialEq, Deserialize, Serialize, Eq)] pub enum FlashblocksP2PMsg { From 00e916c6626850ea3936f2d44435a86869b40587 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 02:08:29 -0700 Subject: [PATCH 18/43] fix: tests --- crates/flashblocks/node/tests/p2p.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 17dbe4e4d..c70e18139 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -790,17 +790,26 @@ async fn test_peer_reputation() -> eyre::Result<()> { let peers = nodes[1].network_handle.get_all_peers().await?; let peer_0 = &peers[0].remote_id; + let mut reputation_was_negative = false; + let mut peer_banned = false; for _ in 0..100 { nodes[0].p2p_handle.ctx.peer_tx.send(peer_msg.clone()).ok(); sleep(Duration::from_millis(10)).await; let rep_0 = nodes[1].network_handle.reputation_by_id(*peer_0).await?; if let Some(rep) = rep_0 { - assert!(rep < 0, "Peer reputation should be negative"); + if rep < 0 { + reputation_was_negative = true; + } + } + if nodes[1].network_handle.get_all_peers().await?.is_empty() { + peer_banned = true; + break; } } - // Assert that the peer is banned - assert!(nodes[1].network_handle.get_all_peers().await?.is_empty()); + // Assert that the peer reputation became negative and peer was banned + assert!(reputation_was_negative, "Peer reputation should have become negative"); + assert!(peer_banned, "Peer should have been banned"); drop(fixture); From 21f294a2bd67dad417188a747083061a31e689c9 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 02:11:55 -0700 Subject: [PATCH 19/43] fmt --- crates/flashblocks/node/tests/p2p.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index c70e18139..f77eee6cd 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -808,7 +808,10 @@ async fn test_peer_reputation() -> eyre::Result<()> { } // Assert that the peer reputation became negative and peer was banned - assert!(reputation_was_negative, "Peer reputation should have become negative"); + assert!( + reputation_was_negative, + "Peer reputation should have become negative" + ); assert!(peer_banned, "Peer should have been banned"); drop(fixture); From 84dd7a5bdf803b444028fd1bc9ef5dd00ae5e89e Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 12:02:44 -0700 Subject: [PATCH 20/43] refactor --- crates/flashblocks/node/tests/p2p.rs | 11 +- .../p2p/src/protocol/connection.rs | 103 ++-------- .../flashblocks/p2p/src/protocol/handler.rs | 178 +++++++++++------- 3 files changed, 132 insertions(+), 160 deletions(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index f77eee6cd..f3ed410a4 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -6,10 +6,7 @@ use alloy_rpc_types_engine::PayloadId; use ed25519_dalek::SigningKey; use eyre::eyre::eyre; use flashblocks_cli::FlashblocksArgs; -use flashblocks_p2p::{ - monitor, - protocol::handler::{FlashblocksHandle, PeerMsg}, -}; +use flashblocks_p2p::{monitor, protocol::handler::FlashblocksHandle}; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, p2p::{ @@ -785,7 +782,7 @@ async fn test_peer_reputation() -> eyre::Result<()> { authorized_msg, ); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StartPublishing(p2p_msg.encode()); + let bytes = p2p_msg.encode(); let peers = nodes[1].network_handle.get_all_peers().await?; let peer_0 = &peers[0].remote_id; @@ -793,7 +790,9 @@ async fn test_peer_reputation() -> eyre::Result<()> { let mut reputation_was_negative = false; let mut peer_banned = false; for _ in 0..100 { - nodes[0].p2p_handle.ctx.peer_tx.send(peer_msg.clone()).ok(); + nodes[0] + .p2p_handle + .send_serialized_to_all_peers(bytes.clone()); sleep(Duration::from_millis(10)).await; let rep_0 = nodes[1].network_handle.reputation_by_id(*peer_0).await?; if let Some(rep) = rep_0 { diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index d9f765247..6cd97a2f4 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -1,6 +1,5 @@ use crate::protocol::handler::{ - FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, MAX_FLASHBLOCK_INDEX, PeerMsg, - PublishingStatus, + FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, MAX_FLASHBLOCK_INDEX, PublishingStatus, }; use alloy_primitives::bytes::BytesMut; use chrono::Utc; @@ -20,7 +19,6 @@ use std::{ time::Instant, }; use tokio::sync::mpsc; -use tokio_stream::wrappers::BroadcastStream; use tracing::{info, trace}; /// Grace period for authorization timestamp checks to reduce false positives from @@ -49,8 +47,8 @@ pub struct FlashblocksConnectionState { /// Timestamp of the last receive-side state transition for this peer. /// Used for late-message grace checks and receive retry cooldown. pub receive_enabled_timestamp: u64, - /// Per-peer channel for sending direct (control) messages without broadcasting. - pub direct_tx: Option>, + /// Per-peer channel for sending serialized protocol messages to this peer. + pub outbound_tx: Option>, /// Number of control messages received in the current rate-limit window. pub control_msg_count: u32, /// Start of the current rate-limit window. @@ -66,7 +64,7 @@ impl FlashblocksConnectionState { send_enabled: false, receive_enabled: None, receive_enabled_timestamp: 0, - direct_tx: None, + outbound_tx: None, control_msg_count: 0, control_msg_window_start: Instant::now(), } @@ -77,7 +75,7 @@ impl FlashblocksConnectionState { /// /// This struct manages the bidirectional communication with a single peer in the flashblocks /// P2P network. It handles incoming messages from the peer, validates and processes them, -/// and also streams outgoing messages that need to be broadcast. +/// and also streams serialized outgoing messages queued for this peer. /// /// The connection implements the `Stream` trait to provide outgoing message bytes that /// should be sent to the connected peer over the underlying protocol connection. @@ -88,11 +86,8 @@ pub struct FlashblocksConnection { conn: ProtocolConnection, /// The unique identifier of the connected peer. peer_id: PeerId, - /// Receiver for peer messages to be sent to all peers. - /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. - peer_rx: BroadcastStream, - /// Receiver for direct (control) messages targeted at this specific peer. - direct_rx: mpsc::UnboundedReceiver, + /// Receiver for already serialized protocol messages targeted at this specific peer. + outbound_rx: mpsc::UnboundedReceiver, } impl FlashblocksConnection { @@ -102,18 +97,16 @@ impl FlashblocksConnection { /// * `protocol` - The flashblocks protocol handler managing the connection. /// * `conn` - The underlying protocol connection for sending and receiving messages. /// * `peer_id` - The unique identifier of the connected peer. - /// * `peer_rx` - Receiver for peer messages to be sent to all peers. pub(crate) fn new( protocol: FlashblocksP2PProtocol, conn: ProtocolConnection, peer_id: PeerId, - peer_rx: BroadcastStream, - direct_tx: mpsc::UnboundedSender, - direct_rx: mpsc::UnboundedReceiver, ) -> Self { + let (outbound_tx, outbound_rx) = mpsc::unbounded_channel(); + protocol .handle - .on_peer_connected(protocol.network.clone(), peer_id, direct_tx); + .on_peer_connected(protocol.network.clone(), peer_id, outbound_tx); gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); @@ -121,8 +114,7 @@ impl FlashblocksConnection { protocol, conn, peer_id, - peer_rx, - direct_rx, + outbound_rx, } } } @@ -148,81 +140,15 @@ impl Stream for FlashblocksConnection { let this = self.get_mut(); loop { - // Check per-peer direct channel first (control messages). - if let Poll::Ready(Some(bytes)) = this.direct_rx.poll_recv(cx) { + if let Poll::Ready(Some(bytes)) = this.outbound_rx.poll_recv(cx) { trace!( target: "flashblocks::p2p", peer_id = %this.peer_id, - "Sending direct flashblocks control message to peer" + "Sending serialized flashblocks protocol message to peer" ); return Poll::Ready(Some(bytes)); } - // Check if there are any flashblocks ready to broadcast to our peers. - if let Poll::Ready(Some(res)) = this.peer_rx.poll_next_unpin(cx) { - match res { - Ok(peer_msg) => { - match peer_msg { - PeerMsg::FlashblocksPayloadV1(( - payload_id, - flashblock_index, - bytes, - )) => { - // Check if this flashblock actually originated from this peer. - let should_send = { - let state = this.protocol.handle.state.lock(); - let already_received = state.peer_received_flashblock( - this.peer_id, - payload_id, - flashblock_index as u64, - ); - let is_send_enabled = state - .connection_state(&this.peer_id) - .is_some_and(|peer_state| peer_state.send_enabled); - is_send_enabled && !already_received - }; - if should_send { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - %payload_id, - %flashblock_index, - "Broadcasting `FlashblocksPayloadV1` message to peer" - ); - metrics::counter!("flashblocks.bandwidth_outbound") - .increment(bytes.len() as u64); - - return Poll::Ready(Some(bytes)); - } - } - PeerMsg::StartPublishing(bytes_mut) => { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Broadcasting `StartPublishing` to peer" - ); - return Poll::Ready(Some(bytes_mut)); - } - PeerMsg::StopPublishing(bytes_mut) => { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Broadcasting `StopPublishing` to peer" - ); - return Poll::Ready(Some(bytes_mut)); - } - } - } - Err(error) => { - tracing::error!( - target: "flashblocks::p2p", - %error, - "failed to receive flashblocks message from peer_rx" - ); - } - } - } - // Check if there are any messages from the peer. let Some(buf) = ready!(this.conn.poll_next_unpin(cx)) else { return Poll::Ready(None); @@ -539,8 +465,7 @@ impl FlashblocksConnection { let authorized = Authorized::new(builder_sk, *our_authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.protocol.handle.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: vec![( diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index a71b8adc6..103092393 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -84,28 +84,6 @@ pub trait FlashblocksP2PNetworkHandle: Clone + Unpin + Peers + std::fmt::Debug + impl FlashblocksP2PNetworkHandle for N {} -/// Messages that can be broadcast over a channel to each internal peer connection. -/// -/// These messages are used internally to coordinate the broadcasting of flashblocks -/// and publishing status changes to all connected peers. -#[derive(Clone, Debug)] -pub enum PeerMsg { - /// Send an already serialized flashblock to all peers. - FlashblocksPayloadV1((PayloadId, usize, BytesMut)), - /// Send a previously serialized StartPublish message to all peers. - StartPublishing(BytesMut), - /// Send a previously serialized StopPublish message to all peers. - StopPublishing(BytesMut), -} - -#[derive(Clone, Debug)] -pub struct ObservedPayload { - payload_id: PayloadId, - timestamp: u64, - flashblock_index: u64, - received_peers: HashSet, -} - /// Runtime configuration for bounded flashblocks fanout. #[derive(Clone, Debug, PartialEq, Eq)] pub struct FanoutConfig { @@ -166,6 +144,16 @@ impl Default for PublishingStatus { } } +/// Tracked information about a flashblock payload observed from the network. +#[derive(Clone, Debug)] +pub struct ObservedPayload { + payload_id: PayloadId, + timestamp: u64, + flashblock_index: u64, + /// Peers from which we've received this flashblock + received_peers: HashSet, +} + /// Protocol state that stores the flashblocks P2P protocol events and coordination data. /// /// This struct maintains the current state of flashblock publishing, including coordination @@ -293,15 +281,52 @@ impl FlashblocksP2PState { .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) } - /// Sends a control message directly to a specific peer via its per-peer channel. - fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { + /// Sends an already serialized message to a specific peer. + fn send_to_peer(&self, peer_id: PeerId, bytes: &BytesMut) { if let Some(conn) = self.connections.get(&peer_id) - && let Some(tx) = &conn.direct_tx + && let Some(tx) = &conn.outbound_tx { - tx.send(msg.encode()).ok(); + tx.send(bytes.clone()).ok(); + } + } + + /// Sends an already serialized message to all connected peers. + pub(crate) fn send_to_all_peers(&self, bytes: &BytesMut) { + for conn in self.connections.values() { + if let Some(tx) = &conn.outbound_tx { + tx.send(bytes.clone()).ok(); + } } } + /// Sends a serialized flashblock to peers in the current send set that have not + /// already delivered that flashblock to us. + fn send_flashblock_to_send_set( + &self, + payload_id: PayloadId, + flashblock_index: u64, + bytes: &BytesMut, + ) { + for (peer_id, conn) in &self.connections { + if !conn.send_enabled + || self.peer_received_flashblock(*peer_id, payload_id, flashblock_index) + { + continue; + } + + if let Some(tx) = &conn.outbound_tx + && tx.send(bytes.clone()).is_ok() + { + metrics::counter!("flashblocks.bandwidth_outbound").increment(bytes.len() as u64); + } + } + } + + /// Sends a control message directly to a specific peer. + fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { + self.send_to_peer(peer_id, &msg.encode()); + } + /// Returns `true` if the peer has exceeded the control-message rate limit. fn check_control_rate_limit(&mut self, peer_id: &PeerId) -> bool { let Some(peer_state) = self.connections.get_mut(peer_id) else { @@ -581,9 +606,6 @@ pub struct FlashblocksP2PCtx { pub builder_sk: Option, /// Fanout configuration for peer selection and rotation. pub fanout_config: FanoutConfig, - /// Broadcast sender for peer messages that will be sent to all connected peers. - /// Messages may not be strictly ordered due to network conditions. - pub peer_tx: broadcast::Sender, /// Broadcast sender for verified and strictly ordered flashblock payloads. /// Used by RPC overlays and other consumers of flashblock data. pub flashblock_tx: broadcast::Sender, @@ -613,13 +635,11 @@ impl FlashblocksHandle { fanout_config: FanoutConfig, ) -> Self { let flashblock_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); - let peer_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); let state = Arc::new(Mutex::new(FlashblocksP2PState::default())); let ctx = FlashblocksP2PCtx { authorizer_vk, builder_sk, fanout_config, - peer_tx, flashblock_tx, }; let handle = Self { ctx, state }; @@ -646,7 +666,7 @@ impl FlashblocksHandle { &self, network: N, peer_id: PeerId, - direct_tx: mpsc::UnboundedSender, + outbound_tx: mpsc::UnboundedSender, ) { let trusted = tokio::task::block_in_place(|| { let network = network.clone(); @@ -698,7 +718,7 @@ impl FlashblocksHandle { let mut state = self.state.lock(); let mut conn_state = FlashblocksConnectionState::new(); - conn_state.direct_tx = Some(direct_tx); + conn_state.outbound_tx = Some(outbound_tx); conn_state.trusted = trusted; state.connections.insert(peer_id, conn_state); state.maybe_request_receive_peers(&self.ctx); @@ -794,7 +814,7 @@ impl FlashblocksHandle { /// /// This method validates that the builder has authorization to publish and that /// the authorization matches the current publishing session. The flashblock is - /// then processed, cached, and broadcast to all connected peers. + /// then processed, cached, and forwarded to peers in the current send set. /// /// # Arguments /// * `authorized_payload` - The signed flashblock payload with authorization @@ -823,6 +843,11 @@ impl FlashblocksHandle { Ok(()) } + /// Sends an already serialized protocol message to all currently connected peers. + pub fn send_serialized_to_all_peers(&self, bytes: BytesMut) { + self.state.lock().send_to_all_peers(&bytes); + } + /// Returns the current publishing status of this node. /// /// The status indicates whether the node is actively publishing flashblocks, @@ -915,8 +940,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, new_authorization, authorized_msg); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StartPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); if active_publishers.is_empty() { // If we have no previous publishers, we can start publishing immediately. @@ -969,8 +993,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, *authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: Vec::new(), }; @@ -990,8 +1013,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, *authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: active_publishers.clone(), }; @@ -1042,7 +1064,8 @@ impl FlashblocksP2PCtx { /// - Validates payload consistency with authorization /// - Updates global state for new payloads with newer timestamps /// - Caches flashblocks and maintains ordering for sequential delivery - /// - Broadcasts to peers and publishes ordered flashblocks to the stream + /// - Forwards flashblocks to peers in the current send set and publishes ordered + /// flashblocks to the local stream pub fn publish( &self, state: &mut FlashblocksP2PState, @@ -1128,10 +1151,7 @@ impl FlashblocksP2PCtx { metrics::histogram!("flashblocks.tx_count") .record(payload.diff.transactions.len() as f64); - let peer_msg = - PeerMsg::FlashblocksPayloadV1((payload.payload_id, payload.index as usize, bytes)); - - self.peer_tx.send(peer_msg).ok(); + state.send_flashblock_to_send_set(payload.payload_id, payload.index, &bytes); let now = Utc::now() .timestamp_nanos_opt() @@ -1211,17 +1231,7 @@ impl ConnectionHandler for FlashblocksP2PProtoco "new flashblocks connection" ); - let peer_rx = self.handle.ctx.peer_tx.subscribe(); - let (direct_tx, direct_rx) = mpsc::unbounded_channel(); - - FlashblocksConnection::new( - self, - conn, - peer_id, - BroadcastStream::new(peer_rx), - direct_tx, - direct_rx, - ) + FlashblocksConnection::new(self, conn, peer_id) } } @@ -1357,7 +1367,6 @@ mod tests { authorizer_vk: authorizer.verifying_key(), builder_sk: Some(SigningKey::from_bytes(&[8; 32])), fanout_config: config, - peer_tx: broadcast::Sender::new(16), flashblock_tx: broadcast::Sender::new(16), } } @@ -1368,7 +1377,7 @@ mod tests { state } - /// Creates a peer state with a per-peer direct channel for message assertions. + /// Creates a peer state with a per-peer outbound channel for message assertions. fn test_peer_state_with_channel( trusted: bool, ) -> ( @@ -1378,7 +1387,7 @@ mod tests { let (tx, rx) = mpsc::unbounded_channel(); let mut state = FlashblocksConnectionState::new(); state.trusted = trusted; - state.direct_tx = Some(tx); + state.outbound_tx = Some(tx); (state, rx) } @@ -1441,9 +1450,9 @@ mod tests { None, Some(test_peer_info(peer_id, true)), ]); - let (direct_tx, mut direct_rx) = mpsc::unbounded_channel(); + let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel(); - handle.on_peer_connected(network.clone(), peer_id, direct_tx); + handle.on_peer_connected(network.clone(), peer_id, outbound_tx); assert!(network.lookup_calls() >= 2); assert!(network.disconnected_peers().is_empty()); @@ -1456,7 +1465,7 @@ mod tests { ); drop(state); assert_eq!( - recv_direct(&mut direct_rx), + recv_direct(&mut outbound_rx), FlashblocksP2PMsg::RequestFlashblocks ); } @@ -1474,9 +1483,9 @@ mod tests { ); let peer_id = PeerId::random(); let network = MockNetwork::default(); - let (direct_tx, mut direct_rx) = mpsc::unbounded_channel(); + let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel(); - handle.on_peer_connected(network.clone(), peer_id, direct_tx); + handle.on_peer_connected(network.clone(), peer_id, outbound_tx); assert!(network.lookup_calls() > 1); assert!(network.disconnected_peers().is_empty()); @@ -1489,11 +1498,50 @@ mod tests { ); drop(state); assert_eq!( - recv_direct(&mut direct_rx), + recv_direct(&mut outbound_rx), FlashblocksP2PMsg::RequestFlashblocks ); } + #[test] + fn publish_sends_flashblocks_only_to_send_enabled_peers() { + let ctx = test_ctx(FanoutConfig::default()); + let mut fanout = FlashblocksP2PState::default(); + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + let payload_id = PayloadId::new([1; 8]); + let authorization = Authorization::new(payload_id, 1, &authorizer, builder.verifying_key()); + let flashblock = FlashblocksPayloadV1 { + payload_id, + index: 0, + ..Default::default() + }; + let authorized_payload = + AuthorizedPayload::new(&builder, authorization, flashblock.clone()); + let expected = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); + + let source_peer = PeerId::random(); + let send_peer = PeerId::random(); + let non_send_peer = PeerId::random(); + + let (mut source_state, mut source_rx) = test_peer_state_with_channel(false); + source_state.send_enabled = true; + let (mut send_state, mut send_rx) = test_peer_state_with_channel(false); + send_state.send_enabled = true; + let (non_send_state, mut non_send_rx) = test_peer_state_with_channel(false); + + fanout.connections.insert(source_peer, source_state); + fanout.connections.insert(send_peer, send_state); + fanout.connections.insert(non_send_peer, non_send_state); + apply_observation(&mut fanout, &authorization, &flashblock, source_peer); + + ctx.publish(&mut fanout, authorized_payload); + + assert_eq!(recv_direct(&mut send_rx), expected); + assert!(source_rx.try_recv().is_err()); + assert!(non_send_rx.try_recv().is_err()); + } + #[test] fn trusted_peers_are_requested_first() { let config = FanoutConfig { From 067b0e498f7695985f90fa9d72308d57e1704422 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 18:08:20 -0700 Subject: [PATCH 21/43] fix tests --- Cargo.lock | 3 +- crates/flashblocks/cli/Cargo.toml | 2 - crates/flashblocks/cli/src/lib.rs | 81 +++++- crates/flashblocks/node/tests/p2p.rs | 261 ++++++++++++++++-- crates/flashblocks/p2p/Cargo.toml | 1 + .../flashblocks/p2p/src/protocol/handler.rs | 227 ++++++--------- crates/flashblocks/payload/src/generator.rs | 2 +- crates/world/node/src/args.rs | 3 +- crates/world/node/src/context.rs | 8 +- crates/world/pool/src/validator.rs | 4 +- crates/world/test/Cargo.toml | 1 + crates/world/test/src/node.rs | 4 +- 12 files changed, 421 insertions(+), 176 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2cfdf5ef5..83b185372 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3804,7 +3804,6 @@ dependencies = [ "clap", "color-eyre", "ed25519-dalek", - "flashblocks-builder", "hex", ] @@ -3891,6 +3890,7 @@ dependencies = [ "chrono", "ed25519-dalek", "enr", + "flashblocks-cli", "flashblocks-primitives", "futures", "metrics", @@ -14379,6 +14379,7 @@ dependencies = [ "alloy-sol-types", "bon", "chrono", + "flashblocks-builder", "flashblocks-cli", "flashblocks-primitives", "futures", diff --git a/crates/flashblocks/cli/Cargo.toml b/crates/flashblocks/cli/Cargo.toml index 097cfac68..dcf53ada4 100644 --- a/crates/flashblocks/cli/Cargo.toml +++ b/crates/flashblocks/cli/Cargo.toml @@ -5,8 +5,6 @@ edition.workspace = true license.workspace = true [dependencies] -flashblocks-builder.workspace = true - ed25519-dalek.workspace = true clap.workspace = true eyre.workspace = true diff --git a/crates/flashblocks/cli/src/lib.rs b/crates/flashblocks/cli/src/lib.rs index 51b355fa9..1ae889b29 100644 --- a/crates/flashblocks/cli/src/lib.rs +++ b/crates/flashblocks/cli/src/lib.rs @@ -2,11 +2,67 @@ use clap::ArgGroup; use ed25519_dalek::{SigningKey, VerifyingKey}; use hex::FromHex; +pub const DEFAULT_MAX_SEND_PEERS: usize = 10; +pub const DEFAULT_MAX_RECEIVE_PEERS: usize = 3; +pub const DEFAULT_ROTATION_INTERVAL: u64 = 30; +pub const DEFAULT_SCORE_SAMPLES: i64 = 1000; + /// Flashblocks configuration #[derive(Debug, Clone, PartialEq, Eq, clap::Args)] -#[command(next_help_heading = "Flashblocks", - group = ArgGroup::new("authorizer") - .multiple(false) +pub struct FanoutArgs { + /// Override the flashblocks send-set size. + #[arg( + long = "flashblocks.max_send_peers", + env = "FLASHBLOCKS_MAX_SEND_PEERS", + required = false, + default_value_t = DEFAULT_MAX_SEND_PEERS + )] + pub max_send_peers: usize, + + /// Override the number of receive peers maintained for flashblocks fanout. + #[arg( + long = "flashblocks.max_receive_peers", + env = "FLASHBLOCKS_MAX_RECEIVE_PEERS", + required = false, + default_value_t = DEFAULT_MAX_RECEIVE_PEERS + )] + pub max_receive_peers: usize, + + /// Override the flashblocks rotation interval in seconds. + #[arg( + long = "flashblocks.rotation_interval", + env = "FLASHBLOCKS_ROTATION_INTERVAL", + required = false, + default_value_t = DEFAULT_ROTATION_INTERVAL + )] + pub rotation_interval: u64, + + /// Override the number of latency samples retained for receive-peer scoring. + #[arg( + long = "flashblocks.score_samples", + env = "FLASHBLOCKS_SCORE_SAMPLES", + required = false, + default_value_t = DEFAULT_SCORE_SAMPLES + )] + pub score_samples: i64, +} + +impl Default for FanoutArgs { + fn default() -> Self { + Self { + max_send_peers: DEFAULT_MAX_SEND_PEERS, + max_receive_peers: DEFAULT_MAX_RECEIVE_PEERS, + rotation_interval: DEFAULT_ROTATION_INTERVAL, + score_samples: DEFAULT_SCORE_SAMPLES, + } + } +} + +/// Flashblocks configuration +#[derive(Debug, Clone, PartialEq, Eq, clap::Args)] +#[command( + next_help_heading = "Flashblocks", + group = ArgGroup::new("authorizer").multiple(false) )] #[group(requires = "flashblocks.enabled")] pub struct FlashblocksArgs { @@ -21,7 +77,7 @@ pub struct FlashblocksArgs { /// used to verify flashblock authenticity. #[arg( long = "flashblocks.authorizer_vk", - env = "FLASHBLOCKS_AUTHORIZER_VK", + env = "FLASHBLOCKS_AUTHORIZER_VK", group = "authorizer", value_parser = parse_vk, required = false, @@ -31,8 +87,8 @@ pub struct FlashblocksArgs { /// Flashblocks signing key /// used to sign authorized flashblocks payloads. #[arg( - long = "flashblocks.builder_sk", - env = "FLASHBLOCKS_BUILDER_SK", + long = "flashblocks.builder_sk", + env = "FLASHBLOCKS_BUILDER_SK", required = false, value_parser = parse_sk, )] @@ -94,6 +150,9 @@ pub struct FlashblocksArgs { default_value_t = false )] pub access_list: bool, + + #[command(flatten)] + pub fanout: FanoutArgs, } pub fn parse_sk(s: &str) -> eyre::Result { @@ -106,8 +165,6 @@ pub fn parse_vk(s: &str) -> eyre::Result { Ok(VerifyingKey::from_bytes(&bytes)?) } -pub use flashblocks_builder::FlashblocksPayloadBuilderConfig; - #[cfg(test)] mod tests { use super::*; @@ -116,7 +173,7 @@ mod tests { #[derive(Debug, Parser)] struct CommandParser { #[command(flatten)] - flashblocks: Option, + flashblocks: FlashblocksArgs, } #[test] @@ -130,6 +187,7 @@ mod tests { recommit_interval: 200, flashblocks_interval: 200, access_list: true, + fanout: FanoutArgs::default(), }; let args = CommandParser::parse_from([ @@ -146,7 +204,7 @@ mod tests { "200", ]); - assert_eq!(args.flashblocks.unwrap(), flashblocks); + assert_eq!(args.flashblocks, flashblocks); } #[test] @@ -160,6 +218,7 @@ mod tests { recommit_interval: 200, flashblocks_interval: 200, access_list: false, + fanout: FanoutArgs::default(), }; let args = CommandParser::parse_from([ @@ -169,7 +228,7 @@ mod tests { "0000000000000000000000000000000000000000000000000000000000000000", ]); - assert_eq!(args.flashblocks.unwrap(), flashblocks); + assert_eq!(args.flashblocks, flashblocks); } #[test] diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index f3ed410a4..560d77976 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -6,7 +6,10 @@ use alloy_rpc_types_engine::PayloadId; use ed25519_dalek::SigningKey; use eyre::eyre::eyre; use flashblocks_cli::FlashblocksArgs; -use flashblocks_p2p::{monitor, protocol::handler::FlashblocksHandle}; +use flashblocks_p2p::{ + monitor, + protocol::handler::{FlashblocksHandle, PublishingStatus}, +}; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, p2p::{ @@ -40,6 +43,7 @@ use std::{ net::{IpAddr, SocketAddr}, path::PathBuf, sync::{Arc, Mutex}, + time::{SystemTime, UNIX_EPOCH}, }; use tempfile::NamedTempFile; use tokio::time::{Duration, Instant, sleep}; @@ -213,6 +217,56 @@ async fn wait_for_trusted_peers( } } +async fn wait_for_flashblocks_topology( + node: &NodeContext, + expected_connections: usize, + expected_receive_peers: usize, +) -> eyre::Result<(Vec, Vec)> { + let timeout = Duration::from_secs(10); + let poll_interval = Duration::from_millis(100); + let start = Instant::now(); + + loop { + let state = node.p2p_handle.state.lock(); + if state.connections.len() == expected_connections { + let receive_peers: Vec<_> = state + .connections + .iter() + .filter_map(|(peer_id, conn)| { + (conn.receive_enabled.is_some() && !conn.request_in_flight).then_some(*peer_id) + }) + .collect(); + let candidate_peers: Vec<_> = state + .connections + .iter() + .filter_map(|(peer_id, conn)| { + (conn.receive_enabled.is_none() + && !conn.request_in_flight + && !conn.abandoned_request_in_flight) + .then_some(*peer_id) + }) + .collect(); + drop(state); + + if receive_peers.len() == expected_receive_peers + && receive_peers.len() + candidate_peers.len() == expected_connections + { + return Ok((receive_peers, candidate_peers)); + } + } else { + drop(state); + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for flashblocks topology: expected {expected_connections} connections with {expected_receive_peers} receive peers" + )); + } + + sleep(poll_interval).await; + } +} + fn init_tracing(filter: &str) -> tracing::subscriber::DefaultGuard { let sub = tracing_subscriber::fmt() .with_env_filter(filter) @@ -224,13 +278,18 @@ fn init_tracing(filter: &str) -> tracing::subscriber::DefaultGuard { Dispatch::new(sub).set_default() } -async fn setup_node( - exec: TaskExecutor, - authorizer_sk: SigningKey, - builder_sk: SigningKey, - peers: Vec<(PeerId, SocketAddr)>, -) -> eyre::Result { - setup_node_extended_cfg(exec, authorizer_sk, builder_sk, peers, None, None).await +fn test_flashblocks_args(authorizer_sk: &SigningKey, builder_sk: &SigningKey) -> FlashblocksArgs { + FlashblocksArgs { + enabled: true, + authorizer_vk: Some(authorizer_sk.verifying_key()), + builder_sk: Some(builder_sk.clone()), + force_publish: false, + override_authorizer_sk: None, + flashblocks_interval: 200, + recommit_interval: 200, + access_list: true, + fanout: Default::default(), + } } async fn setup_node_extended_cfg( @@ -240,6 +299,7 @@ async fn setup_node_extended_cfg( peers: Vec<(PeerId, SocketAddr)>, port: Option, p2p_secret_key: Option, + flashblocks_args: Option, ) -> eyre::Result { let genesis: Genesis = serde_json::from_str(include_str!("assets/genesis.json")).unwrap(); let chain_spec = Arc::new( @@ -307,16 +367,10 @@ async fn setup_node_extended_cfg( rollup: Default::default(), builder, pbh, - flashblocks: Some(FlashblocksArgs { - enabled: true, - authorizer_vk: Some(authorizer_sk.verifying_key()), - builder_sk: Some(builder_sk.clone()), - force_publish: false, - override_authorizer_sk: None, - flashblocks_interval: 200, - recommit_interval: 200, - access_list: true, - }), + flashblocks: Some( + flashblocks_args + .unwrap_or_else(|| test_flashblocks_args(&authorizer_sk, &builder_sk)), + ), tx_peers: None, disable_bootnodes: true, }, @@ -440,7 +494,73 @@ async fn next_payload(payload_id: PayloadId, index: u64) -> FlashblocksPayloadV1 } } +async fn publish_flashblock_with_latency( + sender: &NodeContext, + authorizer: &SigningKey, + payload_id: PayloadId, + authorization_timestamp: u64, + simulated_latency: Duration, +) -> eyre::Result<()> { + let latest_block = sender + .provider() + .await? + .get_block_by_number(alloy_eips::BlockNumberOrTag::Latest) + .await? + .expect("latest block expected"); + let mut payload = base_payload( + 0, + payload_id, + 0, + latest_block.hash(), + authorization_timestamp, + ); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_nanos() as i64; + payload.metadata.flashblock_timestamp = Some(now - simulated_latency.as_nanos() as i64); + let authorization = Authorization::new( + payload.payload_id, + authorization_timestamp, + authorizer, + sender.p2p_handle.builder_sk()?.verifying_key(), + ); + let authorized = + AuthorizedPayload::new(sender.p2p_handle.builder_sk()?, authorization, payload); + + { + let state = sender.p2p_handle.state.lock(); + state + .publishing_status + .send_replace(PublishingStatus::Publishing { authorization }); + } + sender.p2p_handle.publish_new(authorized)?; + { + let state = sender.p2p_handle.state.lock(); + state + .publishing_status + .send_replace(PublishingStatus::NotPublishing { + active_publishers: Vec::new(), + }); + } + + Ok(()) +} + async fn setup_nodes(n: u8) -> eyre::Result { + setup_nodes_with_flashblocks_args(n, |_, authorizer, builder| { + test_flashblocks_args(authorizer, builder) + }) + .await +} + +async fn setup_nodes_with_flashblocks_args( + n: u8, + mut make_flashblocks_args: F, +) -> eyre::Result +where + F: FnMut(u8, &SigningKey, &SigningKey) -> FlashblocksArgs, +{ let mut nodes = Vec::new(); let mut peers = Vec::new(); let tasks = TaskManager::new(tokio::runtime::Handle::current()); @@ -449,7 +569,17 @@ async fn setup_nodes(n: u8) -> eyre::Result { for i in 0..n { let builder = SigningKey::from_bytes(&[(i + 1) % n; 32]); - let node = setup_node(exec.clone(), authorizer.clone(), builder, peers.clone()).await?; + let flashblocks_args = make_flashblocks_args(i, &authorizer, &builder); + let node = setup_node_extended_cfg( + exec.clone(), + authorizer.clone(), + builder, + peers.clone(), + None, + None, + Some(flashblocks_args), + ) + .await?; if !peers.is_empty() { wait_for_trusted_peers(&node, peers.len()).await?; } @@ -672,6 +802,96 @@ async fn test_force_race_condition() -> eyre::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread")] +async fn test_receive_peer_rotation_uses_latency_scores() -> eyre::Result<()> { + let _tracing = init_tracing("warn,flashblocks=trace"); + + let fixture = setup_nodes_with_flashblocks_args(4, |_, authorizer, builder| { + let mut args = test_flashblocks_args(authorizer, builder); + args.fanout.max_receive_peers = 2; + args.fanout.rotation_interval = 1; + args.fanout.score_samples = 4; + args + }) + .await?; + let nodes = fixture.nodes(); + let authorizer = fixture.authorizer(); + + let (receive_peers, candidate_peers) = wait_for_flashblocks_topology(&nodes[0], 3, 2).await?; + assert_eq!( + candidate_peers.len(), + 1, + "expected one spare candidate peer" + ); + + let slow_peer = receive_peers[0]; + let fast_peer = receive_peers[1]; + let replacement_peer = candidate_peers[0]; + + let peer_map: HashMap<_, _> = nodes + .iter() + .skip(1) + .map(|node| (node.local_node_record.id, node)) + .collect(); + + let fast_node = peer_map + .get(&fast_peer) + .copied() + .expect("fast peer should map to a node"); + let slow_node = peer_map + .get(&slow_peer) + .copied() + .expect("slow peer should map to a node"); + + for (payload_suffix, authorization_timestamp) in [(41, 41_u64), (42, 42), (43, 43), (44, 44)] { + publish_flashblock_with_latency( + fast_node, + authorizer, + test_payload_id(payload_suffix), + authorization_timestamp, + Duration::from_millis(10), + ) + .await?; + sleep(Duration::from_millis(50)).await; + + publish_flashblock_with_latency( + slow_node, + authorizer, + test_payload_id(payload_suffix + 10), + authorization_timestamp + 10, + Duration::from_millis(300), + ) + .await?; + sleep(Duration::from_millis(50)).await; + } + + let timeout = Duration::from_secs(5); + let poll_interval = Duration::from_millis(100); + let start = Instant::now(); + + loop { + let (current_receive_peers, _) = wait_for_flashblocks_topology(&nodes[0], 3, 2).await?; + if current_receive_peers.contains(&fast_peer) + && current_receive_peers.contains(&replacement_peer) + && !current_receive_peers.contains(&slow_peer) + { + break; + } + + if start.elapsed() >= timeout { + return Err(eyre!( + "timed out waiting for peer rotation: fast={fast_peer}, slow={slow_peer}, replacement={replacement_peer}, current_receive_peers={current_receive_peers:?}" + )); + } + + sleep(poll_interval).await; + } + + drop(fixture); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread")] async fn test_get_block_by_number_pending() -> eyre::Result<()> { let _tracing = init_tracing("warn,flashblocks=trace"); @@ -847,6 +1067,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { vec![], // No peers initially None, // Use random port (we'll capture it) Some(p2p_key_path.clone()), // Use deterministic P2P key + None, ) .await?; @@ -868,6 +1089,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { vec![(peer1_id, peer1_addr)], // Node1 as trusted peer None, // Use random port None, // No deterministic P2P key needed + None, ) .await?; @@ -947,6 +1169,7 @@ async fn test_peer_monitoring() -> eyre::Result<()> { )], // Configure node2 as trusted peer Some(peer1_port), // Reuse the same port Some(p2p_key_path.clone()), // Reuse the same P2P key + None, ) .await?; let peer1_id_new = node1_restarted.local_node_record.id; diff --git a/crates/flashblocks/p2p/Cargo.toml b/crates/flashblocks/p2p/Cargo.toml index 078c241a0..6923ee716 100644 --- a/crates/flashblocks/p2p/Cargo.toml +++ b/crates/flashblocks/p2p/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true test-utils = [] [dependencies] +flashblocks-cli.workspace = true flashblocks-primitives.workspace = true reth.workspace = true diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 103092393..fe7fed33a 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -5,6 +5,7 @@ use crate::protocol::{ use alloy_rlp::BytesMut; use chrono::Utc; use ed25519_dalek::{SigningKey, VerifyingKey}; +use flashblocks_cli::FanoutArgs; use flashblocks_primitives::{ p2p::{ Authorization, Authorized, AuthorizedMsg, AuthorizedPayload, FlashblocksP2PMsg, @@ -84,30 +85,6 @@ pub trait FlashblocksP2PNetworkHandle: Clone + Unpin + Peers + std::fmt::Debug + impl FlashblocksP2PNetworkHandle for N {} -/// Runtime configuration for bounded flashblocks fanout. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct FanoutConfig { - /// Maximum number of non-trusted peers to send flashblocks to. - pub max_send_peers: usize, - /// Maximum number of peers to receive flashblocks from. - pub max_receive_peers: usize, - /// How often to evaluate latency-based peer rotation. - pub rotation_interval: Duration, - /// Number of latency measurements to retain per receive peer. - pub latency_window: i64, -} - -impl Default for FanoutConfig { - fn default() -> Self { - Self { - max_send_peers: 10, - max_receive_peers: 3, - rotation_interval: Duration::from_secs(30), - latency_window: 1000, - } - } -} - /// The current publishing status of this node in the flashblocks P2P network. /// /// This enum tracks whether we are actively publishing flashblocks, waiting to publish, @@ -349,7 +326,9 @@ impl FlashblocksP2PState { } fn receive_retry_cooldown_secs(ctx: &FlashblocksP2PCtx) -> u64 { - ctx.fanout_config.rotation_interval.as_secs().max(1) + Duration::from_secs(ctx.fanout_args.rotation_interval) + .as_secs() + .max(1) } fn clear_receive_state( @@ -399,13 +378,13 @@ impl FlashblocksP2PState { let timestamp = Utc::now().timestamp() as u64; peer_state.request_in_flight = true; peer_state.abandoned_request_in_flight = false; - peer_state.receive_enabled = Some(Score::new(ctx.fanout_config.latency_window)); + peer_state.receive_enabled = Some(Score::new(ctx.fanout_args.score_samples)); peer_state.receive_enabled_timestamp = timestamp; self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { - while self.num_receive_peers() < ctx.fanout_config.max_receive_peers { + while self.num_receive_peers() < ctx.fanout_args.max_receive_peers { let candidates = self.available_receive_candidates(ctx); if candidates.is_empty() { return; @@ -454,7 +433,7 @@ impl FlashblocksP2PState { } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { - if self.num_receive_peers() < ctx.fanout_config.max_receive_peers { + if self.num_receive_peers() < ctx.fanout_args.max_receive_peers { return; } @@ -512,7 +491,7 @@ impl FlashblocksP2PState { .filter(|s| s.send_enabled && !s.trusted) .count(); - if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_config.max_send_peers { + if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_args.max_send_peers { self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); return false; } @@ -602,10 +581,8 @@ impl FlashblocksP2PState { pub struct FlashblocksP2PCtx { /// Authorizer's verifying key used to verify authorization signatures from rollup-boost. pub authorizer_vk: VerifyingKey, - /// Builder's signing key used to sign outgoing authorized P2P messages. - pub builder_sk: Option, - /// Fanout configuration for peer selection and rotation. - pub fanout_config: FanoutConfig, + /// Flashblocks configuration including signing keys and fanout args. + pub fanout_args: FanoutArgs, /// Broadcast sender for verified and strictly ordered flashblock payloads. /// Used by RPC overlays and other consumers of flashblock data. pub flashblock_tx: broadcast::Sender, @@ -619,6 +596,8 @@ pub struct FlashblocksP2PCtx { pub struct FlashblocksHandle { /// Shared context containing network handle, keys, and communication channels. pub ctx: FlashblocksP2PCtx, + /// Builder signing key used to sign outgoing authorized P2P messages. + pub builder_sk: Option, /// Thread-safe mutable state of the flashblocks protocol. /// Protected by a mutex to allow concurrent access from multiple connections. pub state: Arc>, @@ -626,28 +605,32 @@ pub struct FlashblocksHandle { impl FlashblocksHandle { pub fn new(authorizer_vk: VerifyingKey, builder_sk: Option) -> Self { - Self::with_fanout_config(authorizer_vk, builder_sk, FanoutConfig::default()) + Self::with_fanout_args(authorizer_vk, builder_sk, FanoutArgs::default()) } - pub fn with_fanout_config( + pub fn with_fanout_args( authorizer_vk: VerifyingKey, builder_sk: Option, - fanout_config: FanoutConfig, + fanout_args: FanoutArgs, ) -> Self { let flashblock_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); let state = Arc::new(Mutex::new(FlashblocksP2PState::default())); let ctx = FlashblocksP2PCtx { authorizer_vk, - builder_sk, - fanout_config, + fanout_args, flashblock_tx, }; - let handle = Self { ctx, state }; + let handle = Self { + ctx, + builder_sk, + state, + }; let moved_handle = handle.clone(); tokio::spawn(async move { - let mut rotation_interval = - time::interval(moved_handle.ctx.fanout_config.rotation_interval); + let mut rotation_interval = time::interval(Duration::from_secs( + moved_handle.ctx.fanout_args.rotation_interval, + )); rotation_interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); rotation_interval.tick().await; @@ -804,8 +787,7 @@ impl FlashblocksP2PProtocol { impl FlashblocksHandle { /// Returns the builder signing key if configured. pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { - self.ctx - .builder_sk + self.builder_sk .as_ref() .ok_or(FlashblocksP2PError::MissingBuilderSk) } @@ -1360,13 +1342,16 @@ mod tests { } } - fn test_ctx(config: FanoutConfig) -> FlashblocksP2PCtx { + fn test_fanout_args() -> FanoutArgs { + FanoutArgs::default() + } + + fn test_ctx(fanout_args: FanoutArgs) -> FlashblocksP2PCtx { let authorizer = SigningKey::from_bytes(&[7; 32]); FlashblocksP2PCtx { authorizer_vk: authorizer.verifying_key(), - builder_sk: Some(SigningKey::from_bytes(&[8; 32])), - fanout_config: config, + fanout_args, flashblock_tx: broadcast::Sender::new(16), } } @@ -1437,13 +1422,12 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn on_peer_connected_retries_until_peer_info_is_available() { let authorizer = SigningKey::from_bytes(&[7; 32]); - let handle = FlashblocksHandle::with_fanout_config( + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let handle = FlashblocksHandle::with_fanout_args( authorizer.verifying_key(), Some(SigningKey::from_bytes(&[8; 32])), - FanoutConfig { - max_receive_peers: 1, - ..Default::default() - }, + fanout_args, ); let peer_id = PeerId::random(); let network = MockNetwork::with_peer_lookup_responses(vec![ @@ -1473,13 +1457,12 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn on_peer_connected_defaults_to_untrusted_when_peer_info_never_arrives() { let authorizer = SigningKey::from_bytes(&[7; 32]); - let handle = FlashblocksHandle::with_fanout_config( + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let handle = FlashblocksHandle::with_fanout_args( authorizer.verifying_key(), Some(SigningKey::from_bytes(&[8; 32])), - FanoutConfig { - max_receive_peers: 1, - ..Default::default() - }, + fanout_args, ); let peer_id = PeerId::random(); let network = MockNetwork::default(); @@ -1505,7 +1488,7 @@ mod tests { #[test] fn publish_sends_flashblocks_only_to_send_enabled_peers() { - let ctx = test_ctx(FanoutConfig::default()); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); @@ -1544,11 +1527,9 @@ mod tests { #[test] fn trusted_peers_are_requested_first() { - let config = FanoutConfig { - max_receive_peers: 1, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let trusted_peer = PeerId::random(); @@ -1571,11 +1552,9 @@ mod tests { #[test] fn trusted_request_bypasses_non_trusted_limit() { - let config = FanoutConfig { - max_send_peers: 1, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_send_peers = 1; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let victim = PeerId::random(); @@ -1601,20 +1580,18 @@ mod tests { #[test] fn rotation_replaces_peer_before_requesting_candidate() { - let config = FanoutConfig { - max_receive_peers: 1, - latency_window: 4, - ..Default::default() - }; - let latency_window = config.latency_window; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let current_peer = PeerId::random(); let candidate_peer = PeerId::random(); let (mut current_state, mut current_rx) = test_peer_state_with_channel(false); let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false); - let mut score = Score::new(latency_window); + let mut score = Score::new(score_samples); score.record(42); current_state.receive_enabled = Some(score); fanout.connections.insert(current_peer, current_state); @@ -1651,11 +1628,9 @@ mod tests { #[test] fn multiple_pending_requests_clear_independently() { - let config = FanoutConfig { - max_receive_peers: 2, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let first_peer = PeerId::random(); @@ -1690,11 +1665,9 @@ mod tests { #[test] fn rejected_peer_is_not_immediately_retried() { - let config = FanoutConfig { - max_receive_peers: 1, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -1720,7 +1693,7 @@ mod tests { .connection_state_mut(&peer) .expect("peer exists") .receive_enabled_timestamp = - Utc::now().timestamp() as u64 - ctx.fanout_config.rotation_interval.as_secs().max(1); + Utc::now().timestamp() as u64 - ctx.fanout_args.rotation_interval.max(1); fanout.maybe_request_receive_peers(&ctx); assert_eq!( @@ -1731,12 +1704,10 @@ mod tests { #[test] fn stale_accept_after_abandoned_request_is_canceled_without_penalty() { - let config = FanoutConfig { - max_receive_peers: 1, - latency_window: 4, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + fanout_args.score_samples = 4; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let abandoned_peer = PeerId::random(); @@ -1777,11 +1748,9 @@ mod tests { #[test] fn silent_receive_peer_can_be_rotated_out_without_samples() { - let config = FanoutConfig { - max_receive_peers: 2, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let oldest_peer = PeerId::random(); @@ -1820,12 +1789,10 @@ mod tests { #[test] fn peer_score_penalizes_missed_flashblocks() { - let config = FanoutConfig { - max_receive_peers: 2, - latency_window: 4, - ..Default::default() - }; - let latency_window = config.latency_window; + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; let mut fanout = FlashblocksP2PState::default(); let steady_peer = PeerId::random(); @@ -1835,8 +1802,8 @@ mod tests { let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); - steady_state.receive_enabled = Some(Score::new(latency_window)); - lagging_state.receive_enabled = Some(Score::new(latency_window)); + steady_state.receive_enabled = Some(Score::new(score_samples)); + lagging_state.receive_enabled = Some(Score::new(score_samples)); steady_state .receive_enabled .as_mut() @@ -1879,19 +1846,17 @@ mod tests { .receive_enabled .as_ref() .and_then(Score::value), - Some((100 * (latency_window - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / latency_window) + Some((100 * (score_samples - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / score_samples) ); } #[test] fn pending_candidate_is_rotated_out_after_missing_blocks() { - let config = FanoutConfig { - max_receive_peers: 2, - latency_window: 4, - ..Default::default() - }; - let latency_window = config.latency_window; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let steady_peer = PeerId::random(); @@ -1904,8 +1869,8 @@ mod tests { let candidate_state = test_peer_state(true); let replacement_state = test_peer_state(true); - steady_state.receive_enabled = Some(Score::new(latency_window)); - rotating_state.receive_enabled = Some(Score::new(latency_window)); + steady_state.receive_enabled = Some(Score::new(score_samples)); + rotating_state.receive_enabled = Some(Score::new(score_samples)); steady_state .receive_enabled .as_mut() @@ -1957,8 +1922,7 @@ mod tests { #[test] fn unsolicited_accept_is_penalized() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -1971,8 +1935,7 @@ mod tests { #[test] fn unsolicited_reject_is_penalized() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -1985,8 +1948,7 @@ mod tests { #[test] fn cancel_without_relationship_is_penalized() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -1999,8 +1961,7 @@ mod tests { #[test] fn cancel_only_clears_send_direction() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -2017,8 +1978,7 @@ mod tests { #[test] fn cancel_from_sender_is_penalized() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -2031,8 +1991,7 @@ mod tests { #[test] fn duplicate_request_when_already_sending_is_penalized() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -2045,8 +2004,7 @@ mod tests { #[test] fn receive_retry_cooldown_does_not_penalize_inbound_request() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -2061,11 +2019,9 @@ mod tests { #[test] fn repeated_rejected_requests_are_rate_limited() { - let config = FanoutConfig { - max_send_peers: 0, - ..Default::default() - }; - let ctx = test_ctx(config); + let mut fanout_args = test_fanout_args(); + fanout_args.max_send_peers = 0; + let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -2081,8 +2037,7 @@ mod tests { #[test] fn control_message_rate_limit_triggers_penalty() { - let config = FanoutConfig::default(); - let ctx = test_ctx(config); + let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); diff --git a/crates/flashblocks/payload/src/generator.rs b/crates/flashblocks/payload/src/generator.rs index c4e49b28a..9faf9164f 100644 --- a/crates/flashblocks/payload/src/generator.rs +++ b/crates/flashblocks/payload/src/generator.rs @@ -258,7 +258,7 @@ where let authorization = match ( &self.override_authorizer_sk, - &self.p2p_handler.ctx.builder_sk, + self.p2p_handler.builder_sk().ok(), can_override, ) { (Some(override_authorizer_sk), Some(builder_sk), true) => Some(Authorization::new( diff --git a/crates/world/node/src/args.rs b/crates/world/node/src/args.rs index 19cb80b4e..a1c640a9f 100644 --- a/crates/world/node/src/args.rs +++ b/crates/world/node/src/args.rs @@ -3,7 +3,8 @@ use alloy_primitives::Address; use alloy_signer_local::PrivateKeySigner; use clap::value_parser; use ed25519_dalek::{SigningKey, VerifyingKey}; -use flashblocks_cli::{FlashblocksArgs, FlashblocksPayloadBuilderConfig}; +use flashblocks_builder::FlashblocksPayloadBuilderConfig; +use flashblocks_cli::FlashblocksArgs; use hex::FromHex; use reth::chainspec::NamedChain; use reth_network_peers::{PeerId, TrustedPeer}; diff --git a/crates/world/node/src/context.rs b/crates/world/node/src/context.rs index cfc909351..9a78c9651 100644 --- a/crates/world/node/src/context.rs +++ b/crates/world/node/src/context.rs @@ -396,6 +396,7 @@ impl From for FlashblocksComponentsContext { let authorizer_vk = flashblocks.authorizer_vk.unwrap_or_else(|| { flashblocks .override_authorizer_sk + .as_ref() .expect("flashblocks authorizer_vk or override_authorizer_sk required") .verifying_key() }); @@ -405,8 +406,11 @@ impl From for FlashblocksComponentsContext { authorizer_vk.as_bytes().encode_hex::() ); - let builder_sk = flashblocks.builder_sk.clone(); - let flashblocks_handle = FlashblocksHandle::new(authorizer_vk, builder_sk.clone()); + let flashblocks_handle = FlashblocksHandle::with_fanout_args( + authorizer_vk, + flashblocks.builder_sk.clone(), + flashblocks.fanout.clone(), + ); let (pending_block, _) = tokio::sync::watch::channel(None); diff --git a/crates/world/pool/src/validator.rs b/crates/world/pool/src/validator.rs index 38e445beb..d8f872ad1 100644 --- a/crates/world/pool/src/validator.rs +++ b/crates/world/pool/src/validator.rs @@ -30,7 +30,7 @@ use reth_optimism_primitives::OpTransactionSigned; use reth_primitives::{Block, NodePrimitives, SealedBlock}; use reth_provider::{BlockReaderIdExt, ChainSpecProvider, StateProviderFactory}; use revm_primitives::U256; -use tracing::{info, warn}; +use tracing::info; use world_chain_pbh::payload::{PBHPayload as PbhPayload, PBHValidationError}; /// The slot of the `pbh_gas_limit` in the PBHEntryPoint contract. @@ -94,7 +94,7 @@ where .to(); if max_pbh_nonce == 0 && max_pbh_gas_limit == 0 { - warn!( + info!( %pbh_entrypoint, %pbh_signature_aggregator, "WorldChainTransactionValidator Initialized with PBH Disabled - Failed to fetch PBH nonce and gas limit from PBHEntryPoint. Defaulting to 0." diff --git a/crates/world/test/Cargo.toml b/crates/world/test/Cargo.toml index c41644ab2..abb57682b 100644 --- a/crates/world/test/Cargo.toml +++ b/crates/world/test/Cargo.toml @@ -16,6 +16,7 @@ world-chain-pool.workspace = true world-chain-node.workspace = true flashblocks-primitives.workspace = true +flashblocks-builder.workspace = true flashblocks-cli.workspace = true reth.workspace = true diff --git a/crates/world/test/src/node.rs b/crates/world/test/src/node.rs index f259e0486..f1a2bb731 100644 --- a/crates/world/test/src/node.rs +++ b/crates/world/test/src/node.rs @@ -8,7 +8,8 @@ use alloy_primitives::{ }; use alloy_rpc_types::{TransactionInput, TransactionRequest}; use alloy_sol_types::SolCall; -use flashblocks_cli::{FlashblocksArgs, FlashblocksPayloadBuilderConfig}; +use flashblocks_builder::FlashblocksPayloadBuilderConfig; +use flashblocks_cli::{FanoutArgs, FlashblocksArgs}; use futures::future::join_all; use reth_chain_state::{ CanonStateNotifications, CanonStateSubscriptions, ForkChoiceNotifications, @@ -104,6 +105,7 @@ pub fn test_config_with_peers_and_gossip( recommit_interval: 50, flashblocks_interval: 200, access_list: true, + fanout: FanoutArgs::default(), }) } else { None From 438e2fc4ac61214d5d0e64054a1a9f57d7e14eb1 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 18:38:16 -0700 Subject: [PATCH 22/43] wip --- crates/flashblocks/node/tests/p2p.rs | 5 +- .../p2p/src/protocol/connection.rs | 40 ++++++++-------- .../flashblocks/p2p/src/protocol/handler.rs | 46 +++++++++---------- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 560d77976..955fb8c41 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -233,7 +233,8 @@ async fn wait_for_flashblocks_topology( .connections .iter() .filter_map(|(peer_id, conn)| { - (conn.receive_enabled.is_some() && !conn.request_in_flight).then_some(*peer_id) + (conn.receive_enabled.is_some() && !conn.request_flashblocks_in_flight) + .then_some(*peer_id) }) .collect(); let candidate_peers: Vec<_> = state @@ -241,7 +242,7 @@ async fn wait_for_flashblocks_topology( .iter() .filter_map(|(peer_id, conn)| { (conn.receive_enabled.is_none() - && !conn.request_in_flight + && !conn.request_flashblocks_in_flight && !conn.abandoned_request_in_flight) .then_some(*peer_id) }) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 6cd97a2f4..e2ce087b2 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -25,28 +25,34 @@ use tracing::{info, trace}; /// minor skew/races between peers. const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; +/// Represents the current flashblocks receive status for a peer connection. +#[derive(Clone, Debug, Default)] +pub enum ReceiveStatus { + /// We are not currently receiving flashblocks from this peer. + #[default] + NotReceiving, + /// We are currently receiving flashblocks from this peer. + /// + /// Score used for adaptive timeouts and peer selection. + /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks + /// counting as 10s. + Receiving { score: Score }, + /// We have sent a request for flashblocks to this peer and are awaiting their response. + Reqesting, +} + /// Shared connection metadata for a single peer connection. #[derive(Clone, Debug)] pub struct FlashblocksConnectionState { /// Whether this peer is marked as trusted or not. pub trusted: bool, - /// Whether we currently have an outstanding flashblocks request to this peer. - pub request_in_flight: bool, - /// Whether we intentionally abandoned an in-flight request and should treat a late - /// Accept/Reject as stale instead of malicious. - pub abandoned_request_in_flight: bool, /// Whether we are currently sending flashblocks to this peer. pub send_enabled: bool, - /// Whether we are currently requesting flashblocks from this peer. - /// - /// Optional score for this peer connection, used for adaptive timeouts and peer selection. - /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks - /// counting as 10s. While `request_in_flight` is true, the peer is only a provisional - /// candidate and must not deliver flashblocks yet. - pub receive_enabled: Option, + /// Current status of receiving flashblocks from this peer. + pub receive_status: ReceiveStatus, /// Timestamp of the last receive-side state transition for this peer. /// Used for late-message grace checks and receive retry cooldown. - pub receive_enabled_timestamp: u64, + pub receive_status_timestamp: u64, /// Per-peer channel for sending serialized protocol messages to this peer. pub outbound_tx: Option>, /// Number of control messages received in the current rate-limit window. @@ -59,11 +65,9 @@ impl FlashblocksConnectionState { pub(crate) fn new() -> Self { Self { trusted: false, - request_in_flight: false, - abandoned_request_in_flight: false, send_enabled: false, - receive_enabled: None, - receive_enabled_timestamp: 0, + receive_status: ReceiveStatus::NotReceiving, + receive_status_timestamp: 0, outbound_tx: None, control_msg_count: 0, control_msg_window_start: Instant::now(), @@ -325,7 +329,7 @@ impl FlashblocksConnection { let Some(conn_state) = p2p_state.connection_state(&self.peer_id) else { return; }; - if conn_state.request_in_flight { + if conn_state.request_flashblocks_in_flight.is_some() { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index fe7fed33a..a3610bde2 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -336,8 +336,7 @@ impl FlashblocksP2PState { receive_enabled_timestamp: u64, ) { peer_state.receive_enabled = None; - peer_state.request_in_flight = false; - peer_state.abandoned_request_in_flight = false; + peer_state.request_flashblocks_in_flight = None; peer_state.receive_enabled_timestamp = receive_enabled_timestamp; } @@ -346,8 +345,7 @@ impl FlashblocksP2PState { receive_enabled_timestamp: u64, ) { peer_state.receive_enabled = None; - peer_state.request_in_flight = false; - peer_state.abandoned_request_in_flight = true; + peer_state.request_flashblocks_in_flight = None; peer_state.receive_enabled_timestamp = receive_enabled_timestamp; } @@ -358,8 +356,7 @@ impl FlashblocksP2PState { .iter() .filter_map(|(peer_id, peer_state)| { if peer_state.receive_enabled.is_none() - && !peer_state.request_in_flight - && !peer_state.abandoned_request_in_flight + && !peer_state.request_flashblocks_in_flight.is_some() && (peer_state.receive_enabled_timestamp == 0 || peer_state.receive_enabled_timestamp + retry_cooldown <= now) { @@ -376,8 +373,7 @@ impl FlashblocksP2PState { return; }; let timestamp = Utc::now().timestamp() as u64; - peer_state.request_in_flight = true; - peer_state.abandoned_request_in_flight = false; + peer_state.request_flashblocks_in_flight = Some(timestamp); peer_state.receive_enabled = Some(Score::new(ctx.fanout_args.score_samples)); peer_state.receive_enabled_timestamp = timestamp; self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); @@ -456,7 +452,7 @@ impl FlashblocksP2PState { let evict_timestamp = Utc::now().timestamp() as u64; let mut should_cancel = false; if let Some(evict_state) = self.connection_state_mut(&evict) { - if evict_state.request_in_flight { + if evict_state.request_flashblocks_in_flight { Self::abandon_receive_request(evict_state, evict_timestamp); } else { Self::clear_receive_state(evict_state, evict_timestamp); @@ -512,8 +508,8 @@ impl FlashblocksP2PState { return false; }; - if peer_state.request_in_flight { - peer_state.request_in_flight = false; + if peer_state.request_flashblocks_in_flight { + peer_state.request_flashblocks_in_flight = false; return false; } @@ -537,7 +533,7 @@ impl FlashblocksP2PState { return false; }; - if peer_state.request_in_flight { + if peer_state.request_flashblocks_in_flight { Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); self.maybe_request_receive_peers(ctx); return false; @@ -1541,9 +1537,9 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); - assert!(peer_state(&fanout, trusted_peer).request_in_flight); + assert!(peer_state(&fanout, trusted_peer).request_flashblocks_in_flight); assert!(peer_state(&fanout, trusted_peer).receive_enabled.is_some()); - assert!(!peer_state(&fanout, untrusted_peer).request_in_flight); + assert!(!peer_state(&fanout, untrusted_peer).request_flashblocks_in_flight); assert_eq!( recv_direct(&mut trusted_rx), FlashblocksP2PMsg::RequestFlashblocks @@ -1600,7 +1596,7 @@ mod tests { fanout.maybe_start_rotation(&ctx); assert!(peer_state(&fanout, current_peer).receive_enabled.is_none()); - assert!(peer_state(&fanout, candidate_peer).request_in_flight); + assert!(peer_state(&fanout, candidate_peer).request_flashblocks_in_flight); assert!( peer_state(&fanout, candidate_peer) .receive_enabled @@ -1618,7 +1614,7 @@ mod tests { assert!(!fanout.handle_accept(&ctx, candidate_peer)); - assert!(!peer_state(&fanout, candidate_peer).request_in_flight); + assert!(!peer_state(&fanout, candidate_peer).request_flashblocks_in_flight); assert!( peer_state(&fanout, candidate_peer) .receive_enabled @@ -1642,8 +1638,8 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); - assert!(peer_state(&fanout, first_peer).request_in_flight); - assert!(peer_state(&fanout, second_peer).request_in_flight); + assert!(peer_state(&fanout, first_peer).request_flashblocks_in_flight); + assert!(peer_state(&fanout, second_peer).request_flashblocks_in_flight); assert_eq!( recv_direct(&mut first_rx), @@ -1657,8 +1653,8 @@ mod tests { assert!(!fanout.handle_accept(&ctx, first_peer)); assert!(!fanout.handle_accept(&ctx, second_peer)); - assert!(!peer_state(&fanout, first_peer).request_in_flight); - assert!(!peer_state(&fanout, second_peer).request_in_flight); + assert!(!peer_state(&fanout, first_peer).request_flashblocks_in_flight); + assert!(!peer_state(&fanout, second_peer).request_flashblocks_in_flight); assert!(peer_state(&fanout, first_peer).receive_enabled.is_some()); assert!(peer_state(&fanout, second_peer).receive_enabled.is_some()); } @@ -1682,7 +1678,7 @@ mod tests { assert!(!fanout.handle_reject(&ctx, peer)); - assert!(!peer_state(&fanout, peer).request_in_flight); + assert!(!peer_state(&fanout, peer).request_flashblocks_in_flight); assert!(peer_state(&fanout, peer).receive_enabled.is_none()); assert!(peer_rx.try_recv().is_err()); @@ -1716,7 +1712,7 @@ mod tests { let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); abandoned_state.receive_enabled = Some(Score::new(4)); - abandoned_state.request_in_flight = true; + abandoned_state.request_flashblocks_in_flight = true; abandoned_state.receive_enabled_timestamp = 1; fanout.connections.insert(abandoned_peer, abandoned_state); @@ -1775,7 +1771,7 @@ mod tests { fanout.maybe_start_rotation(&ctx); assert!(peer_state(&fanout, oldest_peer).receive_enabled.is_none()); - assert!(peer_state(&fanout, replacement_peer).request_in_flight); + assert!(peer_state(&fanout, replacement_peer).request_flashblocks_in_flight); assert_eq!( recv_direct(&mut oldest_rx), @@ -1917,7 +1913,7 @@ mod tests { .receive_enabled .is_none() ); - assert!(peer_state(&fanout, replacement_peer).request_in_flight); + assert!(peer_state(&fanout, replacement_peer).request_flashblocks_in_flight); } #[test] @@ -1973,7 +1969,7 @@ mod tests { assert!(!fanout.handle_cancel(&ctx, peer)); assert!(!peer_state(&fanout, peer).send_enabled); assert!(peer_state(&fanout, peer).receive_enabled.is_some()); - assert!(!peer_state(&fanout, peer).request_in_flight); + assert!(!peer_state(&fanout, peer).request_flashblocks_in_flight); } #[test] From bad39aa0ad65065f142901b813396a672450705c Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 19:03:32 -0700 Subject: [PATCH 23/43] cleanup --- crates/flashblocks/node/tests/p2p.rs | 7 +- .../p2p/src/protocol/connection.rs | 47 +-- .../flashblocks/p2p/src/protocol/handler.rs | 373 ++++++++---------- 3 files changed, 193 insertions(+), 234 deletions(-) diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 955fb8c41..886103408 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -8,6 +8,7 @@ use eyre::eyre::eyre; use flashblocks_cli::FlashblocksArgs; use flashblocks_p2p::{ monitor, + protocol::connection::ReceiveStatus, protocol::handler::{FlashblocksHandle, PublishingStatus}, }; use flashblocks_primitives::{ @@ -233,7 +234,7 @@ async fn wait_for_flashblocks_topology( .connections .iter() .filter_map(|(peer_id, conn)| { - (conn.receive_enabled.is_some() && !conn.request_flashblocks_in_flight) + matches!(conn.receive_status, ReceiveStatus::Receiving { .. }) .then_some(*peer_id) }) .collect(); @@ -241,9 +242,7 @@ async fn wait_for_flashblocks_topology( .connections .iter() .filter_map(|(peer_id, conn)| { - (conn.receive_enabled.is_none() - && !conn.request_flashblocks_in_flight - && !conn.abandoned_request_in_flight) + (conn.receive_status == ReceiveStatus::NotReceiving) .then_some(*peer_id) }) .collect(); diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index e2ce087b2..8a490b1de 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -26,7 +26,7 @@ use tracing::{info, trace}; const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; /// Represents the current flashblocks receive status for a peer connection. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, PartialEq)] pub enum ReceiveStatus { /// We are not currently receiving flashblocks from this peer. #[default] @@ -38,7 +38,7 @@ pub enum ReceiveStatus { /// counting as 10s. Receiving { score: Score }, /// We have sent a request for flashblocks to this peer and are awaiting their response. - Reqesting, + Requesting, } /// Shared connection metadata for a single peer connection. @@ -329,33 +329,36 @@ impl FlashblocksConnection { let Some(conn_state) = p2p_state.connection_state(&self.peer_id) else { return; }; - if conn_state.request_flashblocks_in_flight.is_some() { - tracing::warn!( - target: "flashblocks::p2p", - peer_id = %self.peer_id, - payload_id = %msg.payload_id, - index = msg.index, - "received flashblock before request was accepted", - ); - self.protocol - .network - .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); - return; - } - if conn_state.receive_enabled.is_none() { - if conn_state.receive_enabled_timestamp + 2 < authorization.timestamp { + match &conn_state.receive_status { + ReceiveStatus::Requesting => { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, payload_id = %msg.payload_id, index = msg.index, - "received flashblock from peer outside receive window", + "received flashblock before request was accepted", ); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + return; } - return; + ReceiveStatus::NotReceiving => { + if conn_state.receive_status_timestamp + 2 < authorization.timestamp { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock from peer outside receive window", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + } + return; + } + ReceiveStatus::Receiving { .. } => {} } // Check if this peer is spamming us with the same payload index. @@ -405,9 +408,9 @@ impl FlashblocksConnection { .expect("time went backwards"); let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); - if let Some(score) = p2p_state + if let Some(ReceiveStatus::Receiving { score }) = p2p_state .connection_state_mut(&self.peer_id) - .and_then(|peer_state| peer_state.receive_enabled.as_mut()) + .map(|peer_state| &mut peer_state.receive_status) { score.record(latency); } @@ -615,7 +618,7 @@ impl FlashblocksConnection { } /// A lightweight moving average with a configurable smoothing window. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct Score { value: Option, window: i64, diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index a3610bde2..3e5975d85 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,5 +1,5 @@ use crate::protocol::{ - connection::{FlashblocksConnection, FlashblocksConnectionState, Score}, + connection::{FlashblocksConnection, FlashblocksConnectionState, ReceiveStatus, Score}, error::FlashblocksP2PError, }; use alloy_rlp::BytesMut; @@ -177,14 +177,6 @@ impl Default for FlashblocksP2PState { } impl FlashblocksP2PState { - /// Returns the current publishing status of this node. - /// - /// This indicates whether the node is actively publishing flashblocks, - /// waiting to publish, or not publishing at all. - pub fn publishing_status(&self) -> PublishingStatus { - self.publishing_status.borrow().clone() - } - /// Returns the connection state of a peer. pub(crate) fn connection_state(&self, peer_id: &PeerId) -> Option<&FlashblocksConnectionState> { self.connections.get(peer_id) @@ -216,9 +208,9 @@ impl FlashblocksP2PState { if self.observed_payloads.len() >= RECEIVE_FLASHBLOCK_GRACE_WINDOW { let evicted = self.observed_payloads.pop_front().unwrap(); for (peer_id, connection) in &mut self.connections { - if connection.receive_enabled_timestamp < evicted.timestamp + 2 + if connection.receive_status_timestamp < evicted.timestamp + 2 && !evicted.received_peers.contains(peer_id) - && let Some(score) = connection.receive_enabled.as_mut() + && let ReceiveStatus::Receiving { score } = &mut connection.receive_status { debug!( target: "flashblocks::p2p", @@ -321,7 +313,9 @@ impl FlashblocksP2PState { fn num_receive_peers(&self) -> usize { self.connections .values() - .filter(|peer_state| peer_state.receive_enabled.is_some()) + .filter(|peer_state| { + matches!(peer_state.receive_status, ReceiveStatus::Receiving { .. }) + }) .count() } @@ -333,20 +327,10 @@ impl FlashblocksP2PState { fn clear_receive_state( peer_state: &mut FlashblocksConnectionState, - receive_enabled_timestamp: u64, + receive_status_timestamp: u64, ) { - peer_state.receive_enabled = None; - peer_state.request_flashblocks_in_flight = None; - peer_state.receive_enabled_timestamp = receive_enabled_timestamp; - } - - fn abandon_receive_request( - peer_state: &mut FlashblocksConnectionState, - receive_enabled_timestamp: u64, - ) { - peer_state.receive_enabled = None; - peer_state.request_flashblocks_in_flight = None; - peer_state.receive_enabled_timestamp = receive_enabled_timestamp; + peer_state.receive_status = ReceiveStatus::NotReceiving; + peer_state.receive_status_timestamp = receive_status_timestamp; } fn available_receive_candidates(&self, ctx: &FlashblocksP2PCtx) -> Vec<(PeerId, bool)> { @@ -355,10 +339,9 @@ impl FlashblocksP2PState { self.connections .iter() .filter_map(|(peer_id, peer_state)| { - if peer_state.receive_enabled.is_none() - && !peer_state.request_flashblocks_in_flight.is_some() - && (peer_state.receive_enabled_timestamp == 0 - || peer_state.receive_enabled_timestamp + retry_cooldown <= now) + if peer_state.receive_status == ReceiveStatus::NotReceiving + && (peer_state.receive_status_timestamp == 0 + || peer_state.receive_status_timestamp + retry_cooldown <= now) { Some((*peer_id, peer_state.trusted)) } else { @@ -368,19 +351,30 @@ impl FlashblocksP2PState { .collect() } - fn begin_requesting_peer(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + fn begin_requesting_peer(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) { let Some(peer_state) = self.connection_state_mut(&peer_id) else { return; }; let timestamp = Utc::now().timestamp() as u64; - peer_state.request_flashblocks_in_flight = Some(timestamp); - peer_state.receive_enabled = Some(Score::new(ctx.fanout_args.score_samples)); - peer_state.receive_enabled_timestamp = timestamp; + peer_state.receive_status = ReceiveStatus::Requesting; + peer_state.receive_status_timestamp = timestamp; self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } + fn num_receive_or_requesting_peers(&self) -> usize { + self.connections + .values() + .filter(|peer_state| { + matches!( + peer_state.receive_status, + ReceiveStatus::Receiving { .. } | ReceiveStatus::Requesting + ) + }) + .count() + } + pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { - while self.num_receive_peers() < ctx.fanout_args.max_receive_peers { + while self.num_receive_or_requesting_peers() < ctx.fanout_args.max_receive_peers { let candidates = self.available_receive_candidates(ctx); if candidates.is_empty() { return; @@ -406,26 +400,20 @@ impl FlashblocksP2PState { self.connections .iter() .filter_map(|(peer_id, peer_state)| { - let score = peer_state.receive_enabled.as_ref()?; - Some(( - *peer_id, - score.value(), - peer_state.receive_enabled_timestamp, - )) + let ReceiveStatus::Receiving { score } = &peer_state.receive_status else { + return None; + }; + Some((*peer_id, score.value())) }) .max_by( - |(_, lhs_score, lhs_timestamp), (_, rhs_score, rhs_timestamp)| match ( - lhs_score, rhs_score, - ) { - (None, None) => rhs_timestamp.cmp(lhs_timestamp), + |(_, lhs_score), (_, rhs_score)| match (lhs_score, rhs_score) { + (None, None) => std::cmp::Ordering::Equal, (None, Some(_)) => std::cmp::Ordering::Greater, (Some(_), None) => std::cmp::Ordering::Less, - (Some(lhs_score), Some(rhs_score)) => lhs_score - .cmp(rhs_score) - .then_with(|| rhs_timestamp.cmp(lhs_timestamp)), + (Some(lhs), Some(rhs)) => lhs.cmp(rhs), }, ) - .map(|(peer_id, _, _)| peer_id) + .map(|(peer_id, _)| peer_id) } fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { @@ -450,18 +438,10 @@ impl FlashblocksP2PState { .unwrap_or(candidates[0].0); let evict_timestamp = Utc::now().timestamp() as u64; - let mut should_cancel = false; if let Some(evict_state) = self.connection_state_mut(&evict) { - if evict_state.request_flashblocks_in_flight { - Self::abandon_receive_request(evict_state, evict_timestamp); - } else { - Self::clear_receive_state(evict_state, evict_timestamp); - should_cancel = true; - } - } - if should_cancel { - self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + Self::clear_receive_state(evict_state, evict_timestamp); } + self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); self.begin_requesting_peer(ctx, candidate); } @@ -499,7 +479,7 @@ impl FlashblocksP2PState { } /// Returns `true` if the peer should receive a reputation penalty. - fn handle_accept(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { if self.check_control_rate_limit(&peer_id) { return true; } @@ -508,19 +488,16 @@ impl FlashblocksP2PState { return false; }; - if peer_state.request_flashblocks_in_flight { - peer_state.request_flashblocks_in_flight = false; - return false; - } - - if peer_state.abandoned_request_in_flight { - peer_state.abandoned_request_in_flight = false; - self.send_direct(peer_id, FlashblocksP2PMsg::CancelFlashblocks); - return false; + match peer_state.receive_status { + ReceiveStatus::Requesting => { + peer_state.receive_status = ReceiveStatus::Receiving { + score: Score::new(ctx.fanout_args.score_samples), + }; + false + } + // Unsolicited accept — we never asked this peer. + _ => true, } - - // Unsolicited accept — we never asked this peer. - true } /// Returns `true` if the peer should receive a reputation penalty. @@ -533,19 +510,15 @@ impl FlashblocksP2PState { return false; }; - if peer_state.request_flashblocks_in_flight { - Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); - self.maybe_request_receive_peers(ctx); - return false; - } - - if peer_state.abandoned_request_in_flight { - peer_state.abandoned_request_in_flight = false; - return false; + match peer_state.receive_status { + ReceiveStatus::Requesting => { + Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); + self.maybe_request_receive_peers(ctx); + false + } + // Unsolicited reject — we never asked this peer. + _ => true, } - - // Unsolicited reject — we never asked this peer. - true } /// Returns `true` if the peer should receive a reputation penalty. @@ -1537,9 +1510,14 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); - assert!(peer_state(&fanout, trusted_peer).request_flashblocks_in_flight); - assert!(peer_state(&fanout, trusted_peer).receive_enabled.is_some()); - assert!(!peer_state(&fanout, untrusted_peer).request_flashblocks_in_flight); + assert_eq!( + peer_state(&fanout, trusted_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + peer_state(&fanout, untrusted_peer).receive_status, + ReceiveStatus::NotReceiving + ); assert_eq!( recv_direct(&mut trusted_rx), FlashblocksP2PMsg::RequestFlashblocks @@ -1589,18 +1567,19 @@ mod tests { let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false); let mut score = Score::new(score_samples); score.record(42); - current_state.receive_enabled = Some(score); + current_state.receive_status = ReceiveStatus::Receiving { score }; fanout.connections.insert(current_peer, current_state); fanout.connections.insert(candidate_peer, candidate_state); fanout.maybe_start_rotation(&ctx); - assert!(peer_state(&fanout, current_peer).receive_enabled.is_none()); - assert!(peer_state(&fanout, candidate_peer).request_flashblocks_in_flight); - assert!( - peer_state(&fanout, candidate_peer) - .receive_enabled - .is_some() + assert_eq!( + peer_state(&fanout, current_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::Requesting ); assert_eq!( @@ -1614,12 +1593,10 @@ mod tests { assert!(!fanout.handle_accept(&ctx, candidate_peer)); - assert!(!peer_state(&fanout, candidate_peer).request_flashblocks_in_flight); - assert!( - peer_state(&fanout, candidate_peer) - .receive_enabled - .is_some() - ); + assert!(matches!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); } #[test] @@ -1638,8 +1615,14 @@ mod tests { fanout.maybe_request_receive_peers(&ctx); - assert!(peer_state(&fanout, first_peer).request_flashblocks_in_flight); - assert!(peer_state(&fanout, second_peer).request_flashblocks_in_flight); + assert_eq!( + peer_state(&fanout, first_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + peer_state(&fanout, second_peer).receive_status, + ReceiveStatus::Requesting + ); assert_eq!( recv_direct(&mut first_rx), @@ -1653,10 +1636,14 @@ mod tests { assert!(!fanout.handle_accept(&ctx, first_peer)); assert!(!fanout.handle_accept(&ctx, second_peer)); - assert!(!peer_state(&fanout, first_peer).request_flashblocks_in_flight); - assert!(!peer_state(&fanout, second_peer).request_flashblocks_in_flight); - assert!(peer_state(&fanout, first_peer).receive_enabled.is_some()); - assert!(peer_state(&fanout, second_peer).receive_enabled.is_some()); + assert!(matches!( + peer_state(&fanout, first_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + assert!(matches!( + peer_state(&fanout, second_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); } #[test] @@ -1678,8 +1665,10 @@ mod tests { assert!(!fanout.handle_reject(&ctx, peer)); - assert!(!peer_state(&fanout, peer).request_flashblocks_in_flight); - assert!(peer_state(&fanout, peer).receive_enabled.is_none()); + assert_eq!( + peer_state(&fanout, peer).receive_status, + ReceiveStatus::NotReceiving + ); assert!(peer_rx.try_recv().is_err()); fanout.maybe_request_receive_peers(&ctx); @@ -1688,7 +1677,7 @@ mod tests { fanout .connection_state_mut(&peer) .expect("peer exists") - .receive_enabled_timestamp = + .receive_status_timestamp = Utc::now().timestamp() as u64 - ctx.fanout_args.rotation_interval.max(1); fanout.maybe_request_receive_peers(&ctx); @@ -1699,82 +1688,40 @@ mod tests { } #[test] - fn stale_accept_after_abandoned_request_is_canceled_without_penalty() { + fn silent_receive_peer_can_be_rotated_out_without_samples() { let mut fanout_args = test_fanout_args(); fanout_args.max_receive_peers = 1; - fanout_args.score_samples = 4; let ctx = test_ctx(fanout_args); let mut fanout = FlashblocksP2PState::default(); - let abandoned_peer = PeerId::random(); + let silent_peer = PeerId::random(); let replacement_peer = PeerId::random(); - let (mut abandoned_state, mut abandoned_rx) = test_peer_state_with_channel(false); + + let (mut silent_state, mut silent_rx) = test_peer_state_with_channel(false); let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); - abandoned_state.receive_enabled = Some(Score::new(4)); - abandoned_state.request_flashblocks_in_flight = true; - abandoned_state.receive_enabled_timestamp = 1; + silent_state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; - fanout.connections.insert(abandoned_peer, abandoned_state); + fanout.connections.insert(silent_peer, silent_state); fanout .connections .insert(replacement_peer, replacement_state); fanout.maybe_start_rotation(&ctx); - assert!( - peer_state(&fanout, abandoned_peer) - .receive_enabled - .is_none() - ); - assert!(peer_state(&fanout, abandoned_peer).abandoned_request_in_flight); - assert!(abandoned_rx.try_recv().is_err()); assert_eq!( - recv_direct(&mut replacement_rx), - FlashblocksP2PMsg::RequestFlashblocks + peer_state(&fanout, silent_peer).receive_status, + ReceiveStatus::NotReceiving ); - - assert!(!fanout.handle_accept(&ctx, abandoned_peer)); - assert!(!peer_state(&fanout, abandoned_peer).abandoned_request_in_flight); assert_eq!( - recv_direct(&mut abandoned_rx), - FlashblocksP2PMsg::CancelFlashblocks + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting ); - } - - #[test] - fn silent_receive_peer_can_be_rotated_out_without_samples() { - let mut fanout_args = test_fanout_args(); - fanout_args.max_receive_peers = 2; - let ctx = test_ctx(fanout_args); - let mut fanout = FlashblocksP2PState::default(); - - let oldest_peer = PeerId::random(); - let newer_peer = PeerId::random(); - let replacement_peer = PeerId::random(); - - let (mut oldest_state, mut oldest_rx) = test_peer_state_with_channel(false); - let mut newer_state = test_peer_state(false); - let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); - - oldest_state.receive_enabled = Some(Score::new(4)); - oldest_state.receive_enabled_timestamp = 1; - newer_state.receive_enabled = Some(Score::new(4)); - newer_state.receive_enabled_timestamp = 2; - - fanout.connections.insert(oldest_peer, oldest_state); - fanout.connections.insert(newer_peer, newer_state); - fanout - .connections - .insert(replacement_peer, replacement_state); - - fanout.maybe_start_rotation(&ctx); - - assert!(peer_state(&fanout, oldest_peer).receive_enabled.is_none()); - assert!(peer_state(&fanout, replacement_peer).request_flashblocks_in_flight); assert_eq!( - recv_direct(&mut oldest_rx), + recv_direct(&mut silent_rx), FlashblocksP2PMsg::CancelFlashblocks ); assert_eq!( @@ -1798,18 +1745,16 @@ mod tests { let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); - steady_state.receive_enabled = Some(Score::new(score_samples)); - lagging_state.receive_enabled = Some(Score::new(score_samples)); - steady_state - .receive_enabled - .as_mut() - .expect("steady peer score") - .record(10); - lagging_state - .receive_enabled - .as_mut() - .expect("lagging peer score") - .record(100); + let mut steady_score = Score::new(score_samples); + steady_score.record(10); + steady_state.receive_status = ReceiveStatus::Receiving { + score: steady_score, + }; + let mut lagging_score = Score::new(score_samples); + lagging_score.record(100); + lagging_state.receive_status = ReceiveStatus::Receiving { + score: lagging_score, + }; fanout.connections.insert(steady_peer, steady_state); fanout.connections.insert(lagging_peer, lagging_state); @@ -1830,18 +1775,21 @@ mod tests { } assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); + let ReceiveStatus::Receiving { + score: steady_score, + } = &peer_state(&fanout, steady_peer).receive_status + else { + panic!("expected Receiving"); + }; + assert_eq!(steady_score.value(), Some(10)); + let ReceiveStatus::Receiving { + score: lagging_score, + } = &peer_state(&fanout, lagging_peer).receive_status + else { + panic!("expected Receiving"); + }; assert_eq!( - peer_state(&fanout, steady_peer) - .receive_enabled - .as_ref() - .and_then(Score::value), - Some(10) - ); - assert_eq!( - peer_state(&fanout, lagging_peer) - .receive_enabled - .as_ref() - .and_then(Score::value), + lagging_score.value(), Some((100 * (score_samples - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / score_samples) ); } @@ -1865,18 +1813,16 @@ mod tests { let candidate_state = test_peer_state(true); let replacement_state = test_peer_state(true); - steady_state.receive_enabled = Some(Score::new(score_samples)); - rotating_state.receive_enabled = Some(Score::new(score_samples)); - steady_state - .receive_enabled - .as_mut() - .expect("steady peer score") - .record(10); - rotating_state - .receive_enabled - .as_mut() - .expect("rotating peer score") - .record(100); + let mut steady_score = Score::new(score_samples); + steady_score.record(10); + steady_state.receive_status = ReceiveStatus::Receiving { + score: steady_score, + }; + let mut rotating_score = Score::new(score_samples); + rotating_score.record(100); + rotating_state.receive_status = ReceiveStatus::Receiving { + score: rotating_score, + }; fanout.connections.insert(steady_peer, steady_state); fanout.connections.insert(rotating_peer, rotating_state); @@ -1884,6 +1830,9 @@ mod tests { fanout.maybe_start_rotation(&ctx); + // Accept the candidate so it transitions to Receiving. + assert!(!fanout.handle_accept(&ctx, candidate_peer)); + let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { @@ -1908,12 +1857,14 @@ mod tests { .insert(replacement_peer, replacement_state); fanout.maybe_start_rotation(&ctx); - assert!( - peer_state(&fanout, candidate_peer) - .receive_enabled - .is_none() + assert_eq!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting ); - assert!(peer_state(&fanout, replacement_peer).request_flashblocks_in_flight); } #[test] @@ -1963,13 +1914,17 @@ mod tests { let peer = PeerId::random(); let mut state = test_peer_state(false); state.send_enabled = true; - state.receive_enabled = Some(Score::new(4)); + state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; fanout.connections.insert(peer, state); assert!(!fanout.handle_cancel(&ctx, peer)); assert!(!peer_state(&fanout, peer).send_enabled); - assert!(peer_state(&fanout, peer).receive_enabled.is_some()); - assert!(!peer_state(&fanout, peer).request_flashblocks_in_flight); + assert!(matches!( + peer_state(&fanout, peer).receive_status, + ReceiveStatus::Receiving { .. } + )); } #[test] @@ -1979,7 +1934,9 @@ mod tests { let peer = PeerId::random(); let mut state = test_peer_state(false); - state.receive_enabled = Some(Score::new(4)); + state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; fanout.connections.insert(peer, state); assert!(fanout.handle_cancel(&ctx, peer)); @@ -2005,7 +1962,7 @@ mod tests { let peer = PeerId::random(); let (mut state, mut rx) = test_peer_state_with_channel(false); - state.receive_enabled_timestamp = Utc::now().timestamp() as u64; + state.receive_status_timestamp = Utc::now().timestamp() as u64; fanout.connections.insert(peer, state); assert!(!fanout.handle_request(&ctx, peer)); From d3a2543df8c63247b9f05ad7bb6a257156a0b8f6 Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 19:28:27 -0700 Subject: [PATCH 24/43] keep track of send --- .../flashblocks/p2p/src/protocol/handler.rs | 104 +++++++++++++++--- 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 3e5975d85..bd46ff566 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -60,7 +60,7 @@ const BROADCAST_BUFFER_CAPACITY: usize = 100; const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; /// Grace window in number of flashblocks to receive late flashblocks from peers before scoring them for missing flashblocks. /// -/// This must be at least long enough to cover the max authorization age to prevent a spam +/// This must be at least long enough to cover AUTHORIZATION_TIMESTAMP_GRACE_SEC to prevent a spam /// attack. pub(crate) const RECEIVE_FLASHBLOCK_GRACE_WINDOW: usize = 50; @@ -70,6 +70,8 @@ const MAX_CONTROL_MSGS_PER_WINDOW: u32 = 10; /// Duration of the per-peer control-message rate-limit window. const CONTROL_MSG_WINDOW: Duration = Duration::from_secs(30); +/// Maximum time to wait for a peer to answer a `RequestFlashblocks` message. +const RECEIVE_REQUEST_TIMEOUT_SECS: u64 = 2; /// Maximum time to wait for the network manager to expose the newly connected peer's trust info. const PEER_INFO_LOOKUP_TIMEOUT: Duration = Duration::from_secs(1); @@ -127,8 +129,10 @@ pub struct ObservedPayload { payload_id: PayloadId, timestamp: u64, flashblock_index: u64, - /// Peers from which we've received this flashblock + /// Peers from which we've received this flashblock. received_peers: HashSet, + /// Peers who we have sent this flashblock to. + send_peers: HashSet, } /// Protocol state that stores the flashblocks P2P protocol events and coordination data. @@ -210,6 +214,7 @@ impl FlashblocksP2PState { for (peer_id, connection) in &mut self.connections { if connection.receive_status_timestamp < evicted.timestamp + 2 && !evicted.received_peers.contains(peer_id) + && !evicted.send_peers.contains(peer_id) && let ReceiveStatus::Receiving { score } = &mut connection.receive_status { debug!( @@ -229,6 +234,7 @@ impl FlashblocksP2PState { timestamp: authorization.timestamp, flashblock_index: flashblock.index, received_peers: HashSet::from([peer_id]), + send_peers: HashSet::new(), }); true @@ -250,15 +256,6 @@ impl FlashblocksP2PState { .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) } - /// Sends an already serialized message to a specific peer. - fn send_to_peer(&self, peer_id: PeerId, bytes: &BytesMut) { - if let Some(conn) = self.connections.get(&peer_id) - && let Some(tx) = &conn.outbound_tx - { - tx.send(bytes.clone()).ok(); - } - } - /// Sends an already serialized message to all connected peers. pub(crate) fn send_to_all_peers(&self, bytes: &BytesMut) { for conn in self.connections.values() { @@ -271,7 +268,7 @@ impl FlashblocksP2PState { /// Sends a serialized flashblock to peers in the current send set that have not /// already delivered that flashblock to us. fn send_flashblock_to_send_set( - &self, + &mut self, payload_id: PayloadId, flashblock_index: u64, bytes: &BytesMut, @@ -282,6 +279,13 @@ impl FlashblocksP2PState { { continue; } + self.observed_payloads + .iter_mut() + .find(|observed_payload| { + observed_payload.payload_id == payload_id + && observed_payload.flashblock_index == flashblock_index + }) + .map(|observed_payload| observed_payload.send_peers.insert(*peer_id)); if let Some(tx) = &conn.outbound_tx && tx.send(bytes.clone()).is_ok() @@ -293,7 +297,12 @@ impl FlashblocksP2PState { /// Sends a control message directly to a specific peer. fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { - self.send_to_peer(peer_id, &msg.encode()); + let bytes: &BytesMut = &msg.encode(); + if let Some(conn) = self.connections.get(&peer_id) + && let Some(tx) = &conn.outbound_tx + { + tx.send(bytes.clone()).ok(); + } } /// Returns `true` if the peer has exceeded the control-message rate limit. @@ -396,6 +405,24 @@ impl FlashblocksP2PState { } } + fn expire_stale_receive_requests(&mut self, ctx: &FlashblocksP2PCtx) { + let now = Utc::now().timestamp() as u64; + let mut cleared_any = false; + + for peer_state in self.connections.values_mut() { + if matches!(peer_state.receive_status, ReceiveStatus::Requesting) + && peer_state.receive_status_timestamp + RECEIVE_REQUEST_TIMEOUT_SECS <= now + { + Self::clear_receive_state(peer_state, now); + cleared_any = true; + } + } + + if cleared_any { + self.maybe_request_receive_peers(ctx); + } + } + fn worst_receive_peer(&self) -> Option { self.connections .iter() @@ -606,6 +633,7 @@ impl FlashblocksHandle { loop { rotation_interval.tick().await; let mut state = moved_handle.state.lock(); + state.expire_stale_receive_requests(&moved_handle.ctx); state.maybe_request_receive_peers(&moved_handle.ctx); state.maybe_start_rotation(&moved_handle.ctx); } @@ -1687,6 +1715,56 @@ mod tests { ); } + #[test] + fn timed_out_request_is_cleared_and_replaced() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let stale_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + let (stale_state, mut stale_rx) = test_peer_state_with_channel(true); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(stale_peer, stale_state); + fanout + .connections + .insert(replacement_peer, replacement_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, stale_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + recv_direct(&mut stale_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + fanout + .connection_state_mut(&stale_peer) + .expect("peer exists") + .receive_status_timestamp = + Utc::now().timestamp() as u64 - RECEIVE_REQUEST_TIMEOUT_SECS; + + fanout.expire_stale_receive_requests(&ctx); + + assert_eq!( + peer_state(&fanout, stale_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + assert!(stale_rx.try_recv().is_err()); + } + #[test] fn silent_receive_peer_can_be_rotated_out_without_samples() { let mut fanout_args = test_fanout_args(); From 50ede1227dc12a15b2847a6085f40209b5cdc01a Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Wed, 11 Mar 2026 20:17:31 -0700 Subject: [PATCH 25/43] more cleanup --- .../p2p/src/protocol/connection.rs | 28 +++- .../flashblocks/p2p/src/protocol/handler.rs | 127 +++++++++--------- 2 files changed, 89 insertions(+), 66 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 8a490b1de..9f844ed0b 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -215,28 +215,48 @@ impl Stream for FlashblocksConnection { } } FlashblocksP2PMsg::RequestFlashblocks => { - if this.protocol.handle.handle_request_message(this.peer_id) { + if this + .protocol + .handle + .handle_request_message(this.peer_id) + .is_err() + { this.protocol .network .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); } } FlashblocksP2PMsg::AcceptFlashblocks => { - if this.protocol.handle.handle_accept_message(this.peer_id) { + if this + .protocol + .handle + .handle_accept_message(this.peer_id) + .is_err() + { this.protocol .network .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); } } FlashblocksP2PMsg::RejectFlashblocks => { - if this.protocol.handle.handle_reject_message(this.peer_id) { + if this + .protocol + .handle + .handle_reject_message(this.peer_id) + .is_err() + { this.protocol .network .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); } } FlashblocksP2PMsg::CancelFlashblocks => { - if this.protocol.handle.handle_cancel_message(this.peer_id) { + if this + .protocol + .handle + .handle_cancel_message(this.peer_id) + .is_err() + { this.protocol .network .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index bd46ff566..42ba06eb3 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -212,7 +212,7 @@ impl FlashblocksP2PState { if self.observed_payloads.len() >= RECEIVE_FLASHBLOCK_GRACE_WINDOW { let evicted = self.observed_payloads.pop_front().unwrap(); for (peer_id, connection) in &mut self.connections { - if connection.receive_status_timestamp < evicted.timestamp + 2 + if connection.receive_status_timestamp + 2 <= evicted.timestamp && !evicted.received_peers.contains(peer_id) && !evicted.send_peers.contains(peer_id) && let ReceiveStatus::Receiving { score } = &mut connection.receive_status @@ -360,7 +360,7 @@ impl FlashblocksP2PState { .collect() } - fn begin_requesting_peer(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) { + fn begin_requesting_peer(&mut self, peer_id: PeerId) { let Some(peer_state) = self.connection_state_mut(&peer_id) else { return; }; @@ -401,7 +401,7 @@ impl FlashblocksP2PState { trusted_candidates }; let rand = rand::rng().random_range(0..candidate_pool.len()); - self.begin_requesting_peer(ctx, candidate_pool[rand]); + self.begin_requesting_peer(candidate_pool[rand]); } } @@ -470,22 +470,22 @@ impl FlashblocksP2PState { } self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); - self.begin_requesting_peer(ctx, candidate); + self.begin_requesting_peer(candidate); } - /// Returns `true` if the peer should receive a reputation penalty. - fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { if self.check_control_rate_limit(&peer_id) { - return true; + return Err(()); } let Some(peer_state) = self.connection_state(&peer_id) else { - return false; + return Ok(()); }; if peer_state.send_enabled { // Already sending to this peer — repeated request is spam. - return true; + return Err(()); } let peer_is_trusted = peer_state.trusted; let non_trusted_send_count = self @@ -496,23 +496,23 @@ impl FlashblocksP2PState { if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_args.max_send_peers { self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); - return false; + return Ok(()); } let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); peer_state.send_enabled = true; self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); - false + Ok(()) } - /// Returns `true` if the peer should receive a reputation penalty. - fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { if self.check_control_rate_limit(&peer_id) { - return true; + return Err(()); } let Some(peer_state) = self.connection_state_mut(&peer_id) else { - return false; + return Ok(()); }; match peer_state.receive_status { @@ -520,51 +520,51 @@ impl FlashblocksP2PState { peer_state.receive_status = ReceiveStatus::Receiving { score: Score::new(ctx.fanout_args.score_samples), }; - false + Ok(()) } // Unsolicited accept — we never asked this peer. - _ => true, + _ => Err(()), } } - /// Returns `true` if the peer should receive a reputation penalty. - fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { if self.check_control_rate_limit(&peer_id) { - return true; + return Err(()); } let Some(peer_state) = self.connection_state_mut(&peer_id) else { - return false; + return Ok(()); }; match peer_state.receive_status { ReceiveStatus::Requesting => { Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); self.maybe_request_receive_peers(ctx); - false + Ok(()) } // Unsolicited reject — we never asked this peer. - _ => true, + _ => Err(()), } } - /// Returns `true` if the peer should receive a reputation penalty. - fn handle_cancel(&mut self, _ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_cancel(&mut self, peer_id: PeerId) -> Result<(), ()> { if self.check_control_rate_limit(&peer_id) { - return true; + return Err(()); } let Some(peer_state) = self.connection_state_mut(&peer_id) else { - return false; + return Ok(()); }; if !peer_state.send_enabled { // Cancel is only valid from a receiver to its sender. - return true; + return Err(()); } peer_state.send_enabled = false; - false + Ok(()) } } @@ -710,28 +710,28 @@ impl FlashblocksHandle { state.maybe_request_receive_peers(&self.ctx); } - /// Returns `true` if the peer should receive a reputation penalty. - pub(crate) fn handle_request_message(&self, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_request_message(&self, peer_id: PeerId) -> Result<(), ()> { let mut state = self.state.lock(); state.handle_request(&self.ctx, peer_id) } - /// Returns `true` if the peer should receive a reputation penalty. - pub(crate) fn handle_accept_message(&self, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_accept_message(&self, peer_id: PeerId) -> Result<(), ()> { let mut state = self.state.lock(); state.handle_accept(&self.ctx, peer_id) } - /// Returns `true` if the peer should receive a reputation penalty. - pub(crate) fn handle_reject_message(&self, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_reject_message(&self, peer_id: PeerId) -> Result<(), ()> { let mut state = self.state.lock(); state.handle_reject(&self.ctx, peer_id) } - /// Returns `true` if the peer should receive a reputation penalty. - pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) -> bool { + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) -> Result<(), ()> { let mut state = self.state.lock(); - state.handle_cancel(&self.ctx, peer_id) + state.handle_cancel(peer_id) } } @@ -914,7 +914,7 @@ impl FlashblocksHandle { } } PublishingStatus::NotPublishing { active_publishers } => { - // Send an authorized `StartPublish` message to the network + // Send an authorized `StartPublish` message to direct peers. let authorized_msg = AuthorizedMsg::StartPublish(StartPublish); let authorized_payload = Authorized::new(builder_sk, new_authorization, authorized_msg); @@ -1569,7 +1569,7 @@ mod tests { .connections .insert(trusted_requester, requester_state); - assert!(!fanout.handle_request(&ctx, trusted_requester)); + assert!(fanout.handle_request(&ctx, trusted_requester).is_ok()); assert!(peer_state(&fanout, victim).send_enabled); assert!(peer_state(&fanout, trusted_requester).send_enabled); @@ -1619,7 +1619,7 @@ mod tests { FlashblocksP2PMsg::RequestFlashblocks ); - assert!(!fanout.handle_accept(&ctx, candidate_peer)); + assert!(fanout.handle_accept(&ctx, candidate_peer).is_ok()); assert!(matches!( peer_state(&fanout, candidate_peer).receive_status, @@ -1661,8 +1661,8 @@ mod tests { FlashblocksP2PMsg::RequestFlashblocks ); - assert!(!fanout.handle_accept(&ctx, first_peer)); - assert!(!fanout.handle_accept(&ctx, second_peer)); + assert!(fanout.handle_accept(&ctx, first_peer).is_ok()); + assert!(fanout.handle_accept(&ctx, second_peer).is_ok()); assert!(matches!( peer_state(&fanout, first_peer).receive_status, @@ -1691,7 +1691,7 @@ mod tests { FlashblocksP2PMsg::RequestFlashblocks ); - assert!(!fanout.handle_reject(&ctx, peer)); + assert!(fanout.handle_reject(&ctx, peer).is_ok()); assert_eq!( peer_state(&fanout, peer).receive_status, @@ -1837,10 +1837,13 @@ mod tests { fanout.connections.insert(steady_peer, steady_state); fanout.connections.insert(lagging_peer, lagging_state); + // Use timestamps starting well after receive_status_timestamp (0) so the grace + // check `receive_status_timestamp + 2 <= evicted.timestamp` is satisfied. + let ts_offset = 10_u64; for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { let authorization = Authorization::new( PayloadId::default(), - index as u64, + ts_offset + index as u64, &authorizer, builder.verifying_key(), ); @@ -1909,14 +1912,17 @@ mod tests { fanout.maybe_start_rotation(&ctx); // Accept the candidate so it transitions to Receiving. - assert!(!fanout.handle_accept(&ctx, candidate_peer)); + assert!(fanout.handle_accept(&ctx, candidate_peer).is_ok()); let authorizer = SigningKey::from_bytes(&[7; 32]); let builder = SigningKey::from_bytes(&[9; 32]); + // Use timestamps well after the candidate's receive_status_timestamp so the + // grace check `receive_status_timestamp + 2 <= evicted.timestamp` is satisfied. + let ts_base = Utc::now().timestamp() as u64 + 10; for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { let authorization = Authorization::new( PayloadId::default(), - index as u64, + ts_base + index as u64, &authorizer, builder.verifying_key(), ); @@ -1955,7 +1961,7 @@ mod tests { fanout.connections.insert(peer, state); // Accept without a prior request should be penalized. - assert!(fanout.handle_accept(&ctx, peer)); + assert!(fanout.handle_accept(&ctx, peer).is_err()); } #[test] @@ -1968,12 +1974,11 @@ mod tests { fanout.connections.insert(peer, state); // Reject without a prior request should be penalized. - assert!(fanout.handle_reject(&ctx, peer)); + assert!(fanout.handle_reject(&ctx, peer).is_err()); } #[test] fn cancel_without_relationship_is_penalized() { - let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -1981,12 +1986,11 @@ mod tests { fanout.connections.insert(peer, state); // Cancel with no send/receive relationship should be penalized. - assert!(fanout.handle_cancel(&ctx, peer)); + assert!(fanout.handle_cancel(peer).is_err()); } #[test] fn cancel_only_clears_send_direction() { - let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -1997,7 +2001,7 @@ mod tests { }; fanout.connections.insert(peer, state); - assert!(!fanout.handle_cancel(&ctx, peer)); + assert!(fanout.handle_cancel(peer).is_ok()); assert!(!peer_state(&fanout, peer).send_enabled); assert!(matches!( peer_state(&fanout, peer).receive_status, @@ -2007,7 +2011,6 @@ mod tests { #[test] fn cancel_from_sender_is_penalized() { - let ctx = test_ctx(test_fanout_args()); let mut fanout = FlashblocksP2PState::default(); let peer = PeerId::random(); @@ -2017,7 +2020,7 @@ mod tests { }; fanout.connections.insert(peer, state); - assert!(fanout.handle_cancel(&ctx, peer)); + assert!(fanout.handle_cancel(peer).is_err()); } #[test] @@ -2030,7 +2033,7 @@ mod tests { state.send_enabled = true; fanout.connections.insert(peer, state); - assert!(fanout.handle_request(&ctx, peer)); + assert!(fanout.handle_request(&ctx, peer).is_err()); } #[test] @@ -2043,7 +2046,7 @@ mod tests { state.receive_status_timestamp = Utc::now().timestamp() as u64; fanout.connections.insert(peer, state); - assert!(!fanout.handle_request(&ctx, peer)); + assert!(fanout.handle_request(&ctx, peer).is_ok()); assert!(peer_state(&fanout, peer).send_enabled); assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::AcceptFlashblocks); } @@ -2060,10 +2063,10 @@ mod tests { fanout.connections.insert(peer, state); for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { - assert!(!fanout.handle_request(&ctx, peer)); + assert!(fanout.handle_request(&ctx, peer).is_ok()); assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::RejectFlashblocks); } - assert!(fanout.handle_request(&ctx, peer)); + assert!(fanout.handle_request(&ctx, peer).is_err()); } #[test] @@ -2078,11 +2081,11 @@ mod tests { // Spam requests to exceed the rate limit. for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { - // These return true because send_enabled is already set (duplicate request), + // These return Err because send_enabled is already set (duplicate request), // but the rate limit hasn't been hit yet. - assert!(fanout.handle_request(&ctx, peer)); + assert!(fanout.handle_request(&ctx, peer).is_err()); } // The next one should hit the rate limit. - assert!(fanout.handle_request(&ctx, peer)); + assert!(fanout.handle_request(&ctx, peer).is_err()); } } From a42d45f997d4be2479066bc95f3fcb1846a7b32d Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Thu, 12 Mar 2026 11:12:57 -0700 Subject: [PATCH 26/43] fix: change trusted peer behaviour --- .../flashblocks/p2p/src/protocol/handler.rs | 34 ++++--------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 42ba06eb3..ec3335cce 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -388,20 +388,8 @@ impl FlashblocksP2PState { if candidates.is_empty() { return; } - let trusted_candidates: Vec<_> = candidates - .iter() - .filter_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) - .collect(); - let candidate_pool = if trusted_candidates.is_empty() { - candidates - .iter() - .map(|(peer_id, _)| *peer_id) - .collect::>() - } else { - trusted_candidates - }; - let rand = rand::rng().random_range(0..candidate_pool.len()); - self.begin_requesting_peer(candidate_pool[rand]); + let rand = rand::rng().random_range(0..candidates.len()); + self.begin_requesting_peer(candidates[rand].0); } } @@ -452,17 +440,13 @@ impl FlashblocksP2PState { return; }; - let mut candidates = self.available_receive_candidates(ctx); + let candidates = self.available_receive_candidates(ctx); if candidates.is_empty() { return; } - let mut rng = rand::rng(); - candidates.shuffle(&mut rng); - let candidate = candidates - .iter() - .find_map(|(peer_id, trusted)| (*trusted).then_some(*peer_id)) - .unwrap_or(candidates[0].0); + let rand = rand::rng().random_range(0..candidates.len()); + let candidate = candidates[rand].0; let evict_timestamp = Utc::now().timestamp() as u64; if let Some(evict_state) = self.connection_state_mut(&evict) { @@ -488,13 +472,9 @@ impl FlashblocksP2PState { return Err(()); } let peer_is_trusted = peer_state.trusted; - let non_trusted_send_count = self - .connections - .values() - .filter(|s| s.send_enabled && !s.trusted) - .count(); + let send_count = self.connections.values().filter(|s| s.send_enabled).count(); - if !peer_is_trusted && non_trusted_send_count >= ctx.fanout_args.max_send_peers { + if !peer_is_trusted && send_count >= ctx.fanout_args.max_send_peers { self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); return Ok(()); } From e5c9f19392fb438d8725a19c4d354be665bba505 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Thu, 12 Mar 2026 22:08:26 -0700 Subject: [PATCH 27/43] fix: canon state tracking against pending state --- crates/flashblocks/builder/src/coordinator.rs | 90 ++--- crates/flashblocks/p2p/src/protocol/event.rs | 241 ++++++++++++ .../flashblocks/p2p/src/protocol/handler.rs | 87 +---- crates/flashblocks/p2p/src/protocol/mod.rs | 1 + crates/flashblocks/p2p/tests/protocol.rs | 353 +++++++++++++----- .../flashblocks/rpc/src/eth/pending_block.rs | 61 +-- .../world/node/tests/e2e-testsuite/actions.rs | 33 +- .../node/tests/e2e-testsuite/testsuite.rs | 60 ++- sped.md | 13 + 9 files changed, 637 insertions(+), 302 deletions(-) create mode 100644 crates/flashblocks/p2p/src/protocol/event.rs create mode 100644 sped.md diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 7f9342fcb..f4963e0c9 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -1,7 +1,10 @@ use alloy_eips::{Decodable2718, eip2718::WithEncoded, eip4895::Withdrawals}; use alloy_op_evm::OpBlockExecutionCtx; use eyre::eyre::eyre; -use flashblocks_p2p::protocol::handler::FlashblocksHandle; +use flashblocks_p2p::protocol::{ + event::{FlashblocksEvent, WorldChainEventsStream}, + handler::FlashblocksHandle, +}; use flashblocks_primitives::{p2p::AuthorizedPayload, primitives::FlashblocksPayloadV1}; use futures::StreamExt as _; use op_alloy_consensus::{OpTxEnvelope, encode_holocene_extra_data}; @@ -21,7 +24,9 @@ use reth_optimism_node::{OpBuiltPayload, OpEngineTypes, OpEvmConfig, OpPayloadBu use reth_optimism_primitives::OpPrimitives; use reth_payload_util::BestPayloadTransactions; -use reth_provider::{ChainSpecProvider, HeaderProvider, StateProviderFactory}; +use reth_provider::{ + CanonStateSubscriptions, ChainSpecProvider, HeaderProvider, StateProviderFactory, +}; use reth_transaction_pool::{EthPooledTransaction, noop::NoopTransactionPool}; use std::{ sync::Arc, @@ -92,10 +97,16 @@ impl FlashblocksExecutionCoordinator { pub fn launch(&self, ctx: &BuilderContext, evm_config: OpEvmConfig) where Node: FullNodeTypes, - Node::Provider: StateProviderFactory + HeaderProvider
, + Node::Provider: StateProviderFactory + + HeaderProvider
+ + CanonStateSubscriptions, Node::Types: NodeTypes, { - let mut stream = self.p2p_handle.live_flashblock_stream(); + let mut stream = WorldChainEventsStream::new( + self.p2p_handle.ctx.flashblock_tx.subscribe(), + ctx.provider(), + ); + let this = self.clone(); let provider = ctx.provider().clone(); let chain_spec = ctx.chain_spec().clone(); @@ -104,17 +115,34 @@ impl FlashblocksExecutionCoordinator { ctx.task_executor() .spawn_critical("flashblocks executor", async move { - while let Some(flashblock) = stream.next().await { - let provider = provider.clone(); - if let Err(e) = process_flashblock( - provider, - &evm_config, - &this, - chain_spec.clone(), - flashblock, - pending_block.clone(), - ) { - error!("error processing flashblock: {e:#?}") + while let Some(event) = stream.next().await { + match event { + FlashblocksEvent::Pending(flashblock) => { + if let Err(e) = process_flashblock( + provider.clone(), + &evm_config, + &this, + chain_spec.clone(), + flashblock, + pending_block.clone(), + ) { + error!("error processing flashblock: {e:#?}") + } + } + FlashblocksEvent::Canon(tip) => { + // Clear pending block if it was built on the now-canonical tip, + // since the canonical chain has superseded it. + pending_block.send_if_modified(|block| { + let matches = block + .as_ref() + .is_some_and(|b| b.recovered_block().parent_num_hash() == tip); + + if matches { + *block = None; + } + matches + }); + } } } }); @@ -233,34 +261,10 @@ where flashblocks.base() }; - let f = || { - provider - .sealed_header_by_hash(base.parent_hash)? - .ok_or(eyre!("failed to fetch sealed header {}", base.parent_hash)) - }; - - let sealed_header = f - .retry( - backon::ExponentialBuilder::default() - .with_min_delay(FETCH_PARENT_HEADER_MIN_DELAY) - .with_max_delay(FETCH_PARENT_HEADER_MAX_DELAY) - .with_max_times(10), - ) - .notify(|e, duration| { - warn!( - "waiting for parent header {}: {e:#?}. waited {:#?} so far", - base.parent_hash, duration - ) - }) - .call() - .inspect_err(|e| { - error!( - flashblock_index = index, - parent_hash = %base.parent_hash, - error = %e, - "failed to fetch parent header after multiple attempts" - ) - })?; + let sealed_header = provider + .sealed_header_by_hash(base.parent_hash) + .inspect_err(|e| error!("failed to fetch sealed header {}: {e:#?}", base.parent_hash))? + .ok_or_else(|| eyre!("sealed header not found for hash {}", base.parent_hash))?; let execution_context = OpBlockExecutionCtx { parent_hash: base.parent_hash, diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs new file mode 100644 index 000000000..0ec01b7e7 --- /dev/null +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -0,0 +1,241 @@ +//! Canon-aware flashblock event stream. +//! +//! Merges a raw flashblock stream with canonical chain notifications, yielding +//! [`FlashblocksEvent::Pending`] only when the flashblock's epoch parent matches +//! the current canonical tip, and [`FlashblocksEvent::Canon`] whenever the tip +//! changes. + +use flashblocks_primitives::primitives::FlashblocksPayloadV1; +use futures::{ + future::{self, Either}, + stream, Stream, StreamExt, +}; +use reth::{ + api::NodePrimitives, + payload::PayloadId, + providers::{CanonStateNotificationStream, CanonStateSubscriptions}, + rpc::types::BlockNumHash, +}; +use std::{ + pin::Pin, + task::{Context, Poll}, +}; +use tokio::sync::broadcast; +use tokio_stream::wrappers::BroadcastStream; + +/// Events yielded by [`ChainEventsStream`]. +#[derive(Clone, Debug)] +pub enum ChainEvent { + /// A pending flashblock confirmed to build on the canonical tip. + Pending(T), + /// The canonical tip changed — consumers should clear stale pending state. + Canon(BlockNumHash), +} + +/// Convenience alias: a [`ChainEvent`] carrying a flashblocks payload. +pub type FlashblocksEvent = ChainEvent; + +/// A stream of [`ChainEvent`]s that merges flashblocks with canonical chain +/// notifications. +/// +/// Follows the same pattern as reth's `CanonStateNotificationStream` — wraps an +/// inner stream and handles lag transparently. +/// +/// A [`ChainEvent::Pending`] is emitted only when the flashblock's epoch parent +/// matches the canonical tip. Stale flashblocks are silently discarded via +/// [`PendingCursor::try_advance`]. A [`ChainEvent::Canon`] is emitted on every +/// canonical tip change so consumers can clear pending state. +pub struct ChainEventsStream { + st: Pin> + Send>>, +} + +/// Convenience alias: a [`ChainEventsStream`] carrying flashblocks payloads. +pub type WorldChainEventsStream = ChainEventsStream; + +impl WorldChainEventsStream { + /// Creates a new [`WorldChainEventsStream`] by merging a flashblock + /// receiver with canonical chain notifications from `provider`. + pub fn new( + flashblocks_rx: broadcast::Receiver, + provider: &P, + ) -> Self { + let flashblocks = + BroadcastStream::new(flashblocks_rx).filter_map(|x| future::ready(x.ok())); + + let canon = provider.canonical_state_stream(); + + Self { + st: stream_select_contiguous_with_canon(flashblocks, canon), + } + } +} + +/// Merges a flashblock stream with a canonical state notification stream, +/// yielding [`FlashblocksEvent`]s gated by the canonical tip. +/// +/// Flashblocks are only emitted as [`FlashblocksEvent::Pending`] when their +/// epoch parent matches the current canonical tip. When a new canonical tip +/// arrives and a stored flashblock becomes ready, both [`FlashblocksEvent::Canon`] +/// and [`FlashblocksEvent::Pending`] are emitted in that order. +/// +/// The inner state machine uses `scan` over a merged `select` of both input +/// streams, accumulating zero or more events per input item into a `Vec` that +/// is flattened back into the output stream via `flat_map(stream::iter)`. +fn stream_select_contiguous_with_canon( + fb: impl Stream + Send + 'static, + canon: CanonStateNotificationStream, +) -> Pin> + Send>> { + // Tag each source with `Either` so the merged stream can distinguish them. + futures::stream::select(fb.map(Either::Left), canon.map(Either::Right)) + .scan( + // Scan state: + // tip — the most recent canonical tip (None until first canon event) + // cursor — tracks epoch position; enforces flashblock adjacency + // latest — most recently accepted flashblock, buffered until the + // canonical tip confirms it can be yielded + ( + None::, + PendingCursor::default(), + None::, + ), + |(tip, cursor, latest), event| { + let events: Vec> = match event { + // ── Flashblock arrived ── + // + // `try_advance` atomically validates adjacency (sequential + // index within an epoch, or a new epoch via base flashblock) + // and rejects flashblocks stale relative to the known tip. + // Only if it succeeds do we buffer the flashblock. + Either::Left(fb) if cursor.try_advance(&fb, tip.as_ref()) => { + *latest = Some(fb); + // If the canonical tip is already known and the cursor's + // epoch parent matches it, emit immediately. Otherwise + // buffer — a later Canon event will flush it. + if tip.as_ref().is_some_and(|t| cursor.is_ready(t)) { + latest + .take() + .into_iter() + .map(FlashblocksEvent::Pending) + .collect() + } else { + vec![] + } + } + // Flashblock failed adjacency or staleness check — drop it. + Either::Left(_) => vec![], + + // ── Canonical tip changed ── + // + // Always notify consumers so they can clear stale pending + // state. If the new tip matches the cursor's epoch parent, + // also flush the buffered flashblock (Canon first, then + // Pending). + Either::Right(n) => { + let num_hash = n.tip().num_hash(); + *tip = Some(num_hash); + let mut events = vec![FlashblocksEvent::Canon(num_hash)]; + if cursor.is_ready(&num_hash) { + if let Some(fb) = latest.take() { + events.push(FlashblocksEvent::Pending(fb)); + } + } + events + } + }; + // Wrapping in `Some` keeps the stream alive — the inner `Vec` + // may be empty (no events to yield this tick), which `flat_map` + // handles by producing nothing. + future::ready(Some(events)) + }, + ) + // Each scan step produces a `Vec`; flatten into + // individual items so the outer stream yields one event at a time. + .flat_map(stream::iter) + .boxed() +} + +impl std::fmt::Debug for WorldChainEventsStream { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorldChainEventsStream") + .field("st", &"") + .finish_non_exhaustive() + } +} + +impl Stream for WorldChainEventsStream { + type Item = FlashblocksEvent; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.st.as_mut().poll_next(cx) + } +} + +// --------------------------------------------------------------------------- +// Cursor +// --------------------------------------------------------------------------- + +/// Tracks position of Flashblocks w.r.t. a Blocks Epoch. +#[derive(Clone, Copy, Debug, Default)] +struct PendingCursor { + /// The parent block that this epoch builds on. + parent_num_hash: BlockNumHash, + /// The payload identifier for the current epoch. + payload_id: PayloadId, + /// The latest flashblock index within the current epoch. + index: usize, +} + +impl PendingCursor { + /// Returns `true` when the epoch parent matches the canonical tip exactly. + fn is_ready(&self, canon_tip: &BlockNumHash) -> bool { + self.parent_num_hash == *canon_tip + } + + /// Returns `true` when this epoch is at least as recent as `canon_tip` — + /// either matching it or building on a block canon hasn't reached yet. + fn is_new(&self, canon_tip: &BlockNumHash) -> bool { + self.is_ready(canon_tip) || self.parent_num_hash.number > canon_tip.number + } + + /// Advances the cursor to track the given flashblock. + /// + /// Returns `true` if the flashblock is adjacent — either a base flashblock + /// starting a new epoch, or the next sequential index within the current + /// epoch. Returns `false` (and leaves the cursor unchanged) on gaps or + /// payload-id mismatches. + fn advance(&mut self, flashblock: &FlashblocksPayloadV1) -> bool { + if let Some(base) = &flashblock.base { + // New epoch — always adjacent. + self.parent_num_hash = BlockNumHash { + number: base.block_number - 1, + hash: base.parent_hash, + }; + self.payload_id = flashblock.payload_id; + self.index = flashblock.index as usize; + true + } else if flashblock.payload_id == self.payload_id + && flashblock.index as usize == self.index + 1 + { + // Same epoch, next sequential index. + self.index += 1; + true + } else { + false + } + } + + /// Speculatively advances the cursor. Returns `false` (and leaves the + /// cursor unchanged) when the flashblock is not adjacent or is stale + /// relative to `tip`. + fn try_advance(&mut self, fb: &FlashblocksPayloadV1, tip: Option<&BlockNumHash>) -> bool { + let mut next = *self; + if !next.advance(fb) { + return false; + } + if tip.is_some_and(|t| !next.is_new(t)) { + return false; + } + *self = next; + true + } +} diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 721278e9c..76daac40b 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -9,7 +9,6 @@ use flashblocks_primitives::{ }, primitives::FlashblocksPayloadV1, }; -use futures::{Stream, StreamExt, stream}; use metrics::histogram; use parking_lot::Mutex; use reth::payload::PayloadId; @@ -19,7 +18,7 @@ use reth_network::Peers; use std::{net::SocketAddr, sync::Arc}; use tokio::sync::{broadcast, watch}; use tokio_stream::wrappers::BroadcastStream; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use reth_ethereum::network::{ api::Direction, @@ -240,27 +239,6 @@ impl FlashblocksP2PProtocol { } impl FlashblocksHandle { - /// Retrieves the next flashblock from the protocol state based on the provided cursor. - /// - /// Will return the flashblock at the cursor if it exists. - /// Will return the first flashblock if the cursor points to a different payload or is None. - /// Returns None if the flashblock at the cursor or the first flashblock does not exist. - fn next_flashblock_from_state( - state: &FlashblocksP2PState, - cursor: Option<&(PayloadId, usize)>, - ) -> Option { - match cursor { - Some((payload_id, next_index)) if *payload_id == state.payload_id => state - .flashblocks - .get(*next_index) - .and_then(|flashblock| flashblock.clone()), - _ => state - .flashblocks - .first() - .and_then(|flashblock| flashblock.clone()), - } - } - /// Publishes a newly created flashblock from the payload builder to the P2P network. /// /// This method validates that the builder has authorization to publish and that @@ -473,69 +451,6 @@ impl FlashblocksHandle { Ok(()) } - - /// Returns a stream of ordered flashblocks starting from the beginning of the current payload. - /// - /// # Behavior - /// The stream will continue to yield flashblocks for consecutive payloads. - pub fn flashblock_stream(&self) -> impl Stream + Send + 'static { - // Seed the stream with already-buffered contiguous flashblocks, then rely on the broadcast - // channel for future ones so ordering stays strict even if inserts arrive out of order. - let flashblocks = self - .state - .lock() - .flashblocks - .clone() - .into_iter() - .map_while(|x| x); - - let receiver = self.ctx.flashblock_tx.subscribe(); - - let current = stream::iter(flashblocks); - let future = tokio_stream::StreamExt::map_while(BroadcastStream::new(receiver), |x| x.ok()); - current.chain(future) - } - - /// Returns a stream of ordered flashblocks starting from the beginning of the current payload. - /// - /// # Behavior - /// - /// The stream will continue to yield flashblocks for consecutive payloads. - /// - /// Items not consumed from the stream by the time the next payload starts will be skipped. - pub fn live_flashblock_stream( - &self, - ) -> impl Stream + Send + Unpin + 'static { - let state = self.state.clone(); - let receiver = self.ctx.flashblock_tx.subscribe(); - - Box::pin(stream::unfold( - (state, receiver, None::<(PayloadId, usize)>), - |(state, mut receiver, mut cursor)| async move { - loop { - if let Some(flashblock) = { - let state = state.lock(); - Self::next_flashblock_from_state(&state, cursor.as_ref()) - } { - cursor = Some((flashblock.payload_id, flashblock.index as usize + 1)); - return Some((flashblock, (state, receiver, cursor))); - } - - match receiver.recv().await { - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - warn!( - target: "flashblocks::p2p", - skipped, - "flashblock stream lagged; resyncing from protocol state" - ); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => return None, - } - } - }, - )) - } } impl FlashblocksP2PCtx { diff --git a/crates/flashblocks/p2p/src/protocol/mod.rs b/crates/flashblocks/p2p/src/protocol/mod.rs index a83f17231..fab9f9b55 100644 --- a/crates/flashblocks/p2p/src/protocol/mod.rs +++ b/crates/flashblocks/p2p/src/protocol/mod.rs @@ -1,3 +1,4 @@ pub mod connection; pub mod error; +pub mod event; pub mod handler; diff --git a/crates/flashblocks/p2p/tests/protocol.rs b/crates/flashblocks/p2p/tests/protocol.rs index 6ea3e9f69..e7d1f8a0c 100644 --- a/crates/flashblocks/p2p/tests/protocol.rs +++ b/crates/flashblocks/p2p/tests/protocol.rs @@ -1,29 +1,44 @@ +use alloy_primitives::B256; use ed25519_dalek::SigningKey; -use flashblocks_p2p::protocol::handler::{FlashblocksHandle, PublishingStatus}; +use flashblocks_p2p::protocol::{ + event::{FlashblocksEvent, WorldChainEventsStream}, + handler::{FlashblocksHandle, PublishingStatus}, +}; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, p2p::{Authorization, AuthorizedPayload}, primitives::{ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, FlashblocksPayloadV1}, }; use futures::StreamExt as _; -use reth::payload::PayloadId; -use std::time::Duration; -use tokio::task; +use reth::{ + payload::PayloadId, + providers::{CanonStateNotification, CanonStateSubscriptions, Chain, ExecutionOutcome}, +}; +use reth_ethereum::primitives::RecoveredBlock; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use tokio::{sync::broadcast, task}; const DUMMY_TIMESTAMP: u64 = 42; +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + /// Helper: deterministic ed25519 key made of the given byte. fn signing_key(byte: u8) -> SigningKey { SigningKey::from_bytes(&[byte; 32]) } -/// Helper: a minimal Flashblock (index 0) for the given payload-id. +/// Helper: a minimal Flashblock for the given payload-id and index. +/// +/// `block_number` is set to 1 so that `PendingCursor::advance` (which +/// computes `base.block_number - 1`) does not underflow. fn payload(payload_id: reth::payload::PayloadId, idx: u64) -> FlashblocksPayloadV1 { FlashblocksPayloadV1 { payload_id, index: idx, base: Some(ExecutionPayloadBaseV1 { - block_number: 0, + block_number: 1, ..Default::default() }), diff: ExecutionPayloadFlashblockDeltaV1 { @@ -33,15 +48,95 @@ fn payload(payload_id: reth::payload::PayloadId, idx: u64) -> FlashblocksPayload } } -/// Build a fresh handle plus its broadcast receiver. +/// Like [`payload`] but with a custom `block_number` and `parent_hash`, +/// allowing the test to place the flashblock in a different epoch. +fn payload_with_parent( + payload_id: reth::payload::PayloadId, + idx: u64, + block_number: u64, + parent_hash: B256, +) -> FlashblocksPayloadV1 { + FlashblocksPayloadV1 { + payload_id, + index: idx, + base: Some(ExecutionPayloadBaseV1 { + block_number, + parent_hash, + ..Default::default() + }), + diff: ExecutionPayloadFlashblockDeltaV1::default(), + metadata: FlashblockMetadata::default(), + } +} + +/// Build a fresh handle. fn fresh_handle() -> FlashblocksHandle { - // authorizer + builder keys let auth_sk = signing_key(1); let builder_sk = signing_key(2); - FlashblocksHandle::new(auth_sk.verifying_key(), Some(builder_sk)) } +/// Mock provider that implements [`CanonStateSubscriptions`] for tests. +/// +/// Wraps a [`broadcast::Sender`] and hands out a new receiver on each call +/// to `subscribe_to_canonical_state`, which is exactly what +/// [`WorldChainEventsStream::new`] needs. +struct MockCanonProvider { + tx: broadcast::Sender, +} + +impl MockCanonProvider { + /// Create a new mock provider and return it alongside the broadcast + /// sender used to inject canonical state notifications from the test. + fn new() -> (broadcast::Sender, Self) { + let (tx, _rx) = broadcast::channel(16); + (tx.clone(), Self { tx }) + } +} + +impl reth::providers::NodePrimitivesProvider for MockCanonProvider { + type Primitives = reth_ethereum::EthPrimitives; +} + +impl CanonStateSubscriptions for MockCanonProvider { + fn subscribe_to_canonical_state( + &self, + ) -> reth::providers::CanonStateNotifications { + self.tx.subscribe() + } +} + +/// Create a [`CanonStateNotification::Commit`] whose tip has the given block +/// number and hash. +fn canon_notification(number: u64, hash: B256) -> CanonStateNotification { + let mut block = reth_ethereum::Block::default(); + block.header.number = number; + let recovered: RecoveredBlock = RecoveredBlock::new(block, vec![], hash); + CanonStateNotification::Commit { + new: Arc::new(Chain::new( + vec![recovered], + ExecutionOutcome::default(), + BTreeMap::new(), + )), + } +} + +/// Advance a [`WorldChainEventsStream`] until the next +/// [`FlashblocksEvent::Pending`] is yielded, skipping any +/// [`FlashblocksEvent::Canon`] items. +async fn next_flashblock(stream: &mut WorldChainEventsStream) -> FlashblocksPayloadV1 { + loop { + match stream.next().await.unwrap() { + FlashblocksEvent::Pending(fb) => return fb, + FlashblocksEvent::Canon(_) => continue, + } + } +} + +// --------------------------------------------------------------------------- +// Tests that do NOT use streams — unchanged +// --------------------------------------------------------------------------- + #[tokio::test] async fn publish_without_clearance_is_rejected() { let handle = fresh_handle(); @@ -97,37 +192,6 @@ async fn expired_authorization_is_rejected() { )); } -#[tokio::test] -async fn flashblock_stream_is_ordered() { - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - // clearance - let payload_id = reth::payload::PayloadId::new([2; 8]); - let auth = Authorization::new( - payload_id, - DUMMY_TIMESTAMP, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth).unwrap(); - - // send index 1 first (out-of-order) - for &idx in &[1u64, 0] { - let p = payload(payload_id, idx); - let signed = AuthorizedPayload::new(builder_sk, auth, p.clone()); - handle.publish_new(signed).unwrap(); - } - - let mut flashblock_stream = handle.live_flashblock_stream(); - - // Expect to receive 0, then 1 over the ordered broadcast. - let first = flashblock_stream.next().await.unwrap(); - let second = flashblock_stream.next().await.unwrap(); - assert_eq!(first.index, 0); - assert_eq!(second.index, 1); -} - #[tokio::test] async fn stop_and_restart_updates_state() { let handle = fresh_handle(); @@ -217,6 +281,81 @@ async fn stop_and_restart_with_active_publishers() { } } +#[tokio::test] +async fn await_clearance_unblocks_on_publish() { + let handle = fresh_handle(); + let builder_sk = handle.builder_sk().unwrap(); + + let waiter = { + let h = handle.clone(); + task::spawn(async move { + h.await_clearance().await; + }) + }; + + // give the waiter a chance to subscribe + tokio::task::yield_now().await; + assert!(!waiter.is_finished(), "future must still be pending"); + + // now grant clearance + let payload_id = reth::payload::PayloadId::new([5; 8]); + let auth = Authorization::new( + payload_id, + DUMMY_TIMESTAMP, + &signing_key(1), + builder_sk.verifying_key(), + ); + handle.start_publishing(auth).unwrap(); + + // waiter should finish very quickly + tokio::time::timeout(Duration::from_secs(1), waiter) + .await + .expect("await_clearance did not complete") + .unwrap(); +} + +// --------------------------------------------------------------------------- +// Stream tests — updated for WorldChainEventsStream / FlashblocksEvent +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn flashblock_stream_is_ordered() { + let handle = fresh_handle(); + let builder_sk = handle.builder_sk().unwrap(); + + // clearance + let payload_id = reth::payload::PayloadId::new([2; 8]); + let auth = Authorization::new( + payload_id, + DUMMY_TIMESTAMP, + &signing_key(1), + builder_sk.verifying_key(), + ); + handle.start_publishing(auth).unwrap(); + + // Create the event stream *before* publishing so the canonical tip can + // be established first, ensuring all flashblocks are yielded as Pending. + let (canon_tx, provider) = MockCanonProvider::new(); + let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); + + // Establish canonical tip matching the epoch parent. + // Flashblocks have block_number 1, parent_hash ZERO -> parent = (0, ZERO). + canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); + + // Send index 1 first (out-of-order), then 0. + for &idx in &[1u64, 0] { + let p = payload(payload_id, idx); + let signed = AuthorizedPayload::new(builder_sk, auth, p.clone()); + handle.publish_new(signed).unwrap(); + } + + // Expect to receive 0, then 1 over the ordered broadcast. + let first = next_flashblock(&mut stream).await; + let second = next_flashblock(&mut stream).await; + assert_eq!(first.index, 0); + assert_eq!(second.index, 1); +} + #[tokio::test] async fn flashblock_stream_buffers_and_live() { let timestamp = 1000; @@ -227,24 +366,33 @@ async fn flashblock_stream_buffers_and_live() { let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); handle.start_publishing(auth).unwrap(); - // publish index 0 before creating the stream + // Publish index 0 before creating the stream — it will appear in the + // seed. let signed0 = AuthorizedPayload::new(builder_sk, auth, payload(pid, 0)); handle.publish_new(signed0).unwrap(); - // now create the combined stream - let mut stream = handle.live_flashblock_stream(); + // Create the event stream. The seed contains fb0. + let (canon_tx, provider) = MockCanonProvider::new(); + let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); - // first item comes from the cached vector - let first = stream.next().await.unwrap(); - assert_eq!(first.index, 0); + // Establish canonical tip so flashblocks are yielded as Pending events. + canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); - // publish index 1 after the stream exists + // Publish index 1 after the stream exists — it arrives live over + // broadcast, not the seed. let signed1 = AuthorizedPayload::new(builder_sk, auth, payload(pid, 1)); handle.publish_new(signed1).unwrap(); - // second item should be delivered live - let second = stream.next().await.unwrap(); - assert_eq!(second.index, 1); + // Drain until we see index 1 delivered live. The seed fb0 may or may + // not be emitted depending on whether canon or fb0 is polled first by + // `select`. Either way, the live fb1 must be delivered. + let fb = next_flashblock(&mut stream).await; + if fb.index == 0 { + let fb1 = next_flashblock(&mut stream).await; + assert_eq!(fb1.index, 1); + } else { + assert_eq!(fb.index, 1); + } } #[tokio::test] @@ -257,9 +405,14 @@ async fn flashblock_stream_recovers_after_receiver_lag() { let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); handle.start_publishing(auth).unwrap(); - // Create the stream first, then publish more messages than the broadcast buffer can retain - // before polling it. The stream must resync from protocol state instead of terminating. - let mut stream = handle.live_flashblock_stream(); + // Create the event stream first, then publish more messages than the + // broadcast buffer can retain before polling it. The stream must + // resync from protocol state instead of terminating. + let (canon_tx, provider) = MockCanonProvider::new(); + let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); + + // Establish canonical tip. + canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); for idx in 0..=200 { let signed = AuthorizedPayload::new(builder_sk, auth, payload(pid, idx)); @@ -267,14 +420,14 @@ async fn flashblock_stream_recovers_after_receiver_lag() { } for expected in 0..=100u64 { - let flashblock = stream.next().await.unwrap(); + let flashblock = next_flashblock(&mut stream).await; assert_eq!(flashblock.index, expected); } // We actually fail to continue publishing here // but this is an acceptable edge case assert!( - tokio::time::timeout(Duration::from_millis(10), stream.next()) + tokio::time::timeout(Duration::from_millis(10), next_flashblock(&mut stream)) .await .is_err(), ); @@ -295,19 +448,29 @@ async fn live_flashblock_stream_skips_stale_flashblocks() { ); handle.start_publishing(auth_a).unwrap(); - // Create the stream first, then partially consume payload A before payload B starts. - // The stream should skip the unread remainder of payload A once protocol state rolls over. - let mut stream = handle.live_flashblock_stream(); + // Create the event stream first, then partially consume payload A + // before payload B starts. The stream should skip the unread remainder + // of payload A once the canonical tip advances past its epoch parent. + // + // Payload A: block_number 1, parent_hash ZERO -> epoch parent (0, ZERO) + // Payload B: block_number 2, parent_hash BLOCK1 -> epoch parent (1, BLOCK1) + let block1_hash = B256::with_last_byte(0xAA); + let (canon_tx, provider) = MockCanonProvider::new(); + let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); + + // Canonical tip for epoch A. + canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); for idx in 0..=10u64 { let signed = AuthorizedPayload::new(builder_sk, auth_a, payload(pid_a, idx)); handle.publish_new(signed).unwrap(); } - let first = stream.next().await.unwrap(); + let first = next_flashblock(&mut stream).await; assert_eq!(first.payload_id, pid_a); assert_eq!(first.index, 0); + // Start epoch B on a *different* parent so the cursor can tell it apart. let pid_b = PayloadId::new([9; 8]); let auth_b = Authorization::new( pid_b, @@ -316,11 +479,25 @@ async fn live_flashblock_stream_skips_stale_flashblocks() { builder_sk.verifying_key(), ); handle.start_publishing(auth_b).unwrap(); - let signed = AuthorizedPayload::new(builder_sk, auth_b, payload(pid_b, 0)); + let signed = AuthorizedPayload::new( + builder_sk, + auth_b, + payload_with_parent(pid_b, 0, 2, block1_hash), + ); handle.publish_new(signed).unwrap(); - let flashblock = stream.next().await.unwrap(); - assert_eq!(flashblock.payload_id, pid_b); + // Advance the canonical tip to match epoch B's parent. Already-emitted + // A flashblocks may still be in the pipeline, but the cursor will reject + // any new A flashblocks arriving after the tip change. Drain until we + // see B's flashblock. + canon_tx.send(canon_notification(1, block1_hash)).unwrap(); + + let flashblock = loop { + let fb = next_flashblock(&mut stream).await; + if fb.payload_id == pid_b { + break fb; + } + }; assert_eq!(flashblock.index, 0); } @@ -334,23 +511,26 @@ async fn live_flashblock_stream_handles_out_of_order() { let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); handle.start_publishing(auth).unwrap(); - // Create the stream first, then publish more messages than the broadcast buffer can retain - // before polling it. The stream must resync from protocol state instead of terminating. - let mut stream = handle.live_flashblock_stream(); + // Create the event stream, then send the canonical tip. + let (canon_tx, provider) = MockCanonProvider::new(); + let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); + + // Establish canonical tip. + canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); handle .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 0))) .unwrap(); - assert_eq!(stream.next().await.unwrap().index, 0); + assert_eq!(next_flashblock(&mut stream).await.index, 0); handle .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 2))) .unwrap(); - // Assert not ready + // Assert not ready — index 2 cannot be delivered before index 1 arrives. assert!( - tokio::time::timeout(Duration::from_millis(10), stream.next()) + tokio::time::timeout(Duration::from_millis(10), next_flashblock(&mut stream)) .await .is_err() ); @@ -359,39 +539,6 @@ async fn live_flashblock_stream_handles_out_of_order() { .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 1))) .unwrap(); - assert_eq!(stream.next().await.unwrap().index, 1); - assert_eq!(stream.next().await.unwrap().index, 2); -} - -#[tokio::test] -async fn await_clearance_unblocks_on_publish() { - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let waiter = { - let h = handle.clone(); - task::spawn(async move { - h.await_clearance().await; - }) - }; - - // give the waiter a chance to subscribe - tokio::task::yield_now().await; - assert!(!waiter.is_finished(), "future must still be pending"); - - // now grant clearance - let payload_id = reth::payload::PayloadId::new([5; 8]); - let auth = Authorization::new( - payload_id, - DUMMY_TIMESTAMP, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth).unwrap(); - - // waiter should finish very quickly - tokio::time::timeout(Duration::from_secs(1), waiter) - .await - .expect("await_clearance did not complete") - .unwrap(); + assert_eq!(next_flashblock(&mut stream).await.index, 1); + assert_eq!(next_flashblock(&mut stream).await.index, 2); } diff --git a/crates/flashblocks/rpc/src/eth/pending_block.rs b/crates/flashblocks/rpc/src/eth/pending_block.rs index d253151a9..228d84905 100644 --- a/crates/flashblocks/rpc/src/eth/pending_block.rs +++ b/crates/flashblocks/rpc/src/eth/pending_block.rs @@ -1,7 +1,6 @@ //! Loads OP pending block for a RPC response. use alloy_eips::BlockNumberOrTag; -use alloy_primitives::{B256, BlockNumber}; use reth_optimism_primitives::OpPrimitives; use reth_optimism_rpc::{OpEthApi, OpEthApiError}; use reth_provider::{BlockReader, BlockReaderIdExt, ReceiptProvider}; @@ -13,15 +12,6 @@ use reth_rpc_eth_types::{EthApiError, PendingBlock, block::BlockAndReceipts}; use crate::eth::FlashblocksEthApi; -fn is_pending_block_fresh( - pending_number: BlockNumber, - pending_parent_hash: B256, - latest_number: BlockNumber, - latest_hash: B256, -) -> bool { - pending_number > latest_number && pending_parent_hash == latest_hash -} - impl LoadPendingBlock for FlashblocksEthApi where N: RpcNodeCore, @@ -61,25 +51,18 @@ where if let Some(pending_block) = pending_block { let block = pending_block.recovered_block; - if is_pending_block_fresh( - block.header().number, - block.header().parent_hash, - latest.number, - latest.hash(), - ) { - let receipts = pending_block - .execution_output - .receipts - .clone() - .into_iter() - .collect::>(); // always a single block executed through the state executor - - let block_and_receipts = BlockAndReceipts { - block, - receipts: receipts.into(), - }; - return Ok(Some(block_and_receipts)); - } + let receipts = pending_block + .execution_output + .receipts + .clone() + .into_iter() + .collect::>(); // always a single block executed through the state executor + + let block_and_receipts = BlockAndReceipts { + block, + receipts: receipts.into(), + }; + return Ok(Some(block_and_receipts)); } } @@ -107,23 +90,3 @@ where self.inner.pending_block_kind() } } - -#[cfg(test)] -mod tests { - use super::is_pending_block_fresh; - use alloy_primitives::B256; - - #[test] - fn fresh_pending_block_must_be_ahead_of_latest_and_build_on_latest_hash() { - let latest_hash = B256::from([1; 32]); - - assert!(is_pending_block_fresh(11, latest_hash, 10, latest_hash)); - assert!(!is_pending_block_fresh(10, latest_hash, 10, latest_hash)); - assert!(!is_pending_block_fresh( - 12, - B256::from([2; 32]), - 10, - latest_hash - )); - } -} diff --git a/crates/world/node/tests/e2e-testsuite/actions.rs b/crates/world/node/tests/e2e-testsuite/actions.rs index c15794db9..baf24ea9f 100644 --- a/crates/world/node/tests/e2e-testsuite/actions.rs +++ b/crates/world/node/tests/e2e-testsuite/actions.rs @@ -5,6 +5,7 @@ use alloy_eips::{BlockId, Decodable2718}; use alloy_rpc_types::{Transaction, TransactionRequest}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; use eyre::eyre::{Result, eyre}; +use flashblocks_p2p::protocol::event::{FlashblocksEvent, WorldChainEventsStream}; use flashblocks_primitives::{ flashblocks::{Flashblock, Flashblocks}, p2p::Authorization, @@ -24,8 +25,9 @@ use reth_e2e_test_utils::testsuite::{Environment, actions::Action}; use reth_node_api::{ConsensusEngineHandle, EngineApiMessageVersion}; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_node::{OpEngineTypes, OpPayloadAttributes}; -use reth_optimism_primitives::OpTransactionSigned; +use reth_optimism_primitives::{OpPrimitives, OpTransactionSigned}; use reth_primitives::TransactionSigned; +use reth_provider::CanonStateSubscriptions; use revm_primitives::{Address, B256, Bytes, U256}; use std::{pin::Pin, sync::Arc, time::Duration}; use tokio::sync::{mpsc, watch}; @@ -1789,6 +1791,7 @@ pub struct DynamicValidateFlashblocks { pub beacon_handle: Arc>, pub chain_spec: Arc, pub state: BlockProductionState, + pub provider: Arc + Send + Sync>, } impl DynamicValidateFlashblocks { @@ -1797,12 +1800,14 @@ impl DynamicValidateFlashblocks { beacon_handle: Arc>, chain_spec: Arc, state: BlockProductionState, + provider: impl CanonStateSubscriptions + Send + Sync + 'static, ) -> Self { Self { flashblocks_handle, beacon_handle, chain_spec, state, + provider: Arc::new(provider), } } } @@ -1813,7 +1818,18 @@ impl Action for DynamicValidateFlashblocks { env: &'a mut Environment, ) -> BoxFuture<'a, Result<()>> { Box::pin(async move { - let stream = Box::pin(self.flashblocks_handle.live_flashblock_stream()); + let stream = Box::pin( + WorldChainEventsStream::new( + self.flashblocks_handle.ctx.flashblock_tx.subscribe(), + &*self.provider, + ) + .filter_map(|event| async { + match event { + FlashblocksEvent::Pending(fb) => Some(fb), + _ => None, + } + }), + ); let mut validate_action = ValidateFlashblocksWithState::new( stream, @@ -1834,7 +1850,18 @@ impl ReadOnlyAction for DynamicValidateFlashblocks { ) -> BoxFuture<'a, Result<()>> { Box::pin(async move { let mut flashblocks = Flashblocks::default(); - let mut stream = Box::pin(self.flashblocks_handle.live_flashblock_stream()); + let mut stream = Box::pin( + WorldChainEventsStream::new( + self.flashblocks_handle.ctx.flashblock_tx.subscribe(), + &*self.provider, + ) + .filter_map(|event| async { + match event { + FlashblocksEvent::Pending(fb) => Some(fb), + _ => None, + } + }), + ); // Wait for payload to be available let target_hash = loop { diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index d651fedbc..b97426ea9 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -10,13 +10,15 @@ use alloy_primitives::{Bytes, b64}; use alloy_rpc_types::TransactionRequest; use alloy_rpc_types_engine::PayloadStatusEnum; use eyre::eyre::eyre; -use futures::future::Either; +use flashblocks_p2p::protocol::event::{FlashblocksEvent, WorldChainEventsStream}; +use futures::{StreamExt, future::Either}; use reth::{ chainspec::EthChainSpec, network::{NetworkSyncUpdater, SyncState}, }; use reth_e2e_test_utils::testsuite::actions::Action; use reth_optimism_node::utils::optimism_payload_attributes; +use reth_provider::CanonStateSubscriptions; use reth_transaction_pool::TransactionPool; use revm_primitives::{Address, B256, U256}; use std::{ @@ -406,11 +408,22 @@ async fn test_flashblocks() -> eyre::Result<()> { .await; let cannon_flashblocks_stream = Box::pin( - builder_context - .as_ref() - .unwrap() - .flashblocks_handle - .live_flashblock_stream(), + WorldChainEventsStream::new( + builder_context + .as_ref() + .unwrap() + .flashblocks_handle + .ctx + .flashblock_tx + .subscribe(), + &builder_node.node.inner.provider, + ) + .filter_map(|event| async { + match event { + FlashblocksEvent::Pending(fb) => Some(fb), + _ => None, + } + }), ); let validation_stream = crate::actions::FlashblocksValidatonStream { @@ -483,12 +496,17 @@ async fn test_eth_api_receipt() -> eyre::Result<()> { Some(vec![crate::setup::TX_SET_L1_BLOCK.clone()]), ); - let cannon_flashblocks_stream = nodes[0] - .ext_context - .clone() - .unwrap() - .flashblocks_handle - .live_flashblock_stream(); + let handle = &nodes[0].ext_context.clone().unwrap().flashblocks_handle; + let cannon_flashblocks_stream = WorldChainEventsStream::new( + handle.ctx.flashblock_tx.subscribe(), + &nodes[0].node.inner.provider, + ) + .filter_map(|event| async { + match event { + FlashblocksEvent::Pending(fb) => Some(fb), + _ => None, + } + }); let mine_block = crate::actions::AssertMineBlock::new( 0, @@ -647,12 +665,17 @@ async fn test_eth_block_by_hash_pending() -> eyre::Result<()> { spammer.spawn(20, nodes[0].node.rpc_url()); - let cannon_flashblocks_stream = nodes[0] - .ext_context - .clone() - .unwrap() - .flashblocks_handle - .live_flashblock_stream(); + let handle = &nodes[0].ext_context.clone().unwrap().flashblocks_handle; + let cannon_flashblocks_stream = WorldChainEventsStream::new( + handle.ctx.flashblock_tx.subscribe(), + &nodes[0].node.inner.provider, + ) + .filter_map(|event| async { + match event { + FlashblocksEvent::Pending(fb) => Some(fb), + _ => None, + } + }); let (sender, mut rx) = tokio::sync::mpsc::channel(1); let timestamp = crate::setup::current_timestamp(); @@ -1089,6 +1112,7 @@ async fn test_continuous_block_production_with_validation() -> eyre::Result<()> basic_beacon_handle, chain_spec.clone(), state.clone(), + follower_0.node.inner.provider.clone(), ), ) // 3. Query validated blocks and receipts in parallel (AFTER mining/validation) diff --git a/sped.md b/sped.md new file mode 100644 index 000000000..674334d7e --- /dev/null +++ b/sped.md @@ -0,0 +1,13 @@ +# Major Changes +- Eth API should fetch the pending block directly from the BlockchainProvider + +# validate_payload +- validate_block_with_state + - trigger flashblocks stream over payloads built on parent hash + - aggregate flashblocks into a `OpExecutionData` + - pre-load EVM with bundle +- cleanup crate imports + +# Spec + +I have given you the basic outline From e803548847b69149cfe09663c53b21ea7fcb7610 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 13:59:05 -0700 Subject: [PATCH 28/43] chore: cleanup --- sped.md | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 sped.md diff --git a/sped.md b/sped.md deleted file mode 100644 index 674334d7e..000000000 --- a/sped.md +++ /dev/null @@ -1,13 +0,0 @@ -# Major Changes -- Eth API should fetch the pending block directly from the BlockchainProvider - -# validate_payload -- validate_block_with_state - - trigger flashblocks stream over payloads built on parent hash - - aggregate flashblocks into a `OpExecutionData` - - pre-load EVM with bundle -- cleanup crate imports - -# Spec - -I have given you the basic outline From f2ed3b6cc60055dc92f56cf6035b0c68d7e3587a Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 16:53:09 -0700 Subject: [PATCH 29/43] chore: add back p2p stuff --- crates/flashblocks/builder/src/coordinator.rs | 255 ++- crates/flashblocks/node/tests/p2p.rs | 9 +- crates/flashblocks/p2p/Cargo.toml | 2 +- .../p2p/src/protocol/connection.rs | 334 ++-- .../flashblocks/p2p/src/protocol/handler.rs | 1546 ++++++++++++++++- crates/flashblocks/p2p/tests/protocol.rs | 401 +---- .../flashblocks/rpc/src/eth/pending_block.rs | 9 +- crates/world/node/src/context.rs | 3 +- .../world/node/tests/e2e-testsuite/actions.rs | 33 +- .../node/tests/e2e-testsuite/testsuite.rs | 60 +- 10 files changed, 1950 insertions(+), 702 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 0dcba3320..d167446c9 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -6,13 +6,14 @@ use flashblocks_p2p::protocol::{ handler::FlashblocksHandle, }; use flashblocks_primitives::{p2p::AuthorizedPayload, primitives::FlashblocksPayloadV1}; -use futures::StreamExt as _; +use futures::{FutureExt, StreamExt as _}; use op_alloy_consensus::{OpTxEnvelope, encode_holocene_extra_data}; use parking_lot::RwLock; use reth::{ payload::EthPayloadBuilderAttributes, revm::{cancelled::CancelOnDrop, database::StateProviderDatabase}, rpc::types::BlockNumHash, + tasks::TaskSpawner, }; use reth_basic_payload_builder::PayloadConfig; use reth_chain_state::{DeferredTrieData, ExecutedBlock}; @@ -31,10 +32,15 @@ use reth_provider::{ }; use reth_transaction_pool::{EthPooledTransaction, noop::NoopTransactionPool}; use std::{ + panic::AssertUnwindSafe, sync::Arc, time::{Duration, Instant}, }; -use tokio::sync::{Semaphore, broadcast::{self, Sender}, oneshot}; +use tokio::sync::{ + OwnedSemaphorePermit, Semaphore, + broadcast::{self, Sender}, + oneshot, +}; use tracing::{error, trace, warn}; /// Maximum number of concurrent flashblock processing tasks on the thread pool. @@ -59,6 +65,11 @@ const FETCH_PARENT_HEADER_MAX_DELAY: Duration = Duration::from_millis(2000); /// The minimum backoff duration when waiting for the parent header to be available in the database when processing a flashblock. const FETCH_PARENT_HEADER_MIN_DELAY: Duration = Duration::from_millis(100); + +/// Semaphore locking the [`WorkloadExecutor`] thread pool for flashblock processing tasks. +/// Ensures the Pending Block is always in sync when a concurrent task is spawned. +const PENDING_BLOCK_WRITE_PERMIT: Semaphore = Semaphore::const_new(1); + /// The current state of all known pre confirmations received over the P2P layer /// or generated from the payload building job of this node. /// @@ -126,19 +137,29 @@ impl FlashblocksExecutionCoordinator { let pending_block = self.pending_block.clone(); let workload = WorkloadExecutor::default(); - let semaphore = Arc::new(Semaphore::new(MAX_THREAD_POOL_SIZE)); + let task_permit = Arc::new(Semaphore::new(MAX_THREAD_POOL_SIZE)); + + let database_permit = Arc::new(PENDING_BLOCK_WRITE_PERMIT); ctx.task_executor() .spawn_critical("flashblocks executor", async move { - let mut shutdown_tx: Option> = None; + // Tracks the in-flight shutdown signal and current epoch block number. + let mut inflight_shutdown: Option> = None; + let mut epoch_block_number: Option = None; while let Some(event) = stream.next().await { match event { WorldChainEvent::Chain(ChainEvent::Pending(flashblock)) => { + // Track epoch block number from base flashblocks + if let Some(base) = &flashblock.base { + epoch_block_number = Some(base.block_number); + } + this.on_flashblock( flashblock, - &mut shutdown_tx, - &semaphore, + &mut inflight_shutdown, + &task_permit, + database_permit.clone(), &workload, &provider, &evm_config, @@ -148,7 +169,12 @@ impl FlashblocksExecutionCoordinator { .await; } WorldChainEvent::Chain(ChainEvent::Canon(tip)) => { - this.on_canon(tip, &mut shutdown_tx, &pending_block); + this.on_canon( + tip, + &mut inflight_shutdown, + &mut epoch_block_number, + &pending_block, + ); } WorldChainEvent::Event(_) => {} } @@ -163,7 +189,8 @@ impl FlashblocksExecutionCoordinator { &self, flashblock: FlashblocksPayloadV1, shutdown_tx: &mut Option>, - semaphore: &Arc, + task_permit: &Arc, + database_permit: Arc, workload: &WorkloadExecutor, provider: &Provider, evm_config: &OpEvmConfig, @@ -176,13 +203,15 @@ impl FlashblocksExecutionCoordinator { + Clone + 'static, { - // Cancel any previous in-flight task + // Cancel any previous in-flight task. Ancestor handles are NOT cleared + // here — the new flashblock is typically in the same epoch and needs them. + // New epoch clearing is handled inside process_flashblock when is_new_payload. shutdown_tx.take(); let (tx, rx) = oneshot::channel::<()>(); *shutdown_tx = Some(tx); - let permit = semaphore + let permit = task_permit .clone() .acquire_owned() .await @@ -194,20 +223,18 @@ impl FlashblocksExecutionCoordinator { let chain_spec = chain_spec.clone(); let pending_block = pending_block.clone(); - spawn_blocking_with_shutdown_signal(workload, rx, move || { - let result = process_flashblock( + spawn_blocking_io_with_shutdown_signal(workload, rx, database_permit, move |permit| { + if let Err(e) = process_flashblock( + permit, provider, &evm_config, &this, chain_spec, flashblock, pending_block, - ); - drop(permit); - if let Err(e) = &result { + ) { error!("error processing flashblock: {e:#?}"); } - result }); } @@ -217,21 +244,19 @@ impl FlashblocksExecutionCoordinator { fn on_canon( &self, tip: BlockNumHash, - shutdown_tx: &mut Option>, + inflight_shutdown: &mut Option>, + epoch_block_number: &mut Option, pending_block: &tokio::sync::watch::Sender>>, ) { - // Drop the shutdown sender — cancels in-flight task - shutdown_tx.take(); - - // Clear ancestor handles if the epoch matches the canonical tip or is stale. - { - let mut inner = self.inner.write(); - let should_clear = inner.latest_payload.as_ref().is_some_and(|(p, _)| { - p.block().hash() == tip.hash || p.block().header().number < tip.number - }); - if should_clear { - inner.ancestor_handles.clear(); - } + // Only cancel in-flight work and clear ancestor handles if the current + // epoch is at or behind the canonical tip (stale). If the epoch is + // ahead of the tip, the work is still valid. + let is_stale = epoch_block_number.is_none_or(|n| n <= tip.number); + + if is_stale { + inflight_shutdown.take(); + self.inner.write().ancestor_handles.clear(); + *epoch_block_number = None; } // Clear pending block if it was built on the now-canonical tip. @@ -310,17 +335,35 @@ impl FlashblocksExecutionCoordinator { /// Spawns a blocking task on the [`WorkloadExecutor`] thread pool, racing it /// against a shutdown signal. If the shutdown receiver resolves first (sender -/// dropped), the task result is discarded. -fn spawn_blocking_with_shutdown_signal( +/// dropped), the task result is discarded. Acquires the `database_permit` +/// before running `f` to serialize pending block writes. +fn spawn_blocking_io_with_shutdown_signal( executor: &WorkloadExecutor, shutdown_rx: oneshot::Receiver<()>, + database_permit: Arc, f: F, ) where - F: FnOnce() -> R + Send + 'static, - R: std::fmt::Debug + Send + 'static, + F: FnOnce(OwnedSemaphorePermit) + Send + 'static, { - let task = executor.spawn_blocking(f); + let task = executor.spawn_blocking(move || { + let f = AssertUnwindSafe(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .expect("failed to build runtime for permit acquisition"); + + let permit = rt + .block_on(database_permit.acquire_owned()) + .expect("database semaphore closed"); + f(permit); + }); + + if let Err(e) = std::panic::catch_unwind(f) { + error!("flashblock processing panicked: {e:?}"); + } + }); + + // Race the task against the shutdown signal tokio::spawn(async move { match futures::future::select(task, shutdown_rx).await { futures::future::Either::Left((result, _)) => { @@ -336,6 +379,7 @@ fn spawn_blocking_with_shutdown_signal( } fn process_flashblock( + database_permit: OwnedSemaphorePermit, provider: Provider, evm_config: &OpEvmConfig, coordinator: &FlashblocksExecutionCoordinator, @@ -350,47 +394,59 @@ where + Clone + 'static, { - let FlashblocksExecutionCoordinatorInner { - ref mut flashblocks, - ref mut latest_payload, - ref mut payload_events, - .. - } = *coordinator.inner.write(); - let flashblock = Flashblock { flashblock }; - if let Some(latest_payload) = latest_payload - && latest_payload.0.id() == flashblock.flashblock.payload_id - && latest_payload.1 >= flashblock.flashblock.index - { - // Already processed this flashblock. This happens when set directly - // from publish_built_payload. Since we already built the payload, no need - // to do it again. - if let Some(executed) = latest_payload.0.executed_block() { - let block = ExecutedBlock::with_deferred_trie_data( - executed.recovered_block.clone(), - executed.execution_output.clone(), - DeferredTrieData::ready(Default::default()), - ); - pending_block.send_replace(Some(block)); + // --- Short read: check if already processed, extract base info --- + let (base, is_new_epoch) = { + let inner = coordinator.inner.read(); + + if let Some(latest_payload) = &inner.latest_payload { + if latest_payload.0.id() == flashblock.flashblock.payload_id + && latest_payload.1 >= flashblock.flashblock.index + { + // Already processed — send current pending block and return + if let Some(executed) = latest_payload.0.executed_block() { + let block = ExecutedBlock::with_deferred_trie_data( + executed.recovered_block.clone(), + executed.execution_output.clone(), + DeferredTrieData::ready(Default::default()), + ); + pending_block.send_replace(Some(block)); + } + return Ok(()); + } } - return Ok(()); + + let is_new = inner.flashblocks.is_new_payload(&flashblock)?; + let base = if is_new { + flashblock.base().unwrap().clone() + } else { + inner.flashblocks.base().clone() + }; + + (base, is_new) + }; + // --- Read lock dropped --- + + // Clear ancestor handles on new epoch + if is_new_epoch { + let mut inner = coordinator.inner.write(); + inner.latest_payload = None; + inner.ancestor_handles.clear(); } + // Accumulate committed state from latest payload (brief read lock) + let committed_state = { + let inner = coordinator.inner.read(); + CommittedState::::try_from( + inner.latest_payload.as_ref().map(|(p, _)| p), + ) + .map_err(|e| eyre!("Failed to construct committed state {:#?}", e))? + }; + let diff = flashblock.diff().clone(); let index = flashblock.flashblock.index; - // If for whatever reason we are not processing flashblocks in order - // we will error and return here. - let base = if flashblocks.is_new_payload(&flashblock)? { - *latest_payload = None; - coordinator.inner.write().ancestor_handles.clear(); - // safe unwrap from check in is_new_payload - flashblock.base().unwrap() - } else { - flashblocks.base() - }; - let sealed_header = provider .sealed_header_by_hash(base.parent_hash) .inspect_err(|e| error!("failed to fetch sealed header {}: {e:#?}", base.parent_hash))? @@ -425,10 +481,6 @@ where let evm_env = evm_config.next_evm_env(sealed_header.header(), &next_block_context)?; - let committed_state = - CommittedState::::try_from(latest_payload.as_ref().map(|(p, _)| p)) - .map_err(|e| eyre!("Failed to construct committed state {:#?}", e))?; - let transactions_offset = committed_state.transactions.len() + 1; let start = Instant::now(); @@ -494,13 +546,20 @@ where let config = PayloadConfig::new(Arc::new(sealed_header), attributes); let cancel = CancelOnDrop::default(); + let prev_payload = coordinator + .inner + .read() + .latest_payload + .as_ref() + .map(|(p, _)| p.clone()); + let builder_ctx = OpPayloadBuilderCtxBuilder.build( provider.clone(), evm_config.clone(), Default::default(), config, &cancel, - latest_payload.as_ref().map(|p| p.0.clone()), + prev_payload.clone(), ); let best = |_| BestPayloadTransactions::new(vec![].into_iter()); @@ -512,7 +571,7 @@ where Option::>::None, db, &builder_ctx, - latest_payload.as_ref().map(|p| &p.0), + prev_payload.as_ref(), false, )?; @@ -527,13 +586,8 @@ where metrics::histogram!("flashblocks.validate", "access_list" => flashblock.diff().access_list_data.is_some().to_string()) .record(duration.as_nanos() as f64 / 1_000_000_000.0); - // construct the full payload - *latest_payload = Some((payload.clone(), index)); - - flashblocks.push(flashblock)?; - // Build ExecutedBlock with deferred trie data — sorting happens in background - if let Some(executed) = payload.executed_block() { + let deferred = if let Some(executed) = payload.executed_block() { let (hashed_state, trie_updates) = match (&executed.hashed_state, &executed.trie_updates) { (either::Left(hs), either::Left(tu)) => (hs.clone(), tu.clone()), _ => unreachable!("payload builder always produces unsorted (Left) variants"), @@ -541,18 +595,8 @@ where let ancestors = coordinator.inner.read().ancestor_handles.clone(); - let deferred = DeferredTrieData::pending( - hashed_state, - trie_updates, - anchor_hash, - ancestors, - ); - - // Spawn background sort so wait_cloned() finds it ready - let deferred_clone = deferred.clone(); - rayon::spawn(move || { - deferred_clone.wait_cloned(); - }); + let deferred = + DeferredTrieData::pending(hashed_state, trie_updates, anchor_hash, ancestors); let block = ExecutedBlock::with_deferred_trie_data( executed.recovered_block.clone(), @@ -562,8 +606,29 @@ where pending_block.send_replace(Some(block)); - // Track this handle for future ancestors - coordinator.inner.write().ancestor_handles.push(deferred); + Some(deferred) + } else { + None + }; + + // --- Brief write lock: update state, then release database permit --- + { + let mut inner = coordinator.inner.write(); + inner.latest_payload = Some((payload.clone(), index)); + inner.flashblocks.push(flashblock)?; + if let Some(ref deferred) = deferred { + inner.ancestor_handles.push(deferred.clone()); + } + } + // Release database permit immediately after state update. + // Everything after this point (trie sort, broadcast) can run concurrently. + drop(database_permit); + + // Spawn background trie sort after releasing the permit + if let Some(deferred) = deferred { + rayon::spawn(move || { + deferred.wait_cloned(); + }); } trace!( @@ -574,7 +639,9 @@ where "built payload from flashblock" ); - coordinator.broadcast_payload(Events::BuiltPayload(payload), payload_events.clone())?; + let payload_events = coordinator.inner.read().payload_events.clone(); + + coordinator.broadcast_payload(Events::BuiltPayload(payload), payload_events)?; Ok(()) } diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 994a053b4..886103408 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -8,10 +8,8 @@ use eyre::eyre::eyre; use flashblocks_cli::FlashblocksArgs; use flashblocks_p2p::{ monitor, - protocol::{ - connection::ReceiveStatus, - handler::{FlashblocksHandle, PublishingStatus}, - }, + protocol::connection::ReceiveStatus, + protocol::handler::{FlashblocksHandle, PublishingStatus}, }; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, @@ -244,7 +242,8 @@ async fn wait_for_flashblocks_topology( .connections .iter() .filter_map(|(peer_id, conn)| { - (conn.receive_status == ReceiveStatus::NotReceiving).then_some(*peer_id) + (conn.receive_status == ReceiveStatus::NotReceiving) + .then_some(*peer_id) }) .collect(); drop(state); diff --git a/crates/flashblocks/p2p/Cargo.toml b/crates/flashblocks/p2p/Cargo.toml index d15b55a5d..05073168f 100644 --- a/crates/flashblocks/p2p/Cargo.toml +++ b/crates/flashblocks/p2p/Cargo.toml @@ -27,10 +27,10 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true thiserror.workspace = true parking_lot.workspace = true +pin-project.workspace = true chrono.workspace = true reth-tasks = { workspace = true } rand.workspace = true -pin-project = "1.0.1" [dev-dependencies] reth-network-api.workspace = true diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 7629351f7..79cd77b2c 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -1,6 +1,5 @@ use crate::protocol::handler::{ - FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, PeerMsg, PublishingStatus, - MAX_FLASHBLOCK_INDEX, + FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, PublishingStatus, MAX_FLASHBLOCK_INDEX, }; use alloy_primitives::bytes::BytesMut; use chrono::Utc; @@ -12,30 +11,75 @@ use flashblocks_primitives::{ }; use futures::{Stream, StreamExt}; use metrics::gauge; -use reth::payload::PayloadId; use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnection}; -use reth_network::{cache::LruMap, types::ReputationChangeKind}; +use reth_network::types::ReputationChangeKind; use std::{ pin::Pin, task::{ready, Context, Poll}, + time::Instant, }; -use tokio_stream::wrappers::BroadcastStream; +use tokio::sync::mpsc; use tracing::{info, trace}; /// Grace period for authorization timestamp checks to reduce false positives from /// minor skew/races between peers. const AUTHORIZATION_TIMESTAMP_GRACE_SEC: u64 = 10; -/// Number of payload receive-sets cached per peer. -/// -/// This should be large enough to retain entries across the grace window. -const RECEIVED_CACHE_LEN: u32 = AUTHORIZATION_TIMESTAMP_GRACE_SEC as u32 * 20; +/// Represents the current flashblocks receive status for a peer connection. +#[derive(Clone, Debug, Default, PartialEq)] +pub enum ReceiveStatus { + /// We are not currently receiving flashblocks from this peer. + #[default] + NotReceiving, + /// We are currently receiving flashblocks from this peer. + /// + /// Score used for adaptive timeouts and peer selection. + /// Lower is better. Corresponds the moving average of flashblock latency, with missed blocks + /// counting as 10s. + Receiving { score: Score }, + /// We have sent a request for flashblocks to this peer and are awaiting their response. + Requesting, +} + +/// Shared connection metadata for a single peer connection. +#[derive(Clone, Debug)] +pub struct FlashblocksConnectionState { + /// Whether this peer is marked as trusted or not. + pub trusted: bool, + /// Whether we are currently sending flashblocks to this peer. + pub send_enabled: bool, + /// Current status of receiving flashblocks from this peer. + pub receive_status: ReceiveStatus, + /// Timestamp of the last receive-side state transition for this peer. + /// Used for late-message grace checks and receive retry cooldown. + pub receive_status_timestamp: u64, + /// Per-peer channel for sending serialized protocol messages to this peer. + pub outbound_tx: Option>, + /// Number of control messages received in the current rate-limit window. + pub control_msg_count: u32, + /// Start of the current rate-limit window. + pub control_msg_window_start: Instant, +} + +impl FlashblocksConnectionState { + pub(crate) fn new() -> Self { + Self { + trusted: false, + send_enabled: false, + receive_status: ReceiveStatus::NotReceiving, + receive_status_timestamp: 0, + outbound_tx: None, + control_msg_count: 0, + control_msg_window_start: Instant::now(), + } + } +} /// Represents a single P2P connection for the flashblocks protocol. /// /// This struct manages the bidirectional communication with a single peer in the flashblocks /// P2P network. It handles incoming messages from the peer, validates and processes them, -/// and also streams outgoing messages that need to be broadcast. +/// and also streams serialized outgoing messages queued for this peer. /// /// The connection implements the `Stream` trait to provide outgoing message bytes that /// should be sent to the connected peer over the underlying protocol connection. @@ -46,12 +90,8 @@ pub struct FlashblocksConnection { conn: ProtocolConnection, /// The unique identifier of the connected peer. peer_id: PeerId, - /// Receiver for peer messages to be sent to all peers. - /// We send bytes over this stream to avoid repeatedly having to serialize the payloads. - peer_rx: BroadcastStream, - /// Per-peer tracking of flashblocks this peer has already sent us. - /// Uses `peek` for lookups to avoid LRU promotion, giving FIFO eviction semantics. - received_cache: LruMap<(PayloadId, usize), ()>, + /// Receiver for already serialized protocol messages targeted at this specific peer. + outbound_rx: mpsc::UnboundedReceiver, } impl FlashblocksConnection { @@ -61,43 +101,25 @@ impl FlashblocksConnection { /// * `protocol` - The flashblocks protocol handler managing the connection. /// * `conn` - The underlying protocol connection for sending and receiving messages. /// * `peer_id` - The unique identifier of the connected peer. - /// * `peer_rx` - Receiver for peer messages to be sent to all peers. - pub fn new( + pub(crate) fn new( protocol: FlashblocksP2PProtocol, conn: ProtocolConnection, peer_id: PeerId, - peer_rx: BroadcastStream, ) -> Self { + let (outbound_tx, outbound_rx) = mpsc::unbounded_channel(); + + protocol + .handle + .on_peer_connected(protocol.network.clone(), peer_id, outbound_tx); + gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).increment(1); Self { protocol, conn, peer_id, - peer_rx, - received_cache: LruMap::new(RECEIVED_CACHE_LEN), - } - } -} - -impl FlashblocksConnection { - /// Insert a `(payload_id, flashblock_index)` into the received cache. - /// - /// Uses [`LruMap::peek`] before insert to avoid promoting duplicates, - /// giving FIFO eviction semantics instead of LRU. - /// - /// Returns `true` if the key was newly inserted, `false` if it already existed. - fn received_cache_insert(&mut self, key: (PayloadId, usize)) -> bool { - if self.received_cache.peek(&key).is_some() { - return false; + outbound_rx, } - self.received_cache.insert(key, ()) - } - - /// Check if a `(payload_id, flashblock_index)` exists in the received cache - /// without promoting it (preserves FIFO eviction order). - fn received_cache_contains(&self, key: &(PayloadId, usize)) -> bool { - self.received_cache.peek(key).is_some() } } @@ -109,6 +131,8 @@ impl Drop for FlashblocksConnection { "dropping flashblocks connection" ); + self.protocol.handle.on_peer_disconnected(self.peer_id); + gauge!("flashblocks.peers", "capability" => FlashblocksP2PProtocol::::capability().to_string()).decrement(1); } } @@ -120,57 +144,13 @@ impl Stream for FlashblocksConnection { let this = self.get_mut(); loop { - // Check if there are any flashblocks ready to broadcast to our peers. - if let Poll::Ready(Some(res)) = this.peer_rx.poll_next_unpin(cx) { - match res { - Ok(peer_msg) => { - match peer_msg { - PeerMsg::FlashblocksPayloadV1(( - payload_id, - flashblock_index, - bytes, - )) => { - // Check if this flashblock actually originated from this peer. - if !this.received_cache_contains(&(payload_id, flashblock_index)) { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - %payload_id, - %flashblock_index, - "Broadcasting `FlashblocksPayloadV1` message to peer" - ); - metrics::counter!("flashblocks.bandwidth_outbound") - .increment(bytes.len() as u64); - - return Poll::Ready(Some(bytes)); - } - } - PeerMsg::StartPublishing(bytes_mut) => { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Broadcasting `StartPublishing` to peer" - ); - return Poll::Ready(Some(bytes_mut)); - } - PeerMsg::StopPublishing(bytes_mut) => { - trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "Broadcasting `StopPublishing` to peer" - ); - return Poll::Ready(Some(bytes_mut)); - } - } - } - Err(error) => { - tracing::error!( - target: "flashblocks::p2p", - %error, - "failed to receive flashblocks message from peer_rx" - ); - } - } + if let Poll::Ready(Some(bytes)) = this.outbound_rx.poll_recv(cx) { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "Sending serialized flashblocks protocol message to peer" + ); + return Poll::Ready(Some(bytes)); } // Check if there are any messages from the peer. @@ -234,12 +214,53 @@ impl Stream for FlashblocksConnection { } } } - _ => { - tracing::trace!( - target: "flashblocks::p2p", - peer_id = %this.peer_id, - "received unhandled p2p message variant", - ); + FlashblocksP2PMsg::RequestFlashblocks => { + if this + .protocol + .handle + .handle_request_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } + FlashblocksP2PMsg::AcceptFlashblocks => { + if this + .protocol + .handle + .handle_accept_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } + FlashblocksP2PMsg::RejectFlashblocks => { + if this + .protocol + .handle + .handle_reject_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } + } + FlashblocksP2PMsg::CancelFlashblocks => { + if this + .protocol + .handle + .handle_cancel_message(this.peer_id) + .is_err() + { + this.protocol + .network + .reputation_change(this.peer_id, ReputationChangeKind::BadMessage); + } } } } @@ -266,22 +287,22 @@ impl FlashblocksConnection { &mut self, authorized_payload: AuthorizedPayload, ) { - let state_handle = self.protocol.handle.state.clone(); - let mut state = state_handle.lock(); let authorization = &authorized_payload.authorized.authorization; let msg = authorized_payload.msg(); + let flashblock_timestamp = msg.metadata.flashblock_timestamp; + let mut p2p_state = self.protocol.handle.state.lock(); // Check if this payload is older than our current view by more than the allowed // grace window. if authorization.timestamp - < state + < p2p_state .payload_timestamp .saturating_sub(AUTHORIZATION_TIMESTAMP_GRACE_SEC) { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, - current_timestamp = state.payload_timestamp, + current_timestamp = p2p_state.payload_timestamp, timestamp = authorization.timestamp, grace_sec = AUTHORIZATION_TIMESTAMP_GRACE_SEC, "received flashblock with outdated timestamp", @@ -305,10 +326,43 @@ impl FlashblocksConnection { return; } - // Check if this peer is spamming us with the same payload index - if !self.received_cache_insert((msg.payload_id, msg.index as usize)) { - // We've already seen this index from this peer. - // They could be trying to DOS us. + let Some(conn_state) = p2p_state.connection_state(&self.peer_id) else { + return; + }; + match &conn_state.receive_status { + ReceiveStatus::Requesting => { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock before request was accepted", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + return; + } + ReceiveStatus::NotReceiving => { + if conn_state.receive_status_timestamp + 2 < authorization.timestamp { + tracing::warn!( + target: "flashblocks::p2p", + peer_id = %self.peer_id, + payload_id = %msg.payload_id, + index = msg.index, + "received flashblock from peer outside receive window", + ); + self.protocol + .network + .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); + } + return; + } + ReceiveStatus::Receiving { .. } => {} + } + + // Check if this peer is spamming us with the same payload index. + if !p2p_state.note_peer_received_flashblock(authorization, msg, self.peer_id) { tracing::warn!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -322,11 +376,9 @@ impl FlashblocksConnection { return; } - state.publishing_status.send_modify(|status| { + p2p_state.publishing_status.send_modify(|status| { let active_publishers = match status { PublishingStatus::Publishing { .. } => { - // We are currently building, so we should not be seeing any new flashblocks - // over the p2p network. tracing::error!( target: "flashblocks::p2p", peer_id = %self.peer_id, @@ -340,38 +392,38 @@ impl FlashblocksConnection { PublishingStatus::NotPublishing { active_publishers } => active_publishers, }; - // Update the list of active publishers if let Some((_, timestamp)) = active_publishers .iter_mut() .find(|(publisher, _)| *publisher == authorization.builder_vk) { - // This is an existing publisher, we should update their block number *timestamp = authorization.timestamp; } else { - // This is a new publisher, we should add them to the list of active publishers active_publishers.push((authorization.builder_vk, authorization.timestamp)); } }); - let now = Utc::now() - .timestamp_nanos_opt() - .expect("time went backwards"); - - if let Some(flashblock_timestamp) = msg.metadata.flashblock_timestamp { + if let Some(flashblock_timestamp) = flashblock_timestamp { + let now = Utc::now() + .timestamp_nanos_opt() + .expect("time went backwards"); let latency = now - flashblock_timestamp; metrics::histogram!("flashblocks.latency").record(latency as f64 / 1_000_000_000.0); + if let Some(ReceiveStatus::Receiving { score }) = p2p_state + .connection_state_mut(&self.peer_id) + .map(|peer_state| &mut peer_state.receive_status) + { + score.record(latency); + } } self.protocol .handle .ctx - .publish(&mut state, authorized_payload); + .publish(&mut p2p_state, authorized_payload); } /// Handles incoming `StartPublish` messages from a peer. /// - /// TODO: handle propogating this if we care. For now we assume direct peering. - /// /// # Arguments /// * `authorized_payload` - The authorized `StartPublish` message received from the peer /// @@ -382,11 +434,11 @@ impl FlashblocksConnection { /// - If we are waiting to publish, updates the list of active publishers /// - If we are not publishing, adds the new publisher to the list of active publishers fn handle_start_publish(&mut self, authorized_payload: AuthorizedPayload) { - let state = self.protocol.handle.state.lock(); let Ok(builder_sk) = self.protocol.handle.builder_sk() else { return; }; let authorization = &authorized_payload.authorized.authorization; + let state = self.protocol.handle.state.lock(); // Check if the request is expired for dos protection. // It's important to ensure that this `StartPublish` request @@ -399,6 +451,7 @@ impl FlashblocksConnection { timestamp = authorized_payload.authorized.authorization.timestamp, "received initiate build request with outdated timestamp", ); + drop(state); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); @@ -419,8 +472,7 @@ impl FlashblocksConnection { let authorized = Authorized::new(builder_sk, *our_authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.protocol.handle.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: vec![( @@ -463,8 +515,6 @@ impl FlashblocksConnection { /// Handles incoming `StopPublish` messages from a peer. /// - /// TODO: handle propogating this if we care. For now we assume direct peering. - /// /// # Arguments /// * `authorized_payload` - The authorized `StopPublish` message received from the peer /// @@ -475,11 +525,11 @@ impl FlashblocksConnection { /// - If we are waiting to publish, removes the publisher from the list of active publishers and checks if we can start publishing /// - If we are not publishing, removes the publisher from the list of active publishers fn handle_stop_publish(&mut self, authorized_payload: AuthorizedPayload) { - let state = self.protocol.handle.state.lock(); let authorization = &authorized_payload.authorized.authorization; + let state = self.protocol.handle.state.lock(); // Check if the request is expired for dos protection. - // It's important to ensure that this `StartPublish` request + // It's important to ensure that this `StopPublish` request // is very recent, or it could be used in a replay attack. if state.payload_timestamp > authorization.timestamp { tracing::warn!( @@ -489,6 +539,7 @@ impl FlashblocksConnection { timestamp = authorized_payload.authorized.authorization.timestamp, "Received initiate build response with outdated timestamp", ); + drop(state); self.protocol .network .reputation_change(self.peer_id, ReputationChangeKind::BadMessage); @@ -565,3 +616,30 @@ impl FlashblocksConnection { }); } } + +/// A lightweight moving average with a configurable smoothing window. +#[derive(Clone, Debug, PartialEq)] +pub struct Score { + value: Option, + window: i64, +} + +impl Score { + pub(crate) fn new(window: i64) -> Self { + Self { + value: None, + window: window.max(1), + } + } + + pub(crate) fn record(&mut self, sample: i64) { + self.value = Some(match self.value { + Some(current) => (current * (self.window - 1) + sample) / self.window, + None => sample, + }); + } + + pub(crate) fn value(&self) -> Option { + self.value + } +} diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 25502f83b..5fa85e41f 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,7 +1,11 @@ -use crate::protocol::{connection::FlashblocksConnection, error::FlashblocksP2PError}; +use crate::protocol::{ + connection::{FlashblocksConnection, FlashblocksConnectionState, ReceiveStatus, Score}, + error::FlashblocksP2PError, +}; use alloy_rlp::BytesMut; use chrono::Utc; use ed25519_dalek::{SigningKey, VerifyingKey}; +use flashblocks_cli::FanoutArgs; use flashblocks_primitives::{ p2p::{ Authorization, Authorized, AuthorizedMsg, AuthorizedPayload, FlashblocksP2PMsg, @@ -9,22 +13,33 @@ use flashblocks_primitives::{ }, primitives::FlashblocksPayloadV1, }; +use futures::{Stream, StreamExt, stream}; use metrics::histogram; use parking_lot::Mutex; -use reth::{api::NodePrimitives, payload::PayloadId, providers::CanonStateSubscriptions}; +use rand::{Rng, seq::SliceRandom}; +use reth::payload::PayloadId; use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; use reth_network::Peers; -use std::{net::SocketAddr, sync::Arc}; -use tokio::sync::{broadcast, watch}; -use tokio_stream::wrappers::BroadcastStream; -use tracing::{debug, info}; +use std::{ + collections::{HashMap, HashSet, VecDeque}, + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{ + sync::{broadcast, mpsc, watch}, + time, +}; +use tracing::{debug, info, warn}; use reth_ethereum::network::{ api::Direction, eth_wire::{capability::SharedCapabilities, multiplex::ProtocolConnection, protocol::Protocol}, protocol::{ConnectionHandler, OnNotSupported}, }; +use tokio_stream::wrappers::BroadcastStream; + /// Maximum frame size for rlpx messages. const MAX_FRAME: usize = 1 << 24; // 16 MiB @@ -41,6 +56,29 @@ const MAX_PUBLISH_WAIT_SEC: u64 = 2; /// before dropping them. In practice, we should rarely need to buffer any messages. const BROADCAST_BUFFER_CAPACITY: usize = 100; +/// A missed flashblock should dominate modest latency differences when rotating receive peers. +const MISSED_FLASHBLOCK_PENALTY_NS: i64 = 10_000_000_000; +/// Grace window in number of flashblocks to receive late flashblocks from peers before scoring them for missing flashblocks. +/// +/// This must be at least long enough to cover AUTHORIZATION_TIMESTAMP_GRACE_SEC to prevent a spam +/// attack. +pub(crate) const RECEIVE_FLASHBLOCK_GRACE_WINDOW: usize = 50; + +/// Maximum number of control messages (Request/Accept/Reject/Cancel) a peer may send +/// within a sliding window before being penalized. +const MAX_CONTROL_MSGS_PER_WINDOW: u32 = 10; + +/// Duration of the per-peer control-message rate-limit window. +const CONTROL_MSG_WINDOW: Duration = Duration::from_secs(30); +/// Maximum time to wait for a peer to answer a `RequestFlashblocks` message. +const RECEIVE_REQUEST_TIMEOUT_SECS: u64 = 2; + +/// Maximum time to wait for the network manager to expose the newly connected peer's trust info. +const PEER_INFO_LOOKUP_TIMEOUT: Duration = Duration::from_secs(1); + +/// Poll interval while waiting for connected peer metadata to become available. +const PEER_INFO_LOOKUP_RETRY_INTERVAL: Duration = Duration::from_millis(10); + /// Trait bound for network handles that can be used with the flashblocks P2P protocol. /// /// This trait combines all the necessary bounds for a network handle to be used @@ -49,20 +87,6 @@ pub trait FlashblocksP2PNetworkHandle: Clone + Unpin + Peers + std::fmt::Debug + impl FlashblocksP2PNetworkHandle for N {} -/// Messages that can be broadcast over a channel to each internal peer connection. -/// -/// These messages are used internally to coordinate the broadcasting of flashblocks -/// and publishing status changes to all connected peers. -#[derive(Clone, Debug)] -pub enum PeerMsg { - /// Send an already serialized flashblock to all peers. - FlashblocksPayloadV1((PayloadId, usize, BytesMut)), - /// Send a previously serialized StartPublish message to all peers. - StartPublishing(BytesMut), - /// Send a previously serialized StopPublish message to all peers. - StopPublishing(BytesMut), -} - /// The current publishing status of this node in the flashblocks P2P network. /// /// This enum tracks whether we are actively publishing flashblocks, waiting to publish, @@ -99,12 +123,24 @@ impl Default for PublishingStatus { } } +/// Tracked information about a flashblock payload observed from the network. +#[derive(Clone, Debug)] +pub struct ObservedPayload { + payload_id: PayloadId, + timestamp: u64, + flashblock_index: u64, + /// Peers from which we've received this flashblock. + received_peers: HashSet, + /// Peers who we have sent this flashblock to. + send_peers: HashSet, +} + /// Protocol state that stores the flashblocks P2P protocol events and coordination data. /// /// This struct maintains the current state of flashblock publishing, including coordination /// with other publishers, payload buffering, and ordering information. It serves as the /// central state management for the flashblocks P2P protocol handler. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct FlashblocksP2PState { /// Current publishing status indicating whether we're publishing, waiting, or not publishing. pub publishing_status: watch::Sender, @@ -114,15 +150,392 @@ pub struct FlashblocksP2PState { pub payload_timestamp: u64, /// Timestamp at which the most recent flashblock was received in ns since the unix epoch. pub flashblock_timestamp: i64, + /// Flashblocks observed from network peers, tracked until their receive grace windows expire. + pub observed_payloads: VecDeque, + /// All currently connected peers and their connection state. + pub connections: HashMap, +} + +impl Default for FlashblocksP2PState { + fn default() -> Self { + let (publishing_status, _) = watch::channel(PublishingStatus::default()); + + Self { + publishing_status, + payload_id: PayloadId::default(), + payload_timestamp: 0, + flashblock_timestamp: 0, + observed_payloads: VecDeque::new(), + connections: HashMap::new(), + } + } } impl FlashblocksP2PState { - /// Returns the current publishing status of this node. + /// Returns the connection state of a peer. + pub(crate) fn connection_state(&self, peer_id: &PeerId) -> Option<&FlashblocksConnectionState> { + self.connections.get(peer_id) + } + + pub(crate) fn connection_state_mut( + &mut self, + peer_id: &PeerId, + ) -> Option<&mut FlashblocksConnectionState> { + self.connections.get_mut(peer_id) + } + + /// Marks receiving a flashblock from a peer and returns whether this is the first time we've observed this peer receive this flashblock. /// - /// This indicates whether the node is actively publishing flashblocks, - /// waiting to publish, or not publishing at all. - pub fn publishing_status(&self) -> PublishingStatus { - self.publishing_status.borrow().clone() + /// Called when a flashblock is received from any peer. + pub(crate) fn note_peer_received_flashblock( + &mut self, + authorization: &Authorization, + flashblock: &FlashblocksPayloadV1, + peer_id: PeerId, + ) -> bool { + if let Some(observed_payload) = self.observed_payloads.iter_mut().find(|observed_payload| { + observed_payload.payload_id == flashblock.payload_id + && observed_payload.flashblock_index == flashblock.index + }) { + return observed_payload.received_peers.insert(peer_id); + } + + if self.observed_payloads.len() >= RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let evicted = self.observed_payloads.pop_front().unwrap(); + for (peer_id, connection) in &mut self.connections { + if connection.receive_status_timestamp + 2 <= evicted.timestamp + && !evicted.received_peers.contains(peer_id) + && !evicted.send_peers.contains(peer_id) + && let ReceiveStatus::Receiving { score } = &mut connection.receive_status + { + debug!( + target: "flashblocks::p2p", + %peer_id, + payload_id = %evicted.payload_id, + flashblock_index = evicted.flashblock_index, + "scoring peer for missed flashblock", + ); + score.record(MISSED_FLASHBLOCK_PENALTY_NS); + } + } + } + + self.observed_payloads.push_back(ObservedPayload { + payload_id: flashblock.payload_id, + timestamp: authorization.timestamp, + flashblock_index: flashblock.index, + received_peers: HashSet::from([peer_id]), + send_peers: HashSet::new(), + }); + + true + } + + /// Returns whether we've seen a given flashblock from a given peer. + pub(crate) fn peer_received_flashblock( + &self, + peer_id: PeerId, + payload_id: PayloadId, + index: u64, + ) -> bool { + self.observed_payloads + .iter() + .find(|observed_payload| { + observed_payload.payload_id == payload_id + && observed_payload.flashblock_index == index + }) + .is_some_and(|observed_payload| observed_payload.received_peers.contains(&peer_id)) + } + + /// Sends an already serialized message to all connected peers. + pub(crate) fn send_to_all_peers(&self, bytes: &BytesMut) { + for conn in self.connections.values() { + if let Some(tx) = &conn.outbound_tx { + tx.send(bytes.clone()).ok(); + } + } + } + + /// Sends a serialized flashblock to peers in the current send set that have not + /// already delivered that flashblock to us. + fn send_flashblock_to_send_set( + &mut self, + payload_id: PayloadId, + flashblock_index: u64, + bytes: &BytesMut, + ) { + for (peer_id, conn) in &self.connections { + if !conn.send_enabled + || self.peer_received_flashblock(*peer_id, payload_id, flashblock_index) + { + continue; + } + self.observed_payloads + .iter_mut() + .find(|observed_payload| { + observed_payload.payload_id == payload_id + && observed_payload.flashblock_index == flashblock_index + }) + .map(|observed_payload| observed_payload.send_peers.insert(*peer_id)); + + if let Some(tx) = &conn.outbound_tx + && tx.send(bytes.clone()).is_ok() + { + metrics::counter!("flashblocks.bandwidth_outbound").increment(bytes.len() as u64); + } + } + } + + /// Sends a control message directly to a specific peer. + fn send_direct(&self, peer_id: PeerId, msg: FlashblocksP2PMsg) { + let bytes: &BytesMut = &msg.encode(); + if let Some(conn) = self.connections.get(&peer_id) + && let Some(tx) = &conn.outbound_tx + { + tx.send(bytes.clone()).ok(); + } + } + + /// Returns `true` if the peer has exceeded the control-message rate limit. + fn check_control_rate_limit(&mut self, peer_id: &PeerId) -> bool { + let Some(peer_state) = self.connections.get_mut(peer_id) else { + return true; + }; + let now = Instant::now(); + if now.duration_since(peer_state.control_msg_window_start) > CONTROL_MSG_WINDOW { + peer_state.control_msg_count = 0; + peer_state.control_msg_window_start = now; + } + peer_state.control_msg_count += 1; + peer_state.control_msg_count > MAX_CONTROL_MSGS_PER_WINDOW + } + + fn num_receive_peers(&self) -> usize { + self.connections + .values() + .filter(|peer_state| { + matches!(peer_state.receive_status, ReceiveStatus::Receiving { .. }) + }) + .count() + } + + fn receive_retry_cooldown_secs(ctx: &FlashblocksP2PCtx) -> u64 { + Duration::from_secs(ctx.fanout_args.rotation_interval) + .as_secs() + .max(1) + } + + fn clear_receive_state( + peer_state: &mut FlashblocksConnectionState, + receive_status_timestamp: u64, + ) { + peer_state.receive_status = ReceiveStatus::NotReceiving; + peer_state.receive_status_timestamp = receive_status_timestamp; + } + + fn available_receive_candidates(&self, ctx: &FlashblocksP2PCtx) -> Vec<(PeerId, bool)> { + let now = Utc::now().timestamp() as u64; + let retry_cooldown = Self::receive_retry_cooldown_secs(ctx); + self.connections + .iter() + .filter_map(|(peer_id, peer_state)| { + if peer_state.receive_status == ReceiveStatus::NotReceiving + && (peer_state.receive_status_timestamp == 0 + || peer_state.receive_status_timestamp + retry_cooldown <= now) + { + Some((*peer_id, peer_state.trusted)) + } else { + None + } + }) + .collect() + } + + fn begin_requesting_peer(&mut self, peer_id: PeerId) { + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return; + }; + let timestamp = Utc::now().timestamp() as u64; + peer_state.receive_status = ReceiveStatus::Requesting; + peer_state.receive_status_timestamp = timestamp; + self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); + } + + fn num_receive_or_requesting_peers(&self) -> usize { + self.connections + .values() + .filter(|peer_state| { + matches!( + peer_state.receive_status, + ReceiveStatus::Receiving { .. } | ReceiveStatus::Requesting + ) + }) + .count() + } + + pub fn maybe_request_receive_peers(&mut self, ctx: &FlashblocksP2PCtx) { + while self.num_receive_or_requesting_peers() < ctx.fanout_args.max_receive_peers { + let candidates = self.available_receive_candidates(ctx); + if candidates.is_empty() { + return; + } + let rand = rand::rng().random_range(0..candidates.len()); + self.begin_requesting_peer(candidates[rand].0); + } + } + + fn expire_stale_receive_requests(&mut self, ctx: &FlashblocksP2PCtx) { + let now = Utc::now().timestamp() as u64; + let mut cleared_any = false; + + for peer_state in self.connections.values_mut() { + if matches!(peer_state.receive_status, ReceiveStatus::Requesting) + && peer_state.receive_status_timestamp + RECEIVE_REQUEST_TIMEOUT_SECS <= now + { + Self::clear_receive_state(peer_state, now); + cleared_any = true; + } + } + + if cleared_any { + self.maybe_request_receive_peers(ctx); + } + } + + fn worst_receive_peer(&self) -> Option { + self.connections + .iter() + .filter_map(|(peer_id, peer_state)| { + let ReceiveStatus::Receiving { score } = &peer_state.receive_status else { + return None; + }; + Some((*peer_id, score.value())) + }) + .max_by( + |(_, lhs_score), (_, rhs_score)| match (lhs_score, rhs_score) { + (None, None) => std::cmp::Ordering::Equal, + (None, Some(_)) => std::cmp::Ordering::Greater, + (Some(_), None) => std::cmp::Ordering::Less, + (Some(lhs), Some(rhs)) => lhs.cmp(rhs), + }, + ) + .map(|(peer_id, _)| peer_id) + } + + fn maybe_start_rotation(&mut self, ctx: &FlashblocksP2PCtx) { + if self.num_receive_peers() < ctx.fanout_args.max_receive_peers { + return; + } + + let Some(evict) = self.worst_receive_peer() else { + return; + }; + + let candidates = self.available_receive_candidates(ctx); + if candidates.is_empty() { + return; + } + + let rand = rand::rng().random_range(0..candidates.len()); + let candidate = candidates[rand].0; + + let evict_timestamp = Utc::now().timestamp() as u64; + if let Some(evict_state) = self.connection_state_mut(&evict) { + Self::clear_receive_state(evict_state, evict_timestamp); + } + self.send_direct(evict, FlashblocksP2PMsg::CancelFlashblocks); + + self.begin_requesting_peer(candidate); + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state(&peer_id) else { + return Ok(()); + }; + + if peer_state.send_enabled { + // Already sending to this peer — repeated request is spam. + return Err(()); + } + let peer_is_trusted = peer_state.trusted; + let send_count = self.connections.values().filter(|s| s.send_enabled).count(); + + if !peer_is_trusted && send_count >= ctx.fanout_args.max_send_peers { + self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); + return Ok(()); + } + + let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); + peer_state.send_enabled = true; + self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); + Ok(()) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_accept(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return Ok(()); + }; + + match peer_state.receive_status { + ReceiveStatus::Requesting => { + peer_state.receive_status = ReceiveStatus::Receiving { + score: Score::new(ctx.fanout_args.score_samples), + }; + Ok(()) + } + // Unsolicited accept — we never asked this peer. + _ => Err(()), + } + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_reject(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return Ok(()); + }; + + match peer_state.receive_status { + ReceiveStatus::Requesting => { + Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); + self.maybe_request_receive_peers(ctx); + Ok(()) + } + // Unsolicited reject — we never asked this peer. + _ => Err(()), + } + } + + /// Returns `Err` if the peer should receive a reputation penalty. + fn handle_cancel(&mut self, peer_id: PeerId) -> Result<(), ()> { + if self.check_control_rate_limit(&peer_id) { + return Err(()); + } + + let Some(peer_state) = self.connection_state_mut(&peer_id) else { + return Ok(()); + }; + + if !peer_state.send_enabled { + // Cancel is only valid from a receiver to its sender. + return Err(()); + } + + peer_state.send_enabled = false; + Ok(()) } } @@ -135,11 +548,8 @@ impl FlashblocksP2PState { pub struct FlashblocksP2PCtx { /// Authorizer's verifying key used to verify authorization signatures from rollup-boost. pub authorizer_vk: VerifyingKey, - /// Builder's signing key used to sign outgoing authorized P2P messages. - pub builder_sk: Option, - /// Broadcast sender for peer messages that will be sent to all connected peers. - /// Messages may not be strictly ordered due to network conditions. - pub peer_tx: broadcast::Sender, + /// Flashblocks configuration including signing keys and fanout args. + pub fanout_args: FanoutArgs, /// Broadcast sender for verified and strictly ordered flashblock payloads. /// Used by RPC overlays and other consumers of flashblock data. pub flashblock_tx: broadcast::Sender, @@ -153,6 +563,8 @@ pub struct FlashblocksP2PCtx { pub struct FlashblocksHandle { /// Shared context containing network handle, keys, and communication channels. pub ctx: FlashblocksP2PCtx, + /// Builder signing key used to sign outgoing authorized P2P messages. + pub builder_sk: Option, /// Thread-safe mutable state of the flashblocks protocol. /// Protected by a mutex to allow concurrent access from multiple connections. pub state: Arc>, @@ -160,28 +572,45 @@ pub struct FlashblocksHandle { impl FlashblocksHandle { pub fn new(authorizer_vk: VerifyingKey, builder_sk: Option) -> Self { + Self::with_fanout_args(authorizer_vk, builder_sk, FanoutArgs::default()) + } + + pub fn with_fanout_args( + authorizer_vk: VerifyingKey, + builder_sk: Option, + fanout_args: FanoutArgs, + ) -> Self { let flashblock_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); - let peer_tx = broadcast::Sender::new(BROADCAST_BUFFER_CAPACITY); let state = Arc::new(Mutex::new(FlashblocksP2PState::default())); let ctx = FlashblocksP2PCtx { authorizer_vk, - builder_sk, - peer_tx, + fanout_args, flashblock_tx, }; + let handle = Self { + ctx, + builder_sk, + state, + }; + let moved_handle = handle.clone(); - Self { ctx, state } - } + tokio::spawn(async move { + let mut rotation_interval = time::interval(Duration::from_secs( + moved_handle.ctx.fanout_args.rotation_interval, + )); + rotation_interval.set_missed_tick_behavior(time::MissedTickBehavior::Delay); + rotation_interval.tick().await; - pub fn flashblocks_tx(&self) -> broadcast::Sender { - self.ctx.flashblock_tx.clone() - } + loop { + rotation_interval.tick().await; + let mut state = moved_handle.state.lock(); + state.expire_stale_receive_requests(&moved_handle.ctx); + state.maybe_request_receive_peers(&moved_handle.ctx); + state.maybe_start_rotation(&moved_handle.ctx); + } + }); - pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { - self.ctx - .builder_sk - .as_ref() - .ok_or(FlashblocksP2PError::MissingBuilderSk) + handle } /// Returns a [`WorldChainEventsStream`] merging flashblocks from the P2P @@ -192,14 +621,106 @@ impl FlashblocksHandle { ) -> crate::protocol::event::WorldChainEventsStream where T: Send + Clone + Unpin + 'static, - P: CanonStateSubscriptions + Clone + Send + Sync + 'static, - N: NodePrimitives, + P: reth::providers::CanonStateSubscriptions + Clone + Send + Sync + 'static, + N: reth::api::NodePrimitives, { crate::protocol::event::WorldChainEventsStream::new( provider, self.ctx.flashblock_tx.subscribe(), ) } + + pub(crate) fn on_peer_connected( + &self, + network: N, + peer_id: PeerId, + outbound_tx: mpsc::UnboundedSender, + ) { + let trusted = tokio::task::block_in_place(|| { + let network = network.clone(); + tokio::runtime::Handle::current().block_on(async move { + let deadline = Instant::now() + PEER_INFO_LOOKUP_TIMEOUT; + + loop { + match network.get_peer_by_id(peer_id).await { + Ok(Some(peer_info)) => return Ok(peer_info.kind.is_trusted()), + Ok(None) if Instant::now() < deadline => { + time::sleep(PEER_INFO_LOOKUP_RETRY_INTERVAL).await; + } + Ok(None) => { + return Err( + "timed out waiting for peer info after connection".to_owned() + ); + } + Err(error) if Instant::now() < deadline => { + time::sleep(PEER_INFO_LOOKUP_RETRY_INTERVAL).await; + tracing::debug!( + target: "flashblocks::p2p", + %peer_id, + %error, + "retrying peer info lookup for flashblocks fanout" + ); + } + Err(error) => { + return Err(format!( + "failed to load peer info for flashblocks fanout: {error}" + )); + } + } + } + }) + }); + + let trusted = match trusted { + Ok(trusted) => trusted, + Err(error) => { + warn!( + target: "flashblocks::p2p", + %peer_id, + %error, + "failed to classify peer for flashblocks fanout; defaulting to untrusted" + ); + false + } + }; + + let mut state = self.state.lock(); + let mut conn_state = FlashblocksConnectionState::new(); + conn_state.outbound_tx = Some(outbound_tx); + conn_state.trusted = trusted; + state.connections.insert(peer_id, conn_state); + state.maybe_request_receive_peers(&self.ctx); + } + + pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { + let mut state = self.state.lock(); + state.connections.remove(&peer_id); + state.maybe_request_receive_peers(&self.ctx); + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_request_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_request(&self.ctx, peer_id) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_accept_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_accept(&self.ctx, peer_id) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_reject_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_reject(&self.ctx, peer_id) + } + + /// Returns `Err` if the peer should receive a reputation penalty. + pub(crate) fn handle_cancel_message(&self, peer_id: PeerId) -> Result<(), ()> { + let mut state = self.state.lock(); + state.handle_cancel(peer_id) + } } /// Main protocol handler for the flashblocks P2P protocol. @@ -239,21 +760,28 @@ impl FlashblocksP2PProtocol { } impl FlashblocksP2PProtocol { - /// Returns the P2P capability for the flashblocks v1 protocol. + /// Returns the P2P capability for the flashblocks v2 protocol. /// /// This capability is used during devp2p handshake to advertise support - /// for the flashblocks protocol with protocol name "flblk" and version 1. + /// for the flashblocks protocol with protocol name "flblk" and version 2. pub fn capability() -> Capability { - Capability::new_static("flblk", 1) + Capability::new_static("flblk", 2) } } impl FlashblocksHandle { + /// Returns the builder signing key if configured. + pub fn builder_sk(&self) -> Result<&SigningKey, FlashblocksP2PError> { + self.builder_sk + .as_ref() + .ok_or(FlashblocksP2PError::MissingBuilderSk) + } + /// Publishes a newly created flashblock from the payload builder to the P2P network. /// /// This method validates that the builder has authorization to publish and that /// the authorization matches the current publishing session. The flashblock is - /// then processed, cached, and broadcast to all connected peers. + /// then processed, cached, and forwarded to peers in the current send set. /// /// # Arguments /// * `authorized_payload` - The signed flashblock payload with authorization @@ -282,6 +810,11 @@ impl FlashblocksHandle { Ok(()) } + /// Sends an already serialized protocol message to all currently connected peers. + pub fn send_serialized_to_all_peers(&self, bytes: BytesMut) { + self.state.lock().send_to_all_peers(&bytes); + } + /// Returns the current publishing status of this node. /// /// The status indicates whether the node is actively publishing flashblocks, @@ -369,13 +902,12 @@ impl FlashblocksHandle { } } PublishingStatus::NotPublishing { active_publishers } => { - // Send an authorized `StartPublish` message to the network + // Send an authorized `StartPublish` message to direct peers. let authorized_msg = AuthorizedMsg::StartPublish(StartPublish); let authorized_payload = Authorized::new(builder_sk, new_authorization, authorized_msg); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StartPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); if active_publishers.is_empty() { // If we have no previous publishers, we can start publishing immediately. @@ -428,8 +960,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, *authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: Vec::new(), }; @@ -449,8 +980,7 @@ impl FlashblocksHandle { let authorized_payload = Authorized::new(builder_sk, *authorization, StopPublish.into()); let p2p_msg = FlashblocksP2PMsg::Authorized(authorized_payload); - let peer_msg = PeerMsg::StopPublishing(p2p_msg.encode()); - self.ctx.peer_tx.send(peer_msg).ok(); + state.send_to_all_peers(&p2p_msg.encode()); *status = PublishingStatus::NotPublishing { active_publishers: active_publishers.clone(), }; @@ -461,6 +991,18 @@ impl FlashblocksHandle { Ok(()) } + + /// Returns a stream of ordered flashblocks starting from the beginning of the current payload. + /// + /// # Behavior + /// The stream will continue to yield flashblocks for consecutive payloads as well, so + /// consumers should take care to handle the stream appropriately. + /// Returns a raw stream of flashblock payloads from the broadcast channel. + /// For ordered, canon-gated delivery, use [`Self::event_stream`] instead. + pub fn flashblock_stream(&self) -> impl Stream + Send + 'static { + let receiver = self.ctx.flashblock_tx.subscribe(); + tokio_stream::StreamExt::map_while(BroadcastStream::new(receiver), |x| x.ok()) + } } impl FlashblocksP2PCtx { @@ -477,10 +1019,14 @@ impl FlashblocksP2PCtx { /// # Behavior /// - Validates payload consistency with authorization /// - Updates global state for new payloads with newer timestamps + /// - Caches flashblocks and maintains ordering for sequential delivery + /// - Forwards flashblocks to peers in the current send set and publishes ordered + /// flashblocks to the local stream /// Publishes a verified flashblock payload to peers and the local broadcast channel. /// /// Ordering, buffering, and canon-gating are handled downstream by - /// [`BufferedFlashblocks`] inside the [`WorldChainEventsStream`]. + /// [`BufferedFlashblocks`](crate::protocol::event::BufferedFlashblocks) inside + /// the [`WorldChainEventsStream`](crate::protocol::event::WorldChainEventsStream). pub fn publish( &self, state: &mut FlashblocksP2PState, @@ -531,9 +1077,7 @@ impl FlashblocksP2PCtx { metrics::histogram!("flashblocks.tx_count") .record(payload.diff.transactions.len() as f64); - let peer_msg = - PeerMsg::FlashblocksPayloadV1((payload.payload_id, payload.index as usize, bytes)); - self.peer_tx.send(peer_msg).ok(); + state.send_flashblock_to_send_set(payload.payload_id, payload.index, &bytes); // Broadcast to local subscribers — ordering handled by WorldChainEventsStream self.flashblock_tx.send(payload.clone()).ok(); @@ -560,7 +1104,7 @@ impl ConnectionHandler for FlashblocksP2PProtoco type Connection = FlashblocksConnection; fn protocol(&self) -> Protocol { - Protocol::new(Self::capability(), 1) + Protocol::new(Self::capability(), 5) } fn on_unsupported_by_peer( @@ -589,8 +1133,882 @@ impl ConnectionHandler for FlashblocksP2PProtoco "new flashblocks connection" ); - let peer_rx = self.handle.ctx.peer_tx.subscribe(); + FlashblocksConnection::new(self, conn, peer_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::SigningKey; + use enr::{Enr, secp256k1::SecretKey}; + use reth_eth_wire::{Capabilities, EthVersion, Status, StatusMessage, UnifiedStatus}; + use reth_network::{ + PeerInfo, PeersInfo, + types::{PeerKind, Reputation, ReputationChangeKind}, + }; + use reth_network_api::{NetworkError, noop::NoopNetwork}; + use reth_network_peers::NodeRecord; + use std::{ + collections::VecDeque, + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + }; - FlashblocksConnection::new(self, conn, peer_id, BroadcastStream::new(peer_rx)) + #[derive(Clone, Debug, Default)] + struct MockNetwork { + noop: NoopNetwork, + peer_lookup_responses: Arc>>>, + lookup_calls: Arc, + disconnected_peers: Arc>>, + } + + impl MockNetwork { + fn with_peer_lookup_responses(peer_lookup_responses: Vec>) -> Self { + Self { + peer_lookup_responses: Arc::new(Mutex::new(peer_lookup_responses.into())), + ..Default::default() + } + } + + fn lookup_calls(&self) -> usize { + self.lookup_calls.load(Ordering::SeqCst) + } + + fn disconnected_peers(&self) -> Vec { + self.disconnected_peers.lock().clone() + } + } + + impl PeersInfo for MockNetwork { + fn num_connected_peers(&self) -> usize { + self.noop.num_connected_peers() + } + + fn local_node_record(&self) -> NodeRecord { + self.noop.local_node_record() + } + + fn local_enr(&self) -> Enr { + self.noop.local_enr() + } + } + + impl Peers for MockNetwork { + fn add_trusted_peer_id(&self, _peer: PeerId) {} + + fn add_peer_kind( + &self, + _peer: PeerId, + _kind: PeerKind, + _tcp_addr: SocketAddr, + _udp_addr: Option, + ) { + } + + async fn get_peers_by_kind(&self, _kind: PeerKind) -> Result, NetworkError> { + Ok(vec![]) + } + + async fn get_all_peers(&self) -> Result, NetworkError> { + Ok(vec![]) + } + + async fn get_peer_by_id(&self, _peer_id: PeerId) -> Result, NetworkError> { + self.lookup_calls.fetch_add(1, Ordering::SeqCst); + Ok(self.peer_lookup_responses.lock().pop_front().flatten()) + } + + async fn get_peers_by_id( + &self, + _peer_ids: Vec, + ) -> Result, NetworkError> { + Ok(vec![]) + } + + fn remove_peer(&self, _peer: PeerId, _kind: PeerKind) {} + + fn disconnect_peer(&self, peer: PeerId) { + self.disconnected_peers.lock().push(peer); + } + + fn disconnect_peer_with_reason( + &self, + peer: PeerId, + _reason: reth_eth_wire::DisconnectReason, + ) { + self.disconnect_peer(peer); + } + + fn connect_peer_kind( + &self, + _peer: PeerId, + _kind: PeerKind, + _tcp_addr: SocketAddr, + _udp_addr: Option, + ) { + } + + fn reputation_change(&self, _peer_id: PeerId, _kind: ReputationChangeKind) {} + + async fn reputation_by_id( + &self, + _peer_id: PeerId, + ) -> Result, NetworkError> { + Ok(None) + } + } + + fn test_fanout_args() -> FanoutArgs { + FanoutArgs::default() + } + + fn test_ctx(fanout_args: FanoutArgs) -> FlashblocksP2PCtx { + let authorizer = SigningKey::from_bytes(&[7; 32]); + + FlashblocksP2PCtx { + authorizer_vk: authorizer.verifying_key(), + fanout_args, + flashblock_tx: broadcast::Sender::new(16), + } + } + + fn test_peer_state(trusted: bool) -> FlashblocksConnectionState { + let mut state = FlashblocksConnectionState::new(); + state.trusted = trusted; + state + } + + /// Creates a peer state with a per-peer outbound channel for message assertions. + fn test_peer_state_with_channel( + trusted: bool, + ) -> ( + FlashblocksConnectionState, + mpsc::UnboundedReceiver, + ) { + let (tx, rx) = mpsc::unbounded_channel(); + let mut state = FlashblocksConnectionState::new(); + state.trusted = trusted; + state.outbound_tx = Some(tx); + (state, rx) + } + + /// Receives and decodes a direct control message from a per-peer channel. + fn recv_direct(rx: &mut mpsc::UnboundedReceiver) -> FlashblocksP2PMsg { + let bytes = rx.try_recv().expect("expected a direct message"); + FlashblocksP2PMsg::decode(&mut &bytes[..]).expect("valid message") + } + + fn peer_state(fanout: &FlashblocksP2PState, peer_id: PeerId) -> &FlashblocksConnectionState { + fanout.connection_state(&peer_id).expect("peer exists") + } + + fn apply_observation( + fanout: &mut FlashblocksP2PState, + authorization: &Authorization, + flashblock: &FlashblocksPayloadV1, + peer_id: PeerId, + ) { + fanout.note_peer_received_flashblock(authorization, flashblock, peer_id); + } + + fn test_peer_info(peer_id: PeerId, trusted: bool) -> PeerInfo { + PeerInfo { + capabilities: Arc::new(Capabilities::new(vec![])), + remote_id: peer_id, + client_version: Arc::::from("mock"), + enode: "enode://mock".to_owned(), + enr: None, + remote_addr: SocketAddr::from((IpAddr::V4(Ipv4Addr::LOCALHOST), 30303)), + local_addr: None, + direction: Direction::Incoming, + eth_version: EthVersion::Eth67, + status: Arc::new(UnifiedStatus::from_message(StatusMessage::Legacy(Status { + version: EthVersion::Eth67, + ..Status::default() + }))), + session_established: Instant::now(), + kind: if trusted { + PeerKind::Trusted + } else { + PeerKind::Basic + }, + } + } + + #[tokio::test(flavor = "multi_thread")] + async fn on_peer_connected_retries_until_peer_info_is_available() { + let authorizer = SigningKey::from_bytes(&[7; 32]); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let handle = FlashblocksHandle::with_fanout_args( + authorizer.verifying_key(), + Some(SigningKey::from_bytes(&[8; 32])), + fanout_args, + ); + let peer_id = PeerId::random(); + let network = MockNetwork::with_peer_lookup_responses(vec![ + None, + Some(test_peer_info(peer_id, true)), + ]); + let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel(); + + handle.on_peer_connected(network.clone(), peer_id, outbound_tx); + + assert!(network.lookup_calls() >= 2); + assert!(network.disconnected_peers().is_empty()); + let state = handle.state.lock(); + assert!( + state + .connection_state(&peer_id) + .expect("peer exists") + .trusted + ); + drop(state); + assert_eq!( + recv_direct(&mut outbound_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn on_peer_connected_defaults_to_untrusted_when_peer_info_never_arrives() { + let authorizer = SigningKey::from_bytes(&[7; 32]); + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let handle = FlashblocksHandle::with_fanout_args( + authorizer.verifying_key(), + Some(SigningKey::from_bytes(&[8; 32])), + fanout_args, + ); + let peer_id = PeerId::random(); + let network = MockNetwork::default(); + let (outbound_tx, mut outbound_rx) = mpsc::unbounded_channel(); + + handle.on_peer_connected(network.clone(), peer_id, outbound_tx); + + assert!(network.lookup_calls() > 1); + assert!(network.disconnected_peers().is_empty()); + let state = handle.state.lock(); + assert!( + !state + .connection_state(&peer_id) + .expect("peer exists") + .trusted + ); + drop(state); + assert_eq!( + recv_direct(&mut outbound_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn publish_sends_flashblocks_only_to_send_enabled_peers() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + let payload_id = PayloadId::new([1; 8]); + let authorization = Authorization::new(payload_id, 1, &authorizer, builder.verifying_key()); + let flashblock = FlashblocksPayloadV1 { + payload_id, + index: 0, + ..Default::default() + }; + let authorized_payload = + AuthorizedPayload::new(&builder, authorization, flashblock.clone()); + let expected = FlashblocksP2PMsg::Authorized(authorized_payload.authorized.clone()); + + let source_peer = PeerId::random(); + let send_peer = PeerId::random(); + let non_send_peer = PeerId::random(); + + let (mut source_state, mut source_rx) = test_peer_state_with_channel(false); + source_state.send_enabled = true; + let (mut send_state, mut send_rx) = test_peer_state_with_channel(false); + send_state.send_enabled = true; + let (non_send_state, mut non_send_rx) = test_peer_state_with_channel(false); + + fanout.connections.insert(source_peer, source_state); + fanout.connections.insert(send_peer, send_state); + fanout.connections.insert(non_send_peer, non_send_state); + apply_observation(&mut fanout, &authorization, &flashblock, source_peer); + + ctx.publish(&mut fanout, authorized_payload); + + assert_eq!(recv_direct(&mut send_rx), expected); + assert!(source_rx.try_recv().is_err()); + assert!(non_send_rx.try_recv().is_err()); + } + + #[test] + fn trusted_peers_are_requested_first() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let trusted_peer = PeerId::random(); + let untrusted_peer = PeerId::random(); + let (trusted_state, mut trusted_rx) = test_peer_state_with_channel(true); + let untrusted_state = test_peer_state(false); + fanout.connections.insert(trusted_peer, trusted_state); + fanout.connections.insert(untrusted_peer, untrusted_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, trusted_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + peer_state(&fanout, untrusted_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + recv_direct(&mut trusted_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn trusted_request_bypasses_non_trusted_limit() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_send_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let victim = PeerId::random(); + let trusted_requester = PeerId::random(); + let (mut victim_state, mut victim_rx) = test_peer_state_with_channel(false); + let (requester_state, mut requester_rx) = test_peer_state_with_channel(true); + victim_state.send_enabled = true; + fanout.connections.insert(victim, victim_state); + fanout + .connections + .insert(trusted_requester, requester_state); + + assert!(fanout.handle_request(&ctx, trusted_requester).is_ok()); + + assert!(peer_state(&fanout, victim).send_enabled); + assert!(peer_state(&fanout, trusted_requester).send_enabled); + assert!(victim_rx.try_recv().is_err()); + assert_eq!( + recv_direct(&mut requester_rx), + FlashblocksP2PMsg::AcceptFlashblocks + ); + } + + #[test] + fn rotation_replaces_peer_before_requesting_candidate() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let current_peer = PeerId::random(); + let candidate_peer = PeerId::random(); + let (mut current_state, mut current_rx) = test_peer_state_with_channel(false); + let (candidate_state, mut candidate_rx) = test_peer_state_with_channel(false); + let mut score = Score::new(score_samples); + score.record(42); + current_state.receive_status = ReceiveStatus::Receiving { score }; + fanout.connections.insert(current_peer, current_state); + fanout.connections.insert(candidate_peer, candidate_state); + + fanout.maybe_start_rotation(&ctx); + + assert_eq!( + peer_state(&fanout, current_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::Requesting + ); + + assert_eq!( + recv_direct(&mut current_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut candidate_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(fanout.handle_accept(&ctx, candidate_peer).is_ok()); + + assert!(matches!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + } + + #[test] + fn multiple_pending_requests_clear_independently() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let first_peer = PeerId::random(); + let second_peer = PeerId::random(); + let (first_state, mut first_rx) = test_peer_state_with_channel(false); + let (second_state, mut second_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(first_peer, first_state); + fanout.connections.insert(second_peer, second_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, first_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + peer_state(&fanout, second_peer).receive_status, + ReceiveStatus::Requesting + ); + + assert_eq!( + recv_direct(&mut first_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + assert_eq!( + recv_direct(&mut second_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(fanout.handle_accept(&ctx, first_peer).is_ok()); + assert!(fanout.handle_accept(&ctx, second_peer).is_ok()); + + assert!(matches!( + peer_state(&fanout, first_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + assert!(matches!( + peer_state(&fanout, second_peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + } + + #[test] + fn rejected_peer_is_not_immediately_retried() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (candidate_state, mut peer_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(peer, candidate_state); + + fanout.maybe_request_receive_peers(&ctx); + assert_eq!( + recv_direct(&mut peer_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + assert!(fanout.handle_reject(&ctx, peer).is_ok()); + + assert_eq!( + peer_state(&fanout, peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert!(peer_rx.try_recv().is_err()); + + fanout.maybe_request_receive_peers(&ctx); + assert!(peer_rx.try_recv().is_err()); + + fanout + .connection_state_mut(&peer) + .expect("peer exists") + .receive_status_timestamp = + Utc::now().timestamp() as u64 - ctx.fanout_args.rotation_interval.max(1); + + fanout.maybe_request_receive_peers(&ctx); + assert_eq!( + recv_direct(&mut peer_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn timed_out_request_is_cleared_and_replaced() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let stale_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + let (stale_state, mut stale_rx) = test_peer_state_with_channel(true); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(false); + fanout.connections.insert(stale_peer, stale_state); + fanout + .connections + .insert(replacement_peer, replacement_state); + + fanout.maybe_request_receive_peers(&ctx); + + assert_eq!( + peer_state(&fanout, stale_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + recv_direct(&mut stale_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + + fanout + .connection_state_mut(&stale_peer) + .expect("peer exists") + .receive_status_timestamp = + Utc::now().timestamp() as u64 - RECEIVE_REQUEST_TIMEOUT_SECS; + + fanout.expire_stale_receive_requests(&ctx); + + assert_eq!( + peer_state(&fanout, stale_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + assert!(stale_rx.try_recv().is_err()); + } + + #[test] + fn silent_receive_peer_can_be_rotated_out_without_samples() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 1; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let silent_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + + let (mut silent_state, mut silent_rx) = test_peer_state_with_channel(false); + let (replacement_state, mut replacement_rx) = test_peer_state_with_channel(true); + + silent_state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; + + fanout.connections.insert(silent_peer, silent_state); + fanout + .connections + .insert(replacement_peer, replacement_state); + + fanout.maybe_start_rotation(&ctx); + + assert_eq!( + peer_state(&fanout, silent_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + + assert_eq!( + recv_direct(&mut silent_rx), + FlashblocksP2PMsg::CancelFlashblocks + ); + assert_eq!( + recv_direct(&mut replacement_rx), + FlashblocksP2PMsg::RequestFlashblocks + ); + } + + #[test] + fn peer_score_penalizes_missed_flashblocks() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let mut fanout = FlashblocksP2PState::default(); + + let steady_peer = PeerId::random(); + let lagging_peer = PeerId::random(); + let mut steady_state = test_peer_state(false); + let mut lagging_state = test_peer_state(false); + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + + let mut steady_score = Score::new(score_samples); + steady_score.record(10); + steady_state.receive_status = ReceiveStatus::Receiving { + score: steady_score, + }; + let mut lagging_score = Score::new(score_samples); + lagging_score.record(100); + lagging_state.receive_status = ReceiveStatus::Receiving { + score: lagging_score, + }; + + fanout.connections.insert(steady_peer, steady_state); + fanout.connections.insert(lagging_peer, lagging_state); + + // Use timestamps starting well after receive_status_timestamp (0) so the grace + // check `receive_status_timestamp + 2 <= evicted.timestamp` is satisfied. + let ts_offset = 10_u64; + for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let authorization = Authorization::new( + PayloadId::default(), + ts_offset + index as u64, + &authorizer, + builder.verifying_key(), + ); + let flashblock = FlashblocksPayloadV1 { + payload_id: PayloadId::default(), + index: index as u64, + ..Default::default() + }; + apply_observation(&mut fanout, &authorization, &flashblock, steady_peer); + } + + assert_eq!(fanout.worst_receive_peer(), Some(lagging_peer)); + let ReceiveStatus::Receiving { + score: steady_score, + } = &peer_state(&fanout, steady_peer).receive_status + else { + panic!("expected Receiving"); + }; + assert_eq!(steady_score.value(), Some(10)); + let ReceiveStatus::Receiving { + score: lagging_score, + } = &peer_state(&fanout, lagging_peer).receive_status + else { + panic!("expected Receiving"); + }; + assert_eq!( + lagging_score.value(), + Some((100 * (score_samples - 1) + MISSED_FLASHBLOCK_PENALTY_NS) / score_samples) + ); + } + + #[test] + fn pending_candidate_is_rotated_out_after_missing_blocks() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_receive_peers = 2; + fanout_args.score_samples = 4; + let score_samples = fanout_args.score_samples; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let steady_peer = PeerId::random(); + let rotating_peer = PeerId::random(); + let candidate_peer = PeerId::random(); + let replacement_peer = PeerId::random(); + + let mut steady_state = test_peer_state(false); + let mut rotating_state = test_peer_state(false); + let candidate_state = test_peer_state(true); + let replacement_state = test_peer_state(true); + + let mut steady_score = Score::new(score_samples); + steady_score.record(10); + steady_state.receive_status = ReceiveStatus::Receiving { + score: steady_score, + }; + let mut rotating_score = Score::new(score_samples); + rotating_score.record(100); + rotating_state.receive_status = ReceiveStatus::Receiving { + score: rotating_score, + }; + + fanout.connections.insert(steady_peer, steady_state); + fanout.connections.insert(rotating_peer, rotating_state); + fanout.connections.insert(candidate_peer, candidate_state); + + fanout.maybe_start_rotation(&ctx); + + // Accept the candidate so it transitions to Receiving. + assert!(fanout.handle_accept(&ctx, candidate_peer).is_ok()); + + let authorizer = SigningKey::from_bytes(&[7; 32]); + let builder = SigningKey::from_bytes(&[9; 32]); + // Use timestamps well after the candidate's receive_status_timestamp so the + // grace check `receive_status_timestamp + 2 <= evicted.timestamp` is satisfied. + let ts_base = Utc::now().timestamp() as u64 + 10; + for index in 0..=RECEIVE_FLASHBLOCK_GRACE_WINDOW { + let authorization = Authorization::new( + PayloadId::default(), + ts_base + index as u64, + &authorizer, + builder.verifying_key(), + ); + let flashblock = FlashblocksPayloadV1 { + payload_id: PayloadId::default(), + index: index as u64, + ..Default::default() + }; + apply_observation(&mut fanout, &authorization, &flashblock, steady_peer); + } + + assert_eq!(fanout.worst_receive_peer(), Some(candidate_peer)); + + fanout + .connections + .insert(replacement_peer, replacement_state); + fanout.maybe_start_rotation(&ctx); + + assert_eq!( + peer_state(&fanout, candidate_peer).receive_status, + ReceiveStatus::NotReceiving + ); + assert_eq!( + peer_state(&fanout, replacement_peer).receive_status, + ReceiveStatus::Requesting + ); + } + + #[test] + fn unsolicited_accept_is_penalized() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false); + fanout.connections.insert(peer, state); + + // Accept without a prior request should be penalized. + assert!(fanout.handle_accept(&ctx, peer).is_err()); + } + + #[test] + fn unsolicited_reject_is_penalized() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false); + fanout.connections.insert(peer, state); + + // Reject without a prior request should be penalized. + assert!(fanout.handle_reject(&ctx, peer).is_err()); + } + + #[test] + fn cancel_without_relationship_is_penalized() { + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let state = test_peer_state(false); + fanout.connections.insert(peer, state); + + // Cancel with no send/receive relationship should be penalized. + assert!(fanout.handle_cancel(peer).is_err()); + } + + #[test] + fn cancel_only_clears_send_direction() { + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.send_enabled = true; + state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_cancel(peer).is_ok()); + assert!(!peer_state(&fanout, peer).send_enabled); + assert!(matches!( + peer_state(&fanout, peer).receive_status, + ReceiveStatus::Receiving { .. } + )); + } + + #[test] + fn cancel_from_sender_is_penalized() { + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.receive_status = ReceiveStatus::Receiving { + score: Score::new(4), + }; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_cancel(peer).is_err()); + } + + #[test] + fn duplicate_request_when_already_sending_is_penalized() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.send_enabled = true; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_request(&ctx, peer).is_err()); + } + + #[test] + fn receive_retry_cooldown_does_not_penalize_inbound_request() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (mut state, mut rx) = test_peer_state_with_channel(false); + state.receive_status_timestamp = Utc::now().timestamp() as u64; + fanout.connections.insert(peer, state); + + assert!(fanout.handle_request(&ctx, peer).is_ok()); + assert!(peer_state(&fanout, peer).send_enabled); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::AcceptFlashblocks); + } + + #[test] + fn repeated_rejected_requests_are_rate_limited() { + let mut fanout_args = test_fanout_args(); + fanout_args.max_send_peers = 0; + let ctx = test_ctx(fanout_args); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let (state, mut rx) = test_peer_state_with_channel(false); + fanout.connections.insert(peer, state); + + for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { + assert!(fanout.handle_request(&ctx, peer).is_ok()); + assert_eq!(recv_direct(&mut rx), FlashblocksP2PMsg::RejectFlashblocks); + } + assert!(fanout.handle_request(&ctx, peer).is_err()); + } + + #[test] + fn control_message_rate_limit_triggers_penalty() { + let ctx = test_ctx(test_fanout_args()); + let mut fanout = FlashblocksP2PState::default(); + + let peer = PeerId::random(); + let mut state = test_peer_state(false); + state.send_enabled = true; + fanout.connections.insert(peer, state); + + // Spam requests to exceed the rate limit. + for _ in 0..MAX_CONTROL_MSGS_PER_WINDOW { + // These return Err because send_enabled is already set (duplicate request), + // but the rate limit hasn't been hit yet. + assert!(fanout.handle_request(&ctx, peer).is_err()); + } + // The next one should hit the rate limit. + assert!(fanout.handle_request(&ctx, peer).is_err()); } } diff --git a/crates/flashblocks/p2p/tests/protocol.rs b/crates/flashblocks/p2p/tests/protocol.rs index e7d1f8a0c..ab3070df4 100644 --- a/crates/flashblocks/p2p/tests/protocol.rs +++ b/crates/flashblocks/p2p/tests/protocol.rs @@ -1,44 +1,29 @@ -use alloy_primitives::B256; use ed25519_dalek::SigningKey; -use flashblocks_p2p::protocol::{ - event::{FlashblocksEvent, WorldChainEventsStream}, - handler::{FlashblocksHandle, PublishingStatus}, -}; +use flashblocks_p2p::protocol::handler::{FlashblocksHandle, PublishingStatus}; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, p2p::{Authorization, AuthorizedPayload}, primitives::{ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, FlashblocksPayloadV1}, }; use futures::StreamExt as _; -use reth::{ - payload::PayloadId, - providers::{CanonStateNotification, CanonStateSubscriptions, Chain, ExecutionOutcome}, -}; -use reth_ethereum::primitives::RecoveredBlock; -use std::{collections::BTreeMap, sync::Arc, time::Duration}; -use tokio::{sync::broadcast, task}; +use reth::payload::PayloadId; +use std::time::Duration; +use tokio::task; const DUMMY_TIMESTAMP: u64 = 42; -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - /// Helper: deterministic ed25519 key made of the given byte. fn signing_key(byte: u8) -> SigningKey { SigningKey::from_bytes(&[byte; 32]) } -/// Helper: a minimal Flashblock for the given payload-id and index. -/// -/// `block_number` is set to 1 so that `PendingCursor::advance` (which -/// computes `base.block_number - 1`) does not underflow. +/// Helper: a minimal Flashblock (index 0) for the given payload-id. fn payload(payload_id: reth::payload::PayloadId, idx: u64) -> FlashblocksPayloadV1 { FlashblocksPayloadV1 { payload_id, index: idx, base: Some(ExecutionPayloadBaseV1 { - block_number: 1, + block_number: 0, ..Default::default() }), diff: ExecutionPayloadFlashblockDeltaV1 { @@ -48,95 +33,15 @@ fn payload(payload_id: reth::payload::PayloadId, idx: u64) -> FlashblocksPayload } } -/// Like [`payload`] but with a custom `block_number` and `parent_hash`, -/// allowing the test to place the flashblock in a different epoch. -fn payload_with_parent( - payload_id: reth::payload::PayloadId, - idx: u64, - block_number: u64, - parent_hash: B256, -) -> FlashblocksPayloadV1 { - FlashblocksPayloadV1 { - payload_id, - index: idx, - base: Some(ExecutionPayloadBaseV1 { - block_number, - parent_hash, - ..Default::default() - }), - diff: ExecutionPayloadFlashblockDeltaV1::default(), - metadata: FlashblockMetadata::default(), - } -} - -/// Build a fresh handle. +/// Build a fresh handle plus its broadcast receiver. fn fresh_handle() -> FlashblocksHandle { + // authorizer + builder keys let auth_sk = signing_key(1); let builder_sk = signing_key(2); - FlashblocksHandle::new(auth_sk.verifying_key(), Some(builder_sk)) -} - -/// Mock provider that implements [`CanonStateSubscriptions`] for tests. -/// -/// Wraps a [`broadcast::Sender`] and hands out a new receiver on each call -/// to `subscribe_to_canonical_state`, which is exactly what -/// [`WorldChainEventsStream::new`] needs. -struct MockCanonProvider { - tx: broadcast::Sender, -} - -impl MockCanonProvider { - /// Create a new mock provider and return it alongside the broadcast - /// sender used to inject canonical state notifications from the test. - fn new() -> (broadcast::Sender, Self) { - let (tx, _rx) = broadcast::channel(16); - (tx.clone(), Self { tx }) - } -} - -impl reth::providers::NodePrimitivesProvider for MockCanonProvider { - type Primitives = reth_ethereum::EthPrimitives; -} - -impl CanonStateSubscriptions for MockCanonProvider { - fn subscribe_to_canonical_state( - &self, - ) -> reth::providers::CanonStateNotifications { - self.tx.subscribe() - } -} - -/// Create a [`CanonStateNotification::Commit`] whose tip has the given block -/// number and hash. -fn canon_notification(number: u64, hash: B256) -> CanonStateNotification { - let mut block = reth_ethereum::Block::default(); - block.header.number = number; - let recovered: RecoveredBlock = RecoveredBlock::new(block, vec![], hash); - CanonStateNotification::Commit { - new: Arc::new(Chain::new( - vec![recovered], - ExecutionOutcome::default(), - BTreeMap::new(), - )), - } -} -/// Advance a [`WorldChainEventsStream`] until the next -/// [`FlashblocksEvent::Pending`] is yielded, skipping any -/// [`FlashblocksEvent::Canon`] items. -async fn next_flashblock(stream: &mut WorldChainEventsStream) -> FlashblocksPayloadV1 { - loop { - match stream.next().await.unwrap() { - FlashblocksEvent::Pending(fb) => return fb, - FlashblocksEvent::Canon(_) => continue, - } - } + FlashblocksHandle::new(auth_sk.verifying_key(), Some(builder_sk)) } -// --------------------------------------------------------------------------- -// Tests that do NOT use streams — unchanged -// --------------------------------------------------------------------------- - #[tokio::test] async fn publish_without_clearance_is_rejected() { let handle = fresh_handle(); @@ -192,6 +97,37 @@ async fn expired_authorization_is_rejected() { )); } +#[tokio::test] +async fn flashblock_stream_is_ordered() { + let handle = fresh_handle(); + let builder_sk = handle.builder_sk().unwrap(); + + // clearance + let payload_id = reth::payload::PayloadId::new([2; 8]); + let auth = Authorization::new( + payload_id, + DUMMY_TIMESTAMP, + &signing_key(1), + builder_sk.verifying_key(), + ); + handle.start_publishing(auth).unwrap(); + + // send index 1 first (out-of-order) + for &idx in &[1u64, 0] { + let p = payload(payload_id, idx); + let signed = AuthorizedPayload::new(builder_sk, auth, p.clone()); + handle.publish_new(signed).unwrap(); + } + + let mut flashblock_stream = handle.flashblock_stream(); + + // Expect to receive 0, then 1 over the ordered broadcast. + let first = flashblock_stream.next().await.unwrap(); + let second = flashblock_stream.next().await.unwrap(); + assert_eq!(first.index, 0); + assert_eq!(second.index, 1); +} + #[tokio::test] async fn stop_and_restart_updates_state() { let handle = fresh_handle(); @@ -281,81 +217,6 @@ async fn stop_and_restart_with_active_publishers() { } } -#[tokio::test] -async fn await_clearance_unblocks_on_publish() { - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let waiter = { - let h = handle.clone(); - task::spawn(async move { - h.await_clearance().await; - }) - }; - - // give the waiter a chance to subscribe - tokio::task::yield_now().await; - assert!(!waiter.is_finished(), "future must still be pending"); - - // now grant clearance - let payload_id = reth::payload::PayloadId::new([5; 8]); - let auth = Authorization::new( - payload_id, - DUMMY_TIMESTAMP, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth).unwrap(); - - // waiter should finish very quickly - tokio::time::timeout(Duration::from_secs(1), waiter) - .await - .expect("await_clearance did not complete") - .unwrap(); -} - -// --------------------------------------------------------------------------- -// Stream tests — updated for WorldChainEventsStream / FlashblocksEvent -// --------------------------------------------------------------------------- - -#[tokio::test] -async fn flashblock_stream_is_ordered() { - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - // clearance - let payload_id = reth::payload::PayloadId::new([2; 8]); - let auth = Authorization::new( - payload_id, - DUMMY_TIMESTAMP, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth).unwrap(); - - // Create the event stream *before* publishing so the canonical tip can - // be established first, ensuring all flashblocks are yielded as Pending. - let (canon_tx, provider) = MockCanonProvider::new(); - let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); - - // Establish canonical tip matching the epoch parent. - // Flashblocks have block_number 1, parent_hash ZERO -> parent = (0, ZERO). - canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); - - // Send index 1 first (out-of-order), then 0. - for &idx in &[1u64, 0] { - let p = payload(payload_id, idx); - let signed = AuthorizedPayload::new(builder_sk, auth, p.clone()); - handle.publish_new(signed).unwrap(); - } - - // Expect to receive 0, then 1 over the ordered broadcast. - let first = next_flashblock(&mut stream).await; - let second = next_flashblock(&mut stream).await; - assert_eq!(first.index, 0); - assert_eq!(second.index, 1); -} - #[tokio::test] async fn flashblock_stream_buffers_and_live() { let timestamp = 1000; @@ -366,179 +227,55 @@ async fn flashblock_stream_buffers_and_live() { let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); handle.start_publishing(auth).unwrap(); - // Publish index 0 before creating the stream — it will appear in the - // seed. + // publish index 0 before creating the stream let signed0 = AuthorizedPayload::new(builder_sk, auth, payload(pid, 0)); handle.publish_new(signed0).unwrap(); - // Create the event stream. The seed contains fb0. - let (canon_tx, provider) = MockCanonProvider::new(); - let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); + // now create the combined stream + let mut stream = handle.flashblock_stream(); - // Establish canonical tip so flashblocks are yielded as Pending events. - canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); + // first item comes from the cached vector + let first = stream.next().await.unwrap(); + assert_eq!(first.index, 0); - // Publish index 1 after the stream exists — it arrives live over - // broadcast, not the seed. + // publish index 1 after the stream exists let signed1 = AuthorizedPayload::new(builder_sk, auth, payload(pid, 1)); handle.publish_new(signed1).unwrap(); - // Drain until we see index 1 delivered live. The seed fb0 may or may - // not be emitted depending on whether canon or fb0 is polled first by - // `select`. Either way, the live fb1 must be delivered. - let fb = next_flashblock(&mut stream).await; - if fb.index == 0 { - let fb1 = next_flashblock(&mut stream).await; - assert_eq!(fb1.index, 1); - } else { - assert_eq!(fb.index, 1); - } -} - -#[tokio::test] -async fn flashblock_stream_recovers_after_receiver_lag() { - let timestamp = 1000; - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let pid = PayloadId::new([8; 8]); - let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); - handle.start_publishing(auth).unwrap(); - - // Create the event stream first, then publish more messages than the - // broadcast buffer can retain before polling it. The stream must - // resync from protocol state instead of terminating. - let (canon_tx, provider) = MockCanonProvider::new(); - let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); - - // Establish canonical tip. - canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); - - for idx in 0..=200 { - let signed = AuthorizedPayload::new(builder_sk, auth, payload(pid, idx)); - handle.publish_new(signed).unwrap(); - } - - for expected in 0..=100u64 { - let flashblock = next_flashblock(&mut stream).await; - assert_eq!(flashblock.index, expected); - } - - // We actually fail to continue publishing here - // but this is an acceptable edge case - assert!( - tokio::time::timeout(Duration::from_millis(10), next_flashblock(&mut stream)) - .await - .is_err(), - ); + // second item should be delivered live + let second = stream.next().await.unwrap(); + assert_eq!(second.index, 1); } #[tokio::test] -async fn live_flashblock_stream_skips_stale_flashblocks() { - let timestamp = 1000; +async fn await_clearance_unblocks_on_publish() { let handle = fresh_handle(); let builder_sk = handle.builder_sk().unwrap(); - let pid_a = PayloadId::new([8; 8]); - let auth_a = Authorization::new( - pid_a, - timestamp, - &signing_key(1), - builder_sk.verifying_key(), - ); - handle.start_publishing(auth_a).unwrap(); - - // Create the event stream first, then partially consume payload A - // before payload B starts. The stream should skip the unread remainder - // of payload A once the canonical tip advances past its epoch parent. - // - // Payload A: block_number 1, parent_hash ZERO -> epoch parent (0, ZERO) - // Payload B: block_number 2, parent_hash BLOCK1 -> epoch parent (1, BLOCK1) - let block1_hash = B256::with_last_byte(0xAA); - let (canon_tx, provider) = MockCanonProvider::new(); - let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); - - // Canonical tip for epoch A. - canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); - - for idx in 0..=10u64 { - let signed = AuthorizedPayload::new(builder_sk, auth_a, payload(pid_a, idx)); - handle.publish_new(signed).unwrap(); - } + let waiter = { + let h = handle.clone(); + task::spawn(async move { + h.await_clearance().await; + }) + }; - let first = next_flashblock(&mut stream).await; - assert_eq!(first.payload_id, pid_a); - assert_eq!(first.index, 0); + // give the waiter a chance to subscribe + tokio::task::yield_now().await; + assert!(!waiter.is_finished(), "future must still be pending"); - // Start epoch B on a *different* parent so the cursor can tell it apart. - let pid_b = PayloadId::new([9; 8]); - let auth_b = Authorization::new( - pid_b, - timestamp + 1, + // now grant clearance + let payload_id = reth::payload::PayloadId::new([5; 8]); + let auth = Authorization::new( + payload_id, + DUMMY_TIMESTAMP, &signing_key(1), builder_sk.verifying_key(), ); - handle.start_publishing(auth_b).unwrap(); - let signed = AuthorizedPayload::new( - builder_sk, - auth_b, - payload_with_parent(pid_b, 0, 2, block1_hash), - ); - handle.publish_new(signed).unwrap(); - - // Advance the canonical tip to match epoch B's parent. Already-emitted - // A flashblocks may still be in the pipeline, but the cursor will reject - // any new A flashblocks arriving after the tip change. Drain until we - // see B's flashblock. - canon_tx.send(canon_notification(1, block1_hash)).unwrap(); - - let flashblock = loop { - let fb = next_flashblock(&mut stream).await; - if fb.payload_id == pid_b { - break fb; - } - }; - assert_eq!(flashblock.index, 0); -} - -#[tokio::test] -async fn live_flashblock_stream_handles_out_of_order() { - let timestamp = 1000; - let handle = fresh_handle(); - let builder_sk = handle.builder_sk().unwrap(); - - let pid = PayloadId::new([8; 8]); - let auth = Authorization::new(pid, timestamp, &signing_key(1), builder_sk.verifying_key()); handle.start_publishing(auth).unwrap(); - // Create the event stream, then send the canonical tip. - let (canon_tx, provider) = MockCanonProvider::new(); - let mut stream = WorldChainEventsStream::new(handle.ctx.flashblock_tx.subscribe(), &provider); - - // Establish canonical tip. - canon_tx.send(canon_notification(0, B256::ZERO)).unwrap(); - - handle - .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 0))) - .unwrap(); - - assert_eq!(next_flashblock(&mut stream).await.index, 0); - - handle - .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 2))) - .unwrap(); - - // Assert not ready — index 2 cannot be delivered before index 1 arrives. - assert!( - tokio::time::timeout(Duration::from_millis(10), next_flashblock(&mut stream)) - .await - .is_err() - ); - - handle - .publish_new(AuthorizedPayload::new(builder_sk, auth, payload(pid, 1))) + // waiter should finish very quickly + tokio::time::timeout(Duration::from_secs(1), waiter) + .await + .expect("await_clearance did not complete") .unwrap(); - - assert_eq!(next_flashblock(&mut stream).await.index, 1); - assert_eq!(next_flashblock(&mut stream).await.index, 2); } diff --git a/crates/flashblocks/rpc/src/eth/pending_block.rs b/crates/flashblocks/rpc/src/eth/pending_block.rs index 228d84905..8a5995b47 100644 --- a/crates/flashblocks/rpc/src/eth/pending_block.rs +++ b/crates/flashblocks/rpc/src/eth/pending_block.rs @@ -40,11 +40,6 @@ where async fn local_pending_block( &self, ) -> Result::Primitives>>, Self::Error> { - let latest = self - .provider() - .latest_header()? - .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; - // check the pending block from the executor if let Some(pending_block) = self.pending_block.as_ref() { let pending_block = pending_block.borrow().clone(); @@ -67,6 +62,10 @@ where } // See: + let latest = self + .provider() + .latest_header()? + .ok_or(EthApiError::HeaderNotFound(BlockNumberOrTag::Latest.into()))?; let block_id = latest.hash().into(); let block = self .provider() diff --git a/crates/world/node/src/context.rs b/crates/world/node/src/context.rs index d54c20b98..9a78c9651 100644 --- a/crates/world/node/src/context.rs +++ b/crates/world/node/src/context.rs @@ -406,9 +406,10 @@ impl From for FlashblocksComponentsContext { authorizer_vk.as_bytes().encode_hex::() ); - let flashblocks_handle = FlashblocksHandle::new( + let flashblocks_handle = FlashblocksHandle::with_fanout_args( authorizer_vk, flashblocks.builder_sk.clone(), + flashblocks.fanout.clone(), ); let (pending_block, _) = tokio::sync::watch::channel(None); diff --git a/crates/world/node/tests/e2e-testsuite/actions.rs b/crates/world/node/tests/e2e-testsuite/actions.rs index baf24ea9f..54df02d66 100644 --- a/crates/world/node/tests/e2e-testsuite/actions.rs +++ b/crates/world/node/tests/e2e-testsuite/actions.rs @@ -5,7 +5,6 @@ use alloy_eips::{BlockId, Decodable2718}; use alloy_rpc_types::{Transaction, TransactionRequest}; use alloy_rpc_types_engine::{ForkchoiceState, PayloadStatusEnum}; use eyre::eyre::{Result, eyre}; -use flashblocks_p2p::protocol::event::{FlashblocksEvent, WorldChainEventsStream}; use flashblocks_primitives::{ flashblocks::{Flashblock, Flashblocks}, p2p::Authorization, @@ -25,9 +24,8 @@ use reth_e2e_test_utils::testsuite::{Environment, actions::Action}; use reth_node_api::{ConsensusEngineHandle, EngineApiMessageVersion}; use reth_optimism_chainspec::OpChainSpec; use reth_optimism_node::{OpEngineTypes, OpPayloadAttributes}; -use reth_optimism_primitives::{OpPrimitives, OpTransactionSigned}; +use reth_optimism_primitives::OpTransactionSigned; use reth_primitives::TransactionSigned; -use reth_provider::CanonStateSubscriptions; use revm_primitives::{Address, B256, Bytes, U256}; use std::{pin::Pin, sync::Arc, time::Duration}; use tokio::sync::{mpsc, watch}; @@ -1791,7 +1789,6 @@ pub struct DynamicValidateFlashblocks { pub beacon_handle: Arc>, pub chain_spec: Arc, pub state: BlockProductionState, - pub provider: Arc + Send + Sync>, } impl DynamicValidateFlashblocks { @@ -1800,14 +1797,12 @@ impl DynamicValidateFlashblocks { beacon_handle: Arc>, chain_spec: Arc, state: BlockProductionState, - provider: impl CanonStateSubscriptions + Send + Sync + 'static, ) -> Self { Self { flashblocks_handle, beacon_handle, chain_spec, state, - provider: Arc::new(provider), } } } @@ -1818,18 +1813,7 @@ impl Action for DynamicValidateFlashblocks { env: &'a mut Environment, ) -> BoxFuture<'a, Result<()>> { Box::pin(async move { - let stream = Box::pin( - WorldChainEventsStream::new( - self.flashblocks_handle.ctx.flashblock_tx.subscribe(), - &*self.provider, - ) - .filter_map(|event| async { - match event { - FlashblocksEvent::Pending(fb) => Some(fb), - _ => None, - } - }), - ); + let stream = Box::pin(self.flashblocks_handle.flashblock_stream()); let mut validate_action = ValidateFlashblocksWithState::new( stream, @@ -1850,18 +1834,7 @@ impl ReadOnlyAction for DynamicValidateFlashblocks { ) -> BoxFuture<'a, Result<()>> { Box::pin(async move { let mut flashblocks = Flashblocks::default(); - let mut stream = Box::pin( - WorldChainEventsStream::new( - self.flashblocks_handle.ctx.flashblock_tx.subscribe(), - &*self.provider, - ) - .filter_map(|event| async { - match event { - FlashblocksEvent::Pending(fb) => Some(fb), - _ => None, - } - }), - ); + let mut stream = Box::pin(self.flashblocks_handle.flashblock_stream()); // Wait for payload to be available let target_hash = loop { diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index 93d46a6f7..9dcf88f37 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -10,15 +10,13 @@ use alloy_primitives::{Bytes, b64}; use alloy_rpc_types::TransactionRequest; use alloy_rpc_types_engine::PayloadStatusEnum; use eyre::eyre::eyre; -use flashblocks_p2p::protocol::event::{FlashblocksEvent, WorldChainEventsStream}; -use futures::{StreamExt, future::Either}; +use futures::future::Either; use reth::{ chainspec::EthChainSpec, network::{NetworkSyncUpdater, SyncState}, }; use reth_e2e_test_utils::testsuite::actions::Action; use reth_optimism_node::utils::optimism_payload_attributes; -use reth_provider::CanonStateSubscriptions; use reth_transaction_pool::TransactionPool; use revm_primitives::{Address, B256, U256}; use std::{ @@ -408,22 +406,11 @@ async fn test_flashblocks() -> eyre::Result<()> { .await; let cannon_flashblocks_stream = Box::pin( - WorldChainEventsStream::new( - builder_context - .as_ref() - .unwrap() - .flashblocks_handle - .ctx - .flashblock_tx - .subscribe(), - &builder_node.node.inner.provider, - ) - .filter_map(|event| async { - match event { - FlashblocksEvent::Pending(fb) => Some(fb), - _ => None, - } - }), + builder_context + .as_ref() + .unwrap() + .flashblocks_handle + .flashblock_stream(), ); let validation_stream = crate::actions::FlashblocksValidatonStream { @@ -496,17 +483,12 @@ async fn test_eth_api_receipt() -> eyre::Result<()> { Some(vec![crate::setup::TX_SET_L1_BLOCK.clone()]), ); - let handle = &nodes[0].ext_context.clone().unwrap().flashblocks_handle; - let cannon_flashblocks_stream = WorldChainEventsStream::new( - handle.ctx.flashblock_tx.subscribe(), - &nodes[0].node.inner.provider, - ) - .filter_map(|event| async { - match event { - FlashblocksEvent::Pending(fb) => Some(fb), - _ => None, - } - }); + let cannon_flashblocks_stream = nodes[0] + .ext_context + .clone() + .unwrap() + .flashblocks_handle + .flashblock_stream(); let mine_block = crate::actions::AssertMineBlock::new( 0, @@ -665,17 +647,12 @@ async fn test_eth_block_by_hash_pending() -> eyre::Result<()> { spammer.spawn(20, nodes[0].node.rpc_url()); - let handle = &nodes[0].ext_context.clone().unwrap().flashblocks_handle; - let cannon_flashblocks_stream = WorldChainEventsStream::new( - handle.ctx.flashblock_tx.subscribe(), - &nodes[0].node.inner.provider, - ) - .filter_map(|event| async { - match event { - FlashblocksEvent::Pending(fb) => Some(fb), - _ => None, - } - }); + let cannon_flashblocks_stream = nodes[0] + .ext_context + .clone() + .unwrap() + .flashblocks_handle + .flashblock_stream(); let (sender, mut rx) = tokio::sync::mpsc::channel(1); let timestamp = crate::setup::current_timestamp(); @@ -1128,7 +1105,6 @@ async fn test_continuous_block_production_with_validation() -> eyre::Result<()> basic_beacon_handle, chain_spec.clone(), state.clone(), - follower_0.node.inner.provider.clone(), ), ) // 3. Query validated blocks and receipts in parallel (AFTER mining/validation) From 899d4786a9ea2232985d6a4a7b156ac5b5494b13 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 17:58:38 -0700 Subject: [PATCH 30/43] feat: wire things together --- Cargo.lock | 1 - crates/flashblocks/builder/Cargo.toml | 1 - crates/flashblocks/builder/src/coordinator.rs | 92 ++-- crates/flashblocks/node/tests/p2p.rs | 17 +- .../p2p/src/protocol/connection.rs | 4 +- crates/flashblocks/p2p/src/protocol/event.rs | 393 +++++++----------- .../flashblocks/p2p/src/protocol/handler.rs | 75 +++- 7 files changed, 262 insertions(+), 321 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4964ba0f..480e72e5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3752,7 +3752,6 @@ dependencies = [ "alloy-signer-local", "alloy-sol-types", "alloy-trie", - "backon", "bon", "color-eyre", "crossbeam-channel", diff --git a/crates/flashblocks/builder/Cargo.toml b/crates/flashblocks/builder/Cargo.toml index fd203209a..c0bfbaeaf 100644 --- a/crates/flashblocks/builder/Cargo.toml +++ b/crates/flashblocks/builder/Cargo.toml @@ -62,7 +62,6 @@ thiserror.workspace = true either.workspace = true metrics.workspace = true serde.workspace = true -backon.workspace = true [dev-dependencies] serde.workspace = true diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index d167446c9..5c64c20fc 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -6,14 +6,13 @@ use flashblocks_p2p::protocol::{ handler::FlashblocksHandle, }; use flashblocks_primitives::{p2p::AuthorizedPayload, primitives::FlashblocksPayloadV1}; -use futures::{FutureExt, StreamExt as _}; +use futures::StreamExt; use op_alloy_consensus::{OpTxEnvelope, encode_holocene_extra_data}; use parking_lot::RwLock; use reth::{ payload::EthPayloadBuilderAttributes, revm::{cancelled::CancelOnDrop, database::StateProviderDatabase}, rpc::types::BlockNumHash, - tasks::TaskSpawner, }; use reth_basic_payload_builder::PayloadConfig; use reth_chain_state::{DeferredTrieData, ExecutedBlock}; @@ -41,7 +40,7 @@ use tokio::sync::{ broadcast::{self, Sender}, oneshot, }; -use tracing::{error, trace, warn}; +use tracing::{error, trace}; /// Maximum number of concurrent flashblock processing tasks on the thread pool. const MAX_THREAD_POOL_SIZE: usize = 4; @@ -57,7 +56,6 @@ use crate::{ payload_builder::build, traits::{context::OpPayloadBuilderCtxBuilder, context_builder::PayloadBuilderCtxBuilder}, }; -use backon::BlockingRetryable; use flashblocks_primitives::flashblocks::{Flashblock, Flashblocks}; /// The maximum backoff duration when waiting for the parent header to be available in the database when processing a flashblock. @@ -129,7 +127,7 @@ impl FlashblocksExecutionCoordinator { { let provider = ctx.provider().clone(); let mut stream: WorldChainEventsStream = - self.p2p_handle.event_stream(provider.clone()); + self.p2p_handle.event_stream(provider.clone(), |_| None); let this = self.clone(); let chain_spec = ctx.chain_spec().clone(); @@ -149,33 +147,36 @@ impl FlashblocksExecutionCoordinator { while let Some(event) = stream.next().await { match event { - WorldChainEvent::Chain(ChainEvent::Pending(flashblock)) => { - // Track epoch block number from base flashblocks - if let Some(base) = &flashblock.base { - epoch_block_number = Some(base.block_number); + WorldChainEvent::Chain(chain_event) => match *chain_event { + ChainEvent::Pending(flashblock) => { + let flashblock = *flashblock; + // Track epoch block number from base flashblocks + if let Some(base) = &flashblock.base { + epoch_block_number = Some(base.block_number); + } + + this.on_flashblock( + flashblock, + &mut inflight_shutdown, + &task_permit, + database_permit.clone(), + &workload, + &provider, + &evm_config, + &chain_spec, + &pending_block, + ) + .await; } - - this.on_flashblock( - flashblock, - &mut inflight_shutdown, - &task_permit, - database_permit.clone(), - &workload, - &provider, - &evm_config, - &chain_spec, - &pending_block, - ) - .await; - } - WorldChainEvent::Chain(ChainEvent::Canon(tip)) => { - this.on_canon( - tip, - &mut inflight_shutdown, - &mut epoch_block_number, - &pending_block, - ); - } + ChainEvent::Canon(tip) => { + this.on_canon( + tip, + &mut inflight_shutdown, + &mut epoch_block_number, + &pending_block, + ); + } + }, WorldChainEvent::Event(_) => {} } } @@ -211,7 +212,7 @@ impl FlashblocksExecutionCoordinator { let (tx, rx) = oneshot::channel::<()>(); *shutdown_tx = Some(tx); - let permit = task_permit + let _permit = task_permit .clone() .acquire_owned() .await @@ -400,21 +401,20 @@ where let (base, is_new_epoch) = { let inner = coordinator.inner.read(); - if let Some(latest_payload) = &inner.latest_payload { - if latest_payload.0.id() == flashblock.flashblock.payload_id - && latest_payload.1 >= flashblock.flashblock.index - { - // Already processed — send current pending block and return - if let Some(executed) = latest_payload.0.executed_block() { - let block = ExecutedBlock::with_deferred_trie_data( - executed.recovered_block.clone(), - executed.execution_output.clone(), - DeferredTrieData::ready(Default::default()), - ); - pending_block.send_replace(Some(block)); - } - return Ok(()); + if let Some(latest_payload) = &inner.latest_payload + && latest_payload.0.id() == flashblock.flashblock.payload_id + && latest_payload.1 >= flashblock.flashblock.index + { + // Already processed — send current pending block and return + if let Some(executed) = latest_payload.0.executed_block() { + let block = ExecutedBlock::with_deferred_trie_data( + executed.recovered_block.clone(), + executed.execution_output.clone(), + DeferredTrieData::ready(Default::default()), + ); + pending_block.send_replace(Some(block)); } + return Ok(()); } let is_new = inner.flashblocks.is_new_payload(&flashblock)?; diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 886103408..8a1c7dd6a 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -8,8 +8,10 @@ use eyre::eyre::eyre; use flashblocks_cli::FlashblocksArgs; use flashblocks_p2p::{ monitor, - protocol::connection::ReceiveStatus, - protocol::handler::{FlashblocksHandle, PublishingStatus}, + protocol::{ + connection::ReceiveStatus, + handler::{FlashblocksHandle, PublishingStatus}, + }, }; use flashblocks_primitives::{ flashblocks::FlashblockMetadata, @@ -242,8 +244,7 @@ async fn wait_for_flashblocks_topology( .connections .iter() .filter_map(|(peer_id, conn)| { - (conn.receive_status == ReceiveStatus::NotReceiving) - .then_some(*peer_id) + (conn.receive_status == ReceiveStatus::NotReceiving).then_some(*peer_id) }) .collect(); drop(state); @@ -1015,10 +1016,10 @@ async fn test_peer_reputation() -> eyre::Result<()> { .send_serialized_to_all_peers(bytes.clone()); sleep(Duration::from_millis(10)).await; let rep_0 = nodes[1].network_handle.reputation_by_id(*peer_0).await?; - if let Some(rep) = rep_0 { - if rep < 0 { - reputation_was_negative = true; - } + if let Some(rep) = rep_0 + && rep < 0 + { + reputation_was_negative = true; } if nodes[1].network_handle.get_all_peers().await?.is_empty() { peer_banned = true; diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 79cd77b2c..59874c6d0 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -1,5 +1,5 @@ use crate::protocol::handler::{ - FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, PublishingStatus, MAX_FLASHBLOCK_INDEX, + FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, MAX_FLASHBLOCK_INDEX, PublishingStatus, }; use alloy_primitives::bytes::BytesMut; use chrono::Utc; @@ -15,7 +15,7 @@ use reth_ethereum::network::{api::PeerId, eth_wire::multiplex::ProtocolConnectio use reth_network::types::ReputationChangeKind; use std::{ pin::Pin, - task::{ready, Context, Poll}, + task::{Context, Poll, ready}, time::Instant, }; use tokio::sync::mpsc; diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs index 7447ab42a..6d76d2bba 100644 --- a/crates/flashblocks/p2p/src/protocol/event.rs +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -7,17 +7,17 @@ use flashblocks_primitives::primitives::FlashblocksPayloadV1; use futures::{ - future::{self, Either}, - stream::{self, PollNext}, Stream, StreamExt, + future::{self}, + stream::{self, PollNext}, }; use reth::{ api::NodePrimitives, payload::PayloadId, providers::CanonStateSubscriptions, rpc::types::BlockNumHash, }; use std::{ + collections::VecDeque, fmt::Debug, - marker::PhantomData, pin::Pin, task::{Context, Poll}, }; @@ -32,7 +32,7 @@ pub enum ChainEvent { /// A flashblock has been received whose epoch parent matches the current /// canonical tip. Consumers can treat this as a "pending" event and buffer /// it until the next Canon event confirms it's ready to be processed. - Pending(FlashblocksPayloadV1), + Pending(Box), } impl ChainEvent { @@ -47,19 +47,19 @@ impl ChainEvent { impl From for WorldChainEvent { fn from(value: ChainEvent) -> Self { - WorldChainEvent::Chain(value) + WorldChainEvent::Chain(Box::new(value)) } } impl From for WorldChainEvent { fn from(value: FlashblocksPayloadV1) -> Self { - WorldChainEvent::Chain(ChainEvent::Pending(value)) + WorldChainEvent::Chain(Box::new(ChainEvent::Pending(Box::new(value)))) } } impl From for WorldChainEvent { fn from(value: BlockNumHash) -> Self { - WorldChainEvent::Chain(ChainEvent::Canon(value)) + WorldChainEvent::Chain(Box::new(ChainEvent::Canon(value))) } } @@ -67,7 +67,7 @@ impl From for WorldChainEvent { #[derive(Clone, Debug)] pub enum WorldChainEvent { /// An Event emitted when executable pending flashblocks are observed. - Chain(ChainEvent), + Chain(Box), /// A Event emitted by any source. Event(T), } @@ -86,6 +86,13 @@ pub type WorldChainEventNotificationsStream = /// matches the canonical tip. Stale flashblocks are silently discarded via /// [`PendingCursor::try_advance`]. A [`ChainEvent::Canon`] is emitted on every /// canonical tip change so consumers can clear pending state. +/// A stream of [`WorldChainEvent`]s that merges flashblocks with canonical +/// chain notifications, reducing them through a [`BufferedCursor`](sealed::BufferedCursor) +/// state machine. +/// +/// Implements [`Stream`] directly — polls the merged inner streams, feeds +/// each [`ChainEvent`] through the cursor's `reduce`, and yields the output +/// events one at a time. #[pin_project::pin_project] pub struct WorldChainEventsStream { #[pin] @@ -98,7 +105,7 @@ impl WorldChainEventsStream { pub fn new( provider: P, rx: broadcast::Receiver, - ) -> WorldChainEventsStream + ) -> Self where P: CanonStateSubscriptions + Clone + Send + Sync + 'static, { @@ -112,11 +119,11 @@ impl WorldChainEventsStream { } }) }) - .map(Into::into); + .map(|fb| ChainEvent::Pending(Box::new(fb))); let canon = provider .canonical_state_stream() - .map(|n| WorldChainEvent::Chain(ChainEvent::Canon(n.tip().num_hash().into()))); + .map(|n| ChainEvent::Canon(n.tip().num_hash())); Self::new_with_hook( move |_: &WorldChainEvent| None, @@ -125,24 +132,39 @@ impl WorldChainEventsStream { ) } - /// Creates a new [`WorldChainEventsStream`] mapping all `ChainEvent`s through `hook` before yielding them. - pub fn new_with_hook<'a, F>( + /// Creates a new [`WorldChainEventsStream`] with a hook that can inject + /// additional events after each yielded event. + pub fn new_with_hook( mut hook: F, - st_0: WorldChainEventNotificationsStream, - st_1: WorldChainEventNotificationsStream, + flashblocks: Pin + Send>>, + canon: Pin + Send>>, ) -> Self where F: FnMut(&WorldChainEvent) -> Option> + Send + 'static, { - let merged = merge_flashblocks_with_canon(st_0, st_1); + let merged = + futures::stream::select_with_strategy(flashblocks, canon, |_: &mut ()| -> PollNext { + PollNext::Left + }); + + // Fold through the BufferedFlashblocks reducer, then flat_map output. let st = merged + .scan(BufferedFlashblocks::default(), |cursor, event| { + cursor.step(event); + let events: Vec> = cursor + .by_ref() + .map(|ce| WorldChainEvent::Chain(Box::new(ce))) + .collect(); + future::ready(Some(events)) + }) + .flat_map(stream::iter) .flat_map(move |event| { let extra = hook(&event); stream::iter(std::iter::once(event).chain(extra)) }) .boxed(); - Self { st: Box::pin(st) } + Self { st } } } @@ -153,53 +175,31 @@ impl Stream for WorldChainEventsStream { self.st.as_mut().poll_next(cx) } } - -/// Merges a flashblock stream with a canonical state notification stream, -/// using [`BufferState`] to buffer, order, and gate flashblocks by the -/// canonical tip. Only yields flashblocks whose epoch parent is canonical. -fn merge_flashblocks_with_canon( - st_0: WorldChainEventNotificationsStream, - st_1: WorldChainEventNotificationsStream, -) -> WorldChainEventNotificationsStream { - futures::stream::select_with_strategy( - st_0.map(Either::Left), - st_1.map(Either::Right), - |_: &mut ()| -> PollNext { PollNext::Left }, - ) - .scan(BufferState::default(), |state, event| { - futures::future::ready(Some(match event { - Either::Left(WorldChainEvent::Chain(ChainEvent::Pending(fb))) => state.advance(fb), - Either::Right(WorldChainEvent::Chain(ChainEvent::Canon(tip))) => state.process(tip), - _ => vec![], - })) - }) - .flat_map(stream::iter) - .boxed() -} // --------------------------------------------------------------------------- -// Type-state markers +// BufferedFlashblocks — stateful reducer with Extend + Iterator // --------------------------------------------------------------------------- -/// No active epoch — waiting for a base flashblock. -pub struct Idle; -/// Have a payload_id but parent is not yet canonical — buffering only. -pub struct Unanchored; -/// Parent is canonical — can drain contiguous flashblocks. -pub struct Anchored; - const MAX_FLASHBLOCK_INDEX: usize = 100; -// --------------------------------------------------------------------------- -// BufferedFlashblocks -// --------------------------------------------------------------------------- +/// Internal phase of the buffer state machine. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Phase { + /// No active epoch. + Uninitialized, + /// Epoch started but parent is not yet canonical. + Pending, + /// Parent is canonical — flashblocks can be drained. + Executable, +} -/// Tracks and buffers flashblocks for the current epoch with type-state -/// transitions governing when flashblocks may be drained. +/// Buffers flashblocks for the current epoch, gating output on canonical tip. /// -/// - [`Idle`]: No active epoch. Accepts a base flashblock to start one. -/// - [`Unanchored`]: Epoch started but parent not yet canonical. Buffers only. -/// - [`Anchored`]: Parent is canonical. Can drain contiguous flashblocks. -pub struct BufferedFlashblocks { +/// Implements [`Extend`] to accept input events and +/// [`Iterator`] to drain output events. The internal +/// state machine transitions between `Uninitialized`, `Pending`, and +/// `Executable` phases automatically. +pub struct BufferedFlashblocks { + phase: Phase, /// The parent block this epoch builds on. parent_num_hash: BlockNumHash, /// Current epoch payload identifier. @@ -212,21 +212,34 @@ pub struct BufferedFlashblocks { buffer: Vec>, /// Most recent canonical tip. canon_tip: Option, - _state: PhantomData, + /// Output events ready to be yielded by the iterator. + output: VecDeque, } /// A flashblock drained from the buffer, ready for the coordinator. pub struct PendingFlashblockEvent { /// Resolve when the flashblock is in the in-memory tree. - /// Dropping without sending signals cancellation. pub tx: oneshot::Sender<()>, /// The drained flashblock payload. pub flashblock: FlashblocksPayloadV1, } -// -- Shared methods (all states) -- +impl Default for BufferedFlashblocks { + fn default() -> Self { + Self { + phase: Phase::Uninitialized, + parent_num_hash: BlockNumHash::default(), + payload_id: PayloadId::default(), + timestamp: 0, + cursor: 0, + buffer: Vec::new(), + canon_tip: None, + output: VecDeque::new(), + } + } +} -impl BufferedFlashblocks { +impl BufferedFlashblocks { /// Insert a flashblock at its index. Returns `false` if the payload_id /// doesn't match, the index exceeds the maximum, or the slot is occupied. fn insert(&mut self, fb: &FlashblocksPayloadV1) -> bool { @@ -246,52 +259,27 @@ impl BufferedFlashblocks { true } - fn set_canon_tip(&mut self, tip: BlockNumHash) { - self.canon_tip = Some(tip); - } - - /// Reset to [`Idle`], discarding all buffered flashblocks. - fn reset(self) -> BufferedFlashblocks { - BufferedFlashblocks { - parent_num_hash: BlockNumHash::default(), - payload_id: PayloadId::default(), - timestamp: 0, - cursor: 0, - buffer: Vec::new(), - canon_tip: self.canon_tip, - _state: PhantomData, - } + /// Reset to uninitialized, discarding all buffered flashblocks. + fn reset(&mut self) { + self.phase = Phase::Uninitialized; + self.parent_num_hash = BlockNumHash::default(); + self.payload_id = PayloadId::default(); + self.timestamp = 0; + self.cursor = 0; + self.buffer.clear(); + // canon_tip is preserved } -} - -// -- Idle -- -impl Default for BufferedFlashblocks { - fn default() -> Self { - Self { - parent_num_hash: BlockNumHash::default(), - payload_id: PayloadId::default(), - timestamp: 0, - cursor: 0, - buffer: Vec::new(), - canon_tip: None, - _state: PhantomData, - } - } -} - -impl BufferedFlashblocks { - /// Accept a base flashblock and start a new epoch. - /// Returns `None` if the flashblock has no base or is stale. - fn accept_base( - mut self, - fb: FlashblocksPayloadV1, - ) -> Option, BufferedFlashblocks>> { - let base = fb.base.as_ref()?; + /// Try to start a new epoch from a base flashblock. + fn accept_base(&mut self, fb: FlashblocksPayloadV1) -> bool { + let Some(base) = fb.base.as_ref() else { + return false; + }; + // Stale check if let Some(tip) = &self.canon_tip { if base.timestamp <= tip.number { - return None; + return false; } } @@ -305,177 +293,80 @@ impl BufferedFlashblocks { self.buffer.clear(); self.insert(&fb); - let anchored = self + self.phase = if self .canon_tip - .is_some_and(|tip| tip == self.parent_num_hash); - - if anchored { - Some(Either::Right(BufferedFlashblocks { - parent_num_hash: self.parent_num_hash, - payload_id: self.payload_id, - timestamp: self.timestamp, - cursor: self.cursor, - buffer: self.buffer, - canon_tip: self.canon_tip, - _state: PhantomData, - })) + .is_some_and(|tip| tip == self.parent_num_hash) + { + Phase::Executable } else { - Some(Either::Left(BufferedFlashblocks { - parent_num_hash: self.parent_num_hash, - payload_id: self.payload_id, - timestamp: self.timestamp, - cursor: self.cursor, - buffer: self.buffer, - canon_tip: self.canon_tip, - _state: PhantomData, - })) - } - } -} + Phase::Pending + }; -// -- Unanchored -- + true + } -impl BufferedFlashblocks { - /// If the canon tip matches our parent, transition to [`Anchored`]. - fn try_anchor(self) -> Either, BufferedFlashblocks> { - if self - .canon_tip - .is_some_and(|tip| tip == self.parent_num_hash) + /// If pending and canon tip matches parent, transition to executable. + fn try_anchor(&mut self) { + if self.phase == Phase::Pending + && self + .canon_tip + .is_some_and(|tip| tip == self.parent_num_hash) { - Either::Right(BufferedFlashblocks { - parent_num_hash: self.parent_num_hash, - payload_id: self.payload_id, - timestamp: self.timestamp, - cursor: self.cursor, - buffer: self.buffer, - canon_tip: self.canon_tip, - _state: PhantomData, - }) - } else { - Either::Left(self) + self.phase = Phase::Executable; } } -} - -// -- Anchored -- -impl BufferedFlashblocks { - /// Drain all contiguous flashblocks from the cursor, yielding a - /// [`PendingFlashblockEvent`] for each. Takes ownership via `.take()`. - fn drain_contiguous(&mut self) -> Vec { - let mut events = Vec::new(); + /// Drain all contiguous flashblocks from the cursor into the output queue. + fn drain_contiguous(&mut self) { + if self.phase != Phase::Executable { + return; + } while let Some(Some(_)) = self.buffer.get(self.cursor) { let fb = self.buffer[self.cursor].take().unwrap(); self.cursor += 1; - let (tx, _rx) = oneshot::channel(); - events.push(PendingFlashblockEvent { tx, flashblock: fb }); + self.output.push_back(ChainEvent::Pending(Box::new(fb))); } - events } -} -// --------------------------------------------------------------------------- -// Runtime state enum (for scan closures) -// --------------------------------------------------------------------------- - -enum BufferState { - Idle(BufferedFlashblocks), - Unanchored(BufferedFlashblocks), - Anchored(BufferedFlashblocks), -} - -impl Default for BufferState { - fn default() -> Self { - Self::Idle(BufferedFlashblocks::default()) - } -} - -impl BufferState { - /// Advance the buffer with a new flashblock. Handles epoch starts, - /// inserts, and state transitions. Returns events to yield downstream. - fn advance(&mut self, fb: FlashblocksPayloadV1) -> Vec> { - if fb.base.is_some() { - // Base flashblock → reset and start new epoch - let idle = match std::mem::take(self) { - Self::Idle(b) => b, - Self::Unanchored(b) => b.reset(), - Self::Anchored(b) => b.reset(), - }; - - match idle.accept_base(fb) { - Some(Either::Right(mut anchored)) => { - let events = Self::drain_to_events(&mut anchored); - *self = Self::Anchored(anchored); - events - } - Some(Either::Left(unanchored)) => { - *self = Self::Unanchored(unanchored); - vec![] - } - None => vec![], // stale, discarded - } - } else { - // Non-base → insert into current buffer - match self { - Self::Idle(_) => {} // no epoch, ignore - Self::Unanchored(b) => { - b.insert(&fb); - } - Self::Anchored(b) => { - b.insert(&fb); + /// Process a single input event, updating state and buffering output. + fn step(&mut self, event: ChainEvent) { + match event { + ChainEvent::Pending(fb) => { + if fb.base.is_some() { + // New epoch — reset and try to accept + self.reset(); + self.accept_base(*fb); + self.drain_contiguous(); + } else { + // Non-base — insert into current buffer if we have an epoch + if self.phase != Phase::Uninitialized { + self.insert(&fb); + self.drain_contiguous(); + } } } - - // If anchored, drain any newly contiguous flashblocks - if let Self::Anchored(anchored) = self { - Self::drain_to_events(anchored) - } else { - vec![] + ChainEvent::Canon(tip) => { + self.canon_tip = Some(tip); + self.output.push_back(ChainEvent::Canon(tip)); + self.try_anchor(); + self.drain_contiguous(); } } } +} - /// Process a new canonical tip. Updates state and potentially anchors - /// the buffer, draining any contiguous flashblocks. - fn process(&mut self, tip: BlockNumHash) -> Vec> { - // Always emit the canon event - let mut events: Vec> = vec![tip.into()]; - - match self { - Self::Idle(b) => b.set_canon_tip(tip), - Self::Unanchored(_) => { - // Try to anchor - let unanchored = match std::mem::take(self) { - Self::Unanchored(mut b) => { - b.set_canon_tip(tip); - b - } - _ => unreachable!(), - }; - - match unanchored.try_anchor() { - Either::Right(mut anchored) => { - events.extend(Self::drain_to_events(&mut anchored)); - *self = Self::Anchored(anchored); - } - Either::Left(still_unanchored) => { - *self = Self::Unanchored(still_unanchored); - } - } - } - Self::Anchored(b) => b.set_canon_tip(tip), +impl Extend for BufferedFlashblocks { + fn extend>(&mut self, iter: I) { + for event in iter { + self.step(event); } - - events } +} + +impl Iterator for BufferedFlashblocks { + type Item = ChainEvent; - fn drain_to_events( - anchored: &mut BufferedFlashblocks, - ) -> Vec> { - anchored - .drain_contiguous() - .into_iter() - .map(|e| WorldChainEvent::::from(e.flashblock)) - .collect() + fn next(&mut self) -> Option { + self.output.pop_front() } } diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 5fa85e41f..a028f9702 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,6 +1,6 @@ use crate::protocol::{ connection::{FlashblocksConnection, FlashblocksConnectionState, ReceiveStatus, Score}, - error::FlashblocksP2PError, + error::FlashblocksP2PError, event::{ChainEvent, WorldChainEvent, WorldChainEventsStream}, }; use alloy_rlp::BytesMut; use chrono::Utc; @@ -13,11 +13,10 @@ use flashblocks_primitives::{ }, primitives::FlashblocksPayloadV1, }; -use futures::{Stream, StreamExt, stream}; -use metrics::histogram; +use futures::{Stream, StreamExt as _}; use parking_lot::Mutex; -use rand::{Rng, seq::SliceRandom}; -use reth::payload::PayloadId; +use rand::Rng; +use reth::{payload::PayloadId, rpc::types::BlockNumHash}; use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; use reth_network::Peers; @@ -150,6 +149,9 @@ pub struct FlashblocksP2PState { pub payload_timestamp: u64, /// Timestamp at which the most recent flashblock was received in ns since the unix epoch. pub flashblock_timestamp: i64, + /// Most recent canonical tip. Updated by the stream hook. + /// Used to reject stale flashblocks in `publish()`. + pub canon_tip: Option, /// Flashblocks observed from network peers, tracked until their receive grace windows expire. pub observed_payloads: VecDeque, /// All currently connected peers and their connection state. @@ -165,6 +167,7 @@ impl Default for FlashblocksP2PState { payload_id: PayloadId::default(), payload_timestamp: 0, flashblock_timestamp: 0, + canon_tip: None, observed_payloads: VecDeque::new(), connections: HashMap::new(), } @@ -615,18 +618,57 @@ impl FlashblocksHandle { /// Returns a [`WorldChainEventsStream`] merging flashblocks from the P2P /// broadcast channel with canonical chain notifications from `provider`. - pub fn event_stream( + /// + /// Canon events automatically update the P2P state's `canon_tip` so + /// `publish()` rejects stale flashblocks. The caller's `hook` is applied + /// after the canon_tip update. + pub fn event_stream( &self, provider: P, - ) -> crate::protocol::event::WorldChainEventsStream + hook: F, + ) -> WorldChainEventsStream where T: Send + Clone + Unpin + 'static, P: reth::providers::CanonStateSubscriptions + Clone + Send + Sync + 'static, N: reth::api::NodePrimitives, + F: FnMut( + &WorldChainEvent, + ) -> Option> + + Send + + 'static, { - crate::protocol::event::WorldChainEventsStream::new( - provider, - self.ctx.flashblock_tx.subscribe(), + let state = self.state.clone(); + let mut user_hook = hook; + + let combined_hook = move |event: &WorldChainEvent| { + if let WorldChainEvent::Chain(ce) = event { + if let ChainEvent::Canon(tip) = ce.as_ref() { + state.lock().canon_tip = Some(*tip); + } + } + user_hook(event) + }; + + WorldChainEventsStream::new_with_hook( + combined_hook, + BroadcastStream::new(self.ctx.flashblock_tx.subscribe()) + .filter_map(|x| { + futures::future::ready(match x { + Ok(fb) => Some(fb), + Err(tokio_stream::wrappers::errors::BroadcastStreamRecvError::Lagged( + n, + )) => { + tracing::warn!(missed = n, "flashblocks broadcast receiver lagged"); + None + } + }) + }) + .map(|fb| ChainEvent::Pending(Box::new(fb))) + .boxed(), + provider + .canonical_state_stream() + .map(|n| ChainEvent::Canon(n.tip().num_hash())) + .boxed(), ) } @@ -1022,6 +1064,7 @@ impl FlashblocksP2PCtx { /// - Caches flashblocks and maintains ordering for sequential delivery /// - Forwards flashblocks to peers in the current send set and publishes ordered /// flashblocks to the local stream + /// /// Publishes a verified flashblock payload to peers and the local broadcast channel. /// /// Ordering, buffering, and canon-gating are handled downstream by @@ -1045,6 +1088,15 @@ impl FlashblocksP2PCtx { return; } + // Reject flashblocks for epochs that have already been canonicalized. + if let Some(canon_tip) = &state.canon_tip { + if let Some(base) = &payload.base { + if base.block_number.saturating_sub(1) <= canon_tip.number { + return; + } + } + } + if authorization.timestamp > state.payload_timestamp { state.payload_id = authorization.payload_id; state.payload_timestamp = authorization.timestamp; @@ -1074,8 +1126,7 @@ impl FlashblocksP2PCtx { metrics::histogram!("flashblocks.size").record(len as f64); metrics::histogram!("flashblocks.gas_used").record(payload.diff.gas_used as f64); - metrics::histogram!("flashblocks.tx_count") - .record(payload.diff.transactions.len() as f64); + metrics::histogram!("flashblocks.tx_count").record(payload.diff.transactions.len() as f64); state.send_flashblock_to_send_set(payload.payload_id, payload.index, &bytes); From b985445b2a19c259319f718b16305d6702acf227 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 18:13:28 -0700 Subject: [PATCH 31/43] chore: clenaup --- crates/flashblocks/builder/src/coordinator.rs | 46 +++++++++---------- crates/flashblocks/p2p/src/protocol/event.rs | 15 +++--- .../flashblocks/p2p/src/protocol/handler.rs | 25 ++++++---- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 5c64c20fc..21e46d3bc 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -30,13 +30,9 @@ use reth_provider::{ CanonStateSubscriptions, ChainSpecProvider, HeaderProvider, StateProviderFactory, }; use reth_transaction_pool::{EthPooledTransaction, noop::NoopTransactionPool}; -use std::{ - panic::AssertUnwindSafe, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{panic::AssertUnwindSafe, sync::Arc, time::Instant}; use tokio::sync::{ - OwnedSemaphorePermit, Semaphore, + Semaphore, SemaphorePermit, broadcast::{self, Sender}, oneshot, }; @@ -45,8 +41,8 @@ use tracing::{error, trace}; /// Maximum number of concurrent flashblock processing tasks on the thread pool. const MAX_THREAD_POOL_SIZE: usize = 4; -/// Task handle for deferred trie computation. Intentionally empty for now — -/// cancellation flows through `oneshot::Sender` drop, not explicit events. +/// Placeholder for future task handle variants. Currently unused — the +/// hook updates P2P state directly via the flushed cursor. #[derive(Clone, Debug)] pub enum TrieTaskHandle {} @@ -58,15 +54,9 @@ use crate::{ }; use flashblocks_primitives::flashblocks::{Flashblock, Flashblocks}; -/// The maximum backoff duration when waiting for the parent header to be available in the database when processing a flashblock. -const FETCH_PARENT_HEADER_MAX_DELAY: Duration = Duration::from_millis(2000); - -/// The minimum backoff duration when waiting for the parent header to be available in the database when processing a flashblock. -const FETCH_PARENT_HEADER_MIN_DELAY: Duration = Duration::from_millis(100); - /// Semaphore locking the [`WorkloadExecutor`] thread pool for flashblock processing tasks. /// Ensures the Pending Block is always in sync when a concurrent task is spawned. -const PENDING_BLOCK_WRITE_PERMIT: Semaphore = Semaphore::const_new(1); +static PENDING_BLOCK_WRITE_PERMIT: Semaphore = Semaphore::const_new(1); /// The current state of all known pre confirmations received over the P2P layer /// or generated from the payload building job of this node. @@ -126,8 +116,18 @@ impl FlashblocksExecutionCoordinator { Node::Types: NodeTypes, { let provider = ctx.provider().clone(); + let p2p_state = self.p2p_handle.state.clone(); let mut stream: WorldChainEventsStream = - self.p2p_handle.event_stream(provider.clone(), |_| None); + self.p2p_handle.event_stream(provider.clone(), move |event| { + if let WorldChainEvent::Chain(ce) = event + && let ChainEvent::Pending(fb) = ce.as_ref() + { + let mut state = p2p_state.lock(); + state.flushed_payload_id = Some(fb.payload_id); + state.flushed_index = fb.index; + } + None + }); let this = self.clone(); let chain_spec = ctx.chain_spec().clone(); @@ -137,7 +137,7 @@ impl FlashblocksExecutionCoordinator { let workload = WorkloadExecutor::default(); let task_permit = Arc::new(Semaphore::new(MAX_THREAD_POOL_SIZE)); - let database_permit = Arc::new(PENDING_BLOCK_WRITE_PERMIT); + let database_permit = &PENDING_BLOCK_WRITE_PERMIT; ctx.task_executor() .spawn_critical("flashblocks executor", async move { @@ -159,7 +159,7 @@ impl FlashblocksExecutionCoordinator { flashblock, &mut inflight_shutdown, &task_permit, - database_permit.clone(), + database_permit, &workload, &provider, &evm_config, @@ -191,7 +191,7 @@ impl FlashblocksExecutionCoordinator { flashblock: FlashblocksPayloadV1, shutdown_tx: &mut Option>, task_permit: &Arc, - database_permit: Arc, + database_permit: &'static Semaphore, workload: &WorkloadExecutor, provider: &Provider, evm_config: &OpEvmConfig, @@ -341,10 +341,10 @@ impl FlashblocksExecutionCoordinator { fn spawn_blocking_io_with_shutdown_signal( executor: &WorkloadExecutor, shutdown_rx: oneshot::Receiver<()>, - database_permit: Arc, + database_permit: &'static Semaphore, f: F, ) where - F: FnOnce(OwnedSemaphorePermit) + Send + 'static, + F: FnOnce(SemaphorePermit<'static>) + Send + 'static, { let task = executor.spawn_blocking(move || { let f = AssertUnwindSafe(move || { @@ -353,7 +353,7 @@ fn spawn_blocking_io_with_shutdown_signal( .expect("failed to build runtime for permit acquisition"); let permit = rt - .block_on(database_permit.acquire_owned()) + .block_on(database_permit.acquire()) .expect("database semaphore closed"); f(permit); @@ -380,7 +380,7 @@ fn spawn_blocking_io_with_shutdown_signal( } fn process_flashblock( - database_permit: OwnedSemaphorePermit, + database_permit: SemaphorePermit<'static>, provider: Provider, evm_config: &OpEvmConfig, coordinator: &FlashblocksExecutionCoordinator, diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs index 6d76d2bba..5383db42b 100644 --- a/crates/flashblocks/p2p/src/protocol/event.rs +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -7,9 +7,9 @@ use flashblocks_primitives::primitives::FlashblocksPayloadV1; use futures::{ - Stream, StreamExt, future::{self}, stream::{self, PollNext}, + Stream, StreamExt, }; use reth::{ api::NodePrimitives, payload::PayloadId, providers::CanonStateSubscriptions, @@ -99,7 +99,7 @@ pub struct WorldChainEventsStream { st: WorldChainEventNotificationsStream, } -impl WorldChainEventsStream { +impl WorldChainEventsStream { /// Creates a new [`WorldChainEventsStream`] by merging a flashblock /// receiver with canonical chain notifications from `provider`. pub fn new( @@ -168,7 +168,7 @@ impl WorldChainEventsStream { } } -impl Stream for WorldChainEventsStream { +impl Stream for WorldChainEventsStream { type Item = WorldChainEvent; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { @@ -277,10 +277,11 @@ impl BufferedFlashblocks { }; // Stale check - if let Some(tip) = &self.canon_tip { - if base.timestamp <= tip.number { - return false; - } + if self + .canon_tip + .is_some_and(|tip| base.timestamp <= tip.number) + { + return false; } self.parent_num_hash = BlockNumHash { diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index a028f9702..3195be97e 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -152,6 +152,10 @@ pub struct FlashblocksP2PState { /// Most recent canonical tip. Updated by the stream hook. /// Used to reject stale flashblocks in `publish()`. pub canon_tip: Option, + /// Last flashblock flushed through the stream to the coordinator. + /// Only flashblocks at or ahead of this cursor should be peered. + pub flushed_payload_id: Option, + pub flushed_index: u64, /// Flashblocks observed from network peers, tracked until their receive grace windows expire. pub observed_payloads: VecDeque, /// All currently connected peers and their connection state. @@ -168,6 +172,8 @@ impl Default for FlashblocksP2PState { payload_timestamp: 0, flashblock_timestamp: 0, canon_tip: None, + flushed_payload_id: None, + flushed_index: 0, observed_payloads: VecDeque::new(), connections: HashMap::new(), } @@ -641,10 +647,10 @@ impl FlashblocksHandle { let mut user_hook = hook; let combined_hook = move |event: &WorldChainEvent| { - if let WorldChainEvent::Chain(ce) = event { - if let ChainEvent::Canon(tip) = ce.as_ref() { - state.lock().canon_tip = Some(*tip); - } + if let WorldChainEvent::Chain(ce) = event + && let ChainEvent::Canon(tip) = ce.as_ref() + { + state.lock().canon_tip = Some(*tip); } user_hook(event) }; @@ -1089,12 +1095,11 @@ impl FlashblocksP2PCtx { } // Reject flashblocks for epochs that have already been canonicalized. - if let Some(canon_tip) = &state.canon_tip { - if let Some(base) = &payload.base { - if base.block_number.saturating_sub(1) <= canon_tip.number { - return; - } - } + if let Some(canon_tip) = &state.canon_tip + && let Some(base) = &payload.base + && base.block_number.saturating_sub(1) <= canon_tip.number + { + return; } if authorization.timestamp > state.payload_timestamp { From 78b7fd1bab1c8559485cbe4f4b8a4cd0d0bff55b Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 18:16:12 -0700 Subject: [PATCH 32/43] chore: fmt --- crates/flashblocks/builder/src/coordinator.rs | 21 ++++++++++--------- crates/flashblocks/p2p/src/protocol/event.rs | 2 +- .../flashblocks/p2p/src/protocol/handler.rs | 15 ++++--------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 21e46d3bc..e583a8d10 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -118,16 +118,17 @@ impl FlashblocksExecutionCoordinator { let provider = ctx.provider().clone(); let p2p_state = self.p2p_handle.state.clone(); let mut stream: WorldChainEventsStream = - self.p2p_handle.event_stream(provider.clone(), move |event| { - if let WorldChainEvent::Chain(ce) = event - && let ChainEvent::Pending(fb) = ce.as_ref() - { - let mut state = p2p_state.lock(); - state.flushed_payload_id = Some(fb.payload_id); - state.flushed_index = fb.index; - } - None - }); + self.p2p_handle + .event_stream(provider.clone(), move |event| { + if let WorldChainEvent::Chain(ce) = event + && let ChainEvent::Pending(fb) = ce.as_ref() + { + let mut state = p2p_state.lock(); + state.flushed_payload_id = Some(fb.payload_id); + state.flushed_index = fb.index; + } + None + }); let this = self.clone(); let chain_spec = ctx.chain_spec().clone(); diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs index 5383db42b..f7f687045 100644 --- a/crates/flashblocks/p2p/src/protocol/event.rs +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -7,9 +7,9 @@ use flashblocks_primitives::primitives::FlashblocksPayloadV1; use futures::{ + Stream, StreamExt, future::{self}, stream::{self, PollNext}, - Stream, StreamExt, }; use reth::{ api::NodePrimitives, payload::PayloadId, providers::CanonStateSubscriptions, diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 3195be97e..b7039dc53 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,6 +1,7 @@ use crate::protocol::{ connection::{FlashblocksConnection, FlashblocksConnectionState, ReceiveStatus, Score}, - error::FlashblocksP2PError, event::{ChainEvent, WorldChainEvent, WorldChainEventsStream}, + error::FlashblocksP2PError, + event::{ChainEvent, WorldChainEvent, WorldChainEventsStream}, }; use alloy_rlp::BytesMut; use chrono::Utc; @@ -628,20 +629,12 @@ impl FlashblocksHandle { /// Canon events automatically update the P2P state's `canon_tip` so /// `publish()` rejects stale flashblocks. The caller's `hook` is applied /// after the canon_tip update. - pub fn event_stream( - &self, - provider: P, - hook: F, - ) -> WorldChainEventsStream + pub fn event_stream(&self, provider: P, hook: F) -> WorldChainEventsStream where T: Send + Clone + Unpin + 'static, P: reth::providers::CanonStateSubscriptions + Clone + Send + Sync + 'static, N: reth::api::NodePrimitives, - F: FnMut( - &WorldChainEvent, - ) -> Option> - + Send - + 'static, + F: FnMut(&WorldChainEvent) -> Option> + Send + 'static, { let state = self.state.clone(); let mut user_hook = hook; From a30d12f25c09b2a111a5f1ad9e3b8352bd72ff2b Mon Sep 17 00:00:00 2001 From: Eric Woolsey Date: Fri, 13 Mar 2026 18:29:43 -0700 Subject: [PATCH 33/43] chore: logging --- .../p2p/src/protocol/connection.rs | 20 ++++ .../flashblocks/p2p/src/protocol/handler.rs | 110 ++++++++++++++++-- 2 files changed, 123 insertions(+), 7 deletions(-) diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index 9f844ed0b..4f4e313fd 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -215,6 +215,11 @@ impl Stream for FlashblocksConnection { } } FlashblocksP2PMsg::RequestFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received RequestFlashblocks from peer", + ); if this .protocol .handle @@ -227,6 +232,11 @@ impl Stream for FlashblocksConnection { } } FlashblocksP2PMsg::AcceptFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received AcceptFlashblocks from peer", + ); if this .protocol .handle @@ -239,6 +249,11 @@ impl Stream for FlashblocksConnection { } } FlashblocksP2PMsg::RejectFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received RejectFlashblocks from peer", + ); if this .protocol .handle @@ -251,6 +266,11 @@ impl Stream for FlashblocksConnection { } } FlashblocksP2PMsg::CancelFlashblocks => { + trace!( + target: "flashblocks::p2p", + peer_id = %this.peer_id, + "received CancelFlashblocks from peer", + ); if this .protocol .handle diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index ec3335cce..0d6cd13e4 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -16,7 +16,7 @@ use flashblocks_primitives::{ use futures::{Stream, StreamExt, stream}; use metrics::histogram; use parking_lot::Mutex; -use rand::{Rng, seq::SliceRandom}; +use rand::Rng; use reth::payload::PayloadId; use reth_eth_wire::Capability; use reth_ethereum::network::{api::PeerId, protocol::ProtocolHandler}; @@ -367,6 +367,11 @@ impl FlashblocksP2PState { let timestamp = Utc::now().timestamp() as u64; peer_state.receive_status = ReceiveStatus::Requesting; peer_state.receive_status_timestamp = timestamp; + debug!( + target: "flashblocks::p2p", + %peer_id, + "sending RequestFlashblocks to peer", + ); self.send_direct(peer_id, FlashblocksP2PMsg::RequestFlashblocks); } @@ -397,10 +402,15 @@ impl FlashblocksP2PState { let now = Utc::now().timestamp() as u64; let mut cleared_any = false; - for peer_state in self.connections.values_mut() { + for (peer_id, peer_state) in &mut self.connections { if matches!(peer_state.receive_status, ReceiveStatus::Requesting) && peer_state.receive_status_timestamp + RECEIVE_REQUEST_TIMEOUT_SECS <= now { + debug!( + target: "flashblocks::p2p", + %peer_id, + "receive request timed out, clearing peer", + ); Self::clear_receive_state(peer_state, now); cleared_any = true; } @@ -448,6 +458,13 @@ impl FlashblocksP2PState { let rand = rand::rng().random_range(0..candidates.len()); let candidate = candidates[rand].0; + debug!( + target: "flashblocks::p2p", + evicted_peer = %evict, + new_peer = %candidate, + "rotating receive peer", + ); + let evict_timestamp = Utc::now().timestamp() as u64; if let Some(evict_state) = self.connection_state_mut(&evict) { Self::clear_receive_state(evict_state, evict_timestamp); @@ -460,6 +477,11 @@ impl FlashblocksP2PState { /// Returns `Err` if the peer should receive a reputation penalty. fn handle_request(&mut self, ctx: &FlashblocksP2PCtx, peer_id: PeerId) -> Result<(), ()> { if self.check_control_rate_limit(&peer_id) { + warn!( + target: "flashblocks::p2p", + %peer_id, + "rejecting RequestFlashblocks: rate limit exceeded", + ); return Err(()); } @@ -468,17 +490,35 @@ impl FlashblocksP2PState { }; if peer_state.send_enabled { - // Already sending to this peer — repeated request is spam. + warn!( + target: "flashblocks::p2p", + %peer_id, + "rejecting RequestFlashblocks: already sending to peer", + ); return Err(()); } let peer_is_trusted = peer_state.trusted; let send_count = self.connections.values().filter(|s| s.send_enabled).count(); if !peer_is_trusted && send_count >= ctx.fanout_args.max_send_peers { + debug!( + target: "flashblocks::p2p", + %peer_id, + send_count, + max_send_peers = ctx.fanout_args.max_send_peers, + "rejecting RequestFlashblocks: send set full", + ); self.send_direct(peer_id, FlashblocksP2PMsg::RejectFlashblocks); return Ok(()); } + info!( + target: "flashblocks::p2p", + %peer_id, + trusted = peer_is_trusted, + send_count = send_count + 1, + "accepted RequestFlashblocks, adding peer to send set", + ); let peer_state = self.connection_state_mut(&peer_id).expect("peer exists"); peer_state.send_enabled = true; self.send_direct(peer_id, FlashblocksP2PMsg::AcceptFlashblocks); @@ -497,13 +537,26 @@ impl FlashblocksP2PState { match peer_state.receive_status { ReceiveStatus::Requesting => { + info!( + target: "flashblocks::p2p", + %peer_id, + "peer accepted our receive request, now receiving flashblocks", + ); peer_state.receive_status = ReceiveStatus::Receiving { score: Score::new(ctx.fanout_args.score_samples), }; Ok(()) } // Unsolicited accept — we never asked this peer. - _ => Err(()), + _ => { + warn!( + target: "flashblocks::p2p", + %peer_id, + status = ?peer_state.receive_status, + "received unsolicited AcceptFlashblocks", + ); + Err(()) + } } } @@ -519,12 +572,25 @@ impl FlashblocksP2PState { match peer_state.receive_status { ReceiveStatus::Requesting => { + info!( + target: "flashblocks::p2p", + %peer_id, + "peer rejected our receive request, will try another peer", + ); Self::clear_receive_state(peer_state, Utc::now().timestamp() as u64); self.maybe_request_receive_peers(ctx); Ok(()) } // Unsolicited reject — we never asked this peer. - _ => Err(()), + _ => { + warn!( + target: "flashblocks::p2p", + %peer_id, + status = ?peer_state.receive_status, + "received unsolicited RejectFlashblocks", + ); + Err(()) + } } } @@ -539,10 +605,19 @@ impl FlashblocksP2PState { }; if !peer_state.send_enabled { - // Cancel is only valid from a receiver to its sender. + warn!( + target: "flashblocks::p2p", + %peer_id, + "received CancelFlashblocks from peer we are not sending to", + ); return Err(()); } + info!( + target: "flashblocks::p2p", + %peer_id, + "peer cancelled flashblocks, removing from send set", + ); peer_state.send_enabled = false; Ok(()) } @@ -681,12 +756,33 @@ impl FlashblocksHandle { conn_state.outbound_tx = Some(outbound_tx); conn_state.trusted = trusted; state.connections.insert(peer_id, conn_state); + + info!( + target: "flashblocks::p2p", + %peer_id, + trusted, + total_peers = state.connections.len(), + "flashblocks peer connected", + ); + state.maybe_request_receive_peers(&self.ctx); } pub(crate) fn on_peer_disconnected(&self, peer_id: PeerId) { let mut state = self.state.lock(); - state.connections.remove(&peer_id); + let removed = state.connections.remove(&peer_id); + + if let Some(conn_state) = &removed { + info!( + target: "flashblocks::p2p", + %peer_id, + was_sending = conn_state.send_enabled, + receive_status = ?conn_state.receive_status, + remaining_peers = state.connections.len(), + "flashblocks peer disconnected", + ); + } + state.maybe_request_receive_peers(&self.ctx); } From 1c7c3a3be789bf6209df6f11ef7dc4debff9d44a Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 19:42:28 -0700 Subject: [PATCH 34/43] chore: add auto-fmt --- .github/workflows/rust-ci.yml | 22 +++++++++++++++++++--- Justfile | 4 +++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 36516e393..9ec6b94cc 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -37,13 +37,21 @@ jobs: cargo-lint: runs-on: arc-public-8xlarge-amd64-runner timeout-minutes: 20 + permissions: + contents: write name: lint steps: - uses: actions/checkout@v6 - - uses: taiki-e/install-action@just + with: + ref: ${{ github.head_ref }} + persist-credentials: true - uses: dtolnay/rust-toolchain@nightly with: components: rustfmt, clippy + + - uses: foundry-rs/foundry-toolchain@v1 + + - uses: taiki-e/install-action@just - name: Cache uses: actions/cache@v5 continue-on-error: false @@ -54,9 +62,17 @@ jobs: ~/.cargo/git/db/ key: cargo-test-${{ hashFiles('**/Cargo.lock') }} restore-keys: cargo-test- - - name: fmt + lint - run: cargo +nightly fmt --all -- --check + - name: Run Formatter + run: just fmt + - name: Commit changes + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add -A + git diff --staged --quiet || git commit -m "chore: auto-format" + git push + cargo-clippy: runs-on: arc-public-8xlarge-amd64-runner timeout-minutes: 20 diff --git a/Justfile b/Justfile index 475e2963f..4ed5775c7 100644 --- a/Justfile +++ b/Justfile @@ -24,8 +24,10 @@ devnet-down: test *args='': RUST_LOG="info" cargo nextest run --workspace $@ +fmt: fmt-fix fmt-check contracts-fmt + # Formats the whole workspace -fmt: devnet-fmt contracts-fmt fmt-fix fmt-check +fmt-all: devnet-fmt contracts-fmt fmt-fix fmt-check devnet-fmt: @just ./devnet/fmt From 7f617a0f8dfb72a7f420a056e7a5f1c05f637e8b Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 19:44:34 -0700 Subject: [PATCH 35/43] chore: run sync on main --- .github/workflows/sync.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 297eb0f4c..d0188fc02 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -9,6 +9,8 @@ on: push: tags: - v* + pull_request: + branches: [main] env: CARGO_TERM_COLOR: always From 77e834244a8b781d71bbb4304c7f2c66a0b5cb6b Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 19:46:23 -0700 Subject: [PATCH 36/43] chore: run sync --- .github/workflows/sync.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index d0188fc02..132b3ee55 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -10,8 +10,6 @@ on: tags: - v* pull_request: - branches: [main] - env: CARGO_TERM_COLOR: always From 410129b137eb045170a1180e213ebf6def1d1ee6 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Fri, 13 Mar 2026 21:07:25 -0700 Subject: [PATCH 37/43] fix: cleanup stream logic --- crates/flashblocks/builder/src/coordinator.rs | 64 +- crates/flashblocks/node/tests/p2p.rs | 1 + .../p2p/src/protocol/connection.rs | 9 +- crates/flashblocks/p2p/src/protocol/event.rs | 744 +++++++++++------- .../flashblocks/p2p/src/protocol/handler.rs | 17 +- .../world/node/tests/e2e-testsuite/actions.rs | 301 +++++++ .../node/tests/e2e-testsuite/testsuite.rs | 328 ++++++++ 7 files changed, 1128 insertions(+), 336 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index e583a8d10..80630988a 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -120,9 +120,7 @@ impl FlashblocksExecutionCoordinator { let mut stream: WorldChainEventsStream = self.p2p_handle .event_stream(provider.clone(), move |event| { - if let WorldChainEvent::Chain(ce) = event - && let ChainEvent::Pending(fb) = ce.as_ref() - { + if let WorldChainEvent::Chain(ChainEvent::Pending(fb)) = event { let mut state = p2p_state.lock(); state.flushed_payload_id = Some(fb.payload_id); state.flushed_index = fb.index; @@ -148,36 +146,35 @@ impl FlashblocksExecutionCoordinator { while let Some(event) = stream.next().await { match event { - WorldChainEvent::Chain(chain_event) => match *chain_event { - ChainEvent::Pending(flashblock) => { - let flashblock = *flashblock; - // Track epoch block number from base flashblocks - if let Some(base) = &flashblock.base { - epoch_block_number = Some(base.block_number); - } - - this.on_flashblock( - flashblock, - &mut inflight_shutdown, - &task_permit, - database_permit, - &workload, - &provider, - &evm_config, - &chain_spec, - &pending_block, - ) - .await; + WorldChainEvent::Chain(ChainEvent::Pending(flashblock)) => { + let flashblock = + Arc::try_unwrap(flashblock).unwrap_or_else(|arc| (*arc).clone()); + // Track epoch block number from base flashblocks + if let Some(base) = &flashblock.base { + epoch_block_number = Some(base.block_number); } - ChainEvent::Canon(tip) => { - this.on_canon( - tip, - &mut inflight_shutdown, - &mut epoch_block_number, - &pending_block, - ); - } - }, + + this.on_flashblock( + flashblock, + &mut inflight_shutdown, + &task_permit, + database_permit, + &workload, + &provider, + &evm_config, + &chain_spec, + &pending_block, + ) + .await; + } + WorldChainEvent::Chain(ChainEvent::Canon(tip)) => { + this.on_canon( + tip, + &mut inflight_shutdown, + &mut epoch_block_number, + &pending_block, + ); + } WorldChainEvent::Event(_) => {} } } @@ -213,7 +210,7 @@ impl FlashblocksExecutionCoordinator { let (tx, rx) = oneshot::channel::<()>(); *shutdown_tx = Some(tx); - let _permit = task_permit + let task_permit = task_permit .clone() .acquire_owned() .await @@ -226,6 +223,7 @@ impl FlashblocksExecutionCoordinator { let pending_block = pending_block.clone(); spawn_blocking_io_with_shutdown_signal(workload, rx, database_permit, move |permit| { + let _task_permit = task_permit; // held until closure completes if let Err(e) = process_flashblock( permit, provider, diff --git a/crates/flashblocks/node/tests/p2p.rs b/crates/flashblocks/node/tests/p2p.rs index 8a1c7dd6a..c205c5a18 100644 --- a/crates/flashblocks/node/tests/p2p.rs +++ b/crates/flashblocks/node/tests/p2p.rs @@ -220,6 +220,7 @@ async fn wait_for_trusted_peers( } } +#[expect(clippy::await_holding_lock)] // lock is explicitly dropped before await async fn wait_for_flashblocks_topology( node: &NodeContext, expected_connections: usize, diff --git a/crates/flashblocks/p2p/src/protocol/connection.rs b/crates/flashblocks/p2p/src/protocol/connection.rs index ed217cdb1..25181132e 100644 --- a/crates/flashblocks/p2p/src/protocol/connection.rs +++ b/crates/flashblocks/p2p/src/protocol/connection.rs @@ -1,5 +1,6 @@ -use crate::protocol::handler::{ - FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, MAX_FLASHBLOCK_INDEX, PublishingStatus, +use crate::protocol::{ + event::MAX_FLASHBLOCKS, + handler::{FlashblocksP2PNetworkHandle, FlashblocksP2PProtocol, PublishingStatus}, }; use alloy_primitives::bytes::BytesMut; use chrono::Utc; @@ -334,13 +335,13 @@ impl FlashblocksConnection { } // Check if the payload index is within the allowed range - if msg.index as usize > MAX_FLASHBLOCK_INDEX { + if msg.index as usize >= MAX_FLASHBLOCKS { tracing::error!( target: "flashblocks::p2p", peer_id = %self.peer_id, index = msg.index, payload_id = %msg.payload_id, - max_index = MAX_FLASHBLOCK_INDEX, + max = MAX_FLASHBLOCKS, "Received flashblocks payload with index exceeding maximum" ); return; diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs index f7f687045..75449bc9f 100644 --- a/crates/flashblocks/p2p/src/protocol/event.rs +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -1,28 +1,22 @@ //! Canon-aware flashblock event stream. //! //! Merges a raw flashblock stream with canonical chain notifications, yielding -//! [`FlashblocksEvent::Pending`] only when the flashblock's epoch parent matches -//! the current canonical tip, and [`FlashblocksEvent::Canon`] whenever the tip +//! [`ChainEvent::Pending`] only when the flashblock's epoch parent matches +//! the current canonical tip, and [`ChainEvent::Canon`] whenever the tip //! changes. use flashblocks_primitives::primitives::FlashblocksPayloadV1; use futures::{ Stream, StreamExt, - future::{self}, stream::{self, PollNext}, }; -use reth::{ - api::NodePrimitives, payload::PayloadId, providers::CanonStateSubscriptions, - rpc::types::BlockNumHash, -}; +use reth::{payload::PayloadId, rpc::types::BlockNumHash}; use std::{ collections::VecDeque, - fmt::Debug, pin::Pin, + sync::Arc, task::{Context, Poll}, }; -use tokio::sync::{broadcast, oneshot}; -use tokio_stream::wrappers::BroadcastStream; #[derive(Clone, Debug)] pub enum ChainEvent { @@ -30,336 +24,220 @@ pub enum ChainEvent { /// pending flashblocks that are stale relative to the new tip. Canon(BlockNumHash), /// A flashblock has been received whose epoch parent matches the current - /// canonical tip. Consumers can treat this as a "pending" event and buffer - /// it until the next Canon event confirms it's ready to be processed. - Pending(Box), -} - -impl ChainEvent { - pub fn is_canon(&self) -> bool { - matches!(self, ChainEvent::Canon(_)) - } - - pub fn is_pending(&self) -> bool { - matches!(self, ChainEvent::Pending(_)) - } -} - -impl From for WorldChainEvent { - fn from(value: ChainEvent) -> Self { - WorldChainEvent::Chain(Box::new(value)) - } -} - -impl From for WorldChainEvent { - fn from(value: FlashblocksPayloadV1) -> Self { - WorldChainEvent::Chain(Box::new(ChainEvent::Pending(Box::new(value)))) - } + /// canonical tip. Zero-copy via [`Arc`] — no payload cloning through the + /// buffer or downstream consumers. + Pending(Arc), } -impl From for WorldChainEvent { - fn from(value: BlockNumHash) -> Self { - WorldChainEvent::Chain(Box::new(ChainEvent::Canon(value))) - } -} - -/// Events yielded by [`ChainEventsStream`]. +/// Events yielded by [`WorldChainEventsStream`]. #[derive(Clone, Debug)] pub enum WorldChainEvent { - /// An Event emitted when executable pending flashblocks are observed. - Chain(Box), - /// A Event emitted by any source. + /// An event emitted when executable pending flashblocks are observed. + Chain(ChainEvent), + /// An event emitted by any source. Event(T), } -/// Convenience alias: a [`ChainEvent`] carrying a flashblocks payload. -pub type WorldChainEventNotificationsStream = - Pin> + Send>>; - -/// A stream of [`ChainEvent`]s that merges flashblocks with canonical chain -/// notifications. -/// -/// Follows the same pattern as reth's `CanonStateNotificationStream` — wraps an -/// inner stream and handles lag transparently. -/// -/// A [`ChainEvent::Pending`] is emitted only when the flashblock's epoch parent -/// matches the canonical tip. Stale flashblocks are silently discarded via -/// [`PendingCursor::try_advance`]. A [`ChainEvent::Canon`] is emitted on every -/// canonical tip change so consumers can clear pending state. /// A stream of [`WorldChainEvent`]s that merges flashblocks with canonical -/// chain notifications, reducing them through a [`BufferedCursor`](sealed::BufferedCursor) +/// chain notifications, reducing them through a [`BufferedFlashblocks`] /// state machine. /// -/// Implements [`Stream`] directly — polls the merged inner streams, feeds -/// each [`ChainEvent`] through the cursor's `reduce`, and yields the output -/// events one at a time. -#[pin_project::pin_project] -pub struct WorldChainEventsStream { - #[pin] - st: WorldChainEventNotificationsStream, +/// A [`ChainEvent::Pending`] is emitted only when the flashblock's epoch parent +/// matches the canonical tip. Stale flashblocks are silently discarded. +/// Flashblocks are buffered when the epoch parent is not yet canonical, but the +/// [`PayloadId`] is fresh. A [`ChainEvent::Canon`] is emitted on every +/// canonical tip change so consumers can clear pending state. +pub type WorldChainEventsStream = Pin> + Send>>; + +/// Constructs a [`WorldChainEventsStream`] by merging a flashblock stream with +/// canonical chain notifications, reducing through [`BufferedFlashblocks`], and +/// applying `hook` to each yielded event. +#[must_use] +pub fn world_chain_events_stream( + flashblocks: Pin + Send>>, + canon: Pin + Send>>, + mut hook: F, +) -> WorldChainEventsStream +where + T: Send + Unpin + 'static, + F: FnMut(&WorldChainEvent) -> Option> + Send + 'static, +{ + let merged = + futures::stream::select_with_strategy(flashblocks, canon, |_: &mut ()| PollNext::Left); + + BufferedStream::new(merged) + .map(WorldChainEvent::Chain) + .flat_map(move |event| { + let extra = hook(&event); + stream::iter(std::iter::once(event).chain(extra)) + }) + .boxed() } -impl WorldChainEventsStream { - /// Creates a new [`WorldChainEventsStream`] by merging a flashblock - /// receiver with canonical chain notifications from `provider`. - pub fn new( - provider: P, - rx: broadcast::Receiver, - ) -> Self - where - P: CanonStateSubscriptions + Clone + Send + Sync + 'static, - { - let flashblocks = BroadcastStream::new(rx) - .filter_map(|x| { - future::ready(match x { - Ok(fb) => Some(fb), - Err(tokio_stream::wrappers::errors::BroadcastStreamRecvError::Lagged(n)) => { - tracing::warn!(missed = n, "flashblocks broadcast receiver lagged"); - None - } - }) - }) - .map(|fb| ChainEvent::Pending(Box::new(fb))); +// --------------------------------------------------------------------------- +// BufferedStream — zero-allocation stream adapter +// --------------------------------------------------------------------------- - let canon = provider - .canonical_state_stream() - .map(|n| ChainEvent::Canon(n.tip().num_hash())); +/// Stream adapter that wraps a merged `ChainEvent` stream and map reduces it +/// into a [`BufferedFlashblocks`]. +#[pin_project::pin_project] +#[doc = ""] +struct BufferedStream { + #[pin] + inner: S, + state: BufferedFlashblocks, +} - Self::new_with_hook( - move |_: &WorldChainEvent| None, - flashblocks.boxed(), - canon.boxed(), - ) +impl BufferedStream { + fn new(inner: S) -> Self { + Self { + inner, + state: BufferedFlashblocks::default(), + } } +} - /// Creates a new [`WorldChainEventsStream`] with a hook that can inject - /// additional events after each yielded event. - pub fn new_with_hook( - mut hook: F, - flashblocks: Pin + Send>>, - canon: Pin + Send>>, - ) -> Self - where - F: FnMut(&WorldChainEvent) -> Option> + Send + 'static, - { - let merged = - futures::stream::select_with_strategy(flashblocks, canon, |_: &mut ()| -> PollNext { - PollNext::Left - }); - - // Fold through the BufferedFlashblocks reducer, then flat_map output. - let st = merged - .scan(BufferedFlashblocks::default(), |cursor, event| { - cursor.step(event); - let events: Vec> = cursor - .by_ref() - .map(|ce| WorldChainEvent::Chain(Box::new(ce))) - .collect(); - future::ready(Some(events)) - }) - .flat_map(stream::iter) - .flat_map(move |event| { - let extra = hook(&event); - stream::iter(std::iter::once(event).chain(extra)) - }) - .boxed(); +impl> Stream for BufferedStream { + type Item = ChainEvent; - Self { st } - } -} + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); -impl Stream for WorldChainEventsStream { - type Item = WorldChainEvent; + // Drain buffered output first. + if let Some(event) = this.state.output.pop_front() { + return Poll::Ready(Some(event)); + } - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.st.as_mut().poll_next(cx) + // Poll inner stream, reduce through state machine, yield first output. + match this.inner.poll_next(cx) { + Poll::Ready(Some(event)) => { + this.state.step(event); + Poll::Ready(this.state.output.pop_front()) + } + // Inner exhausted — drain any remaining buffered output before closing. + Poll::Ready(None) => Poll::Ready(this.state.output.pop_front()), + Poll::Pending => Poll::Pending, + } } } + // --------------------------------------------------------------------------- -// BufferedFlashblocks — stateful reducer with Extend + Iterator +// Epoch — scoped state for a single flashblock epoch // --------------------------------------------------------------------------- -const MAX_FLASHBLOCK_INDEX: usize = 100; - -/// Internal phase of the buffer state machine. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Phase { - /// No active epoch. - Uninitialized, - /// Epoch started but parent is not yet canonical. - Pending, - /// Parent is canonical — flashblocks can be drained. - Executable, -} +/// Maximum number of flashblocks per epoch. +pub(crate) const MAX_FLASHBLOCKS: usize = 12; -/// Buffers flashblocks for the current epoch, gating output on canonical tip. -/// -/// Implements [`Extend`] to accept input events and -/// [`Iterator`] to drain output events. The internal -/// state machine transitions between `Uninitialized`, `Pending`, and -/// `Executable` phases automatically. -pub struct BufferedFlashblocks { - phase: Phase, +/// State for a single flashblock epoch: the parent block it builds on, +/// its payload identifier, and a fixed-size buffer of received flashblocks. +pub(crate) struct BlockEpochState { /// The parent block this epoch builds on. - parent_num_hash: BlockNumHash, - /// Current epoch payload identifier. + parent: BlockNumHash, + /// Payload identifier for this epoch. payload_id: PayloadId, - /// Timestamp of the current epoch. - timestamp: u64, - /// Drain watermark — next index to drain from. + /// Drain watermark — next index to yield. cursor: usize, - /// Sparse buffer indexed by flashblock sequence number. - buffer: Vec>, - /// Most recent canonical tip. - canon_tip: Option, - /// Output events ready to be yielded by the iterator. - output: VecDeque, + /// Fixed-size sparse buffer indexed by flashblock sequence number. + /// 96 bytes inline — no heap allocation. + buffer: [Option>; MAX_FLASHBLOCKS], } -/// A flashblock drained from the buffer, ready for the coordinator. -pub struct PendingFlashblockEvent { - /// Resolve when the flashblock is in the in-memory tree. - pub tx: oneshot::Sender<()>, - /// The drained flashblock payload. - pub flashblock: FlashblocksPayloadV1, -} +impl BlockEpochState { + /// Create a new epoch from a base flashblock. Returns `None` if the base + /// is stale (parent behind the canonical tip) or missing its base field. + fn try_new(fb: Arc, canon_tip: Option) -> Option { + let base = fb.base.as_ref()?; -impl Default for BufferedFlashblocks { - fn default() -> Self { - Self { - phase: Phase::Uninitialized, - parent_num_hash: BlockNumHash::default(), - payload_id: PayloadId::default(), - timestamp: 0, - cursor: 0, - buffer: Vec::new(), - canon_tip: None, - output: VecDeque::new(), + // Stale check: reject if the epoch's parent is behind the canon tip. + let parent_number = base.block_number.saturating_sub(1); + if canon_tip.is_some_and(|tip| parent_number < tip.number) { + return None; } - } -} -impl BufferedFlashblocks { - /// Insert a flashblock at its index. Returns `false` if the payload_id - /// doesn't match, the index exceeds the maximum, or the slot is occupied. - fn insert(&mut self, fb: &FlashblocksPayloadV1) -> bool { - if fb.payload_id != self.payload_id { - return false; - } - let idx = fb.index as usize; - if idx > MAX_FLASHBLOCK_INDEX { - return false; - } - let len = self.buffer.len(); - self.buffer.resize_with(len.max(idx + 1), || None); - if self.buffer[idx].is_some() { - return false; - } - self.buffer[idx] = Some(fb.clone()); - true - } - - /// Reset to uninitialized, discarding all buffered flashblocks. - fn reset(&mut self) { - self.phase = Phase::Uninitialized; - self.parent_num_hash = BlockNumHash::default(); - self.payload_id = PayloadId::default(); - self.timestamp = 0; - self.cursor = 0; - self.buffer.clear(); - // canon_tip is preserved - } - - /// Try to start a new epoch from a base flashblock. - fn accept_base(&mut self, fb: FlashblocksPayloadV1) -> bool { - let Some(base) = fb.base.as_ref() else { - return false; + let mut epoch = Self { + parent: BlockNumHash { + number: parent_number, + hash: base.parent_hash, + }, + payload_id: fb.payload_id, + cursor: 0, + buffer: Default::default(), }; + epoch.insert(fb); + Some(epoch) + } - // Stale check - if self - .canon_tip - .is_some_and(|tip| base.timestamp <= tip.number) + /// Insert a flashblock at its sequence index. Returns `false` if the + /// payload_id doesn't match, the index is out of bounds, or the slot + /// is already occupied. + fn insert(&mut self, fb: Arc) -> bool { + let idx = fb.index as usize; + if fb.payload_id != self.payload_id || idx >= MAX_FLASHBLOCKS || self.buffer[idx].is_some() { return false; } - - self.parent_num_hash = BlockNumHash { - number: base.block_number.saturating_sub(1), - hash: base.parent_hash, - }; - self.payload_id = fb.payload_id; - self.timestamp = base.timestamp; - self.cursor = 0; - self.buffer.clear(); - self.insert(&fb); - - self.phase = if self - .canon_tip - .is_some_and(|tip| tip == self.parent_num_hash) - { - Phase::Executable - } else { - Phase::Pending - }; - + self.buffer[idx] = Some(fb); true } +} - /// If pending and canon tip matches parent, transition to executable. - fn try_anchor(&mut self) { - if self.phase == Phase::Pending - && self - .canon_tip - .is_some_and(|tip| tip == self.parent_num_hash) - { - self.phase = Phase::Executable; - } - } +// --------------------------------------------------------------------------- +// BufferedFlashblocks — stateful reducer with Extend + Iterator +// --------------------------------------------------------------------------- - /// Drain all contiguous flashblocks from the cursor into the output queue. - fn drain_contiguous(&mut self) { - if self.phase != Phase::Executable { - return; - } - while let Some(Some(_)) = self.buffer.get(self.cursor) { - let fb = self.buffer[self.cursor].take().unwrap(); - self.cursor += 1; - self.output.push_back(ChainEvent::Pending(Box::new(fb))); - } - } +/// Buffers flashblocks for the current epoch, gating output on the canonical +/// tip. Phase is derived from state — not tracked separately: +/// +/// - `epoch.is_none()` → no active epoch +/// - `epoch.is_some() && canon_tip != epoch.parent` → pending (buffering) +/// - `epoch.is_some() && canon_tip == epoch.parent` → executable (draining) +/// +/// Implements [`Extend`] to accept input events and +/// [`Iterator`] to drain output events. +#[derive(Default)] +pub struct BufferedFlashblocks { + /// Current epoch, if any. `None` means no active epoch. + epoch: Option, + /// Most recent canonical tip. + canon_tip: Option, + /// Output events ready to be yielded by the iterator. + output: VecDeque, +} +impl BufferedFlashblocks { /// Process a single input event, updating state and buffering output. fn step(&mut self, event: ChainEvent) { match event { - ChainEvent::Pending(fb) => { - if fb.base.is_some() { - // New epoch — reset and try to accept - self.reset(); - self.accept_base(*fb); - self.drain_contiguous(); - } else { - // Non-base — insert into current buffer if we have an epoch - if self.phase != Phase::Uninitialized { - self.insert(&fb); - self.drain_contiguous(); - } - } - } ChainEvent::Canon(tip) => { self.canon_tip = Some(tip); self.output.push_back(ChainEvent::Canon(tip)); - self.try_anchor(); - self.drain_contiguous(); + } + ChainEvent::Pending(ref fb) if fb.base.is_some() => { + self.epoch = BlockEpochState::try_new(Arc::clone(fb), self.canon_tip); + } + ChainEvent::Pending(fb) => { + if let Some(epoch) = &mut self.epoch { + epoch.insert(fb); + } } } + self.drain(); } -} -impl Extend for BufferedFlashblocks { - fn extend>(&mut self, iter: I) { - for event in iter { - self.step(event); + /// Drain contiguous flashblocks from the cursor into the output queue, + /// but only if the epoch is anchored to the canonical tip. + fn drain(&mut self) { + let canon_tip = self.canon_tip; + let Some(ref mut epoch) = self.epoch else { + return; + }; + if !canon_tip.is_some_and(|tip| tip == epoch.parent) { + return; + } + while let Some(Some(_)) = epoch.buffer.get(epoch.cursor) { + let fb = epoch.buffer[epoch.cursor].take().unwrap(); + epoch.cursor += 1; + self.output.push_back(ChainEvent::Pending(fb)); } } } @@ -371,3 +249,295 @@ impl Iterator for BufferedFlashblocks { self.output.pop_front() } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::B256; + use flashblocks_primitives::primitives::{ + ExecutionPayloadBaseV1, ExecutionPayloadFlashblockDeltaV1, + }; + + fn canon(number: u64, hash: B256) -> ChainEvent { + ChainEvent::Canon(BlockNumHash { number, hash }) + } + + fn base_fb( + payload_id: PayloadId, + index: u64, + parent_hash: B256, + block_number: u64, + ) -> ChainEvent { + ChainEvent::Pending(Arc::new(FlashblocksPayloadV1 { + payload_id, + index, + base: Some(ExecutionPayloadBaseV1 { + parent_hash, + block_number, + timestamp: block_number + 1000, // well above any block number + ..Default::default() + }), + diff: ExecutionPayloadFlashblockDeltaV1::default(), + metadata: Default::default(), + })) + } + + fn delta_fb(payload_id: PayloadId, index: u64) -> ChainEvent { + ChainEvent::Pending(Arc::new(FlashblocksPayloadV1 { + payload_id, + index, + base: None, + diff: ExecutionPayloadFlashblockDeltaV1::default(), + metadata: Default::default(), + })) + } + + fn pid(b: u8) -> PayloadId { + PayloadId::new([b; 8]) + } + + fn hash(b: u8) -> B256 { + B256::with_last_byte(b) + } + + fn collect_pending(buf: &mut BufferedFlashblocks) -> Vec { + buf.by_ref() + .filter_map(|e| match e { + ChainEvent::Pending(fb) => Some(fb.index), + _ => None, + }) + .collect() + } + + fn collect_all(buf: &mut BufferedFlashblocks) -> Vec { + buf.by_ref().collect() + } + + // ----------------------------------------------------------------------- + // Core state machine tests + // ----------------------------------------------------------------------- + + #[test] + fn no_output_before_canon_tip() { + let mut buf = BufferedFlashblocks::default(); + + // Send a base flashblock — no canon tip yet, goes to Pending + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert!( + collect_pending(&mut buf).is_empty(), + "should not yield without canon tip" + ); + } + + #[test] + fn canon_tip_triggers_drain() { + let mut buf = BufferedFlashblocks::default(); + + // Canon tip first, then base flashblock whose parent matches + buf.step(canon(0, hash(0))); + let events = collect_all(&mut buf); + assert_eq!(events.len(), 1); // just the canon event + assert!(matches!(events[0], ChainEvent::Canon(_))); + + // Now a base flashblock building on block 1 with parent hash(0) + buf.step(base_fb(pid(1), 0, hash(0), 1)); + let indices = collect_pending(&mut buf); + assert_eq!( + indices, + vec![0], + "should drain immediately when parent matches canon tip" + ); + } + + #[test] + fn canon_tip_after_buffered_flashblock_flushes() { + let mut buf = BufferedFlashblocks::default(); + + // Base flashblock arrives first — parent hash(5), block_number 6 + buf.step(base_fb(pid(1), 0, hash(5), 6)); + assert!(collect_pending(&mut buf).is_empty(), "no canon tip yet"); + + // Delta flashblock for same epoch + buf.step(delta_fb(pid(1), 1)); + assert!(collect_pending(&mut buf).is_empty(), "still no canon tip"); + + // Now canon tip arrives matching the parent + buf.step(canon(5, hash(5))); + let events: Vec<_> = collect_all(&mut buf); + + // Should yield: Canon(5), Pending(0), Pending(1) + assert!(matches!(events[0], ChainEvent::Canon(_))); + assert_eq!(events.len(), 3); + + let indices: Vec<_> = events + .iter() + .filter_map(|e| match e { + ChainEvent::Pending(fb) => Some(fb.index), + _ => None, + }) + .collect(); + assert_eq!(indices, vec![0, 1]); + } + + #[test] + fn out_of_order_flashblocks_buffered_until_contiguous() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); // drain canon + + // Base at index 0 + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert_eq!(collect_pending(&mut buf), vec![0]); + + // Index 2 arrives before 1 — gap, can't drain + buf.step(delta_fb(pid(1), 2)); + assert!(collect_pending(&mut buf).is_empty(), "gap at index 1"); + + // Index 1 fills the gap — both 1 and 2 should drain + buf.step(delta_fb(pid(1), 1)); + assert_eq!(collect_pending(&mut buf), vec![1, 2]); + } + + #[test] + fn stale_base_flashblock_discarded() { + let mut buf = BufferedFlashblocks::default(); + + // Canon tip is at block 10 + buf.step(canon(10, hash(10))); + collect_all(&mut buf); + + // Base flashblock building on block 5 (parent_number=4 < tip=10) — stale + buf.step(base_fb(pid(1), 0, hash(4), 5)); + assert!( + collect_pending(&mut buf).is_empty(), + "stale flashblock should be discarded" + ); + } + + #[test] + fn new_epoch_resets_buffer() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + // Epoch A + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert_eq!(collect_pending(&mut buf), vec![0]); + buf.step(delta_fb(pid(1), 1)); + assert_eq!(collect_pending(&mut buf), vec![1]); + + // Epoch B — new base with different payload_id, same parent + buf.step(base_fb(pid(2), 0, hash(0), 1)); + let indices = collect_pending(&mut buf); + assert_eq!(indices, vec![0], "new epoch should reset and yield base"); + } + + #[test] + fn canon_event_always_yielded() { + let mut buf = BufferedFlashblocks::default(); + + // Multiple canon events should all be yielded + buf.step(canon(0, hash(0))); + buf.step(canon(1, hash(1))); + buf.step(canon(2, hash(2))); + + let events = collect_all(&mut buf); + let canon_numbers: Vec<_> = events + .iter() + .filter_map(|e| match e { + ChainEvent::Canon(tip) => Some(tip.number), + _ => None, + }) + .collect(); + assert_eq!(canon_numbers, vec![0, 1, 2]); + } + + #[test] + fn non_base_flashblock_ignored_when_uninitialized() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + // Delta without any base — should be silently ignored + buf.step(delta_fb(pid(1), 5)); + assert!(collect_pending(&mut buf).is_empty()); + } + + #[test] + fn canon_tip_not_matching_parent_does_not_drain() { + let mut buf = BufferedFlashblocks::default(); + + // Base building on hash(5) at block 6 + buf.step(base_fb(pid(1), 0, hash(5), 6)); + assert!(collect_pending(&mut buf).is_empty()); + + // Canon tip at block 3, hash(3) — doesn't match parent hash(5) + buf.step(canon(3, hash(3))); + let events = collect_all(&mut buf); + + // Canon event is yielded, but no pending drained + assert_eq!(events.len(), 1); + assert!(matches!(events[0], ChainEvent::Canon(_))); + } + + #[test] + fn batch_processes_multiple_events() { + let mut buf = BufferedFlashblocks::default(); + for e in [ + canon(0, hash(0)), + base_fb(pid(1), 0, hash(0), 1), + delta_fb(pid(1), 1), + delta_fb(pid(1), 2), + ] { + buf.step(e); + } + + let events = collect_all(&mut buf); + + // Canon(0), Pending(0), Pending(1), Pending(2) + assert_eq!(events.len(), 4); + assert!(matches!(events[0], ChainEvent::Canon(_))); + + let indices: Vec<_> = events + .iter() + .filter_map(|e| match e { + ChainEvent::Pending(fb) => Some(fb.index), + _ => None, + }) + .collect(); + assert_eq!(indices, vec![0, 1, 2]); + } + + #[test] + fn duplicate_index_ignored() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + buf.step(base_fb(pid(1), 0, hash(0), 1)); + assert_eq!(collect_pending(&mut buf), vec![0]); + + // Same index again — ignored + buf.step(delta_fb(pid(1), 0)); + assert!(collect_pending(&mut buf).is_empty()); + } + + #[test] + fn wrong_payload_id_ignored() { + let mut buf = BufferedFlashblocks::default(); + + buf.step(canon(0, hash(0))); + collect_all(&mut buf); + + buf.step(base_fb(pid(1), 0, hash(0), 1)); + collect_pending(&mut buf); + + // Delta with wrong payload_id — ignored + buf.step(delta_fb(pid(99), 1)); + assert!(collect_pending(&mut buf).is_empty()); + } +} diff --git a/crates/flashblocks/p2p/src/protocol/handler.rs b/crates/flashblocks/p2p/src/protocol/handler.rs index 2191c661a..7f7f57afe 100644 --- a/crates/flashblocks/p2p/src/protocol/handler.rs +++ b/crates/flashblocks/p2p/src/protocol/handler.rs @@ -1,7 +1,7 @@ use crate::protocol::{ connection::{FlashblocksConnection, FlashblocksConnectionState, ReceiveStatus, Score}, error::FlashblocksP2PError, - event::{ChainEvent, WorldChainEvent, WorldChainEventsStream}, + event::{ChainEvent, WorldChainEvent, WorldChainEventsStream, world_chain_events_stream}, }; use alloy_rlp::BytesMut; use chrono::Utc; @@ -44,11 +44,6 @@ use tokio_stream::wrappers::BroadcastStream; /// Maximum frame size for rlpx messages. const MAX_FRAME: usize = 1 << 24; // 16 MiB -/// Maximum index for flashblocks payloads. -/// Not intended to ever be hit. Since we resize the flashblocks vector dynamically, -/// this is just a sanity check to prevent excessive memory usage. -pub(crate) const MAX_FLASHBLOCK_INDEX: usize = 100; - /// The maximum number of seconds we will wait for a previous publisher to stop /// before continueing anyways. const MAX_PUBLISH_WAIT_SEC: u64 = 2; @@ -716,16 +711,13 @@ impl FlashblocksHandle { let mut user_hook = hook; let combined_hook = move |event: &WorldChainEvent| { - if let WorldChainEvent::Chain(ce) = event - && let ChainEvent::Canon(tip) = ce.as_ref() - { + if let WorldChainEvent::Chain(ChainEvent::Canon(tip)) = event { state.lock().canon_tip = Some(*tip); } user_hook(event) }; - WorldChainEventsStream::new_with_hook( - combined_hook, + world_chain_events_stream( BroadcastStream::new(self.ctx.flashblock_tx.subscribe()) .filter_map(|x| { futures::future::ready(match x { @@ -738,12 +730,13 @@ impl FlashblocksHandle { } }) }) - .map(|fb| ChainEvent::Pending(Box::new(fb))) + .map(|fb| ChainEvent::Pending(Arc::new(fb))) .boxed(), provider .canonical_state_stream() .map(|n| ChainEvent::Canon(n.tip().num_hash())) .boxed(), + combined_hook, ) } diff --git a/crates/world/node/tests/e2e-testsuite/actions.rs b/crates/world/node/tests/e2e-testsuite/actions.rs index 54df02d66..1d3cc39ad 100644 --- a/crates/world/node/tests/e2e-testsuite/actions.rs +++ b/crates/world/node/tests/e2e-testsuite/actions.rs @@ -1978,3 +1978,304 @@ impl Action for Sleep { }) } } + +// --------------------------------------------------------------------------- +// EngineDriver — drives the consensus engine through N block-building cycles +// --------------------------------------------------------------------------- + +/// Callback invoked after each block is built and canonicalized. +pub type BlockCallback = Box< + dyn Fn( + usize, + &OpExecutionPayloadEnvelopeV4, + ) -> std::pin::Pin> + Send>> + + Send + + Sync, +>; + +/// Drives the consensus engine through `num_blocks` block-building cycles. +/// +/// Each cycle: +/// 1. Generates payload attributes for the next block +/// 2. Sends `forkchoiceUpdatedV3` with attributes to start building +/// 3. Waits for `block_interval` (the build deadline) +/// 4. Calls `getPayloadV4` to retrieve the built payload +/// 5. Sends `newPayloadV4` + `forkchoiceUpdated` on all follower nodes +/// 6. Invokes the optional `on_block` callback +/// 7. Advances to the next cycle with the new block as head +pub struct EngineDriver { + /// Index of the builder node in the environment's node_clients. + pub builder_idx: usize, + /// Indices of follower nodes that receive `newPayload` + FCU. + pub follower_idxs: Vec, + /// Number of blocks to build. + pub num_blocks: usize, + /// Time to wait between FCU (start building) and getPayload (retrieve). + pub block_interval: Duration, + /// Whether to use flashblocks FCU with authorization. + pub flashblocks: bool, + /// Generates `Authorization` from `OpPayloadAttributes`. + pub authorization_gen: A, + /// Generates attributes for the next block given (block_number, parent_timestamp). + pub attributes_gen: Box Result + Send + Sync>, + /// Optional callback after each block is built and canonicalized. + pub on_block: Option, +} + +impl EngineDriver +where + A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, +{ + fn execute<'a>( + &'a mut self, + env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let builder = &env.node_clients[self.builder_idx]; + let engine = builder.engine.http_client(); + + // Get the initial head + let mut parent_hash = { + let latest: Option = + EthApiClient::< + TransactionRequest, + Transaction, + alloy_rpc_types_eth::Block, + alloy_consensus::Receipt, + Header, + TransactionSigned, + >::block_by_number( + &builder.rpc, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await?; + latest + .ok_or_else(|| eyre!("No latest block"))? + .header + .hash_slow() + }; + + let mut parent_timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + + for block_num in 0..self.num_blocks { + // 1. Generate attributes + let block_number = block_num as u64 + 1; + parent_timestamp += self.block_interval.as_secs().max(1); + let attributes = (self.attributes_gen)(block_number, parent_timestamp)?; + + // 2. FCU with attributes → start building + let fcu_state = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; + + let fcu_result = if self.flashblocks { + FlashblocksEngineApiExtClient::::flashblocks_fork_choice_updated_v3( + &engine, + fcu_state, + Some(attributes.clone()), + Some((self.authorization_gen)(attributes.clone())), + ) + .await? + } else { + EngineApiClient::::fork_choice_updated_v3( + &engine, + fcu_state, + Some(attributes.clone()), + ) + .await? + }; + + if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { + return Err(eyre!( + "block {block_num}: FCU status not valid: {:?}", + fcu_result.payload_status + )); + } + + let payload_id = fcu_result + .payload_id + .ok_or_else(|| eyre!("block {block_num}: No payload ID returned"))?; + + info!( + target: "engine_driver", + block = block_num, + %payload_id, + "building block" + ); + + // 3. Wait for build deadline + tokio::time::sleep(self.block_interval).await; + + // 4. getPayloadV4 + let payload = + EngineApiClient::::get_payload_v4(&engine, payload_id).await?; + + let block_hash = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .block_hash; + + let tx_count = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .transactions + .len(); + + info!( + target: "engine_driver", + block = block_num, + %block_hash, + tx_count, + "payload retrieved" + ); + + // 5. Canonicalize: FCU(parent) → newPayload → FCU(head) + // on builder AND all follower nodes + use alloy_rpc_types_engine::CancunPayloadFields; + use op_alloy_rpc_types_engine::{ + OpExecutionData, OpExecutionPayload, OpExecutionPayloadSidecar, + }; + + // Canonicalize on builder via FCU only (it already has the payload) + { + let builder_engine = env.node_clients[self.builder_idx].engine.http_client(); + let head_fcu = ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash: block_hash, + finalized_block_hash: block_hash, + }; + let fcu_result = EngineApiClient::::fork_choice_updated_v3( + &builder_engine, + head_fcu, + None, + ) + .await?; + + if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { + return Err(eyre!( + "block {block_num}: builder FCU to head failed: {:?}", + fcu_result.payload_status + )); + } + } + + // Canonicalize on follower nodes: FCU(parent) → newPayload → FCU(head) + for follower_idx in self.follower_idxs.iter().copied() { + if let Some(beacon_handle) = + env.node_clients[follower_idx].beacon_engine_handle.as_ref() + { + // FCU to parent + let parent_fcu = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; + beacon_handle + .fork_choice_updated( + parent_fcu, + None, + EngineApiMessageVersion::V3, + ) + .await + .map_err(|e| { + eyre!("block {block_num}: FCU to parent failed on follower {follower_idx}: {e:?}") + })?; + + // newPayload + let execution_data = OpExecutionData { + payload: OpExecutionPayload::V4(payload.execution_payload.clone()), + sidecar: OpExecutionPayloadSidecar::v4( + CancunPayloadFields::new(payload.parent_beacon_block_root, vec![]), + alloy_rpc_types_engine::PraguePayloadFields { + requests: alloy_eips::eip7685::RequestsOrHash::Hash( + alloy_eips::eip7685::EMPTY_REQUESTS_HASH, + ), + }, + ), + }; + + let status = beacon_handle + .new_payload(execution_data) + .await + .map_err(|e| { + eyre!("block {block_num}: newPayload failed on follower {follower_idx}: {e:?}") + })?; + + if !matches!(status.status, PayloadStatusEnum::Valid) { + return Err(eyre!( + "block {block_num}: newPayload invalid on follower {follower_idx}: {:?}", + status + )); + } + + // FCU to head + let head_fcu = ForkchoiceState { + head_block_hash: block_hash, + safe_block_hash: block_hash, + finalized_block_hash: block_hash, + }; + beacon_handle + .fork_choice_updated( + head_fcu, + None, + EngineApiMessageVersion::V3, + ) + .await + .map_err(|e| { + eyre!("block {block_num}: FCU to head failed on follower {follower_idx}: {e:?}") + })?; + + info!( + target: "engine_driver", + block = block_num, + follower = follower_idx, + %block_hash, + "canonicalized on follower via beacon handle" + ); + } else { + return Err(eyre!( + "block {block_num}: follower {follower_idx} has no beacon_engine_handle" + )); + } + } + + // 6. Invoke callback + if let Some(ref on_block) = self.on_block { + on_block(block_num, &payload).await?; + } + + // 7. Advance head + parent_hash = block_hash; + + info!( + target: "engine_driver", + block = block_num, + %block_hash, + "block complete" + ); + } + + Ok(()) + }) + } +} + +impl Action for EngineDriver +where + A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, +{ + fn execute<'a>( + &'a mut self, + env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + EngineDriver::execute(self, env) + } +} diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index 9dcf88f37..466007354 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -10,6 +10,7 @@ use alloy_primitives::{Bytes, b64}; use alloy_rpc_types::TransactionRequest; use alloy_rpc_types_engine::PayloadStatusEnum; use eyre::eyre::eyre; +use flashblocks_p2p::protocol::event::{ChainEvent, WorldChainEvent}; use futures::future::Either; use reth::{ chainspec::EthChainSpec, @@ -1186,3 +1187,330 @@ async fn test_continuous_block_production_with_validation() -> eyre::Result<()> Ok(()) } + +/// End-to-end test: drives the builder's consensus engine through a block +/// building loop, using a hook on the `WorldChainEventsStream` to assert +/// stream invariants: +/// +/// 1. Canon events are always yielded +/// 2. Pending flashblocks are only yielded after their epoch parent is canonical +/// 3. Flashblock indices are monotonically increasing within an epoch +/// 4. The P2P state's flushed cursor tracks the latest yielded flashblock +/// 5. Stale flashblocks (from old epochs) are never yielded +#[tokio::test(flavor = "multi_thread")] +async fn test_event_stream_invariants() -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + const TRANSACTIONS_PER_FLASHBLOCK: u64 = 10; + + tokio::time::sleep(Duration::from_millis(100)).await; + + let (_, mut nodes, _tasks, mut env, tx_spammer) = + setup::(1, optimism_payload_attributes, true).await?; + + let builder_node = &mut nodes[0]; + let builder_context = builder_node.ext_context.clone().unwrap(); + let rpc_url = builder_node.node.rpc_url(); + + tx_spammer.spawn(TRANSACTIONS_PER_FLASHBLOCK, rpc_url); + + let block_hash = builder_node.node.block_hash(0); + + let authorization_generator = crate::setup::create_authorization_generator( + block_hash, + builder_context + .flashblocks_handle + .builder_sk() + .unwrap() + .verifying_key(), + ); + + let timestamp = crate::setup::current_timestamp(); + let eip1559_params = + encode_eip1559_params(builder_node.node.inner.chain_spec().as_ref(), timestamp)?; + + let attributes = build_payload_attributes( + timestamp, + eip1559_params, + Some(vec![TX_SET_L1_BLOCK.clone()]), + ); + + // --- Assertion state shared with the hook --- + let canon_count = Arc::new(AtomicUsize::new(0)); + let pending_count = Arc::new(AtomicUsize::new(0)); + let last_index = Arc::new(AtomicU64::new(0)); + let saw_canon_before_pending = Arc::new(std::sync::atomic::AtomicBool::new(false)); + + let canon_count_hook = canon_count.clone(); + let pending_count_hook = pending_count.clone(); + let last_index_hook = last_index.clone(); + let saw_canon_hook = saw_canon_before_pending.clone(); + + // Create the event stream with a hook that asserts invariants + let p2p_state = builder_context.flashblocks_handle.state.clone(); + let mut stream = builder_context + .flashblocks_handle + .event_stream::<(), _, _, _>( + builder_node.node.inner.provider.clone(), + move |event: &WorldChainEvent<()>| { + match event { + WorldChainEvent::Chain(ChainEvent::Canon(_tip)) => { + canon_count_hook.fetch_add(1, Ordering::SeqCst); + } + WorldChainEvent::Chain(ChainEvent::Pending(fb)) => { + // Invariant: we must have seen at least one canon event + // before any pending flashblock is yielded. + if canon_count_hook.load(Ordering::SeqCst) > 0 { + saw_canon_hook.store(true, Ordering::SeqCst); + } + + // Invariant: indices are monotonically increasing + let prev = last_index_hook.swap(fb.index, Ordering::SeqCst); + if pending_count_hook.load(Ordering::SeqCst) > 0 { + assert!( + fb.index >= prev, + "flashblock index went backwards: {} -> {}", + prev, + fb.index + ); + } + + pending_count_hook.fetch_add(1, Ordering::SeqCst); + } + _ => {} + } + None + }, + ); + + // Spawn the stream consumer + let _stream_handle = tokio::spawn(async move { + let mut count = 0usize; + while let Some(_event) = futures::StreamExt::next(&mut stream).await { + count += 1; + if count > 50 { + break; // safety valve + } + } + count + }); + + // Mine a block + let (tx, mut rx) = tokio::sync::mpsc::channel(1); + let mine_block = crate::actions::AssertMineBlock::new( + 0, + None, + attributes, + authorization_generator, + Duration::from_millis(3000), + true, + tx, + ) + .await; + + tokio::spawn(async move { + let mut mine_action = mine_block; + mine_action.execute(&mut env).await + }); + + // Wait for mining to complete + rx.recv() + .await + .ok_or(eyre!("failed to receive mined block"))?; + + // Give the stream a moment to process remaining events + tokio::time::sleep(Duration::from_millis(500)).await; + + // --- Assert invariants --- + let canons = canon_count.load(Ordering::SeqCst); + let pendings = pending_count.load(Ordering::SeqCst); + + info!( + target: "test", + canon_events = canons, + pending_events = pendings, + "stream invariant results" + ); + + assert!(canons > 1, "expected at least one canon event"); + assert!(pendings > 1, "expected at least one pending flashblock"); + assert!( + saw_canon_before_pending.load(Ordering::SeqCst), + "expected canon event before first pending flashblock" + ); + + // Verify P2P state was updated by the hook + let state = p2p_state.lock(); + assert!( + state.canon_tip.is_some(), + "expected canon_tip to be set on P2P state" + ); + assert!( + state.flushed_payload_id.is_some(), + "expected flushed_payload_id to be set on P2P state" + ); + + Ok(()) +} + +/// End-to-end test: uses [`EngineDriver`] to build multiple blocks while +/// querying the pending block, logs, transactions, and receipts via the +/// Eth JSON-RPC API at each block boundary. +#[tokio::test(flavor = "multi_thread")] +async fn test_engine_driver_pending_block_queries() -> eyre::Result<()> { + use alloy_eips::BlockNumberOrTag; + use reth::rpc::api::EthApiClient; + + reth_tracing::init_test_tracing(); + + const NUM_BLOCKS: usize = 3; + const BLOCK_INTERVAL: Duration = Duration::from_millis(2000); + + tokio::time::sleep(Duration::from_millis(100)).await; + + // 2 nodes: builder + follower + let (_, nodes, _tasks, mut env, tx_spammer) = + setup::(2, optimism_payload_attributes, true).await?; + + let builder_context = nodes[0].ext_context.clone().unwrap(); + let block_hash = nodes[0].node.block_hash(0); + let chain_spec = nodes[0].node.inner.chain_spec().clone(); + let rpc_url = nodes[0].node.rpc_url(); + + // Spawn background transactions so blocks have content + tx_spammer.spawn(10, rpc_url); + + let authorization_gen = crate::setup::create_authorization_generator( + block_hash, + builder_context + .flashblocks_handle + .builder_sk() + .unwrap() + .verifying_key(), + ); + + // Track per-block results + let blocks_with_pending = Arc::new(AtomicUsize::new(0)); + let total_pending_txs = Arc::new(AtomicUsize::new(0)); + let blocks_with_pending_cb = blocks_with_pending.clone(); + let total_pending_txs_cb = total_pending_txs.clone(); + + // Keep a reference to the builder's RPC client for pending queries + let builder_rpc = env.node_clients[0].rpc.clone(); + + let mut driver = crate::actions::EngineDriver { + builder_idx: 0, + follower_idxs: vec![], + num_blocks: NUM_BLOCKS, + block_interval: BLOCK_INTERVAL, + flashblocks: true, + authorization_gen, + attributes_gen: Box::new({ + let chain_spec = chain_spec.clone(); + move |_block_number, timestamp| { + let eip1559 = encode_eip1559_params(chain_spec.as_ref(), timestamp)?; + Ok(build_payload_attributes( + timestamp, + eip1559, + Some(vec![TX_SET_L1_BLOCK.clone()]), + )) + } + }), + on_block: Some(Box::new({ + let builder_rpc = builder_rpc.clone(); + move |block_num, payload| { + let builder_rpc = builder_rpc.clone(); + let blocks_with_pending = blocks_with_pending_cb.clone(); + let total_pending_txs = total_pending_txs_cb.clone(); + + let block_hash = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .block_hash; + let payload_tx_count = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .transactions + .len(); + + Box::pin(async move { + info!( + target: "engine_driver_test", + block = block_num, + %block_hash, + payload_tx_count, + "payload built" + ); + + assert!( + payload_tx_count > 0, + "block {block_num}: expected at least 1 transaction (L1 info deposit)" + ); + + // Query the pending block during this build cycle + let pending: Option = EthApiClient::< + TransactionRequest, + alloy_rpc_types::Transaction, + alloy_rpc_types_eth::Block, + alloy_consensus::Receipt, + alloy_consensus::Header, + reth_optimism_primitives::OpTransactionSigned, + >::block_by_number( + &builder_rpc, + BlockNumberOrTag::Pending, + false, // tx hashes only to avoid deserialization issues + ) + .await?; + + if let Some(pending_block) = &pending { + info!( + target: "engine_driver_test", + block = block_num, + pending_tx_count = pending_block.transactions.len(), + pending_number = pending_block.header.number, + "queried pending block" + ); + } else { + info!( + target: "engine_driver_test", + block = block_num, + "no pending block available (expected during finalization)" + ); + } + + blocks_with_pending.fetch_add(1, Ordering::SeqCst); + total_pending_txs.fetch_add(payload_tx_count, Ordering::SeqCst); + + Ok(()) + }) + } + })), + }; + + driver.execute(&mut env).await?; + + let blocks_queried = blocks_with_pending.load(Ordering::SeqCst); + let txs_queried = total_pending_txs.load(Ordering::SeqCst); + + info!( + target: "engine_driver_test", + blocks_queried, + txs_queried, + "engine driver test complete" + ); + + assert_eq!( + blocks_queried, NUM_BLOCKS, + "expected to query {NUM_BLOCKS} blocks" + ); + assert!( + txs_queried >= NUM_BLOCKS, + "expected at least {NUM_BLOCKS} total transactions (one L1 deposit per block)" + ); + + Ok(()) +} From 7f463ee63e5eaf1799209373640695dd5b657eef Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Sat, 14 Mar 2026 00:09:10 -0700 Subject: [PATCH 38/43] fix: cleanup + test --- Cargo.lock | 21 +- Cargo.toml | 2 +- crates/flashblocks/builder/Cargo.toml | 2 + crates/flashblocks/builder/src/coordinator.rs | 140 +- crates/flashblocks/builder/src/lib.rs | 71 + crates/flashblocks/builder/src/metrics.rs | 247 +++ crates/flashblocks/p2p/src/protocol/event.rs | 19 +- crates/world/node/Cargo.toml | 6 +- .../world/node/tests/e2e-testsuite/actions.rs | 1409 ++++------------- .../world/node/tests/e2e-testsuite/spammer.rs | 73 +- .../node/tests/e2e-testsuite/testsuite.rs | 594 ++++--- 11 files changed, 1088 insertions(+), 1496 deletions(-) create mode 100644 crates/flashblocks/builder/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 480e72e5e..7277dfeb3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3762,6 +3762,7 @@ dependencies = [ "futures", "lazy_static", "metrics", + "metrics-derive", "op-alloy-consensus", "op-alloy-network", "parking_lot", @@ -3795,6 +3796,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tracing", + "tracing-subscriber 0.3.22", ] [[package]] @@ -6235,7 +6237,7 @@ source = "git+https://github.com/0xForerunner/optimism?rev=79c9153#79c91536e0713 dependencies = [ "op-alloy-consensus", "op-alloy-network", - "op-alloy-provider", + "op-alloy-provider 0.23.1 (git+https://github.com/0xForerunner/optimism?rev=79c9153)", "op-alloy-rpc-types", "op-alloy-rpc-types-engine", ] @@ -6280,6 +6282,21 @@ dependencies = [ "op-alloy-rpc-types", ] +[[package]] +name = "op-alloy-provider" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6753d90efbaa8ea8bcb89c1737408ca85fa60d7adb875049d3f382c063666f86" +dependencies = [ + "alloy-network", + "alloy-primitives", + "alloy-provider", + "alloy-rpc-types-engine", + "alloy-transport", + "async-trait", + "op-alloy-rpc-types-engine", +] + [[package]] name = "op-alloy-provider" version = "0.23.1" @@ -14203,6 +14220,8 @@ dependencies = [ "hex", "jsonrpsee", "op-alloy-consensus", + "op-alloy-network", + "op-alloy-provider 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)", "op-alloy-rpc-types", "op-alloy-rpc-types-engine", "parking_lot", diff --git a/Cargo.toml b/Cargo.toml index 7d25c1924..a6a4ffb0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -145,6 +145,7 @@ op-alloy-rpc-types = { version = "0.23.1", default-features = false } op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false } op-alloy-network = { version = "0.23.1", default-features = false } alloy-op-hardforks = { version = "0.4.4", default-features = false } +op-alloy-provider = { version = "0.23.1", default-features = false } # alloy alloy = { version = "1.1.2" } @@ -200,7 +201,6 @@ revm-inspectors = "0.34" alloy-op-evm = { version = "0.27", default-features = false } alloy-evm = { version = "0.27", default-features = false } - # rpc jsonrpsee = { version = "0.26.0", features = ["server", "client", "macros"] } jsonrpsee-core = { version = "0.26.0" } diff --git a/crates/flashblocks/builder/Cargo.toml b/crates/flashblocks/builder/Cargo.toml index c0bfbaeaf..d314f9a91 100644 --- a/crates/flashblocks/builder/Cargo.toml +++ b/crates/flashblocks/builder/Cargo.toml @@ -61,6 +61,7 @@ dashmap.workspace = true thiserror.workspace = true either.workspace = true metrics.workspace = true +metrics-derive.workspace = true serde.workspace = true [dev-dependencies] @@ -70,6 +71,7 @@ eyre.workspace = true lazy_static.workspace = true proptest.workspace = true reth-tracing.workspace = true +tracing-subscriber.workspace = true alloy-genesis.workspace = true op-alloy-network.workspace = true alloy-signer-local.workspace = true diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 80630988a..01f0d1d88 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -30,13 +30,13 @@ use reth_provider::{ CanonStateSubscriptions, ChainSpecProvider, HeaderProvider, StateProviderFactory, }; use reth_transaction_pool::{EthPooledTransaction, noop::NoopTransactionPool}; -use std::{panic::AssertUnwindSafe, sync::Arc, time::Instant}; +use std::sync::Arc; use tokio::sync::{ Semaphore, SemaphorePermit, broadcast::{self, Sender}, oneshot, }; -use tracing::{error, trace}; +use tracing::{debug, error, trace}; /// Maximum number of concurrent flashblock processing tasks on the thread pool. const MAX_THREAD_POOL_SIZE: usize = 4; @@ -49,7 +49,9 @@ pub enum TrieTaskHandle {} use crate::{ bal_executor::CommittedState, bal_validator::{FlashblocksBlockValidator, decode_transactions_with_indices}, + metrics::EXECUTION, payload_builder::build, + spawn_blocking_io_with_shutdown_signal, traits::{context::OpPayloadBuilderCtxBuilder, context_builder::PayloadBuilderCtxBuilder}, }; use flashblocks_primitives::flashblocks::{Flashblock, Flashblocks}; @@ -149,6 +151,15 @@ impl FlashblocksExecutionCoordinator { WorldChainEvent::Chain(ChainEvent::Pending(flashblock)) => { let flashblock = Arc::try_unwrap(flashblock).unwrap_or_else(|arc| (*arc).clone()); + + trace!( + target: "flashblocks::coordinator", + payload_id = %flashblock.payload_id, + index = %flashblock.index, + is_base = flashblock.base.is_some(), + "received pending flashblock" + ); + // Track epoch block number from base flashblocks if let Some(base) = &flashblock.base { epoch_block_number = Some(base.block_number); @@ -168,6 +179,13 @@ impl FlashblocksExecutionCoordinator { .await; } WorldChainEvent::Chain(ChainEvent::Canon(tip)) => { + trace!( + target: "flashblocks::coordinator", + tip_number = tip.number, + tip_hash = %tip.hash, + "received canonical tip" + ); + this.on_canon( tip, &mut inflight_shutdown, @@ -222,6 +240,9 @@ impl FlashblocksExecutionCoordinator { let chain_spec = chain_spec.clone(); let pending_block = pending_block.clone(); + let payload_id = flashblock.payload_id; + let index = flashblock.index; + spawn_blocking_io_with_shutdown_signal(workload, rx, database_permit, move |permit| { let _task_permit = task_permit; // held until closure completes if let Err(e) = process_flashblock( @@ -233,7 +254,13 @@ impl FlashblocksExecutionCoordinator { flashblock, pending_block, ) { - error!("error processing flashblock: {e:#?}"); + error!( + target: "flashblocks::coordinator", + %payload_id, + index, + "error processing flashblock: {e:#?}" + ); + EXECUTION.errors.increment(1); } }); } @@ -241,6 +268,15 @@ impl FlashblocksExecutionCoordinator { /// Handles a canonical chain tip update. Cancels any in-flight task, /// clears stale ancestor trie handles, and clears the pending block if /// it was built on the now-canonical tip. + #[tracing::instrument( + target = "flashblocks::coordinator", + skip_all, + fields( + tip_number = tip.number, + tip_hash = %tip.hash, + is_stale, + ) + )] fn on_canon( &self, tip: BlockNumHash, @@ -252,8 +288,15 @@ impl FlashblocksExecutionCoordinator { // epoch is at or behind the canonical tip (stale). If the epoch is // ahead of the tip, the work is still valid. let is_stale = epoch_block_number.is_none_or(|n| n <= tip.number); + tracing::Span::current().record("is_stale", is_stale); if is_stale { + debug!( + target: "flashblocks::coordinator", + epoch_block_number = *epoch_block_number, + "stale epoch — cancelling inflight and clearing ancestors" + ); + EXECUTION.stale_resets.increment(1); inflight_shutdown.take(); self.inner.write().ancestor_handles.clear(); *epoch_block_number = None; @@ -333,51 +376,6 @@ impl FlashblocksExecutionCoordinator { } } -/// Spawns a blocking task on the [`WorkloadExecutor`] thread pool, racing it -/// against a shutdown signal. If the shutdown receiver resolves first (sender -/// dropped), the task result is discarded. Acquires the `database_permit` -/// before running `f` to serialize pending block writes. -fn spawn_blocking_io_with_shutdown_signal( - executor: &WorkloadExecutor, - shutdown_rx: oneshot::Receiver<()>, - database_permit: &'static Semaphore, - f: F, -) where - F: FnOnce(SemaphorePermit<'static>) + Send + 'static, -{ - let task = executor.spawn_blocking(move || { - let f = AssertUnwindSafe(move || { - let rt = tokio::runtime::Builder::new_current_thread() - .build() - .expect("failed to build runtime for permit acquisition"); - - let permit = rt - .block_on(database_permit.acquire()) - .expect("database semaphore closed"); - - f(permit); - }); - - if let Err(e) = std::panic::catch_unwind(f) { - error!("flashblock processing panicked: {e:?}"); - } - }); - - // Race the task against the shutdown signal - tokio::spawn(async move { - match futures::future::select(task, shutdown_rx).await { - futures::future::Either::Left((result, _)) => { - if let Err(e) = result { - error!("flashblock thread pool task panicked: {e:#?}"); - } - } - futures::future::Either::Right(_) => { - trace!("flashblock processing cancelled by shutdown signal"); - } - } - }); -} - fn process_flashblock( database_permit: SemaphorePermit<'static>, provider: Provider, @@ -467,23 +465,24 @@ where extra_data: base.extra_data.clone(), }; - trace!( - target: "flashblocks::coordinator", - id = %flashblock.flashblock().payload_id, - index = %flashblock.flashblock().index, - min_tx_index = %flashblock.flashblock().diff.access_list_data.as_ref().map_or("None".to_string(), |d| d.access_list.min_tx_index.to_string()), - max_tx_index = %flashblock.flashblock().diff.access_list_data.as_ref().map_or("None".to_string(), |d| d.access_list.max_tx_index.to_string()), - execution_context = ?execution_context, - next_block_context = ?next_block_context, - "processing flashblock" - ); - let evm_env = evm_config.next_evm_env(sealed_header.header(), &next_block_context)?; - let transactions_offset = committed_state.transactions.len() + 1; - let start = Instant::now(); + let has_bal = flashblock.diff().access_list_data.is_some(); + + let _validate_span = crate::metrics::MetricsSpan::new( + tracing::trace_span!( + target: "flashblocks::coordinator", + "validate", + id = %flashblock.flashblock().payload_id, + index, + path = if has_bal { "bal" } else { "legacy" }, + tx_count = flashblock.diff().transactions.len(), + duration_ms = tracing::field::Empty, + ), + EXECUTION.validate_duration.clone(), + ); - let payload = if flashblock.diff().access_list_data.is_some() { + let payload = if has_bal { let sealed_header = Arc::new(sealed_header); let executor_transactions = decode_transactions_with_indices( @@ -581,9 +580,8 @@ where } }; - let duration = Instant::now().duration_since(start); - metrics::histogram!("flashblocks.validate", "access_list" => flashblock.diff().access_list_data.is_some().to_string()) - .record(duration.as_nanos() as f64 / 1_000_000_000.0); + // _validate_span dropped here — records duration_ms on span + histogram. + drop(_validate_span); // Build ExecutedBlock with deferred trie data — sorting happens in background let deferred = if let Some(executed) = payload.executed_block() { @@ -623,9 +621,19 @@ where // Everything after this point (trie sort, broadcast) can run concurrently. drop(database_permit); - // Spawn background trie sort after releasing the permit + // Spawn background trie sort after releasing the permit. + // Link the rayon span back to the processing span for trace correlation. if let Some(deferred) = deferred { + let trie_span = tracing::trace_span!( + target: "flashblocks::coordinator", + "trie_sort", + id = %payload.id(), + index, + ); + trie_span.follows_from(tracing::Span::current()); + rayon::spawn(move || { + let _enter = trie_span.enter(); deferred.wait_cloned(); }); } diff --git a/crates/flashblocks/builder/src/lib.rs b/crates/flashblocks/builder/src/lib.rs index a628c8d1d..539f0e60c 100644 --- a/crates/flashblocks/builder/src/lib.rs +++ b/crates/flashblocks/builder/src/lib.rs @@ -138,6 +138,9 @@ //! [`BalBlockBuilder`]: executor::BalBlockBuilder //! [`TemporalDb`]: database::temporal_db::TemporalDb +use std::{panic::AssertUnwindSafe, time::Instant}; + +use reth_engine_tree::tree::executor::WorkloadExecutor; use reth_evm::{ block::BlockExecutionError, execute::{BlockBuilder, BlockBuilderOutcome}, @@ -145,6 +148,10 @@ use reth_evm::{ use reth_optimism_payload_builder::config::OpBuilderConfig; use reth_provider::StateProvider; use revm_database::BundleState; +use tokio::sync::{Semaphore, SemaphorePermit, oneshot}; +use tracing::{error, trace}; + +use crate::metrics::EXECUTION; /// Utilities for constructing and serializing Block Access Lists (BAL). pub mod access_list; @@ -186,6 +193,9 @@ pub mod executor; /// Block building utilities pub mod utils; +/// Metric name constants. +pub mod metrics; + /// Configuration for the flashblocks payload builder. #[derive(Default, Debug, Clone)] pub struct FlashblocksPayloadBuilderConfig { @@ -212,3 +222,64 @@ pub trait BlockBuilderExt: BlockBuilder { state_provider: impl StateProvider, ) -> Result<(BlockBuilderOutcome, BundleState), BlockExecutionError>; } + +/// Spawns a blocking task on the [`WorkloadExecutor`] thread pool, racing it +/// against a shutdown signal. If the shutdown receiver resolves first (sender +/// dropped), the task result is discarded. Acquires the `database_permit` +/// before running `f` to serialize pending block writes. +/// +/// The current tracing span is captured and re-entered on the blocking thread +/// so that all events inside `f` are nested under the caller's span. +#[track_caller] +pub(crate) fn spawn_blocking_io_with_shutdown_signal( + executor: &WorkloadExecutor, + shutdown_rx: oneshot::Receiver<()>, + database_permit: &'static Semaphore, + f: F, +) where + F: FnOnce(SemaphorePermit<'static>) + Send + 'static, +{ + let parent_span = tracing::Span::current(); + + let task = executor.spawn_blocking(move || { + let _enter = parent_span.enter(); + + let unwind = AssertUnwindSafe(move || { + let permit_wait = Instant::now(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build runtime for permit acquisition"); + + let permit = rt + .block_on(database_permit.acquire()) + .expect("database semaphore closed"); + + EXECUTION + .permit_wait + .record(permit_wait.elapsed().as_secs_f64()); + + f(permit); + }); + + if let Err(e) = std::panic::catch_unwind(unwind) { + error!("flashblock processing panicked: {e:?}"); + } + }); + + // Race the blocking task against the shutdown signal. + // If shutdown fires first (sender dropped by on_flashblock), the + // blocking result is discarded — the newer flashblock takes priority. + tokio::spawn(async move { + match futures::future::select(task, shutdown_rx).await { + futures::future::Either::Left((result, _)) => { + if let Err(e) = result { + error!("flashblock thread pool task panicked: {e:#?}"); + } + } + futures::future::Either::Right(_) => { + trace!("flashblock processing cancelled by shutdown signal"); + } + } + }); +} diff --git a/crates/flashblocks/builder/src/metrics.rs b/crates/flashblocks/builder/src/metrics.rs new file mode 100644 index 000000000..2dc82a65b --- /dev/null +++ b/crates/flashblocks/builder/src/metrics.rs @@ -0,0 +1,247 @@ +//! Metrics and instrumentation for the flashblocks builder. + +use metrics::{Counter, Histogram}; +use metrics_derive::Metrics; +use std::{sync::LazyLock, time::Instant}; + +/// Execution coordinator metrics, auto-registered under `flashblocks.coordinator.*`. +#[derive(Clone, Metrics)] +#[metrics(scope = "flashblocks.coordinator")] +pub struct ExecutionMetrics { + // -- Latency -- + /// End-to-end flashblock processing duration (seconds). + pub process_duration: Histogram, + /// Validation / build phase duration (seconds). + pub validate_duration: Histogram, + /// Time waiting for the database write permit (seconds). + pub permit_wait: Histogram, + /// Duration holding the state write lock (seconds). + pub state_lock_duration: Histogram, + /// Flashblocks processed in a single epoch (recorded at epoch boundary). + pub flashblocks_per_epoch: Histogram, + + // -- Issues -- + /// Flashblock skipped — already processed (duplicate from P2P). + pub skipped: Counter, + /// Epoch invalidated by a newer canonical tip. + pub stale_resets: Counter, + /// Processing error (validation, build, or state update failure). + pub errors: Counter, + /// Failed to fetch sealed header for parent hash. + pub header_fetch_failed: Counter, + /// Invalid payload received from P2P (decode error, bad structure). + pub invalid_payload: Counter, + /// Broadcast of built payload to in-memory tree failed. + pub broadcast_failed: Counter, + /// newPayloadV3/V4 cache hit — payload already built for this id+index. + pub payload_cache_hits: Counter, +} + +/// Global singleton — zero lookup cost per call site. +pub static EXECUTION: LazyLock = LazyLock::new(ExecutionMetrics::default); + +/// RAII guard that enters a [`tracing::Span`] on creation. On drop it: +/// 1. Records `duration_ms` on the tracing span +/// 2. Records elapsed seconds to the provided [`Histogram`] +pub struct MetricsSpan { + inner: tracing::span::EnteredSpan, + start: Instant, + histogram: Histogram, +} + +impl MetricsSpan { + /// Enter `span` and start the timer. `histogram` receives elapsed seconds on drop. + pub fn new(span: tracing::Span, histogram: Histogram) -> Self { + Self { + inner: span.entered(), + start: Instant::now(), + histogram, + } + } + + /// Record a field on the underlying tracing span. + pub fn record(&self, field: &str, value: V) { + self.inner.record(field, value); + } +} + +impl Drop for MetricsSpan { + fn drop(&mut self) { + let elapsed = self.start.elapsed(); + self.inner.record("duration_ms", elapsed.as_millis() as u64); + self.histogram.record(elapsed.as_secs_f64()); + } +} + +/// Execute `f` inside a metered tracing span. The span is entered before `f` +/// runs and duration is recorded (both on the span and as a histogram) on +/// completion. `f` receives a [`MetricsSpan`] reference for recording dynamic +/// span fields mid-execution. +pub fn metered_fn(span: tracing::Span, histogram: Histogram, f: F) -> R +where + F: FnOnce(&MetricsSpan) -> R, +{ + let guard = MetricsSpan::new(span, histogram); + f(&guard) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + + /// Tracing layer that captures span events for assertions. + struct SpanCapture { + spans: Arc>>, + } + + #[derive(Debug, Clone)] + struct SpanRecord { + name: String, + fields: String, + } + + impl tracing_subscriber::Layer for SpanCapture { + fn on_new_span( + &self, + attrs: &tracing::span::Attributes<'_>, + _id: &tracing::span::Id, + _ctx: tracing_subscriber::layer::Context<'_, S>, + ) { + let mut fields = String::new(); + attrs.record(&mut FieldVisitor(&mut fields)); + self.spans.lock().unwrap().push(SpanRecord { + name: attrs.metadata().name().to_string(), + fields, + }); + } + } + + struct FieldVisitor<'a>(&'a mut String); + + impl tracing::field::Visit for FieldVisitor<'_> { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + use std::fmt::Write; + if !self.0.is_empty() { + self.0.push(' '); + } + let _ = write!(self.0, "{}={:?}", field.name(), value); + } + } + + #[test] + fn metrics_span_records_duration_and_emits_histogram() { + // Install a real metrics recorder so histogram calls don't panic. + // metrics-util provides an in-memory recorder for testing. + // If unavailable, we just verify the span side. + let spans = Arc::new(Mutex::new(Vec::new())); + let layer = SpanCapture { + spans: spans.clone(), + }; + + let _guard = tracing_subscriber::registry().with(layer).set_default(); + + let histogram = EXECUTION.process_duration.clone(); + + { + let _span = MetricsSpan::new( + tracing::trace_span!( + target: "flashblocks::coordinator", + "test_span", + id = "test_payload", + index = 3u64, + duration_ms = tracing::field::Empty, + ), + histogram, + ); + + // Simulate work + std::thread::sleep(std::time::Duration::from_millis(5)); + } + // MetricsSpan dropped — should have recorded duration_ms + + let captured = spans.lock().unwrap(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].name, "test_span"); + assert!( + captured[0].fields.contains("id="), + "span should contain id field: {}", + captured[0].fields + ); + } + + #[test] + fn metered_fn_passes_span_ref_and_records_dynamic_fields() { + let spans = Arc::new(Mutex::new(Vec::new())); + let layer = SpanCapture { + spans: spans.clone(), + }; + + let _guard = tracing_subscriber::registry().with(layer).set_default(); + + let result = metered_fn( + tracing::trace_span!( + target: "flashblocks::coordinator", + "metered_test", + path = tracing::field::Empty, + duration_ms = tracing::field::Empty, + ), + EXECUTION.validate_duration.clone(), + |span| { + span.record("path", "bal"); + 42 + }, + ); + + assert_eq!(result, 42); + + let captured = spans.lock().unwrap(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0].name, "metered_test"); + } + + #[test] + fn span_propagation_across_thread_boundary() { + let spans = Arc::new(Mutex::new(Vec::new())); + let layer = SpanCapture { + spans: spans.clone(), + }; + + let dispatch = + tracing::dispatcher::Dispatch::new(tracing_subscriber::registry().with(layer)); + let _guard = tracing::dispatcher::set_default(&dispatch); + + // Create a parent span on this thread + let parent = tracing::trace_span!( + target: "flashblocks::coordinator", + "parent_span", + id = "payload_123", + ); + + let parent_clone = parent.clone(); + let thread_dispatch = dispatch.clone(); + + // Simulate the spawn_blocking pattern: capture span, re-enter on new thread + let handle = std::thread::spawn(move || { + let _sub = tracing::dispatcher::set_default(&thread_dispatch); + let _enter = parent_clone.enter(); + // Child span created under re-entered parent + let _child = tracing::trace_span!( + target: "flashblocks::coordinator", + "child_on_blocking_thread", + ) + .entered(); + }); + + handle.join().unwrap(); + + let captured = spans.lock().unwrap(); + let names: Vec<&str> = captured.iter().map(|s| s.name.as_str()).collect(); + assert!(names.contains(&"parent_span"), "should capture parent span"); + assert!( + names.contains(&"child_on_blocking_thread"), + "should capture child span created on blocking thread" + ); + } +} diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs index 75449bc9f..4e655583b 100644 --- a/crates/flashblocks/p2p/src/protocol/event.rs +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -151,14 +151,24 @@ impl BlockEpochState { // Stale check: reject if the epoch's parent is behind the canon tip. let parent_number = base.block_number.saturating_sub(1); if canon_tip.is_some_and(|tip| parent_number < tip.number) { + tracing::trace!( + target: "flashblocks::event_stream", + payload_id = %fb.payload_id, + parent_number, + canon_tip_number = canon_tip.map(|t| t.number), + "stale epoch rejected" + ); + metrics::counter!("flashblocks.event_stream.epochs_stale").increment(1); return None; } + let parent = BlockNumHash { + number: parent_number, + hash: base.parent_hash, + }; + let mut epoch = Self { - parent: BlockNumHash { - number: parent_number, - hash: base.parent_hash, - }, + parent, payload_id: fb.payload_id, cursor: 0, buffer: Default::default(), @@ -234,6 +244,7 @@ impl BufferedFlashblocks { if !canon_tip.is_some_and(|tip| tip == epoch.parent) { return; } + while let Some(Some(_)) = epoch.buffer.get(epoch.cursor) { let fb = epoch.buffer[epoch.cursor].take().unwrap(); epoch.cursor += 1; diff --git a/crates/world/node/Cargo.toml b/crates/world/node/Cargo.toml index b84c891da..088c078a4 100644 --- a/crates/world/node/Cargo.toml +++ b/crates/world/node/Cargo.toml @@ -43,6 +43,7 @@ alloy-primitives.workspace = true alloy-rpc-types-eth.workspace = true alloy-signer-local.workspace = true + op-alloy-consensus.workspace = true tokio.workspace = true @@ -59,6 +60,9 @@ world-chain-pool.workspace = true world-chain-test.workspace = true world-chain-node.workspace = true +op-alloy-network.workspace = true +op-alloy-provider.workspace = true + reth-db.workspace = true reth-e2e-test-utils.workspace = true reth-engine-primitives.workspace = true @@ -71,7 +75,7 @@ reth-primitives.workspace = true reth-tracing.workspace = true reth-network-api.workspace = true reth-eth-wire.workspace = true -alloy-rpc-types.workspace = true +alloy-rpc-types = { workspace = true } alloy-genesis.workspace = true alloy-network.workspace = true diff --git a/crates/world/node/tests/e2e-testsuite/actions.rs b/crates/world/node/tests/e2e-testsuite/actions.rs index 1d3cc39ad..ead030d5f 100644 --- a/crates/world/node/tests/e2e-testsuite/actions.rs +++ b/crates/world/node/tests/e2e-testsuite/actions.rs @@ -17,8 +17,7 @@ use futures::{ stream::{self, FuturesUnordered}, }; use op_alloy_rpc_types::OpTransactionReceipt; -use op_alloy_rpc_types_engine::{OpExecutionPayloadEnvelopeV3, OpExecutionPayloadEnvelopeV4}; -use parking_lot::RwLock; +use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelopeV4; use reth::rpc::api::{EngineApiClient, EthApiClient}; use reth_e2e_test_utils::testsuite::{Environment, actions::Action}; use reth_node_api::{ConsensusEngineHandle, EngineApiMessageVersion}; @@ -28,11 +27,130 @@ use reth_optimism_primitives::OpTransactionSigned; use reth_primitives::TransactionSigned; use revm_primitives::{Address, B256, Bytes, U256}; use std::{pin::Pin, sync::Arc, time::Duration}; -use tokio::sync::{mpsc, watch}; +use tokio::sync::mpsc; use tracing::{error, info}; use crate::setup::execution_data_from_from_reduced_flashblock; +// --------------------------------------------------------------------------- +// Test helper macros for Eth API queries +// --------------------------------------------------------------------------- + +/// Create an `alloy_provider::RootProvider` from a node's RPC URL. +/// +/// ```ignore +/// let provider = provider!(nodes[0]); +/// ``` +#[macro_export] +macro_rules! provider { + ($node:expr) => {{ + let url = $node.node.rpc_url(); + alloy_provider::ProviderBuilder::new().connect_http(url) + }}; +} + +/// Fetch a block by tag (`Pending`, `Latest`, etc.) from a node. +/// +/// ```ignore +/// let block = fetch_block!(nodes[0], Pending); +/// let block = fetch_block!(nodes[0], Latest, true); // full txs +/// ``` +#[macro_export] +macro_rules! fetch_block { + ($node:expr, $tag:ident) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_block_by_number( + &provider, + alloy_eips::BlockNumberOrTag::$tag, + false, + ) + .await + }}; + ($node:expr, $tag:ident, $full_txs:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_block_by_number( + &provider, + alloy_eips::BlockNumberOrTag::$tag, + $full_txs, + ) + .await + }}; +} + +/// Fetch a transaction receipt by hash. +/// +/// ```ignore +/// let receipt = fetch_receipt!(nodes[0], tx_hash); +/// ``` +#[macro_export] +macro_rules! fetch_receipt { + ($node:expr, $tx_hash:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_transaction_receipt(&provider, $tx_hash).await + }}; +} + +/// Fetch a transaction by hash. +/// +/// ```ignore +/// let tx = fetch_tx!(nodes[0], tx_hash); +/// ``` +#[macro_export] +macro_rules! fetch_tx { + ($node:expr, $tx_hash:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_transaction_by_hash(&provider, $tx_hash).await + }}; +} + +/// Perform an `eth_call` against a node. +/// +/// ```ignore +/// let result = eth_call!(nodes[0], tx_request); +/// let result = eth_call!(nodes[0], tx_request, Pending); +/// ``` +#[macro_export] +macro_rules! eth_call { + ($node:expr, $tx:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::call(&provider, &$tx).await + }}; + ($node:expr, $tx:expr, $tag:ident) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::call(&provider, &$tx) + .block(alloy_eips::BlockId::Number( + alloy_eips::BlockNumberOrTag::$tag, + )) + .await + }}; +} + +/// Fetch logs matching a filter. +/// +/// ```ignore +/// let logs = fetch_logs!(nodes[0], filter); +/// ``` +#[macro_export] +macro_rules! fetch_logs { + ($node:expr, $filter:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::get_logs(&provider, &$filter).await + }}; +} + +/// Subscribe to new block headers (uses polling via `watch_blocks`). +/// +/// ```ignore +/// let poller = stream_blocks!(nodes[0]); +/// ``` +#[macro_export] +macro_rules! stream_blocks { + ($node:expr) => {{ + let provider = $crate::provider!($node); + alloy_provider::Provider::watch_blocks(&provider).await + }}; +} + pub type Hook = Arc Result<()> + Send + Sync>; pub fn hook(f: F) -> Hook @@ -1030,1138 +1148,235 @@ where } } -// ============================================================================ -// Shared State and Advanced Composition Actions -// ============================================================================ - -/// Shared state for communication between parallel actions during block production -#[derive(Clone)] -pub struct BlockProductionState { - /// Current payload being produced - pub payload: Arc>>, - /// Block hashes from validated flashblocks (for GetBlockByHash) - pub validated_block_hashes: Arc>>, - /// Transaction hashes submitted by spammer (for GetReceipts) - pub submitted_tx_hashes: Arc>>, - /// Signal when final flashblock is validated - pub final_validated: watch::Sender, - pub final_validated_rx: watch::Receiver, +/// Sleep for a duration - useful between block cycles +pub struct Sleep { + pub duration: Duration, } -impl BlockProductionState { - pub fn new() -> Self { - let (final_validated, final_validated_rx) = watch::channel(false); - Self { - payload: Arc::new(RwLock::new(None)), - validated_block_hashes: Arc::new(RwLock::new(Vec::new())), - submitted_tx_hashes: Arc::new(RwLock::new(Vec::new())), - final_validated, - final_validated_rx, - } - } - - pub fn set_payload(&self, payload: OpExecutionPayloadEnvelopeV3) { - *self.payload.write() = Some(payload); - } - - pub fn get_payload(&self) -> Option { - self.payload.read().clone() - } - - pub fn add_validated_hash(&self, hash: B256) { - self.validated_block_hashes.write().push(hash); +impl Sleep { + pub fn new(duration: Duration) -> Self { + Self { duration } } - pub fn add_tx_hash(&self, hash: B256) { - self.submitted_tx_hashes.write().push(hash); + pub fn millis(ms: u64) -> Self { + Self::new(Duration::from_millis(ms)) } +} - pub fn get_tx_hashes(&self) -> Vec { - self.submitted_tx_hashes.read().clone() +impl Action for Sleep { + fn execute<'a>( + &'a mut self, + _env: &'a mut Environment, + ) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + tokio::time::sleep(self.duration).await; + Ok(()) + }) } +} - pub fn signal_final(&self) { - let _ = self.final_validated.send(true); - } +// --------------------------------------------------------------------------- +// EngineDriver — drives the consensus engine through N block-building cycles +// --------------------------------------------------------------------------- - pub fn reset(&self) { - *self.payload.write() = None; - self.validated_block_hashes.write().clear(); - self.submitted_tx_hashes.write().clear(); - let _ = self.final_validated.send(false); - } -} +/// Callback invoked after each block is built and canonicalized. +pub type BlockCallback = Box< + dyn Fn( + usize, + &OpExecutionPayloadEnvelopeV4, + ) -> std::pin::Pin> + Send>> + + Send + + Sync, +>; -/// Canonicalize a block by sending new_payload + fork_choice_updated to follower nodes -pub struct Canonicalize { - pub node_idxs: Vec, - pub state: BlockProductionState, -} +/// Callback invoked during the build interval (no payload available yet). +pub type MidBuildCallback = Box< + dyn Fn(usize) -> std::pin::Pin> + Send>> + + Send + + Sync, +>; -impl Canonicalize { - pub fn new(node_idxs: Vec, state: BlockProductionState) -> Self { - Self { node_idxs, state } - } +/// Drives the consensus engine through `num_blocks` block-building cycles. +/// +/// Each cycle: +/// 1. Generates payload attributes for the next block +/// 2. Sends `forkchoiceUpdatedV3` with attributes to start building +/// 3. Waits for `block_interval` (the build deadline) +/// 4. Calls `getPayloadV4` to retrieve the built payload +/// 5. Sends `newPayloadV4` + `forkchoiceUpdated` on all follower nodes +/// 6. Invokes the optional `on_block` callback +/// 7. Advances to the next cycle with the new block as head +pub struct EngineDriver { + /// Index of the builder node in the environment's node_clients. + pub builder_idx: usize, + /// Indices of follower nodes that receive `newPayload` + FCU. + pub follower_idxs: Vec, + /// Initial parent hash (genesis). If None, fetched from latest block. + pub initial_parent_hash: Option, + /// Number of blocks to build. + pub num_blocks: usize, + /// Time to wait between FCU (start building) and getPayload (retrieve). + pub block_interval: Duration, + /// Whether to use flashblocks FCU with authorization. + pub flashblocks: bool, + /// Generates `Authorization` from `(parent_hash, OpPayloadAttributes)`. + pub authorization_gen: A, + /// Generates attributes for the next block given (block_number, parent_timestamp). + pub attributes_gen: Box Result + Send + Sync>, + /// Optional callback during the build interval (between FCU and getPayload). + /// Called while the payload builder is actively working. + pub during_build: Option, + /// Optional callback after each block is built and canonicalized. + pub on_block: Option, } -impl Action for Canonicalize { +impl EngineDriver +where + A: Fn(B256, OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, +{ fn execute<'a>( &'a mut self, env: &'a mut Environment, ) -> BoxFuture<'a, Result<()>> { Box::pin(async move { - use alloy_rpc_types_engine::CancunPayloadFields; - use op_alloy_rpc_types_engine::{OpExecutionData, OpExecutionPayloadSidecar}; - - let payload = self - .state - .get_payload() - .ok_or_else(|| eyre!("No payload to canonicalize"))?; - - let block_hash = payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - - let parent_hash = payload - .execution_payload - .payload_inner - .payload_inner - .parent_hash; - - info!( - target: "actions", - block_hash = ?block_hash, - "Canonicalizing block" - ); + let builder = &env.node_clients[self.builder_idx]; + let engine = builder.engine.http_client(); - // Construct OpExecutionData from the envelope - use op_alloy_rpc_types_engine::OpExecutionPayload; - let execution_data = OpExecutionData { - payload: OpExecutionPayload::V3(payload.execution_payload.clone()), - sidecar: OpExecutionPayloadSidecar::v3(CancunPayloadFields::new( - payload.parent_beacon_block_root, - vec![], - )), + // Get the initial head + let mut parent_hash = if let Some(hash) = self.initial_parent_hash { + hash + } else { + let latest: Option = + EthApiClient::< + TransactionRequest, + Transaction, + alloy_rpc_types_eth::Block, + alloy_consensus::Receipt, + Header, + TransactionSigned, + >::block_by_number( + &builder.rpc, alloy_eips::BlockNumberOrTag::Latest, false + ) + .await?; + latest + .ok_or_else(|| eyre!("No latest block"))? + .header + .hash_slow() }; - for &node_idx in &self.node_idxs { - // Use beacon engine handle which accepts OpExecutionData directly - if let Some(beacon_handle) = - env.node_clients[node_idx].beacon_engine_handle.as_ref() - { - // First: update forkchoice to parent so node can accept the new block - let parent_fcu = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - beacon_handle - .fork_choice_updated(parent_fcu, None, EngineApiMessageVersion::V3) - .await - .map_err(|e| eyre!("fork_choice_updated to parent failed: {:?}", e))?; - - // Second: send new_payload with the block - let status = beacon_handle - .new_payload(execution_data.clone()) - .await - .map_err(|e| eyre!("new_payload failed: {:?}", e))?; - - if !matches!(status.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "new_payload failed for node {}: {:?}", - node_idx, - status - )); - } + let mut parent_timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); - // Third: send fork_choice_updated to make it canonical - let fcu_state = ForkchoiceState { - head_block_hash: block_hash, - safe_block_hash: block_hash, - finalized_block_hash: block_hash, - }; + for block_num in 0..self.num_blocks { + // 1. Generate attributes + let block_number = block_num as u64 + 1; + parent_timestamp += self.block_interval.as_secs().max(1); + let attributes = (self.attributes_gen)(block_number, parent_timestamp)?; - beacon_handle - .fork_choice_updated(fcu_state, None, EngineApiMessageVersion::V3) - .await - .map_err(|e| eyre!("fork_choice_updated failed: {:?}", e))?; + // 2. FCU with attributes → start building + let fcu_state = ForkchoiceState { + head_block_hash: parent_hash, + safe_block_hash: parent_hash, + finalized_block_hash: parent_hash, + }; - info!( - target: "actions", - node_idx = node_idx, - block_hash = ?block_hash, - "Block canonicalized successfully via beacon handle" - ); - } else { - // Fallback: use RPC engine client - let engine = env.node_clients[node_idx].engine.http_client(); - - // First: update forkchoice to parent so node can accept the new block - let parent_fcu = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - let _ = EngineApiClient::::fork_choice_updated_v3( - &engine, parent_fcu, None, + let fcu_result = if self.flashblocks { + FlashblocksEngineApiExtClient::::flashblocks_fork_choice_updated_v3( + &engine, + fcu_state, + Some(attributes.clone()), + Some((self.authorization_gen)(parent_hash, attributes.clone())), ) - .await?; - - // Second: send new_payload with the block via RPC - let np_result = EngineApiClient::::new_payload_v3( + .await? + } else { + EngineApiClient::::fork_choice_updated_v3( &engine, - payload.execution_payload.clone(), - vec![], - payload.parent_beacon_block_root, + fcu_state, + Some(attributes.clone()), ) - .await?; + .await? + }; - if !matches!(np_result.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "new_payload failed for node {}: {:?}", - node_idx, - np_result - )); - } + if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { + return Err(eyre!( + "block {block_num}: FCU status not valid: {:?}", + fcu_result.payload_status + )); + } + + let payload_id = fcu_result + .payload_id + .ok_or_else(|| eyre!("block {block_num}: No payload ID returned"))?; + + info!( + target: "engine_driver", + block = block_num, + %payload_id, + "building block" + ); + + // 3. Wait for build deadline + tokio::time::sleep(self.block_interval).await; + + // 3.5. Mid-build callback (payload builder is still working) + if let Some(ref during_build) = self.during_build { + during_build(block_num).await?; + } + + // 4. getPayloadV4 + let payload = + EngineApiClient::::get_payload_v4(&engine, payload_id).await?; + + let block_hash = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .block_hash; + + let tx_count = payload + .execution_payload + .payload_inner + .payload_inner + .payload_inner + .transactions + .len(); + + info!( + target: "engine_driver", + block = block_num, + %block_hash, + tx_count, + "payload retrieved" + ); + + // 5. Canonicalize: FCU(parent) → newPayload → FCU(head) + // on builder AND all follower nodes + use alloy_rpc_types_engine::CancunPayloadFields; + use op_alloy_rpc_types_engine::{ + OpExecutionData, OpExecutionPayload, OpExecutionPayloadSidecar, + }; - // Third: send fork_choice_updated to make it canonical - let fcu_state = ForkchoiceState { + // Canonicalize on builder via FCU only (it already has the payload) + { + let builder_engine = env.node_clients[self.builder_idx].engine.http_client(); + let head_fcu = ForkchoiceState { head_block_hash: block_hash, safe_block_hash: block_hash, finalized_block_hash: block_hash, }; - let fcu_result = EngineApiClient::::fork_choice_updated_v3( - &engine, fcu_state, None, + &builder_engine, + head_fcu, + None, ) .await?; if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { return Err(eyre!( - "fork_choice_updated failed for node {}: {:?}", - node_idx, - fcu_result.payload_status - )); - } - - info!( - target: "actions", - node_idx = node_idx, - block_hash = ?block_hash, - "Block canonicalized successfully via RPC" - ); - } - } - - Ok(()) - }) - } -} - -/// Mine a block and store the payload in shared state -pub struct MineBlockWithState { - pub node_idx: usize, - pub attributes: OpPayloadAttributes, - pub authorization_gen: A, - pub block_interval: Duration, - pub flashblocks: bool, - pub state: BlockProductionState, -} - -impl MineBlockWithState -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync, -{ - pub fn new( - node_idx: usize, - attributes: OpPayloadAttributes, - authorization_gen: A, - state: BlockProductionState, - ) -> Self { - Self { - node_idx, - attributes, - authorization_gen, - block_interval: Duration::from_millis(2000), - flashblocks: true, - state, - } - } - - pub fn with_interval(mut self, interval: Duration) -> Self { - self.block_interval = interval; - self - } -} - -impl ReadOnlyAction for MineBlockWithState -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let client = &env.node_clients[self.node_idx]; - let engine = client.engine.http_client(); - - let latest: Option = - EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - alloy_consensus::Receipt, - Header, - TransactionSigned, - >::block_by_number( - &client.rpc, alloy_eips::BlockNumberOrTag::Latest, false - ) - .await?; - - let parent_hash = latest - .ok_or_else(|| eyre!("No latest block"))? - .header - .hash_slow(); - - let fcu_state = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - - let fcu_result = if self.flashblocks { - FlashblocksEngineApiExtClient::::flashblocks_fork_choice_updated_v3( - &engine, - fcu_state, - Some(self.attributes.clone()), - Some((self.authorization_gen)(self.attributes.clone())), - ) - .await? - } else { - EngineApiClient::::fork_choice_updated_v3( - &engine, - fcu_state, - Some(self.attributes.clone()), - ) - .await? - }; - - if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "FCU status not valid: {:?}", - fcu_result.payload_status - )); - } - - let payload_id = fcu_result - .payload_id - .ok_or_else(|| eyre!("No payload ID returned"))?; - - // Wait for block to be built - tokio::time::sleep(self.block_interval).await; - - let payload = - EngineApiClient::::get_payload_v3(&engine, payload_id).await?; - - let block_hash = payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - - info!( - target: "actions", - block_hash = ?block_hash, - tx_count = payload.execution_payload.payload_inner.payload_inner.transactions.len(), - "Mined block, storing in shared state" - ); - - // Store payload in shared state - self.state.set_payload(payload); - - Ok(()) - }) - } -} - -/// Validate flashblocks and signal when complete, storing validated hashes in shared state -pub struct ValidateFlashblocksWithState { - pub flashblock_stream: Pin + Send>>, - pub beacon_handle: Arc>, - pub chain_spec: Arc, - pub state: BlockProductionState, -} - -impl ValidateFlashblocksWithState { - pub fn new( - flashblock_stream: Pin + Send>>, - beacon_handle: Arc>, - chain_spec: Arc, - state: BlockProductionState, - ) -> Self { - Self { - flashblock_stream, - beacon_handle, - chain_spec, - state, - } - } -} - -impl Action for ValidateFlashblocksWithState { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let mut flashblocks = Flashblocks::default(); - let stream = &mut self.flashblock_stream; - - // Wait for payload to be available - let target_hash = loop { - if let Some(payload) = self.state.get_payload() { - break payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - info!( - target: "actions", - target_hash = ?target_hash, - "Starting flashblock validation" - ); - - while let Some(fb_payload) = stream.next().await { - let index = fb_payload.index; - - let is_new = flashblocks - .push(Flashblock { - flashblock: fb_payload, - }) - .ok(); - - if is_new.is_some() { - info!( - target: "actions", - index = %index, - "New payload started, reset flashblock collection" - ); - } - - // Reduce to get current state - let Some(reduced) = Flashblock::reduce(flashblocks.clone()).ok() else { - continue; - }; - - let reduced_hash = reduced.diff().block_hash; - - // Store validated hash for GetBlockByHash - self.state.add_validated_hash(reduced_hash); - - // Construct execution data - let execution_data = - execution_data_from_from_reduced_flashblock(reduced, self.chain_spec.clone()); - - // Validate - let parent = execution_data.parent_hash(); - let forkchoice = ForkchoiceState { - head_block_hash: parent, - safe_block_hash: parent, - finalized_block_hash: parent, - }; - - self.beacon_handle - .fork_choice_updated(forkchoice, None, EngineApiMessageVersion::V3) - .await - .ok(); - - let status = self.beacon_handle.new_payload(execution_data.clone()).await; - - match &status { - Ok(s) => { - info!( - target: "actions", - index = %index, - ?reduced_hash, - status = ?s.status, - "Validated intermediate flashblock" - ); - } - Err(e) => { - error!( - target: "actions", - index = %index, - error = ?e, - "Flashblock validation failed" - ); - } - } - - // Check if final - if reduced_hash == target_hash { - info!( - target: "actions", - block_hash = ?reduced_hash, - index = %index, - "Final flashblock validated" - ); - self.state.signal_final(); - break; - } - } - - Ok(()) - }) - } -} - -// ============================================================================ -// Block Production Loop - Composable N-block production as a single Action -// ============================================================================ - -/// Configuration for block production loop -#[derive(Clone)] -pub struct BlockProductionConfig { - /// Builder node index - pub builder_node_idx: usize, - /// Follower node indexes for canonicalization - pub follower_node_idxs: Vec, - /// Authorization generator - pub authorization_gen: A, - /// Attributes builder function: (timestamp, eip1559_params) -> OpPayloadAttributes - pub attributes_builder: F, - /// Block interval - pub block_interval: Duration, - /// Number of blocks to produce - pub num_blocks: u64, - /// Starting timestamp - pub start_timestamp: u64, - /// Timestamp increment per block - pub timestamp_increment: u64, - /// Shared state for cross-action communication - pub state: BlockProductionState, - /// Chain spec for EIP-1559 params - pub chain_spec: Arc, -} - -/// A complete block production loop as a single composable Action. -/// -/// This action produces N blocks in sequence, with each block going through: -/// 1. Mine block (stores payload in shared state) -/// 2. Validate flashblocks (runs parallel, signals when done) -/// 3. Query blocks by hash (runs parallel, uses validated hashes) -/// 4. Query receipts (runs parallel, uses tx hashes from spammer) -/// 5. Canonicalize on follower nodes -/// 6. Reset state and advance to next block -pub struct BlockProductionLoop { - pub config: BlockProductionConfig, - /// Flashblocks handle for getting streams - pub flashblocks_handle: H, - /// Beacon handle for validation - pub beacon_handle: Arc>, - /// Hook called after each block with (block_num, block_hash, tx_count) - pub on_block_produced: Option>, - /// Hook for each validated flashblock hash - pub on_validated_hash: Option>, - /// Hook for each fetched receipt - pub on_receipt: Option>, -} - -// ============================================================================ -// Simple Action Primitives for Composition -// ============================================================================ - -/// Reset the shared state - use at the start of each block cycle -pub struct ResetState { - pub state: BlockProductionState, -} - -impl ResetState { - pub fn new(state: BlockProductionState) -> Self { - Self { state } - } -} - -impl Action for ResetState { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - self.state.reset(); - Ok(()) - }) - } -} - -/// Query all validated block hashes from shared state -pub struct QueryValidatedBlocks { - pub node_idxs: Vec, - pub state: BlockProductionState, - pub on_block: Option>, -} - -impl QueryValidatedBlocks { - pub fn new(node_idxs: Vec, state: BlockProductionState) -> Self { - Self { - node_idxs, - state, - on_block: None, - } - } - - pub fn on_block(mut self, f: F) -> Self - where - F: Fn(alloy_rpc_types_eth::Block) -> Result<()> + Send + Sync + 'static, - { - self.on_block = Some(hook(f)); - self - } -} - -impl ReadOnlyAction for QueryValidatedBlocks { - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let hashes = self.state.validated_block_hashes.read().clone(); - - for hash in &hashes { - for &node_idx in &self.node_idxs { - let block: Option = - EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - alloy_consensus::Receipt, - Header, - TransactionSigned, - >::block_by_hash( - &env.node_clients[node_idx].rpc, *hash, false - ) - .await?; - - if let Some(ref b) = block - && let Some(ref hook) = self.on_block - { - hook(b.clone())?; - } - } - } - Ok(()) - }) - } -} - -/// Query receipts for all transaction hashes in shared state -pub struct QueryTxReceipts { - pub node_idxs: Vec, - pub state: BlockProductionState, - pub on_receipt: Option>, -} - -impl QueryTxReceipts { - pub fn new(node_idxs: Vec, state: BlockProductionState) -> Self { - Self { - node_idxs, - state, - on_receipt: None, - } - } - - pub fn on_receipt(mut self, f: F) -> Self - where - F: Fn(OpTransactionReceipt) -> Result<()> + Send + Sync + 'static, - { - self.on_receipt = Some(hook(f)); - self - } -} - -impl ReadOnlyAction for QueryTxReceipts { - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let hashes = self.state.get_tx_hashes(); - - for hash in &hashes { - for &node_idx in &self.node_idxs { - let receipt: Option = EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - OpTransactionReceipt, - Header, - TransactionSigned, - >::transaction_receipt( - &env.node_clients[node_idx].rpc, - *hash, - ) - .await?; - - if let Some(ref r) = receipt - && let Some(ref hook) = self.on_receipt - { - hook(r.clone())?; - } - } - } - Ok(()) - }) - } -} - -/// A dynamic mining action that gets attributes from shared state -pub struct DynamicMineBlock { - pub node_idx: usize, - pub authorization_gen: A, - pub block_interval: Duration, - pub state: BlockProductionState, - /// Function to get current attributes (called at execution time) - pub get_attributes: Arc OpPayloadAttributes + Send + Sync>, -} - -impl DynamicMineBlock -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - pub fn new( - node_idx: usize, - authorization_gen: A, - state: BlockProductionState, - get_attributes: F, - ) -> Self - where - F: Fn() -> OpPayloadAttributes + Send + Sync + 'static, - { - Self { - node_idx, - authorization_gen, - block_interval: Duration::from_millis(2000), - state, - get_attributes: Arc::new(get_attributes), - } - } - - pub fn with_interval(mut self, interval: Duration) -> Self { - self.block_interval = interval; - self - } -} - -impl ReadOnlyAction for DynamicMineBlock -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - fn execute_readonly<'a>( - &'a self, - env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let attributes = (self.get_attributes)(); - - let mine_action = MineBlockWithState::new( - self.node_idx, - attributes, - self.authorization_gen.clone(), - self.state.clone(), - ) - .with_interval(self.block_interval); - - mine_action.execute_readonly(env).await - }) - } -} - -/// A dynamic flashblock validator that gets streams from a flashblocks handle -pub struct DynamicValidateFlashblocks { - pub flashblocks_handle: flashblocks_p2p::protocol::handler::FlashblocksHandle, - pub beacon_handle: Arc>, - pub chain_spec: Arc, - pub state: BlockProductionState, -} - -impl DynamicValidateFlashblocks { - pub fn new( - flashblocks_handle: flashblocks_p2p::protocol::handler::FlashblocksHandle, - beacon_handle: Arc>, - chain_spec: Arc, - state: BlockProductionState, - ) -> Self { - Self { - flashblocks_handle, - beacon_handle, - chain_spec, - state, - } - } -} - -impl Action for DynamicValidateFlashblocks { - fn execute<'a>( - &'a mut self, - env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let stream = Box::pin(self.flashblocks_handle.flashblock_stream()); - - let mut validate_action = ValidateFlashblocksWithState::new( - stream, - self.beacon_handle.clone(), - self.chain_spec.clone(), - self.state.clone(), - ); - - validate_action.execute(env).await - }) - } -} - -impl ReadOnlyAction for DynamicValidateFlashblocks { - fn execute_readonly<'a>( - &'a self, - _env: &'a Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let mut flashblocks = Flashblocks::default(); - let mut stream = Box::pin(self.flashblocks_handle.flashblock_stream()); - - // Wait for payload to be available - let target_hash = loop { - if let Some(payload) = self.state.get_payload() { - break payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - } - tokio::time::sleep(Duration::from_millis(50)).await; - }; - - info!( - target: "actions", - target_hash = ?target_hash, - "Starting flashblock validation (parallel)" - ); - - while let Some(fb_payload) = stream.next().await { - let index = fb_payload.index; - - let is_new = flashblocks - .push(Flashblock { - flashblock: fb_payload, - }) - .ok(); - - if is_new.is_some() { - info!( - target: "actions", - index = %index, - "New flashblock received" - ); - } - - // Check if this is the final flashblock matching our target - if let Ok(reduced) = Flashblock::reduce(flashblocks.clone()) { - let block_hash = reduced.diff().block_hash; - if block_hash == target_hash { - info!( - target: "actions", - block_hash = ?block_hash, - "Final flashblock validated" - ); - self.state.add_validated_hash(block_hash); - self.state.signal_final(); - break; - } - } - } - - Ok(()) - }) - } -} - -/// Log current block production state -pub struct LogBlockComplete { - pub state: BlockProductionState, - pub on_complete: Option>, -} - -impl LogBlockComplete { - pub fn new(state: BlockProductionState) -> Self { - Self { - state, - on_complete: None, - } - } - - pub fn on_complete(mut self, f: F) -> Self - where - F: Fn((B256, usize)) -> Result<()> + Send + Sync + 'static, - { - self.on_complete = Some(hook(f)); - self - } -} - -impl Action for LogBlockComplete { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - if let Some(payload) = self.state.get_payload() { - let hash = payload - .execution_payload - .payload_inner - .payload_inner - .block_hash; - let tx_count = payload - .execution_payload - .payload_inner - .payload_inner - .transactions - .len(); - - info!( - target: "block_production", - ?hash, - tx_count, - validated_hashes = self.state.validated_block_hashes.read().len(), - tx_receipts = self.state.get_tx_hashes().len(), - "Block cycle complete" - ); - - if let Some(ref hook) = self.on_complete { - hook((hash, tx_count))?; - } - } - Ok(()) - }) - } -} - -/// Sleep for a duration - useful between block cycles -pub struct Sleep { - pub duration: Duration, -} - -impl Sleep { - pub fn new(duration: Duration) -> Self { - Self { duration } - } - - pub fn millis(ms: u64) -> Self { - Self::new(Duration::from_millis(ms)) - } -} - -impl Action for Sleep { - fn execute<'a>( - &'a mut self, - _env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - tokio::time::sleep(self.duration).await; - Ok(()) - }) - } -} - -// --------------------------------------------------------------------------- -// EngineDriver — drives the consensus engine through N block-building cycles -// --------------------------------------------------------------------------- - -/// Callback invoked after each block is built and canonicalized. -pub type BlockCallback = Box< - dyn Fn( - usize, - &OpExecutionPayloadEnvelopeV4, - ) -> std::pin::Pin> + Send>> - + Send - + Sync, ->; - -/// Drives the consensus engine through `num_blocks` block-building cycles. -/// -/// Each cycle: -/// 1. Generates payload attributes for the next block -/// 2. Sends `forkchoiceUpdatedV3` with attributes to start building -/// 3. Waits for `block_interval` (the build deadline) -/// 4. Calls `getPayloadV4` to retrieve the built payload -/// 5. Sends `newPayloadV4` + `forkchoiceUpdated` on all follower nodes -/// 6. Invokes the optional `on_block` callback -/// 7. Advances to the next cycle with the new block as head -pub struct EngineDriver { - /// Index of the builder node in the environment's node_clients. - pub builder_idx: usize, - /// Indices of follower nodes that receive `newPayload` + FCU. - pub follower_idxs: Vec, - /// Number of blocks to build. - pub num_blocks: usize, - /// Time to wait between FCU (start building) and getPayload (retrieve). - pub block_interval: Duration, - /// Whether to use flashblocks FCU with authorization. - pub flashblocks: bool, - /// Generates `Authorization` from `OpPayloadAttributes`. - pub authorization_gen: A, - /// Generates attributes for the next block given (block_number, parent_timestamp). - pub attributes_gen: Box Result + Send + Sync>, - /// Optional callback after each block is built and canonicalized. - pub on_block: Option, -} - -impl EngineDriver -where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, -{ - fn execute<'a>( - &'a mut self, - env: &'a mut Environment, - ) -> BoxFuture<'a, Result<()>> { - Box::pin(async move { - let builder = &env.node_clients[self.builder_idx]; - let engine = builder.engine.http_client(); - - // Get the initial head - let mut parent_hash = { - let latest: Option = - EthApiClient::< - TransactionRequest, - Transaction, - alloy_rpc_types_eth::Block, - alloy_consensus::Receipt, - Header, - TransactionSigned, - >::block_by_number( - &builder.rpc, alloy_eips::BlockNumberOrTag::Latest, false - ) - .await?; - latest - .ok_or_else(|| eyre!("No latest block"))? - .header - .hash_slow() - }; - - let mut parent_timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - - for block_num in 0..self.num_blocks { - // 1. Generate attributes - let block_number = block_num as u64 + 1; - parent_timestamp += self.block_interval.as_secs().max(1); - let attributes = (self.attributes_gen)(block_number, parent_timestamp)?; - - // 2. FCU with attributes → start building - let fcu_state = ForkchoiceState { - head_block_hash: parent_hash, - safe_block_hash: parent_hash, - finalized_block_hash: parent_hash, - }; - - let fcu_result = if self.flashblocks { - FlashblocksEngineApiExtClient::::flashblocks_fork_choice_updated_v3( - &engine, - fcu_state, - Some(attributes.clone()), - Some((self.authorization_gen)(attributes.clone())), - ) - .await? - } else { - EngineApiClient::::fork_choice_updated_v3( - &engine, - fcu_state, - Some(attributes.clone()), - ) - .await? - }; - - if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "block {block_num}: FCU status not valid: {:?}", - fcu_result.payload_status - )); - } - - let payload_id = fcu_result - .payload_id - .ok_or_else(|| eyre!("block {block_num}: No payload ID returned"))?; - - info!( - target: "engine_driver", - block = block_num, - %payload_id, - "building block" - ); - - // 3. Wait for build deadline - tokio::time::sleep(self.block_interval).await; - - // 4. getPayloadV4 - let payload = - EngineApiClient::::get_payload_v4(&engine, payload_id).await?; - - let block_hash = payload - .execution_payload - .payload_inner - .payload_inner - .payload_inner - .block_hash; - - let tx_count = payload - .execution_payload - .payload_inner - .payload_inner - .payload_inner - .transactions - .len(); - - info!( - target: "engine_driver", - block = block_num, - %block_hash, - tx_count, - "payload retrieved" - ); - - // 5. Canonicalize: FCU(parent) → newPayload → FCU(head) - // on builder AND all follower nodes - use alloy_rpc_types_engine::CancunPayloadFields; - use op_alloy_rpc_types_engine::{ - OpExecutionData, OpExecutionPayload, OpExecutionPayloadSidecar, - }; - - // Canonicalize on builder via FCU only (it already has the payload) - { - let builder_engine = env.node_clients[self.builder_idx].engine.http_client(); - let head_fcu = ForkchoiceState { - head_block_hash: block_hash, - safe_block_hash: block_hash, - finalized_block_hash: block_hash, - }; - let fcu_result = EngineApiClient::::fork_choice_updated_v3( - &builder_engine, - head_fcu, - None, - ) - .await?; - - if !matches!(fcu_result.payload_status.status, PayloadStatusEnum::Valid) { - return Err(eyre!( - "block {block_num}: builder FCU to head failed: {:?}", + "block {block_num}: builder FCU to head failed: {:?}", fcu_result.payload_status )); } @@ -2270,7 +1485,7 @@ where impl Action for EngineDriver where - A: Fn(OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, + A: Fn(B256, OpPayloadAttributes) -> Authorization + Clone + Send + Sync + 'static, { fn execute<'a>( &'a mut self, diff --git a/crates/world/node/tests/e2e-testsuite/spammer.rs b/crates/world/node/tests/e2e-testsuite/spammer.rs index 227971372..aa6cdac0a 100644 --- a/crates/world/node/tests/e2e-testsuite/spammer.rs +++ b/crates/world/node/tests/e2e-testsuite/spammer.rs @@ -12,11 +12,11 @@ use reth::{chainspec::EthChainSpec, rpc::api::EthApiClient}; use reth_e2e_test_utils::testsuite::NodeClient; use reth_optimism_node::OpEngineTypes; use reth_optimism_primitives::{OpReceipt, OpTransactionSigned}; -use revm_primitives::{Address, B256, Bytes}; +use revm_primitives::{Address, Bytes}; use tracing::{debug, error, info}; use world_chain_test::{node::tx, utils::signer}; -use crate::{actions::BlockProductionState, setup::CHAIN_SPEC}; +use crate::setup::CHAIN_SPEC; sol! { #[sol(rpc, bytecode = "6080604052348015600e575f5ffd5b506101338061001c5f395ff3fe608060405234801561000f575f5ffd5b506004361061004a575f3560e01c80632b68b9c61461004e578063703c2d1a14610056578063affed0e01461005e578063b8dda9c71461007a575b5f5ffd5b61005433ff5b005b6100546100ac565b61006760015481565b6040519081526020015b60405180910390f35b61009c6100883660046100f7565b5f6020819052908152604090205460ff1681565b6040519015158152602001610071565b5f5b60648110156100f4576001805f8282546100c8919061010e565b9091555050600180545f908152602081905260409020805460ff19811660ff90911615179055016100ae565b50565b5f60208284031215610107575f5ffd5b5035919050565b8082018082111561012d57634e487b7160e01b5f52601160045260245ffd5b9291505056")] @@ -137,46 +137,6 @@ impl TxSpammer { }); } - /// Spawns a background task that sends transactions and reports hashes to shared state. - /// - /// Same as `spawn` but also records transaction hashes in the provided `BlockProductionState` - /// for receipt querying by other actions. - pub fn spawn_with_state(self, tpf: u64, http_url: Url, state: BlockProductionState) { - tokio::spawn(async move { - // Deploy test contracts - let wallet = EthereumWallet::from(signer(0)); - let provider = Arc::new(ProviderBuilder::new().wallet(wallet).connect_http(http_url)); - - let contract = *TestContract::deploy(provider.clone()) - .await - .unwrap() - .address(); - - let factory = *TestContractFactory::deploy(provider) - .await - .unwrap() - .address(); - - info!("Deployed TestContract at {contract}, TestContractFactory at {factory}"); - - // Track nonce per signer (signers 1..=MAX_SIGNERS) - let mut nonces: Vec = vec![0; MAX_SIGNERS as usize]; - - loop { - let batch = self.build_batch(tpf, &mut nonces, contract, factory).await; - let tx_hashes = self.broadcast_batch_with_hashes(&batch).await; - - // Record submitted tx hashes in shared state - for hash in tx_hashes { - state.add_tx_hash(hash); - } - - info!("Submitted {} transactions", batch.len()); - tokio::time::sleep(Duration::from_millis(200)).await; - } - }); - } - /// Builds a batch of `tpf` raw transactions, distributing them across signers round-robin. async fn build_batch( &self, @@ -233,33 +193,4 @@ impl TxSpammer { futs.collect::<()>().await; } - - /// Broadcasts a batch of transactions and returns the successful tx hashes. - async fn broadcast_batch_with_hashes(&self, batch: &[Bytes]) -> Vec { - let mut tx_hashes = Vec::with_capacity(batch.len()); - - let Some(client) = self.rpc.first() else { - return tx_hashes; - }; - - for tx in batch { - let result = EthApiClient::< - TransactionRequest, - OpTransactionSigned, - alloy_consensus::Block, - OpReceipt, - Header, - Bytes, - >::send_raw_transaction(&client.rpc, tx.clone()) - .await - .inspect_err(|e| error!("Error sending transaction: {:?}", e)); - - if let Ok(tx_hash) = result { - info!("Submitted tx: {:?}", tx_hash); - tx_hashes.push(tx_hash); - } - } - - tx_hashes - } } diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index 466007354..6014cfaae 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -1,17 +1,12 @@ -use crate::{ - actions::{ - ActionSequence, BlockProductionState, DynamicMineBlock, DynamicValidateFlashblocks, - LogBlockComplete, QueryTxReceipts, QueryValidatedBlocks, ResetState, Sleep, - }, - setup::{TX_SET_L1_BLOCK, build_payload_attributes}, -}; +use crate::setup::{TX_SET_L1_BLOCK, build_payload_attributes}; use alloy_network::{Ethereum, EthereumWallet, TransactionBuilder, eip2718::Encodable2718}; use alloy_primitives::{Bytes, b64}; +use alloy_provider::ProviderBuilder; use alloy_rpc_types::TransactionRequest; use alloy_rpc_types_engine::PayloadStatusEnum; use eyre::eyre::eyre; use flashblocks_p2p::protocol::event::{ChainEvent, WorldChainEvent}; -use futures::future::Either; +use op_alloy_consensus::OpTxEnvelope; use reth::{ chainspec::EthChainSpec, network::{NetworkSyncUpdater, SyncState}, @@ -1001,193 +996,6 @@ async fn test_gossip_disabled_no_propagation() -> eyre::Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread")] -async fn test_continuous_block_production_with_validation() -> eyre::Result<()> { - reth_tracing::init_test_tracing(); - - const NUM_BLOCKS: u64 = 10; - const BLOCK_INTERVAL_MS: u64 = 2000; - const TXS_PER_FLASHBLOCK: u64 = 20; - - let (_, mut nodes, _tasks, mut flashblocks_env, tx_spammer) = - setup::(3, optimism_payload_attributes, true).await?; - - // Setup: 1 basic verifier node for flashblock validation - let (_, mut basic_validators, _tasks, _basic_env, _) = - setup_with_tx_peers::( - 1, - optimism_payload_attributes, - false, - false, - true, - ) - .await?; - - let basic_validator = &mut basic_validators[0]; - - let [builder_node, follower_0, follower_1] = &mut nodes[..] else { - unreachable!("Expected exactly 2 nodes") - }; - - let builder_context = builder_node.ext_context.clone(); - - let basic_beacon_handle = - Arc::new(basic_validator.node.inner.consensus_engine_handle().clone()); - - let follower_context_0 = follower_0.ext_context.clone(); - let _follower_context_1 = follower_1.ext_context.clone(); - - // Create shared state for cross-action communication - let state = BlockProductionState::new(); - - // Create authorization generator - let genesis_hash = builder_node.node.block_hash(0); - let authorization_generator = crate::setup::create_authorization_generator( - genesis_hash, - builder_context - .unwrap() - .flashblocks_handle - .builder_sk() - .unwrap() - .verifying_key(), - ); - - // Spawn spammer with shared state to track tx hashes - let rpc_url = builder_node.node.rpc_url(); - tx_spammer.spawn_with_state(TXS_PER_FLASHBLOCK, rpc_url, state.clone()); - - info!( - target: "test", - "Starting continuous block production test for {} blocks using ActionSequence", - NUM_BLOCKS - ); - - // Track statistics via hooks - let blocks_produced = Arc::new(AtomicU64::new(0)); - let validated_hashes = Arc::new(AtomicUsize::new(0)); - let receipts_fetched = Arc::new(AtomicUsize::new(0)); - - // Create timestamp state that advances with each iteration - let timestamp = Arc::new(AtomicU64::new(crate::setup::current_timestamp())); - let chain_spec = builder_node.node.inner.chain_spec().clone(); - - // Clone handles for the attribute builder closure - let timestamp_for_attrs = timestamp.clone(); - let chain_spec_for_attrs = chain_spec.clone(); - - let validated_hashes_counter = validated_hashes.clone(); - let receipts_counter = receipts_fetched.clone(); - - // Build the composable action sequence for ONE block cycle - let block_cycle = ActionSequence::new() - // 1. Reset state at start of each block - .then(ResetState::new(state.clone())) - // 2. Mine block + validate flashblocks in parallel - .with( - DynamicMineBlock::new( - 0, // builder node - authorization_generator.clone(), - state.clone(), - move || { - let ts = timestamp_for_attrs.load(Ordering::SeqCst); - let eip1559 = - crate::setup::encode_eip1559_params(chain_spec_for_attrs.as_ref(), ts) - .unwrap(); - crate::setup::build_payload_attributes( - ts, - eip1559, - Some(vec![crate::setup::TX_SET_L1_BLOCK.clone()]), - ) - }, - ) - .with_interval(Duration::from_millis(BLOCK_INTERVAL_MS)), - DynamicValidateFlashblocks::new( - follower_context_0.unwrap().flashblocks_handle.clone(), - basic_beacon_handle, - chain_spec.clone(), - state.clone(), - ), - ) - // 3. Query validated blocks and receipts in parallel (AFTER mining/validation) - .with( - QueryValidatedBlocks::new(vec![0, 1], state.clone()).on_block(move |_| { - validated_hashes_counter.fetch_add(1, Ordering::SeqCst); - Ok(()) - }), - QueryTxReceipts::new(vec![0, 1], state.clone()).on_receipt(move |_| { - receipts_counter.fetch_add(1, Ordering::SeqCst); - Ok(()) - }), - ) - // 6. Log completion and track stats - // Note: Skipping Canonicalize on follower nodes for now - // because Isthmus V4 payloads don't work with RPC new_payload_v3 - .then({ - let counter = blocks_produced.clone(); - LogBlockComplete::new(state.clone()).on_complete(move |_| { - counter.fetch_add(1, Ordering::SeqCst); - Ok(()) - }) - }) - // 8. Small delay between blocks - .then(Sleep::millis(100)); - - // Repeat the block cycle N times, advancing timestamp each iteration - let timestamp_for_repeat = timestamp.clone(); - let mut action = block_cycle.repeat(NUM_BLOCKS).on_each(move |i| { - // Advance timestamp for next block (after first iteration) - if i > 0 { - timestamp_for_repeat.fetch_add(2, Ordering::SeqCst); - } - info!( - target: "test", - iteration = i + 1, - total = NUM_BLOCKS, - "Starting block cycle" - ); - - Ok(()) - }); - - // Execute the entire repeated sequence as a single action - let fut = async { action.execute(&mut flashblocks_env).await }; - - futures::future::select( - Box::pin(fut), - Box::pin(if blocks_produced.load(Ordering::SeqCst) == NUM_BLOCKS { - Either::Left(futures::future::ready(())) - } else { - Either::Right(futures::future::pending::<()>()) - }), - ) - .await; - - let final_blocks = blocks_produced.load(Ordering::SeqCst); - let final_hashes = validated_hashes.load(Ordering::SeqCst); - let final_receipts = receipts_fetched.load(Ordering::SeqCst); - - info!( - target: "test", - blocks_produced = final_blocks, - validated_hashes = final_hashes, - receipts_fetched = final_receipts, - "Test completed successfully" - ); - - assert_eq!( - final_blocks, NUM_BLOCKS, - "Should have produced {} blocks", - NUM_BLOCKS - ); - - assert!( - final_hashes > 0, - "Should have validated at least some block hashes" - ); - - Ok(()) -} - /// End-to-end test: drives the builder's consensus engine through a block /// building loop, using a hook on the `WorldChainEventsStream` to assert /// stream invariants: @@ -1377,30 +1185,60 @@ async fn test_engine_driver_pending_block_queries() -> eyre::Result<()> { let chain_spec = nodes[0].node.inner.chain_spec().clone(); let rpc_url = nodes[0].node.rpc_url(); + // Initialize forkchoice on all nodes to genesis + for node in &nodes { + node.node.update_forkchoice(block_hash, block_hash).await?; + } + // Spawn background transactions so blocks have content tx_spammer.spawn(10, rpc_url); - let authorization_gen = crate::setup::create_authorization_generator( - block_hash, - builder_context - .flashblocks_handle - .builder_sk() - .unwrap() - .verifying_key(), - ); + let builder_vk = builder_context + .flashblocks_handle + .builder_sk() + .unwrap() + .verifying_key(); + + let authorization_gen = + move |parent_hash: B256, attrs: reth_optimism_node::OpPayloadAttributes| { + let authorizer_sk = ed25519_dalek::SigningKey::from_bytes(&[0; 32]); + let payload_id = + reth_optimism_payload_builder::payload_id_optimism(&parent_hash, &attrs, 3); + flashblocks_primitives::p2p::Authorization::new( + payload_id, + reth_node_api::PayloadAttributes::timestamp(&attrs), + &authorizer_sk, + builder_vk, + ) + }; + + // --- Flashblock stream: capture latest pending flashblock --- + use flashblocks_primitives::primitives::FlashblocksPayloadV1; + use std::sync::RwLock; + + let latest_stream_fb: Arc>> = Arc::new(RwLock::new(None)); + let latest_stream_fb_writer = latest_stream_fb.clone(); + + // Use the raw flashblock_stream (no buffering) to capture flashblocks + // as they're broadcast, independent of canon state. + let mut fb_stream = builder_context.flashblocks_handle.flashblock_stream(); + + let _stream_task = tokio::spawn(async move { + while let Some(fb) = futures::StreamExt::next(&mut fb_stream).await { + *latest_stream_fb_writer.write().unwrap() = Some(fb); + } + }); // Track per-block results - let blocks_with_pending = Arc::new(AtomicUsize::new(0)); - let total_pending_txs = Arc::new(AtomicUsize::new(0)); - let blocks_with_pending_cb = blocks_with_pending.clone(); - let total_pending_txs_cb = total_pending_txs.clone(); + let blocks_verified = Arc::new(AtomicUsize::new(0)); + let blocks_verified_cb = blocks_verified.clone(); - // Keep a reference to the builder's RPC client for pending queries let builder_rpc = env.node_clients[0].rpc.clone(); let mut driver = crate::actions::EngineDriver { builder_idx: 0, follower_idxs: vec![], + initial_parent_hash: Some(block_hash), num_blocks: NUM_BLOCKS, block_interval: BLOCK_INTERVAL, flashblocks: true, @@ -1416,19 +1254,15 @@ async fn test_engine_driver_pending_block_queries() -> eyre::Result<()> { )) } }), + during_build: None, on_block: Some(Box::new({ let builder_rpc = builder_rpc.clone(); + let latest_stream_fb = latest_stream_fb.clone(); move |block_num, payload| { let builder_rpc = builder_rpc.clone(); - let blocks_with_pending = blocks_with_pending_cb.clone(); - let total_pending_txs = total_pending_txs_cb.clone(); + let blocks_verified = blocks_verified_cb.clone(); + let latest_stream_fb = latest_stream_fb.clone(); - let block_hash = payload - .execution_payload - .payload_inner - .payload_inner - .payload_inner - .block_hash; let payload_tx_count = payload .execution_payload .payload_inner @@ -1438,53 +1272,71 @@ async fn test_engine_driver_pending_block_queries() -> eyre::Result<()> { .len(); Box::pin(async move { - info!( - target: "engine_driver_test", - block = block_num, - %block_hash, - payload_tx_count, - "payload built" - ); - assert!( payload_tx_count > 0, - "block {block_num}: expected at least 1 transaction (L1 info deposit)" + "block {block_num}: expected at least 1 transaction" ); - // Query the pending block during this build cycle - let pending: Option = EthApiClient::< - TransactionRequest, - alloy_rpc_types::Transaction, - alloy_rpc_types_eth::Block, - alloy_consensus::Receipt, - alloy_consensus::Header, - reth_optimism_primitives::OpTransactionSigned, - >::block_by_number( - &builder_rpc, - BlockNumberOrTag::Pending, - false, // tx hashes only to avoid deserialization issues - ) - .await?; - - if let Some(pending_block) = &pending { + // Query the pending block from the Eth API + let pending: Option = + EthApiClient::< + TransactionRequest, + alloy_rpc_types::Transaction, + alloy_rpc_types_eth::Block, + alloy_consensus::Receipt, + alloy_consensus::Header, + reth_optimism_primitives::OpTransactionSigned, + >::block_by_number( + &builder_rpc, BlockNumberOrTag::Pending, false + ) + .await?; + + // Get the latest flashblock from the event stream + let stream_fb = latest_stream_fb.read().unwrap().clone(); + + if let (Some(pending_block), Some(stream_fb)) = (&pending, &stream_fb) { info!( target: "engine_driver_test", block = block_num, - pending_tx_count = pending_block.transactions.len(), pending_number = pending_block.header.number, - "queried pending block" + pending_tx_count = pending_block.transactions.len(), + stream_payload_id = %stream_fb.payload_id, + stream_index = stream_fb.index, + stream_tx_count = stream_fb.diff.transactions.len(), + "comparing pending block vs event stream flashblock" + ); + + // The pending block from the Eth API should be for the + // same payload as the stream flashblock. + assert_eq!( + stream_fb.payload_id, + stream_fb.payload_id, // sanity + "block {block_num}: stream flashblock should have a valid payload_id" + ); + + // The pending block tx count should be >= the stream + // flashblock's cumulative tx count (pending block + // includes all transactions, stream fb has the diff). + assert!( + !pending_block.transactions.is_empty() + || stream_fb.diff.transactions.is_empty(), + "block {block_num}: pending block should have transactions if stream flashblock does" + ); + assert!( + pending_block.hash() == stream_fb.diff.block_hash, + "block {block_num}: pending block hash should match stream flashblock hash" ); } else { info!( target: "engine_driver_test", block = block_num, - "no pending block available (expected during finalization)" + pending_available = pending.is_some(), + stream_available = stream_fb.is_some(), + "pending block or stream flashblock not yet available" ); } - blocks_with_pending.fetch_add(1, Ordering::SeqCst); - total_pending_txs.fetch_add(payload_tx_count, Ordering::SeqCst); - + blocks_verified.fetch_add(1, Ordering::SeqCst); Ok(()) }) } @@ -1493,24 +1345,256 @@ async fn test_engine_driver_pending_block_queries() -> eyre::Result<()> { driver.execute(&mut env).await?; - let blocks_queried = blocks_with_pending.load(Ordering::SeqCst); - let txs_queried = total_pending_txs.load(Ordering::SeqCst); + let verified = blocks_verified.load(Ordering::SeqCst); info!( target: "engine_driver_test", - blocks_queried, - txs_queried, + verified, "engine driver test complete" ); assert_eq!( - blocks_queried, NUM_BLOCKS, - "expected to query {NUM_BLOCKS} blocks" + verified, NUM_BLOCKS, + "expected to verify {NUM_BLOCKS} blocks" ); + + // Verify the event stream captured flashblocks + let final_fb = latest_stream_fb.read().unwrap().clone(); assert!( - txs_queried >= NUM_BLOCKS, - "expected at least {NUM_BLOCKS} total transactions (one L1 deposit per block)" + final_fb.is_some(), + "expected the event stream to have captured at least one flashblock" + ); + + Ok(()) +} + +/// Large block production loop using [`EngineDriver`] that sanity-checks +/// all helper macros in the `on_block` hook: `provider!`, `fetch_block!`, +/// `fetch_tx!`, `fetch_receipt!`, `eth_call!`, `fetch_logs!`. +#[tokio::test(flavor = "multi_thread")] +async fn test_eth_api_assertions() -> eyre::Result<()> { + use crate::setup::encode_eip1559_params; + use alloy_provider::Provider; + use alloy_rpc_types::Filter; + + reth_tracing::init_test_tracing(); + tokio::time::sleep(Duration::from_millis(100)).await; + + const NUM_BLOCKS: usize = 5; + const BLOCK_INTERVAL: Duration = Duration::from_millis(2000); + + let (_, nodes, _tasks, mut env, tx_spammer) = + setup::(1, optimism_payload_attributes, true).await?; + + let builder_context = nodes[0].ext_context.clone().unwrap(); + let block_hash = nodes[0].node.block_hash(0); + let chain_spec = nodes[0].node.inner.chain_spec().clone(); + let rpc_url = nodes[0].node.rpc_url(); + + for node in &nodes { + node.node.update_forkchoice(block_hash, block_hash).await?; + } + + tx_spammer.spawn(10, rpc_url); + + let builder_vk = builder_context + .flashblocks_handle + .builder_sk() + .unwrap() + .verifying_key(); + + let authorization_gen = + move |parent_hash: B256, attrs: reth_optimism_node::OpPayloadAttributes| { + let authorizer_sk = ed25519_dalek::SigningKey::from_bytes(&[0; 32]); + let payload_id = + reth_optimism_payload_builder::payload_id_optimism(&parent_hash, &attrs, 3); + flashblocks_primitives::p2p::Authorization::new( + payload_id, + reth_node_api::PayloadAttributes::timestamp(&attrs), + &authorizer_sk, + builder_vk, + ) + }; + + let checks_passed = Arc::new(AtomicUsize::new(0)); + let checks_passed_cb = checks_passed.clone(); + + let mut driver = crate::actions::EngineDriver { + builder_idx: 0, + follower_idxs: vec![], + initial_parent_hash: Some(block_hash), + num_blocks: NUM_BLOCKS, + block_interval: BLOCK_INTERVAL, + flashblocks: true, + authorization_gen, + attributes_gen: Box::new({ + let chain_spec = chain_spec.clone(); + move |_block_number, timestamp| { + let eip1559 = encode_eip1559_params(chain_spec.as_ref(), timestamp)?; + Ok(build_payload_attributes( + timestamp, + eip1559, + Some(vec![TX_SET_L1_BLOCK.clone()]), + )) + } + }), + during_build: Some(Box::new({ + let url = nodes[0].node.rpc_url(); + move |block_num| { + let url = url.clone(); + Box::pin(async move { + use alloy_provider::Provider; + let provider = ProviderBuilder::<_, _, op_alloy_network::Optimism>::default() + .network::() + .with_recommended_fillers() + .connect_http(url); + + let pending = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) + .await?; + + let latest = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Latest) + .await?; + + if let (Some(pending_block), Some(latest_block)) = (&pending, &latest) { + assert_eq!( + pending_block.header.parent_hash, latest_block.header.hash, + "block {block_num}: pending.parent_hash must equal latest.hash" + ); + assert_ne!( + pending_block.header.hash, latest_block.header.hash, + "block {block_num}: pending must differ from latest" + ); + info!( + target: "macro_sanity", + block = block_num, + pending_number = pending_block.header.number, + latest_number = latest_block.header.number, + "pending != latest verified during build" + ); + } + + Ok(()) + }) + } + })), + on_block: Some(Box::new({ + let nodes_0 = nodes[0].node.rpc_url(); + move |block_num, _payload| { + let checks_passed = checks_passed_cb.clone(); + let url = nodes_0.clone(); + + Box::pin(async move { + let provider = ProviderBuilder::<_, _, op_alloy_network::Optimism>::default() + .network::() + .with_recommended_fillers() + .connect_http(url); + + // --- fetch_block!(Pending) --- + let pending = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Pending) + .await?; + info!( + target: "macro_sanity", + block = block_num, + pending = pending.is_some(), + "pending block query" + ); + + // --- fetch_block!(Latest) --- + let latest: Option< + alloy_rpc_types::Block>, + > = provider + .get_block_by_number(alloy_eips::BlockNumberOrTag::Latest) + .full() + .await?; + + let latest = latest.unwrap(); + let latest_number = latest.header.number; + let tx_count = latest.transactions.len(); + info!( + target: "macro_sanity", + block = block_num, + latest_number, + tx_count, + "latest block query" + ); + + // --- fetch_tx! (first tx in latest block) --- + let tx_hashes: Vec<_> = latest.transactions.hashes().collect(); + if let Some(&tx_hash) = tx_hashes.first() { + let tx: Option> = + provider.get_transaction_by_hash(tx_hash).await?; + assert!( + tx.is_some(), + "block {block_num}: fetch_tx for {tx_hash} should return a result" + ); + + // --- fetch_receipt! --- + let receipt = provider.get_transaction_receipt(tx_hash).await?; + assert!( + receipt.is_some(), + "block {block_num}: fetch_receipt for {tx_hash} should return a result" + ); + + info!( + target: "macro_sanity", + block = block_num, + %tx_hash, + "tx + receipt queries passed" + ); + } + + // --- eth_call --- + let call_tx = alloy_rpc_types::TransactionRequest::default() + .to(Address::ZERO) + .value(U256::ZERO); + let call_result = provider.call(call_tx.into()).await; + assert!( + call_result.is_ok(), + "block {block_num}: eth_call should succeed: {:?}", + call_result.err() + ); + info!(target: "macro_sanity", block = block_num, "eth_call passed"); + + // --- fetch_logs --- + let filter = Filter::new() + .from_block(latest_number) + .to_block(latest_number); + let logs = provider.get_logs(&filter).await?; + info!( + target: "macro_sanity", + block = block_num, + log_count = logs.len(), + "fetch_logs passed" + ); + + // --- chain_id sanity --- + let chain_id = provider.get_chain_id().await?; + assert!(chain_id > 0, "block {block_num}: chain_id should be > 0"); + + checks_passed.fetch_add(1, Ordering::SeqCst); + info!( + target: "macro_sanity", + block = block_num, + "all checks passed" + ); + + Ok(()) + }) + } + })), + }; + + driver.execute(&mut env).await?; + + let passed = checks_passed.load(Ordering::SeqCst); + assert_eq!( + passed, NUM_BLOCKS, + "expected all {NUM_BLOCKS} blocks to pass macro sanity checks, got {passed}" ); + info!(target: "macro_sanity", passed, "all blocks verified"); Ok(()) } From 8507b4d3f2083084943f88ff302629c4feb5a978 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Sat, 14 Mar 2026 00:13:57 -0700 Subject: [PATCH 39/43] fix: tasks --- crates/flashblocks/builder/src/coordinator.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 01f0d1d88..f18df2ead 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -38,9 +38,6 @@ use tokio::sync::{ }; use tracing::{debug, error, trace}; -/// Maximum number of concurrent flashblock processing tasks on the thread pool. -const MAX_THREAD_POOL_SIZE: usize = 4; - /// Placeholder for future task handle variants. Currently unused — the /// hook updates P2P state directly via the flushed cursor. #[derive(Clone, Debug)] @@ -136,7 +133,6 @@ impl FlashblocksExecutionCoordinator { let pending_block = self.pending_block.clone(); let workload = WorkloadExecutor::default(); - let task_permit = Arc::new(Semaphore::new(MAX_THREAD_POOL_SIZE)); let database_permit = &PENDING_BLOCK_WRITE_PERMIT; @@ -168,7 +164,6 @@ impl FlashblocksExecutionCoordinator { this.on_flashblock( flashblock, &mut inflight_shutdown, - &task_permit, database_permit, &workload, &provider, @@ -206,7 +201,6 @@ impl FlashblocksExecutionCoordinator { &self, flashblock: FlashblocksPayloadV1, shutdown_tx: &mut Option>, - task_permit: &Arc, database_permit: &'static Semaphore, workload: &WorkloadExecutor, provider: &Provider, @@ -228,12 +222,6 @@ impl FlashblocksExecutionCoordinator { let (tx, rx) = oneshot::channel::<()>(); *shutdown_tx = Some(tx); - let task_permit = task_permit - .clone() - .acquire_owned() - .await - .expect("semaphore closed"); - let provider = provider.clone(); let evm_config = evm_config.clone(); let this = self.clone(); @@ -244,7 +232,6 @@ impl FlashblocksExecutionCoordinator { let index = flashblock.index; spawn_blocking_io_with_shutdown_signal(workload, rx, database_permit, move |permit| { - let _task_permit = task_permit; // held until closure completes if let Err(e) = process_flashblock( permit, provider, From d5f590c436453464e2e323e44e624d693d91e5e9 Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Sat, 14 Mar 2026 00:22:48 -0700 Subject: [PATCH 40/43] Apply suggestion from @0xOsiris --- crates/flashblocks/p2p/src/protocol/event.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/flashblocks/p2p/src/protocol/event.rs b/crates/flashblocks/p2p/src/protocol/event.rs index 4e655583b..ad1be62bd 100644 --- a/crates/flashblocks/p2p/src/protocol/event.rs +++ b/crates/flashblocks/p2p/src/protocol/event.rs @@ -81,7 +81,6 @@ where /// Stream adapter that wraps a merged `ChainEvent` stream and map reduces it /// into a [`BufferedFlashblocks`]. #[pin_project::pin_project] -#[doc = ""] struct BufferedStream { #[pin] inner: S, From d1b35914f78a7f7bb7c3e1123df03e595f83dbfd Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Sat, 14 Mar 2026 12:34:52 -0700 Subject: [PATCH 41/43] chore: cleanup metrics --- crates/flashblocks/builder/src/coordinator.rs | 13 +---- crates/flashblocks/builder/src/lib.rs | 4 -- crates/flashblocks/builder/src/metrics.rs | 53 ------------------- 3 files changed, 1 insertion(+), 69 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index f18df2ead..927f5600d 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -247,7 +247,6 @@ impl FlashblocksExecutionCoordinator { index, "error processing flashblock: {e:#?}" ); - EXECUTION.errors.increment(1); } }); } @@ -255,15 +254,6 @@ impl FlashblocksExecutionCoordinator { /// Handles a canonical chain tip update. Cancels any in-flight task, /// clears stale ancestor trie handles, and clears the pending block if /// it was built on the now-canonical tip. - #[tracing::instrument( - target = "flashblocks::coordinator", - skip_all, - fields( - tip_number = tip.number, - tip_hash = %tip.hash, - is_stale, - ) - )] fn on_canon( &self, tip: BlockNumHash, @@ -274,8 +264,7 @@ impl FlashblocksExecutionCoordinator { // Only cancel in-flight work and clear ancestor handles if the current // epoch is at or behind the canonical tip (stale). If the epoch is // ahead of the tip, the work is still valid. - let is_stale = epoch_block_number.is_none_or(|n| n <= tip.number); - tracing::Span::current().record("is_stale", is_stale); + let is_stale = epoch_block_number.is_some_and(|n| n <= tip.number); if is_stale { debug!( diff --git a/crates/flashblocks/builder/src/lib.rs b/crates/flashblocks/builder/src/lib.rs index 539f0e60c..0a13ecdfd 100644 --- a/crates/flashblocks/builder/src/lib.rs +++ b/crates/flashblocks/builder/src/lib.rs @@ -255,10 +255,6 @@ pub(crate) fn spawn_blocking_io_with_shutdown_signal( .block_on(database_permit.acquire()) .expect("database semaphore closed"); - EXECUTION - .permit_wait - .record(permit_wait.elapsed().as_secs_f64()); - f(permit); }); diff --git a/crates/flashblocks/builder/src/metrics.rs b/crates/flashblocks/builder/src/metrics.rs index 2dc82a65b..b5af3539b 100644 --- a/crates/flashblocks/builder/src/metrics.rs +++ b/crates/flashblocks/builder/src/metrics.rs @@ -9,26 +9,14 @@ use std::{sync::LazyLock, time::Instant}; #[metrics(scope = "flashblocks.coordinator")] pub struct ExecutionMetrics { // -- Latency -- - /// End-to-end flashblock processing duration (seconds). - pub process_duration: Histogram, /// Validation / build phase duration (seconds). pub validate_duration: Histogram, - /// Time waiting for the database write permit (seconds). - pub permit_wait: Histogram, - /// Duration holding the state write lock (seconds). - pub state_lock_duration: Histogram, /// Flashblocks processed in a single epoch (recorded at epoch boundary). pub flashblocks_per_epoch: Histogram, // -- Issues -- - /// Flashblock skipped — already processed (duplicate from P2P). - pub skipped: Counter, /// Epoch invalidated by a newer canonical tip. pub stale_resets: Counter, - /// Processing error (validation, build, or state update failure). - pub errors: Counter, - /// Failed to fetch sealed header for parent hash. - pub header_fetch_failed: Counter, /// Invalid payload received from P2P (decode error, bad structure). pub invalid_payload: Counter, /// Broadcast of built payload to in-memory tree failed. @@ -130,47 +118,6 @@ mod tests { } } - #[test] - fn metrics_span_records_duration_and_emits_histogram() { - // Install a real metrics recorder so histogram calls don't panic. - // metrics-util provides an in-memory recorder for testing. - // If unavailable, we just verify the span side. - let spans = Arc::new(Mutex::new(Vec::new())); - let layer = SpanCapture { - spans: spans.clone(), - }; - - let _guard = tracing_subscriber::registry().with(layer).set_default(); - - let histogram = EXECUTION.process_duration.clone(); - - { - let _span = MetricsSpan::new( - tracing::trace_span!( - target: "flashblocks::coordinator", - "test_span", - id = "test_payload", - index = 3u64, - duration_ms = tracing::field::Empty, - ), - histogram, - ); - - // Simulate work - std::thread::sleep(std::time::Duration::from_millis(5)); - } - // MetricsSpan dropped — should have recorded duration_ms - - let captured = spans.lock().unwrap(); - assert_eq!(captured.len(), 1); - assert_eq!(captured[0].name, "test_span"); - assert!( - captured[0].fields.contains("id="), - "span should contain id field: {}", - captured[0].fields - ); - } - #[test] fn metered_fn_passes_span_ref_and_records_dynamic_fields() { let spans = Arc::new(Mutex::new(Vec::new())); From 7a14d1b77e959a4cb767dca028bcd29d6e459b8c Mon Sep 17 00:00:00 2001 From: 0xOsiris Date: Sat, 14 Mar 2026 13:10:14 -0700 Subject: [PATCH 42/43] chore: cleanup --- crates/flashblocks/builder/src/coordinator.rs | 9 +- crates/flashblocks/builder/src/lib.rs | 1 - crates/flashblocks/builder/src/metrics.rs | 122 +----------------- .../node/tests/e2e-testsuite/testsuite.rs | 12 +- 4 files changed, 12 insertions(+), 132 deletions(-) diff --git a/crates/flashblocks/builder/src/coordinator.rs b/crates/flashblocks/builder/src/coordinator.rs index 927f5600d..ac3a5981b 100644 --- a/crates/flashblocks/builder/src/coordinator.rs +++ b/crates/flashblocks/builder/src/coordinator.rs @@ -278,11 +278,10 @@ impl FlashblocksExecutionCoordinator { *epoch_block_number = None; } - // Clear pending block if it was built on the now-canonical tip. pending_block.send_if_modified(|block| { let matches = block - .as_ref() - .is_some_and(|b| b.recovered_block().parent_num_hash() == tip); + .as_ref() // We want to remove the pending block immediately when the canonical tip matches + .is_some_and(|b| b.recovered_block().hash() == tip.hash); if matches { *block = None; @@ -399,7 +398,6 @@ where (base, is_new) }; - // --- Read lock dropped --- // Clear ancestor handles on new epoch if is_new_epoch { @@ -420,10 +418,12 @@ where let diff = flashblock.diff().clone(); let index = flashblock.flashblock.index; + // this should never fail. if it does there's a bug in our streaming. let sealed_header = provider .sealed_header_by_hash(base.parent_hash) .inspect_err(|e| error!("failed to fetch sealed header {}: {e:#?}", base.parent_hash))? .ok_or_else(|| eyre!("sealed header not found for hash {}", base.parent_hash))?; + let anchor_hash = sealed_header.hash(); let execution_context = OpBlockExecutionCtx { @@ -556,7 +556,6 @@ where } }; - // _validate_span dropped here — records duration_ms on span + histogram. drop(_validate_span); // Build ExecutedBlock with deferred trie data — sorting happens in background diff --git a/crates/flashblocks/builder/src/lib.rs b/crates/flashblocks/builder/src/lib.rs index 0a13ecdfd..a10b464db 100644 --- a/crates/flashblocks/builder/src/lib.rs +++ b/crates/flashblocks/builder/src/lib.rs @@ -245,7 +245,6 @@ pub(crate) fn spawn_blocking_io_with_shutdown_signal( let _enter = parent_span.enter(); let unwind = AssertUnwindSafe(move || { - let permit_wait = Instant::now(); let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/crates/flashblocks/builder/src/metrics.rs b/crates/flashblocks/builder/src/metrics.rs index b5af3539b..7756a8426 100644 --- a/crates/flashblocks/builder/src/metrics.rs +++ b/crates/flashblocks/builder/src/metrics.rs @@ -71,124 +71,4 @@ where { let guard = MetricsSpan::new(span, histogram); f(&guard) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::{Arc, Mutex}; - use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; - - /// Tracing layer that captures span events for assertions. - struct SpanCapture { - spans: Arc>>, - } - - #[derive(Debug, Clone)] - struct SpanRecord { - name: String, - fields: String, - } - - impl tracing_subscriber::Layer for SpanCapture { - fn on_new_span( - &self, - attrs: &tracing::span::Attributes<'_>, - _id: &tracing::span::Id, - _ctx: tracing_subscriber::layer::Context<'_, S>, - ) { - let mut fields = String::new(); - attrs.record(&mut FieldVisitor(&mut fields)); - self.spans.lock().unwrap().push(SpanRecord { - name: attrs.metadata().name().to_string(), - fields, - }); - } - } - - struct FieldVisitor<'a>(&'a mut String); - - impl tracing::field::Visit for FieldVisitor<'_> { - fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { - use std::fmt::Write; - if !self.0.is_empty() { - self.0.push(' '); - } - let _ = write!(self.0, "{}={:?}", field.name(), value); - } - } - - #[test] - fn metered_fn_passes_span_ref_and_records_dynamic_fields() { - let spans = Arc::new(Mutex::new(Vec::new())); - let layer = SpanCapture { - spans: spans.clone(), - }; - - let _guard = tracing_subscriber::registry().with(layer).set_default(); - - let result = metered_fn( - tracing::trace_span!( - target: "flashblocks::coordinator", - "metered_test", - path = tracing::field::Empty, - duration_ms = tracing::field::Empty, - ), - EXECUTION.validate_duration.clone(), - |span| { - span.record("path", "bal"); - 42 - }, - ); - - assert_eq!(result, 42); - - let captured = spans.lock().unwrap(); - assert_eq!(captured.len(), 1); - assert_eq!(captured[0].name, "metered_test"); - } - - #[test] - fn span_propagation_across_thread_boundary() { - let spans = Arc::new(Mutex::new(Vec::new())); - let layer = SpanCapture { - spans: spans.clone(), - }; - - let dispatch = - tracing::dispatcher::Dispatch::new(tracing_subscriber::registry().with(layer)); - let _guard = tracing::dispatcher::set_default(&dispatch); - - // Create a parent span on this thread - let parent = tracing::trace_span!( - target: "flashblocks::coordinator", - "parent_span", - id = "payload_123", - ); - - let parent_clone = parent.clone(); - let thread_dispatch = dispatch.clone(); - - // Simulate the spawn_blocking pattern: capture span, re-enter on new thread - let handle = std::thread::spawn(move || { - let _sub = tracing::dispatcher::set_default(&thread_dispatch); - let _enter = parent_clone.enter(); - // Child span created under re-entered parent - let _child = tracing::trace_span!( - target: "flashblocks::coordinator", - "child_on_blocking_thread", - ) - .entered(); - }); - - handle.join().unwrap(); - - let captured = spans.lock().unwrap(); - let names: Vec<&str> = captured.iter().map(|s| s.name.as_str()).collect(); - assert!(names.contains(&"parent_span"), "should capture parent span"); - assert!( - names.contains(&"child_on_blocking_thread"), - "should capture child span created on blocking thread" - ); - } -} +} \ No newline at end of file diff --git a/crates/world/node/tests/e2e-testsuite/testsuite.rs b/crates/world/node/tests/e2e-testsuite/testsuite.rs index 6014cfaae..b731ded82 100644 --- a/crates/world/node/tests/e2e-testsuite/testsuite.rs +++ b/crates/world/node/tests/e2e-testsuite/testsuite.rs @@ -1458,13 +1458,15 @@ async fn test_eth_api_assertions() -> eyre::Result<()> { .await?; if let (Some(pending_block), Some(latest_block)) = (&pending, &latest) { - assert_eq!( - pending_block.header.parent_hash, latest_block.header.hash, - "block {block_num}: pending.parent_hash must equal latest.hash" - ); assert_ne!( pending_block.header.hash, latest_block.header.hash, - "block {block_num}: pending must differ from latest" + "block {block_num}: pending must differ from latest during build" + ); + assert!( + pending_block.header.number > latest_block.header.number, + "block {block_num}: pending number ({}) must be > latest ({})", + pending_block.header.number, + latest_block.header.number, ); info!( target: "macro_sanity", From b07117ff143a60e528b844f68a93d55b7d26e50f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 14 Mar 2026 20:17:48 +0000 Subject: [PATCH 43/43] chore: auto-format --- crates/flashblocks/builder/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/flashblocks/builder/src/metrics.rs b/crates/flashblocks/builder/src/metrics.rs index 7756a8426..c06fa4119 100644 --- a/crates/flashblocks/builder/src/metrics.rs +++ b/crates/flashblocks/builder/src/metrics.rs @@ -71,4 +71,4 @@ where { let guard = MetricsSpan::new(span, histogram); f(&guard) -} \ No newline at end of file +}