Fixes for Embers

.cargo/config.toml

@@ -3,13 +3,16 @@
 # This file contains build configuration and compiler flags for the entire workspace.
 
 [env]
-# Set minimum thread stack size to 8MB for test threads.
+# Set minimum thread stack size to 32MB for test threads.
 # The Rholang interpreter uses deep async recursion (eval → produce/consume → dispatch → eval)
 # via Box::pin patterns. In debug builds, each recursion level consumes significantly more
 # stack space (~1-2KB) than in release builds (~100-200 bytes) due to lack of inlining and
-# unoptimized async state machines. The default 2MB stack overflows during normal test execution.
-# See: https://github.com/F1R3FLY-io/f1r3node/issues/305
-RUST_MIN_STACK = "8388608"
+# unoptimized async state machines. With receives-first evaluation ordering, genesis contracts
+# create deeper COMM cascades (50+ levels), requiring more thread stack than the stacker crate
+# can compensate for via heap-allocated segments.
+# History: 2MB (default) → 8MB (issue #305) → 32MB (receives-first evaluation).
+# Long-term fix: convert recursive async calls to trampolines.
+RUST_MIN_STACK = "33554432"
 
 [build]
 # Enable native CPU features for gxhash (requires AES and SSE2 intrinsics)

casper/Cargo.toml

@@ -52,6 +52,7 @@ k256 = { version = "0.13.4", features = ["ecdsa"] }
 serial_test = "3.1"
 env_logger = "0.11.8"
 tempfile = "3.24.0"
+libc = "0.2"
 
 
 

casper/src/rust/api/block_api.rs

@@ -361,7 +361,18 @@ impl BlockAPI {
                                     }
                                 }
                                 ProposerResult::Empty => {
-                                    tracing::debug!("Propose already in progress");
+                                    if attempt < max_attempts {
+                                        tracing::debug!(
+                                            "Propose already in progress (attempt {}/{}); retrying in {:?}",
+                                            attempt,
+                                            max_attempts,
+                                            retry_delay
+                                        );
+                                        attempt += 1;
+                                        tokio::time::sleep(retry_delay).await;
+                                        continue;
+                                    }
+                                    tracing::debug!("Propose already in progress; max retries exhausted");
                                 }
                                 ProposerResult::Started(seq_number) => {
                                     tracing::debug!("Propose started (seqNum {})", seq_number);
@@ -1471,6 +1482,7 @@ impl BlockAPI {
                                 &snapshot,
                                 &runtime_guard,
                                 Some(true), // disable_late_block_filtering = true for exploratory deploy
+                                None,
                             )?;
                         merged_state_hash
                     };
@@ -1507,11 +1519,30 @@ impl BlockAPI {
 
                 match target_block {
                     Some(b) => {
+                        tracing::info!(
+                            target: "f1r3fly.rholang.diag",
+                            state_hash = %PrettyPrinter::build_string_bytes(&state_hash),
+                            block_number = b.body.state.block_number,
+                            block_hash = %PrettyPrinter::build_string_bytes(&b.block_hash),
+                            parent_hash = %b.header.parents_hash_list.first()
+                                .map(|h| PrettyPrinter::build_string_bytes(h))
+                                .unwrap_or_default(),
+                            deploy_count = b.body.deploys.len(),
+                            "exploratory_deploy: LFB details for state selection"
+                        );
                         let res = runtime_manager
                             .lock()
                             .await
                             .play_exploratory_deploy(term, &state_hash)
                             .await?;
+                        tracing::info!(
+                            target: "f1r3fly.rholang.diag",
+                            state_hash = %PrettyPrinter::build_string_bytes(&state_hash),
+                            block_number = b.body.state.block_number,
+                            result_count = res.len(),
+                            "exploratory_deploy: play_exploratory_deploy returned {} result pars",
+                            res.len()
+                        );
                         let light_block_info =
                             Self::get_light_block_info(casper.as_ref(), &b).await?;
                         Ok((res, light_block_info))

casper/src/rust/blocks/block_processor.rs

@@ -37,8 +37,10 @@ use shared::rust::env;
 
 use crate::rust::block_status::BlockError;
 use crate::rust::engine::block_retriever::{AdmitHashReason, BlockRetriever};
+#[cfg(all(target_os = "linux", target_env = "gnu"))]
+use crate::rust::metrics_constants::ALLOCATOR_TRIM_TOTAL_METRIC;
 use crate::rust::metrics_constants::{
-    ALLOCATOR_TRIM_TOTAL_METRIC, BLOCK_PROCESSING_STORAGE_TIME_METRIC,
+    BLOCK_PROCESSING_STORAGE_TIME_METRIC,
     BLOCK_PROCESSING_VALIDATION_SETUP_TIME_METRIC, BLOCK_PROCESSOR_METRICS_SOURCE,
     BLOCK_SIZE_METRIC, BLOCK_VALIDATION_FAILED_METRIC, BLOCK_VALIDATION_SUCCESS_METRIC,
     BLOCK_VALIDATION_TIME_METRIC,

casper/src/rust/engine/block_approver_protocol.rs

@@ -240,8 +240,12 @@ impl<T: TransportLayer + Send + Sync + 'static> BlockApproverProtocol<T> {
             ));
         }
 
-        // State hash checks
-        let empty_state_hash = RuntimeManager::empty_state_hash_fixed();
+        // State hash checks — use the genesis block's own pre-state hash
+        // rather than a hardcoded constant. The pre-state hash is dynamically
+        // computed during genesis creation and is guaranteed to be in the
+        // roots store. A hardcoded constant breaks when evaluation order or
+        // trie hashing changes.
+        let empty_state_hash = block.body.state.pre_state_hash.clone();
         let state_hash = runtime_manager
             .replay_compute_state(
                 &empty_state_hash,

casper/src/rust/engine/initializing.rs

@@ -790,8 +790,9 @@ impl<T: TransportLayer + Send + Sync + Clone> Initializing<T> {
         let system_deploys = proto_util::system_deploys(block);
         let block_data = rholang::rust::interpreter::system_processes::BlockData::from_block(block);
 
-        // Genesis starts from empty state
-        let pre_state_hash = RuntimeManager::empty_state_hash_fixed();
+        // Use the genesis block's own pre-state hash rather than a hardcoded
+        // constant — see block_approver_protocol.rs for rationale.
+        let pre_state_hash = block.body.state.pre_state_hash.clone();
 
         // Replay genesis - this will save mergeable channels to the store
         let mut runtime_manager = self.runtime_manager.lock().await;

casper/src/rust/rholang/replay_runtime.rs

@@ -165,6 +165,11 @@ impl ReplayRuntimeOps {
         let checkpoint_start = Instant::now();
         tracing::debug!(target: "f1r3fly.casper.replay-rho-runtime", "create-checkpoint-started");
         let checkpoint = self.runtime_ops.runtime.create_checkpoint();
+        tracing::info!(
+            target: "f1r3fly.rspace",
+            replay_root = %hex::encode(checkpoint.root.bytes()),
+            "replay_deploys: checkpoint completed"
+        );
         tracing::debug!(target: "f1r3fly.casper.replay-rho-runtime", "create-checkpoint-finished");
         metrics::histogram!(BLOCK_REPLAY_PHASE_CREATE_CHECKPOINT_TIME_METRIC, "source" => CASPER_METRICS_SOURCE)
             .record(checkpoint_start.elapsed().as_secs_f64());
@@ -324,6 +329,7 @@ impl ReplayRuntimeOps {
         let deploy_data = SystemProcessDeployData::from_deploy(&processed_deploy.deploy);
         self.runtime_ops.runtime.set_deploy_data(deploy_data).await;
 
+        rholang::rust::interpreter::storage::charging_rspace::reset_cost_trace_seq();
         let mut user_eval_result = self.runtime_ops.evaluate(&processed_deploy.deploy).await?;
         self.discard_event_log("user-deploy", false);
 
@@ -347,6 +353,17 @@ impl ReplayRuntimeOps {
         }
 
         if processed_deploy.cost.cost != user_eval_result.cost.value as u64 {
+            tracing::error!(
+                target: "f1r3fly.rspace.cost_trace",
+                initial_cost = processed_deploy.cost.cost,
+                replay_cost = user_eval_result.cost.value,
+                cost_diff = processed_deploy.cost.cost as i64 - user_eval_result.cost.value,
+                deploy_sig = %hex::encode(&processed_deploy.deploy.sig),
+                "COST_MISMATCH_DETAIL: validator_cost={} observer_cost={} diff={}",
+                processed_deploy.cost.cost,
+                user_eval_result.cost.value,
+                processed_deploy.cost.cost as i64 - user_eval_result.cost.value
+            );
             return Err(CasperError::ReplayFailure(
                 ReplayFailure::replay_cost_mismatch(
                     processed_deploy.cost.cost,