diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f5fca135e..6e78883f55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Perf +### 2025-07-16 + +- Improve levm memory model [#3564](https://github.com/lambdaclass/ethrex/pull/3564) +- ### 2025-07-15 - Add sstore bench [#3552](https://github.com/lambdaclass/ethrex/pull/3552) diff --git a/crates/vm/levm/Cargo.toml b/crates/vm/levm/Cargo.toml index 9935149890..19f5f75d92 100644 --- a/crates/vm/levm/Cargo.toml +++ b/crates/vm/levm/Cargo.toml @@ -45,7 +45,7 @@ ethereum_foundation_tests = [] debug = [] [lints.rust] -unsafe_code = "forbid" +unsafe_code = "warn" warnings = "warn" rust_2018_idioms = "warn" diff --git a/crates/vm/levm/bench/revm_comparison/src/levm_bench.rs b/crates/vm/levm/bench/revm_comparison/src/levm_bench.rs index fd1a56aa6f..f843fc24cc 100644 --- a/crates/vm/levm/bench/revm_comparison/src/levm_bench.rs +++ b/crates/vm/levm/bench/revm_comparison/src/levm_bench.rs @@ -35,7 +35,8 @@ pub fn run_with_levm(contract_code: &str, runs: u64, calldata: &str) { } let mut vm = init_vm(&mut db, 0, calldata.clone()); let tx_report = black_box(vm.stateless_execute().unwrap()); - assert!(tx_report.is_success()); + + assert!(tx_report.is_success(), "{:?}", tx_report.result); match tx_report.result { TxResult::Success => { diff --git a/crates/vm/levm/src/call_frame.rs b/crates/vm/levm/src/call_frame.rs index f1c6389a9e..b2153375bb 100644 --- a/crates/vm/levm/src/call_frame.rs +++ b/crates/vm/levm/src/call_frame.rs @@ -28,29 +28,34 @@ pub struct Stack { } impl Stack { + #[inline] pub fn pop(&mut self) -> Result<&[U256; N], ExceptionalHalt> { // Compile-time check for stack underflow. - if N > STACK_LIMIT { - return Err(ExceptionalHalt::StackUnderflow); + const { + assert!(N <= STACK_LIMIT); } // The following operation can never overflow as both `self.offset` and N are within // STACK_LIMIT (1024). - #[expect(clippy::arithmetic_side_effects)] - let next_offset = self.offset + N; + let next_offset = self.offset.wrapping_add(N); // The index cannot fail because `self.offset` is known to be valid. The `first_chunk()` // method will ensure that `next_offset` is within `STACK_LIMIT`, so there's no need to // check it again. - #[expect(clippy::indexing_slicing)] - let values = self.values[self.offset..] - .first_chunk::() - .ok_or(ExceptionalHalt::StackUnderflow)?; + #[expect(unsafe_code)] + let values = unsafe { + self.values + .get_unchecked(self.offset..) + .first_chunk::() + .ok_or(ExceptionalHalt::StackUnderflow)? + }; + // Due to previous error check in first_chunk, next_offset is guaranteed to be < STACK_LIMIT self.offset = next_offset; Ok(values) } + #[inline] pub fn push(&mut self, values: &[U256; N]) -> Result<(), ExceptionalHalt> { // Since the stack grows downwards, when an offset underflow is detected the stack is // overflowing. @@ -61,8 +66,19 @@ impl Stack { // The following index cannot fail because `next_offset` has already been checked and // `self.offset` is known to be within `STACK_LIMIT`. - #[expect(clippy::indexing_slicing)] - self.values[next_offset..self.offset].copy_from_slice(values); + #[expect( + unsafe_code, + reason = "self.offset < STACK_LIMIT and next_offset == self.offset - N >= 0" + )] + unsafe { + std::ptr::copy_nonoverlapping( + values.as_ptr(), + self.values + .get_unchecked_mut(next_offset..self.offset) + .as_mut_ptr(), + N, + ); + } self.offset = next_offset; Ok(()) @@ -84,12 +100,16 @@ impl Stack { pub fn get(&self, index: usize) -> Result<&U256, ExceptionalHalt> { // The following index cannot fail because `self.offset` is known to be within // `STACK_LIMIT`. - #[expect(clippy::indexing_slicing)] - self.values[self.offset..] - .get(index) - .ok_or(ExceptionalHalt::StackUnderflow) + #[expect(unsafe_code)] + unsafe { + self.values + .get_unchecked(self.offset..) + .get(index) + .ok_or(ExceptionalHalt::StackUnderflow) + } } + #[inline(always)] pub fn swap(&mut self, index: usize) -> Result<(), ExceptionalHalt> { let index = self .offset @@ -134,7 +154,7 @@ impl fmt::Debug for Stack { } } -#[derive(Debug, Clone, Default, PartialEq)] +#[derive(Debug, Clone, Default)] /// A call frame, or execution environment, is the context in which /// the EVM is currently executing. /// One context can trigger another with opcodes like CALL or CREATE. @@ -176,7 +196,7 @@ pub struct CallFrame { /// Everytime we want to write an account during execution of a callframe we store the pre-write state so that we can restore if it reverts pub call_frame_backup: CallFrameBackup, /// Return data offset - pub ret_offset: U256, + pub ret_offset: usize, /// Return data size pub ret_size: usize, /// If true then transfer value from caller to callee @@ -221,6 +241,9 @@ impl CallFrameBackup { impl CallFrame { #[allow(clippy::too_many_arguments)] + // Force inline, due to lot of arguments, inlining must be forced, and it is actually beneficial + // because passing so much data is costly. Verified with samply. + #[inline(always)] pub fn new( msg_sender: Address, to: Address, @@ -233,12 +256,14 @@ impl CallFrame { depth: usize, should_transfer_value: bool, is_create: bool, - ret_offset: U256, + ret_offset: usize, ret_size: usize, stack: Stack, + memory: Memory, ) -> Self { let invalid_jump_destinations = get_invalid_jump_destinations(&bytecode).unwrap_or_default(); + // Note: Do not use ..Default::default() because it has runtime cost. Self { gas_limit, gas_remaining: gas_limit, @@ -256,7 +281,11 @@ impl CallFrame { ret_offset, ret_size, stack, - ..Default::default() + memory, + call_frame_backup: CallFrameBackup::default(), + output: Bytes::default(), + pc: 0, + sub_return_data: Bytes::default(), } } diff --git a/crates/vm/levm/src/execution_handlers.rs b/crates/vm/levm/src/execution_handlers.rs index d23ec72303..9d9fb934c2 100644 --- a/crates/vm/levm/src/execution_handlers.rs +++ b/crates/vm/levm/src/execution_handlers.rs @@ -40,8 +40,22 @@ impl<'a> VM<'a> { } pub fn execute_opcode(&mut self, opcode: Opcode) -> Result { + // The match order matters because it's evaluated top down. So the most frequent opcodes should be near the top. + // A future improvement could be to use a function pointer table. match opcode { - Opcode::STOP => Ok(OpcodeResult::Halt), + Opcode::PUSH32 => self.op_push::<32>(), + Opcode::MLOAD => self.op_mload(), + Opcode::MSTORE => self.op_mstore(), + Opcode::MSTORE8 => self.op_mstore8(), + Opcode::JUMP => self.op_jump(), + Opcode::SLOAD => self.op_sload(), + Opcode::SSTORE => self.op_sstore(), + Opcode::MSIZE => self.op_msize(), + Opcode::GAS => self.op_gas(), + Opcode::MCOPY => self.op_mcopy(), + Opcode::PUSH0 => self.op_push0(), + Opcode::PUSH1 => self.op_push::<1>(), + Opcode::POP => self.op_pop(), Opcode::ADD => self.op_add(), Opcode::MUL => self.op_mul(), Opcode::SUB => self.op_sub(), @@ -52,6 +66,22 @@ impl<'a> VM<'a> { Opcode::ADDMOD => self.op_addmod(), Opcode::MULMOD => self.op_mulmod(), Opcode::EXP => self.op_exp(), + Opcode::CALL => self.op_call(), + Opcode::CALLCODE => self.op_callcode(), + Opcode::RETURN => self.op_return(), + Opcode::DELEGATECALL => self.op_delegatecall(), + Opcode::STATICCALL => self.op_staticcall(), + Opcode::CREATE => self.op_create(), + Opcode::CREATE2 => self.op_create2(), + Opcode::JUMPI => self.op_jumpi(), + Opcode::JUMPDEST => self.op_jumpdest(), + Opcode::ADDRESS => self.op_address(), + Opcode::ORIGIN => self.op_origin(), + Opcode::BALANCE => self.op_balance(), + Opcode::CALLER => self.op_caller(), + Opcode::CALLVALUE => self.op_callvalue(), + Opcode::CODECOPY => self.op_codecopy(), + Opcode::STOP => Ok(OpcodeResult::Halt), Opcode::SIGNEXTEND => self.op_signextend(), Opcode::LT => self.op_lt(), Opcode::GT => self.op_gt(), @@ -65,9 +95,6 @@ impl<'a> VM<'a> { Opcode::CALLDATACOPY => self.op_calldatacopy(), Opcode::RETURNDATASIZE => self.op_returndatasize(), Opcode::RETURNDATACOPY => self.op_returndatacopy(), - Opcode::JUMP => self.op_jump(), - Opcode::JUMPI => self.op_jumpi(), - Opcode::JUMPDEST => self.op_jumpdest(), Opcode::PC => self.op_pc(), Opcode::BLOCKHASH => self.op_blockhash(), Opcode::COINBASE => self.op_coinbase(), @@ -79,8 +106,6 @@ impl<'a> VM<'a> { Opcode::BASEFEE => self.op_basefee(), Opcode::BLOBHASH => self.op_blobhash(), Opcode::BLOBBASEFEE => self.op_blobbasefee(), - Opcode::PUSH0 => self.op_push0(), - Opcode::PUSH1 => self.op_push::<1>(), Opcode::PUSH2 => self.op_push::<2>(), Opcode::PUSH3 => self.op_push::<3>(), Opcode::PUSH4 => self.op_push::<4>(), @@ -111,7 +136,6 @@ impl<'a> VM<'a> { Opcode::PUSH29 => self.op_push::<29>(), Opcode::PUSH30 => self.op_push::<30>(), Opcode::PUSH31 => self.op_push::<31>(), - Opcode::PUSH32 => self.op_push::<32>(), Opcode::AND => self.op_and(), Opcode::OR => self.op_or(), Opcode::XOR => self.op_xor(), @@ -132,36 +156,16 @@ impl<'a> VM<'a> { #[expect(clippy::arithmetic_side_effects, clippy::as_conversions)] self.op_swap(op as usize - Opcode::SWAP1 as usize + 1) } - Opcode::POP => self.op_pop(), + Opcode::LOG0 => self.op_log::<0>(), Opcode::LOG1 => self.op_log::<1>(), Opcode::LOG2 => self.op_log::<2>(), Opcode::LOG3 => self.op_log::<3>(), Opcode::LOG4 => self.op_log::<4>(), - Opcode::MLOAD => self.op_mload(), - Opcode::MSTORE => self.op_mstore(), - Opcode::MSTORE8 => self.op_mstore8(), - Opcode::SLOAD => self.op_sload(), - Opcode::SSTORE => self.op_sstore(), - Opcode::MSIZE => self.op_msize(), - Opcode::GAS => self.op_gas(), - Opcode::MCOPY => self.op_mcopy(), - Opcode::CALL => self.op_call(), - Opcode::CALLCODE => self.op_callcode(), - Opcode::RETURN => self.op_return(), - Opcode::DELEGATECALL => self.op_delegatecall(), - Opcode::STATICCALL => self.op_staticcall(), - Opcode::CREATE => self.op_create(), - Opcode::CREATE2 => self.op_create2(), Opcode::TLOAD => self.op_tload(), Opcode::TSTORE => self.op_tstore(), Opcode::SELFBALANCE => self.op_selfbalance(), - Opcode::ADDRESS => self.op_address(), - Opcode::ORIGIN => self.op_origin(), - Opcode::BALANCE => self.op_balance(), - Opcode::CALLER => self.op_caller(), - Opcode::CALLVALUE => self.op_callvalue(), - Opcode::CODECOPY => self.op_codecopy(), + Opcode::CODESIZE => self.op_codesize(), Opcode::GASPRICE => self.op_gasprice(), Opcode::EXTCODESIZE => self.op_extcodesize(), @@ -175,6 +179,7 @@ impl<'a> VM<'a> { } } + #[cold] // used in the hot path loop, called only really once. pub fn handle_opcode_result(&mut self) -> Result { // On successful create check output validity if self.is_create()? { @@ -218,6 +223,7 @@ impl<'a> VM<'a> { }) } + #[cold] // used in the hot path loop, called only really once. pub fn handle_opcode_error(&mut self, error: VMError) -> Result { if error.should_propagate() { return Err(error); diff --git a/crates/vm/levm/src/memory.rs b/crates/vm/levm/src/memory.rs index ae88580eb7..7a4970a1c6 100644 --- a/crates/vm/levm/src/memory.rs +++ b/crates/vm/levm/src/memory.rs @@ -1,198 +1,289 @@ +use std::{cell::RefCell, rc::Rc}; + use crate::{ constants::{MEMORY_EXPANSION_QUOTIENT, WORD_SIZE_IN_BYTES_USIZE}, errors::{ExceptionalHalt, InternalError, VMError}, }; use ExceptionalHalt::OutOfBounds; use ExceptionalHalt::OutOfGas; -use ethrex_common::{U256, utils::u256_from_big_endian}; +use ethrex_common::{U256, utils::u256_from_big_endian_const}; -/// Memory of the EVM, a volatile byte array. -pub type Memory = Vec; +/// A cheaply clonable callframe-shared memory buffer. +/// +/// When a new callframe is created a RC clone of this memory is made, with the current base offset at the length of the buffer at that time. +#[derive(Debug, Clone)] +pub struct Memory { + buffer: Rc>>, + len: usize, + current_base: usize, +} -pub fn try_resize(memory: &mut Memory, unchecked_new_size: usize) -> Result<(), VMError> { - if unchecked_new_size == 0 || unchecked_new_size <= memory.len() { - return Ok(()); +impl Memory { + #[inline] + pub fn new() -> Self { + Self { + buffer: Rc::new(RefCell::new(Vec::new())), + len: 0, + current_base: 0, + } } - let new_size = unchecked_new_size - .checked_next_multiple_of(WORD_SIZE_IN_BYTES_USIZE) - .ok_or(OutOfBounds)?; - - if new_size > memory.len() { - let additional_size = new_size - .checked_sub(memory.len()) - .ok_or(InternalError::Underflow)?; - memory - .try_reserve(additional_size) - .map_err(|_err| InternalError::MemorySizeOverflow)?; - memory.resize(new_size, 0); + /// Gets the Memory for the next children callframe. + #[inline] + pub fn next_memory(&self) -> Memory { + let mut mem = self.clone(); + mem.current_base = mem.buffer.borrow().len(); + mem.len = 0; + mem } - Ok(()) -} + /// Cleans the memory from base onwards, this must be used in callframes when handling returns. + /// + /// On the callframe that is about to be dropped. + #[inline] + pub fn clean_from_base(&self) { + #[expect(unsafe_code)] + unsafe { + self.buffer + .borrow_mut() + .get_unchecked_mut(self.current_base..(self.current_base.wrapping_add(self.len))) + .fill(0); + } + } -pub fn load_word(memory: &mut Memory, offset: U256) -> Result { - load_range(memory, offset, WORD_SIZE_IN_BYTES_USIZE).map(u256_from_big_endian) -} + /// Returns the len of the current memory, from the current base. + #[inline] + pub fn len(&self) -> usize { + self.len + } -pub fn load_range(memory: &mut Memory, offset: U256, size: usize) -> Result<&[u8], VMError> { - if size == 0 { - return Ok(&[]); + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 } - let offset: usize = offset - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + /// Resizes the from the current base to fit the memory specified at new_memory_size. + /// + /// Note: new_memory_size is increased to the next 32 byte multiple. + #[inline(always)] + pub fn resize(&mut self, new_memory_size: usize) -> Result<(), VMError> { + if new_memory_size == 0 { + return Ok(()); + } - try_resize(memory, offset.checked_add(size).ok_or(OutOfBounds)?)?; + let new_memory_size = new_memory_size + .checked_next_multiple_of(WORD_SIZE_IN_BYTES_USIZE) + .ok_or(OutOfBounds)?; - memory - .get(offset..offset.checked_add(size).ok_or(OutOfBounds)?) - .ok_or(OutOfBounds.into()) -} + let current_len = self.len(); -pub fn try_store_word(memory: &mut Memory, offset: U256, word: U256) -> Result<(), VMError> { - let new_size: usize = offset - .checked_add(WORD_SIZE_IN_BYTES_USIZE.into()) - .ok_or(OutOfBounds)? - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; - - try_resize(memory, new_size)?; - try_store( - memory, - &word.to_big_endian(), - offset, - WORD_SIZE_IN_BYTES_USIZE, - ) -} + if new_memory_size <= current_len { + return Ok(()); + } -pub fn try_store_data(memory: &mut Memory, offset: U256, data: &[u8]) -> Result<(), VMError> { - let new_size = offset - .checked_add(data.len().into()) - .ok_or(OutOfBounds)? - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; - try_resize(memory, new_size)?; - try_store(memory, data, offset, data.len()) -} + self.len = new_memory_size; -pub fn try_store_range( - memory: &mut Memory, - offset: U256, - size: usize, - data: &[u8], -) -> Result<(), VMError> { - if size == 0 { - return Ok(()); + let mut buffer = self.buffer.borrow_mut(); + + #[allow(clippy::arithmetic_side_effects)] + let real_new_memory_size = new_memory_size + self.current_base; + + if real_new_memory_size > buffer.len() { + // when resizing, resize by allocating entire pages instead of small memory sizes. + let new_size = real_new_memory_size.next_multiple_of(4096); + buffer.resize(new_size, 0); + } + + Ok(()) } - let new_size = offset - .checked_add(size.into()) - .ok_or(OutOfBounds)? - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; - try_resize(memory, new_size)?; - try_store(memory, data, offset, size) -} + /// Load `size` bytes from the given offset. + #[inline] + pub fn load_range(&mut self, offset: usize, size: usize) -> Result, VMError> { + if size == 0 { + return Ok(Vec::new()); + } + + let new_size = offset.checked_add(size).ok_or(OutOfBounds)?; + self.resize(new_size)?; + + let true_offset = offset.wrapping_add(self.current_base); -fn try_store( - memory: &mut Memory, - data: &[u8], - at_offset: U256, - data_size: usize, -) -> Result<(), VMError> { - if data_size == 0 { - return Ok(()); + let buf = self.buffer.borrow(); + + // SAFETY: resize already makes sure bounds are correct. + #[allow(unsafe_code)] + unsafe { + Ok(buf + .get_unchecked(true_offset..(true_offset.wrapping_add(size))) + .to_vec()) + } } - let at_offset: usize = at_offset - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; - - for (byte_to_store, memory_slot) in data.iter().zip( - memory - .get_mut( - at_offset - ..at_offset - .checked_add(data_size) - .ok_or(InternalError::Overflow)?, - ) - .ok_or(OutOfBounds)? - .iter_mut(), - ) { - *memory_slot = *byte_to_store; - } - Ok(()) -} + /// Load N bytes from the given offset. + #[inline(always)] + pub fn load_range_const(&mut self, offset: usize) -> Result<[u8; N], VMError> { + let new_size = offset.checked_add(N).ok_or(OutOfBounds)?; + self.resize(new_size)?; -pub fn try_copy_within( - memory: &mut Memory, - from_offset: U256, - to_offset: U256, - size: usize, -) -> Result<(), VMError> { - if size == 0 { - return Ok(()); + let true_offset = offset.checked_add(self.current_base).ok_or(OutOfBounds)?; + + let buf = self.buffer.borrow(); + // SAFETY: resize already makes sure bounds are correct. + #[allow(unsafe_code)] + unsafe { + Ok(*buf + .get_unchecked(true_offset..(true_offset.wrapping_add(N))) + .as_ptr() + .cast::<[u8; N]>()) + } } - let from_offset: usize = from_offset - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; - let to_offset: usize = to_offset - .try_into() - .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; - try_resize( - memory, - to_offset - .max(from_offset) - .checked_add(size) - .ok_or(InternalError::Overflow)?, - )?; + /// Load a word from at the given offset. + #[inline(always)] + pub fn load_word(&mut self, offset: usize) -> Result { + let value: [u8; 32] = self.load_range_const(offset)?; + Ok(u256_from_big_endian_const(value)) + } - let mut temporary_buffer = vec![0u8; size]; - for i in 0..size { - if let Some(temporary_buffer_byte) = temporary_buffer.get_mut(i) { - *temporary_buffer_byte = *memory - .get(from_offset.checked_add(i).ok_or(InternalError::Overflow)?) - .unwrap_or(&0u8); + /// Stores the given data and data size at the given offset. + /// + /// Internal use. + #[inline(always)] + fn store(&self, data: &[u8], at_offset: usize, data_size: usize) -> Result<(), VMError> { + if data_size == 0 { + return Ok(()); } + + let real_offset = self.current_base.wrapping_add(at_offset); + + let mut buffer = self.buffer.borrow_mut(); + + let real_data_size = data_size.min(data.len()); + + // SAFETY: Used internally, resize always called before this function. + #[allow(clippy::indexing_slicing, clippy::arithmetic_side_effects)] + #[allow(unsafe_code)] + unsafe { + std::ptr::copy_nonoverlapping( + data.get_unchecked(..real_data_size).as_ptr(), + buffer + .get_unchecked_mut(real_offset..(real_offset + real_data_size)) + .as_mut_ptr(), + real_data_size, + ); + } + + Ok(()) } - for i in 0..size { - if let Some(memory_byte) = memory.get_mut(to_offset.checked_add(i).ok_or(OutOfBounds)?) { - *memory_byte = *temporary_buffer.get(i).unwrap_or(&0u8); + /// Stores the given data at the given offset. + #[inline(always)] + pub fn store_data(&mut self, offset: usize, data: &[u8]) -> Result<(), VMError> { + let new_size = offset.checked_add(data.len()).ok_or(OutOfBounds)?; + self.resize(new_size)?; + self.store(data, offset, data.len()) + } + + /// Stores the given data and data size at the given offset. + /// + /// Resizes memory to fit the given data. + #[inline(always)] + pub fn store_range(&mut self, offset: usize, size: usize, data: &[u8]) -> Result<(), VMError> { + if size == 0 { + return Ok(()); } + + let new_size = offset.checked_add(size).ok_or(OutOfBounds)?; + self.resize(new_size)?; + self.store(data, offset, size) + } + + /// Stores a word at the given offset, resizing memory if needed. + #[inline(always)] + pub fn store_word(&mut self, offset: usize, word: U256) -> Result<(), VMError> { + let new_size: usize = offset + .checked_add(WORD_SIZE_IN_BYTES_USIZE) + .ok_or(OutOfBounds)?; + + self.resize(new_size)?; + self.store(&word.to_big_endian(), offset, WORD_SIZE_IN_BYTES_USIZE)?; + Ok(()) } - Ok(()) + /// Copies memory within 2 offsets. Like a memmove. + /// + /// Resizes if needed, because one can copy from "expanded memory", which is initialized with zeroes. + pub fn copy_within( + &mut self, + from_offset: usize, + to_offset: usize, + size: usize, + ) -> Result<(), VMError> { + if size == 0 { + return Ok(()); + } + + self.resize( + to_offset + .max(from_offset) + .checked_add(size) + .ok_or(InternalError::Overflow)?, + )?; + + let true_from_offset = from_offset + .checked_add(self.current_base) + .ok_or(OutOfBounds)?; + + let true_to_offset = to_offset + .checked_add(self.current_base) + .ok_or(OutOfBounds)?; + let mut buffer = self.buffer.borrow_mut(); + + buffer.copy_within( + true_from_offset + ..(true_from_offset + .checked_add(size) + .ok_or(InternalError::Overflow)?), + true_to_offset, + ); + + Ok(()) + } +} + +impl Default for Memory { + fn default() -> Self { + Self::new() + } } /// When a memory expansion is triggered, only the additional bytes of memory /// must be paid for. +#[inline] pub fn expansion_cost(new_memory_size: usize, current_memory_size: usize) -> Result { let cost = if new_memory_size <= current_memory_size { 0 } else { - cost(new_memory_size)? - .checked_sub(cost(current_memory_size)?) - .ok_or(InternalError::Underflow)? + // We already know new_memory_size > current_memory_size, + // and cost(x) > cost(y) where x > y, so cost should not underflow. + cost(new_memory_size)?.wrapping_sub(cost(current_memory_size)?) }; Ok(cost) } /// The total cost for a given memory size. +#[inline] fn cost(memory_size: usize) -> Result { let memory_size_word = memory_size - .checked_add( - WORD_SIZE_IN_BYTES_USIZE - .checked_sub(1) - .ok_or(InternalError::Underflow)?, - ) + .checked_add(WORD_SIZE_IN_BYTES_USIZE.wrapping_sub(1)) .ok_or(OutOfGas)? / WORD_SIZE_IN_BYTES_USIZE; - let gas_cost = (memory_size_word.checked_pow(2).ok_or(OutOfGas)? / MEMORY_EXPANSION_QUOTIENT) + let gas_cost = (memory_size_word + .checked_mul(memory_size_word) + .ok_or(OutOfGas)? + / MEMORY_EXPANSION_QUOTIENT) .checked_add(3usize.checked_mul(memory_size_word).ok_or(OutOfGas)?) .ok_or(OutOfGas)?; @@ -201,15 +292,85 @@ fn cost(memory_size: usize) -> Result { .map_err(|_| ExceptionalHalt::VeryLargeNumber.into()) } -pub fn calculate_memory_size(offset: U256, size: usize) -> Result { +#[inline] +pub fn calculate_memory_size(offset: usize, size: usize) -> Result { if size == 0 { return Ok(0); } - let offset: usize = offset.try_into().map_err(|_err| OutOfGas)?; - offset .checked_add(size) .and_then(|sum| sum.checked_next_multiple_of(WORD_SIZE_IN_BYTES_USIZE)) .ok_or(OutOfBounds.into()) } + +#[cfg(test)] +mod test { + #![allow(clippy::indexing_slicing, clippy::arithmetic_side_effects)] + use ethrex_common::U256; + + use crate::memory::Memory; + + #[test] + fn test_basic_store_data() { + let mut mem = Memory::new(); + + mem.store_data(0, &[1, 2, 3, 4, 0, 0, 0, 0, 0, 0]).unwrap(); + + assert_eq!(&mem.buffer.borrow()[0..10], &[1, 2, 3, 4, 0, 0, 0, 0, 0, 0]); + assert_eq!(mem.len(), 32); + } + + #[test] + fn test_words() { + let mut mem = Memory::new(); + + mem.store_word(0, U256::from(4)).unwrap(); + + assert_eq!(mem.load_word(0).unwrap(), U256::from(4)); + assert_eq!(mem.len(), 32); + } + + #[test] + fn test_copy_word_within() { + { + let mut mem = Memory::new(); + + mem.store_word(0, U256::from(4)).unwrap(); + mem.copy_within(0, 32, 32).unwrap(); + + assert_eq!(mem.load_word(32).unwrap(), U256::from(4)); + assert_eq!(mem.len(), 64); + } + + { + let mut mem = Memory::new(); + + mem.store_word(32, U256::from(4)).unwrap(); + mem.copy_within(32, 0, 32).unwrap(); + + assert_eq!(mem.load_word(0).unwrap(), U256::from(4)); + assert_eq!(mem.len(), 64); + } + + { + let mut mem = Memory::new(); + + mem.store_word(0, U256::from(4)).unwrap(); + mem.copy_within(0, 0, 32).unwrap(); + + assert_eq!(mem.load_word(0).unwrap(), U256::from(4)); + assert_eq!(mem.len(), 32); + } + + { + let mut mem = Memory::new(); + + mem.store_word(0, U256::from(4)).unwrap(); + mem.copy_within(32, 0, 32).unwrap(); + + assert_eq!(mem.load_word(0).unwrap(), U256::zero()); + assert_eq!(mem.len(), 64); + } + } +} diff --git a/crates/vm/levm/src/opcode_handlers/environment.rs b/crates/vm/levm/src/opcode_handlers/environment.rs index 52747a15c6..4afdde2881 100644 --- a/crates/vm/levm/src/opcode_handlers/environment.rs +++ b/crates/vm/levm/src/opcode_handlers/environment.rs @@ -1,7 +1,7 @@ use crate::{ errors::{ExceptionalHalt, InternalError, OpcodeResult, VMError}, gas_cost::{self}, - memory::{self, calculate_memory_size}, + memory::calculate_memory_size, utils::word_to_address, vm::VM, }; @@ -138,6 +138,17 @@ impl<'a> VM<'a> { .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let dest_offset: usize = match dest_offset.try_into() { + Ok(x) => x, + Err(_) if size == 0 => 0, + Err(_) => return Err(ExceptionalHalt::VeryLargeNumber.into()), + }; + + let calldata_offset: usize = match calldata_offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; + let new_memory_size = calculate_memory_size(dest_offset, size)?; current_call_frame.increase_consumed_gas(gas_cost::calldatacopy( @@ -151,27 +162,22 @@ impl<'a> VM<'a> { } let mut data = vec![0u8; size]; - if calldata_offset > current_call_frame.calldata.len().into() { - memory::try_store_data(&mut current_call_frame.memory, dest_offset, &data)?; - return Ok(OpcodeResult::Continue { pc_increment: 1 }); - } - - let calldata_offset: usize = calldata_offset - .try_into() - .map_err(|_err| InternalError::TypeConversion)?; - for (i, byte) in current_call_frame - .calldata - .iter() - .skip(calldata_offset) - .take(size) - .enumerate() - { - if let Some(data_byte) = data.get_mut(i) { - *data_byte = *byte; + if calldata_offset <= current_call_frame.calldata.len() { + for (i, byte) in current_call_frame + .calldata + .iter() + .skip(calldata_offset) + .take(size) + .enumerate() + { + if let Some(data_byte) = data.get_mut(i) { + *data_byte = *byte; + } } } - memory::try_store_data(&mut current_call_frame.memory, dest_offset, &data)?; + + current_call_frame.memory.store_data(dest_offset, &data)?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } @@ -198,6 +204,16 @@ impl<'a> VM<'a> { .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; + let destination_offset: usize = match destination_offset.try_into() { + Ok(x) => x, + Err(_) if size == 0 => 0, + Err(_) => return Err(ExceptionalHalt::VeryLargeNumber.into()), + }; + let code_offset: usize = match code_offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; + let new_memory_size = calculate_memory_size(destination_offset, size)?; current_call_frame.increase_consumed_gas(gas_cost::codecopy( @@ -211,11 +227,7 @@ impl<'a> VM<'a> { } let mut data = vec![0u8; size]; - if code_offset < current_call_frame.bytecode.len().into() { - let code_offset: usize = code_offset - .try_into() - .map_err(|_| InternalError::TypeConversion)?; - + if code_offset < current_call_frame.bytecode.len() { for (i, byte) in current_call_frame .bytecode .iter() @@ -229,7 +241,9 @@ impl<'a> VM<'a> { } } - memory::try_store_data(&mut current_call_frame.memory, destination_offset, &data)?; + current_call_frame + .memory + .store_data(destination_offset, &data)?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } @@ -272,6 +286,12 @@ impl<'a> VM<'a> { let address_was_cold = self.substate.accessed_addresses.insert(address); + let dest_offset: usize = match dest_offset.try_into() { + Ok(x) => x, + Err(_) if size == 0 => 0, + Err(_) => return Err(ExceptionalHalt::VeryLargeNumber.into()), + }; + let new_memory_size = calculate_memory_size(dest_offset, size)?; self.current_call_frame_mut()? @@ -302,11 +322,9 @@ impl<'a> VM<'a> { } } - memory::try_store_data( - &mut self.current_call_frame_mut()?.memory, - dest_offset, - &data, - )?; + self.current_call_frame_mut()? + .memory + .store_data(dest_offset, &data)?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } @@ -334,6 +352,12 @@ impl<'a> VM<'a> { .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; + let dest_offset: usize = match dest_offset.try_into() { + Ok(x) => x, + Err(_) if size == 0 => 0, + Err(_) => return Err(ExceptionalHalt::VeryLargeNumber.into()), + }; + let new_memory_size = calculate_memory_size(dest_offset, size)?; current_call_frame.increase_consumed_gas(gas_cost::returndatacopy( @@ -371,7 +395,7 @@ impl<'a> VM<'a> { } } - memory::try_store_data(&mut current_call_frame.memory, dest_offset, &data)?; + current_call_frame.memory.store_data(dest_offset, &data)?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } diff --git a/crates/vm/levm/src/opcode_handlers/keccak.rs b/crates/vm/levm/src/opcode_handlers/keccak.rs index 5337111185..640614d7e2 100644 --- a/crates/vm/levm/src/opcode_handlers/keccak.rs +++ b/crates/vm/levm/src/opcode_handlers/keccak.rs @@ -1,7 +1,7 @@ use crate::{ errors::{ExceptionalHalt, OpcodeResult, VMError}, gas_cost, - memory::{self, calculate_memory_size}, + memory::calculate_memory_size, vm::VM, }; use ethrex_common::utils::u256_from_big_endian; @@ -17,6 +17,10 @@ impl<'a> VM<'a> { let size: usize = size .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; + let offset: usize = match offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; let new_memory_size = calculate_memory_size(offset, size)?; @@ -27,11 +31,7 @@ impl<'a> VM<'a> { )?)?; let mut hasher = Keccak256::new(); - hasher.update(memory::load_range( - &mut current_call_frame.memory, - offset, - size, - )?); + hasher.update(current_call_frame.memory.load_range(offset, size)?); current_call_frame .stack .push(&[u256_from_big_endian(&hasher.finalize())])?; diff --git a/crates/vm/levm/src/opcode_handlers/logging.rs b/crates/vm/levm/src/opcode_handlers/logging.rs index f556c120fa..6e62736f3e 100644 --- a/crates/vm/levm/src/opcode_handlers/logging.rs +++ b/crates/vm/levm/src/opcode_handlers/logging.rs @@ -1,7 +1,7 @@ use crate::{ errors::{ExceptionalHalt, OpcodeResult, VMError}, gas_cost, - memory::{self, calculate_memory_size}, + memory::calculate_memory_size, vm::VM, }; use bytes::Bytes; @@ -22,6 +22,10 @@ impl<'a> VM<'a> { let size = size .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; + let offset = match offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; let topics = current_call_frame .stack .pop::()? @@ -39,9 +43,7 @@ impl<'a> VM<'a> { let log = Log { address: current_call_frame.to, topics: topics.to_vec(), - data: Bytes::from( - memory::load_range(&mut current_call_frame.memory, offset, size)?.to_vec(), - ), + data: Bytes::from(current_call_frame.memory.load_range(offset, size)?), }; self.tracer.log(&log)?; diff --git a/crates/vm/levm/src/opcode_handlers/push.rs b/crates/vm/levm/src/opcode_handlers/push.rs index 3e444f7038..0eec648cf7 100644 --- a/crates/vm/levm/src/opcode_handlers/push.rs +++ b/crates/vm/levm/src/opcode_handlers/push.rs @@ -1,10 +1,8 @@ use crate::{ - call_frame::CallFrame, errors::{ExceptionalHalt, InternalError, OpcodeResult, VMError}, gas_cost, vm::VM, }; -use ExceptionalHalt::OutOfBounds; use ethrex_common::{U256, types::Fork, utils::u256_from_big_endian_const}; // Push Operations @@ -16,9 +14,35 @@ impl<'a> VM<'a> { let current_call_frame = self.current_call_frame_mut()?; current_call_frame.increase_consumed_gas(gas_cost::PUSHN)?; - let read_n_bytes = read_bytcode_slice::(current_call_frame)?; + let current_pc = current_call_frame.pc; + + // Check to avoid multiple checks. + if current_pc.checked_add(N.wrapping_add(1)).is_none() { + Err(InternalError::Overflow)?; + } + + let pc_offset = current_pc + // Add 1 to the PC because we don't want to include the + // Bytecode of the current instruction in the data we're about + // to read. We only want to read the data _NEXT_ to that + // bytecode + .wrapping_add(1); + + let value = if let Some(slice) = current_call_frame + .bytecode + .get(pc_offset..pc_offset.wrapping_add(N)) + { + u256_from_big_endian_const( + // SAFETY: If the get succeeded, we got N elements so the cast is safe. + #[expect(unsafe_code)] + unsafe { + *slice.as_ptr().cast::<[u8; N]>() + }, + ) + } else { + U256::zero() + }; - let value = u256_from_big_endian_const(read_n_bytes); current_call_frame.stack.push(&[value])?; // The n_bytes that you push to the stack + 1 for the next instruction @@ -44,26 +68,3 @@ impl<'a> VM<'a> { Ok(OpcodeResult::Continue { pc_increment: 1 }) } } - -// Like `read_bytcode_slice` but using a const generic and returning a fixed size array. -fn read_bytcode_slice(current_call_frame: &CallFrame) -> Result<[u8; N], VMError> { - let current_pc = current_call_frame.pc; - let pc_offset = current_pc - // Add 1 to the PC because we don't want to include the - // Bytecode of the current instruction in the data we're about - // to read. We only want to read the data _NEXT_ to that - // bytecode - .checked_add(1) - .ok_or(InternalError::Overflow)?; - - if let Some(slice) = current_call_frame - .bytecode - .get(pc_offset..pc_offset.checked_add(N).ok_or(OutOfBounds)?) - { - Ok(slice - .try_into() - .map_err(|_| VMError::Internal(InternalError::TypeConversion))?) - } else { - Ok([0; N]) - } -} diff --git a/crates/vm/levm/src/opcode_handlers/stack_memory_storage_flow.rs b/crates/vm/levm/src/opcode_handlers/stack_memory_storage_flow.rs index f304bb652e..36f3568f3b 100644 --- a/crates/vm/levm/src/opcode_handlers/stack_memory_storage_flow.rs +++ b/crates/vm/levm/src/opcode_handlers/stack_memory_storage_flow.rs @@ -3,7 +3,7 @@ use crate::{ constants::{WORD_SIZE, WORD_SIZE_IN_BYTES_USIZE}, errors::{ExceptionalHalt, InternalError, OpcodeResult, VMError}, gas_cost::{self, SSTORE_STIPEND}, - memory::{self, calculate_memory_size}, + memory::calculate_memory_size, opcodes::Opcode, vm::VM, }; @@ -12,6 +12,8 @@ use ethrex_common::{H256, U256, types::Fork}; // Stack, Memory, Storage and Flow Operations (15) // Opcodes: POP, MLOAD, MSTORE, MSTORE8, SLOAD, SSTORE, JUMP, JUMPI, PC, MSIZE, GAS, JUMPDEST, TLOAD, TSTORE, MCOPY +pub const OUT_OF_BOUNDS: U256 = U256([u64::MAX, 0, 0, 0]); + impl<'a> VM<'a> { // POP operation pub fn op_pop(&mut self) -> Result { @@ -73,6 +75,10 @@ impl<'a> VM<'a> { let current_call_frame = self.current_call_frame_mut()?; let [offset] = *current_call_frame.stack.pop()?; + let offset: usize = offset + .try_into() + .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let new_memory_size = calculate_memory_size(offset, WORD_SIZE_IN_BYTES_USIZE)?; current_call_frame.increase_consumed_gas(gas_cost::mload( @@ -82,7 +88,7 @@ impl<'a> VM<'a> { current_call_frame .stack - .push(&[memory::load_word(&mut current_call_frame.memory, offset)?])?; + .push(&[current_call_frame.memory.load_word(offset)?])?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } @@ -96,6 +102,10 @@ impl<'a> VM<'a> { return Ok(OpcodeResult::Continue { pc_increment: 1 }); } + let offset: usize = offset + .try_into() + .map_err(|_err| ExceptionalHalt::OutOfGas)?; + let current_call_frame = self.current_call_frame_mut()?; let new_memory_size = calculate_memory_size(offset, WORD_SIZE_IN_BYTES_USIZE)?; @@ -105,11 +115,7 @@ impl<'a> VM<'a> { current_call_frame.memory.len(), )?)?; - memory::try_store_data( - &mut current_call_frame.memory, - offset, - &value.to_big_endian(), - )?; + current_call_frame.memory.store_word(offset, value)?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } @@ -120,6 +126,10 @@ impl<'a> VM<'a> { let [offset] = *current_call_frame.stack.pop()?; + let offset: usize = offset + .try_into() + .map_err(|_err| ExceptionalHalt::OutOfGas)?; + let new_memory_size = calculate_memory_size(offset, 1)?; current_call_frame.increase_consumed_gas(gas_cost::mstore8( @@ -129,11 +139,9 @@ impl<'a> VM<'a> { let [value] = current_call_frame.stack.pop()?; - memory::try_store_data( - &mut current_call_frame.memory, - offset, - &value.to_big_endian()[WORD_SIZE - 1..WORD_SIZE], - )?; + current_call_frame + .memory + .store_data(offset, &value.to_big_endian()[WORD_SIZE - 1..WORD_SIZE])?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } @@ -270,10 +278,23 @@ impl<'a> VM<'a> { } let current_call_frame = self.current_call_frame_mut()?; let [dest_offset, src_offset, size] = *current_call_frame.stack.pop()?; + let size: usize = size .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; + let dest_offset: usize = match dest_offset.try_into() { + Ok(x) => x, + Err(_) if size == 0 => 0, + Err(_) => return Err(ExceptionalHalt::VeryLargeNumber.into()), + }; + + let src_offset: usize = match src_offset.try_into() { + Ok(x) => x, + Err(_) if size == 0 => 0, + Err(_) => return Err(ExceptionalHalt::VeryLargeNumber.into()), + }; + let new_memory_size_for_dest = calculate_memory_size(dest_offset, size)?; let new_memory_size_for_src = calculate_memory_size(src_offset, size)?; @@ -284,12 +305,9 @@ impl<'a> VM<'a> { size, )?)?; - memory::try_copy_within( - &mut current_call_frame.memory, - src_offset, - dest_offset, - size, - )?; + current_call_frame + .memory + .copy_within(src_offset, dest_offset, size)?; Ok(OpcodeResult::Continue { pc_increment: 1 }) } diff --git a/crates/vm/levm/src/opcode_handlers/system.rs b/crates/vm/levm/src/opcode_handlers/system.rs index d6001789e0..b24e645b76 100644 --- a/crates/vm/levm/src/opcode_handlers/system.rs +++ b/crates/vm/levm/src/opcode_handlers/system.rs @@ -3,7 +3,7 @@ use crate::{ constants::{FAIL, INIT_CODE_MAX_SIZE, SUCCESS}, errors::{ContextResult, ExceptionalHalt, InternalError, OpcodeResult, TxResult, VMError}, gas_cost::{self, max_message_call_gas}, - memory::{self, calculate_memory_size}, + memory::calculate_memory_size, utils::{address_to_word, word_to_address, *}, vm::VM, }; @@ -40,9 +40,15 @@ impl<'a> VM<'a> { return_data_size, ] = *current_call_frame.stack.pop()?; let callee: Address = word_to_address(callee); + let args_start_offset = args_start_offset + .try_into() + .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; let args_size = args_size .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; + let return_data_start_offset: usize = return_data_start_offset + .try_into() + .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; let return_data_size: usize = return_data_size .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; @@ -145,6 +151,13 @@ impl<'a> VM<'a> { let args_size = args_size .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let args_start_offset = match args_start_offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; + let return_data_start_offset = return_data_start_offset + .try_into() + .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; let return_data_size = return_data_size .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; @@ -227,16 +240,18 @@ impl<'a> VM<'a> { return Ok(OpcodeResult::Halt); } + let offset: usize = match offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; + let new_memory_size = calculate_memory_size(offset, size)?; let current_memory_size = current_call_frame.memory.len(); current_call_frame .increase_consumed_gas(gas_cost::exit_opcode(new_memory_size, current_memory_size)?)?; - current_call_frame.output = - memory::load_range(&mut current_call_frame.memory, offset, size)? - .to_vec() - .into(); + current_call_frame.output = current_call_frame.memory.load_range(offset, size)?.into(); Ok(OpcodeResult::Halt) } @@ -263,9 +278,15 @@ impl<'a> VM<'a> { return_data_size, ] = *current_call_frame.stack.pop()?; let address = word_to_address(address); + let args_start_offset = args_start_offset + .try_into() + .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; let args_size = args_size .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let return_data_start_offset = return_data_start_offset + .try_into() + .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; let return_data_size = return_data_size .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; @@ -358,9 +379,15 @@ impl<'a> VM<'a> { return_data_size, ] = *current_call_frame.stack.pop()?; let address = word_to_address(address); + let args_start_offset = args_start_offset + .try_into() + .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; let args_size = args_size .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let return_data_start_offset = return_data_start_offset + .try_into() + .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; let return_data_size = return_data_size .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; @@ -442,6 +469,10 @@ impl<'a> VM<'a> { let code_size_in_memory: usize = code_size_in_memory .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let code_offset_in_memory: usize = match code_offset_in_memory.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; let new_size = calculate_memory_size(code_offset_in_memory, code_size_in_memory)?; @@ -473,6 +504,10 @@ impl<'a> VM<'a> { let code_size_in_memory: usize = code_size_in_memory .try_into() .map_err(|_err| ExceptionalHalt::VeryLargeNumber)?; + let code_offset_in_memory: usize = match code_offset_in_memory.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; let new_size = calculate_memory_size(code_offset_in_memory, code_size_in_memory)?; @@ -500,6 +535,12 @@ impl<'a> VM<'a> { let current_call_frame = self.current_call_frame_mut()?; let [offset, size] = *current_call_frame.stack.pop()?; + + let offset = match offset.try_into() { + Ok(x) => x, + Err(_) => usize::MAX, + }; + let size = size .try_into() .map_err(|_| ExceptionalHalt::VeryLargeNumber)?; @@ -510,10 +551,7 @@ impl<'a> VM<'a> { current_call_frame .increase_consumed_gas(gas_cost::exit_opcode(new_memory_size, current_memory_size)?)?; - current_call_frame.output = - memory::load_range(&mut current_call_frame.memory, offset, size)? - .to_vec() - .into(); + current_call_frame.output = current_call_frame.memory.load_range(offset, size)?.into(); Err(VMError::RevertOpcode) } @@ -589,7 +627,7 @@ impl<'a> VM<'a> { pub fn generic_create( &mut self, value: U256, - code_offset_in_memory: U256, + code_offset_in_memory: usize, code_size_in_memory: usize, salt: Option, ) -> Result { @@ -614,12 +652,9 @@ impl<'a> VM<'a> { // Load code from memory let code = Bytes::from( - memory::load_range( - &mut self.current_call_frame_mut()?.memory, - code_offset_in_memory, - code_size_in_memory, - )? - .to_vec(), + self.current_call_frame_mut()? + .memory + .load_range(code_offset_in_memory, code_size_in_memory)?, ); // Get account info of deployer @@ -683,6 +718,8 @@ impl<'a> VM<'a> { let mut stack = self.stack_pool.pop().unwrap_or_default(); stack.clear(); + let next_memory = self.current_call_frame()?.memory.next_memory(); + let new_call_frame = CallFrame::new( deployer, new_address, @@ -695,9 +732,10 @@ impl<'a> VM<'a> { new_depth, true, true, - U256::zero(), + 0, 0, stack, + next_memory, ); self.call_frames.push(new_call_frame); @@ -716,6 +754,10 @@ impl<'a> VM<'a> { /// This (should) be the only function where gas is used as a /// U256. This is because we have to use the values that are /// pushed to the stack. + /// + // Force inline, due to lot of arguments, inlining must be forced, and it is actually beneficial + // because passing so much data is costly. Verified with samply. + #[inline(always)] pub fn generic_call( &mut self, gas_limit: u64, @@ -726,7 +768,7 @@ impl<'a> VM<'a> { should_transfer_value: bool, is_static: bool, calldata: Bytes, - ret_offset: U256, + ret_offset: usize, ret_size: usize, bytecode: Bytes, is_delegation_7702: bool, @@ -755,6 +797,8 @@ impl<'a> VM<'a> { let mut stack = self.stack_pool.pop().unwrap_or_default(); stack.clear(); + let next_memory = self.current_call_frame_mut()?.memory.next_memory(); + let new_call_frame = CallFrame::new( msg_sender, to, @@ -770,6 +814,7 @@ impl<'a> VM<'a> { ret_offset, ret_size, stack, + next_memory, ); self.call_frames.push(new_call_frame); @@ -828,9 +873,12 @@ impl<'a> VM<'a> { gas_limit, ret_offset, ret_size, + memory: old_callframe_memory, .. } = executed_call_frame; + old_callframe_memory.clean_from_base(); + let parent_call_frame = self.current_call_frame_mut()?; // Return gas left from subcontext @@ -843,12 +891,25 @@ impl<'a> VM<'a> { .ok_or(InternalError::Overflow)?; // Store return data of sub-context - memory::try_store_range( - &mut parent_call_frame.memory, + if ret_size > 0 && ctx_result.output.len() < ret_size { + parent_call_frame.memory.resize( + ret_offset + .checked_add(ret_size) + .ok_or(ExceptionalHalt::OutOfBounds)?, + )?; + } + parent_call_frame.memory.store_data( ret_offset, - ret_size, - &ctx_result.output, + if ctx_result.output.len() >= ret_size { + ctx_result + .output + .get(..ret_size) + .ok_or(ExceptionalHalt::OutOfBounds)? + } else { + &ctx_result.output + }, )?; + parent_call_frame.sub_return_data = ctx_result.output.clone(); // What to do, depending on TxResult @@ -880,8 +941,12 @@ impl<'a> VM<'a> { gas_limit, to, call_frame_backup, + memory: old_callframe_memory, .. } = executed_call_frame; + + old_callframe_memory.clean_from_base(); + let parent_call_frame = self.current_call_frame_mut()?; // Return unused gas @@ -921,9 +986,9 @@ impl<'a> VM<'a> { /// Obtains the values needed for CALL, CALLCODE, DELEGATECALL and STATICCALL opcodes to calculate total gas cost fn get_call_gas_params( &mut self, - args_start_offset: U256, + args_start_offset: usize, args_size: usize, - return_data_start_offset: U256, + return_data_start_offset: usize, return_data_size: usize, eip7702_gas_consumed: u64, address: Address, @@ -952,9 +1017,11 @@ impl<'a> VM<'a> { )) } - fn get_calldata(&mut self, offset: U256, size: usize) -> Result { + fn get_calldata(&mut self, offset: usize, size: usize) -> Result { Ok(Bytes::from( - memory::load_range(&mut self.current_call_frame_mut()?.memory, offset, size)?.to_vec(), + self.current_call_frame_mut()? + .memory + .load_range(offset, size)?, )) } diff --git a/crates/vm/levm/src/vm.rs b/crates/vm/levm/src/vm.rs index 80aa077e3c..d55c72df9f 100644 --- a/crates/vm/levm/src/vm.rs +++ b/crates/vm/levm/src/vm.rs @@ -9,7 +9,9 @@ use crate::{ backup_hook::BackupHook, hook::{Hook, get_hooks}, }, - l2_precompiles, precompiles, + l2_precompiles, + memory::Memory, + precompiles, tracing::LevmCallTracer, }; use bytes::Bytes; @@ -112,9 +114,10 @@ impl<'a> VM<'a> { 0, true, is_create, - U256::zero(), + 0, 0, Stack::default(), + Memory::default(), ); self.call_frames.push(initial_call_frame); @@ -188,6 +191,7 @@ impl<'a> VM<'a> { self.increment_pc_by(pc_increment)?; continue; } + Ok(OpcodeResult::Halt) => self.handle_opcode_result()?, Err(error) => self.handle_opcode_error(error)?, };