diff --git a/Cargo.lock b/Cargo.lock index 40f2b8295..142e2333d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2990,6 +2990,7 @@ dependencies = [ "thiserror 1.0.69", "tracing", "tracing-subscriber", + "twox-hash", ] [[package]] @@ -5507,7 +5508,7 @@ dependencies = [ [[package]] name = "solana-account" version = "2.2.1" -source = "git+https://github.com/magicblock-labs/solana-account.git?rev=6eae52b#6eae52bde25e90b3c79d4935ce2b267e35338945" +source = "git+https://github.com/magicblock-labs/solana-account.git?rev=3ff1b2ea#3ff1b2eaf16fd8b82dd8e625673df9798b07db22" dependencies = [ "bincode", "qualifier_attr", @@ -8936,6 +8937,12 @@ dependencies = [ "webpki-roots 0.24.0", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index 6c72a1d8d..b8f6b13b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,7 +140,7 @@ serde_json = "1.0" serde_with = "3.16" serial_test = "3.2" sha3 = "0.10.8" -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" } solana-account-decoder = { version = "2.2" } solana-account-decoder-client-types = { version = "2.2" } solana-account-info = { version = "2.2" } @@ -211,6 +211,10 @@ tonic-build = "0.9.2" tracing = "0.1" tracing-log = { version = "0.2", features = ["log-tracer"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +twox-hash = { version = "2.1", default-features = false, features = [ + "xxhash3_64", + "alloc", +] } url = "2.5.0" # SPL Token crates used across the workspace @@ -231,7 +235,7 @@ version = "0.22.0" # some solana dependencies have solana-storage-proto as dependency # we need to patch them with our version, because they use protobuf-src v1.1.0 # and we use protobuf-src v2.1.1. Otherwise compilation fails -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" } solana-storage-proto = { path = "./storage-proto" } solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "bdbaac86" } # Fork is used to enable `disable_manual_compaction` usage diff --git a/magicblock-accounts-db/Cargo.toml b/magicblock-accounts-db/Cargo.toml index 981ef200f..2a854cd11 100644 --- a/magicblock-accounts-db/Cargo.toml +++ b/magicblock-accounts-db/Cargo.toml @@ -23,6 +23,9 @@ thiserror = { workspace = true } tracing = { workspace = true } magicblock-config = { workspace = true } +# misc +twox-hash = { workspace = true } + [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/magicblock-accounts-db/src/lib.rs b/magicblock-accounts-db/src/lib.rs index b1437a24a..f2826bc80 100644 --- a/magicblock-accounts-db/src/lib.rs +++ b/magicblock-accounts-db/src/lib.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path, sync::Arc, thread}; +use std::{fs, hash::Hasher, path::Path, sync::Arc, thread}; use error::{AccountsDbError, LogErr}; use index::{ @@ -13,6 +13,7 @@ use solana_account::{ use solana_pubkey::Pubkey; use storage::AccountsStorage; use tracing::{error, info, warn}; +use twox_hash::xxhash3_64; use crate::{snapshot::SnapshotManager, traits::AccountsBank}; @@ -374,6 +375,26 @@ impl AccountsDb { pub fn write_lock(&self) -> GlobalSyncLock { self.write_lock.clone() } + + /// Computes a deterministic checksum of all active accounts. + /// + /// Iterates all accounts in key-sorted order (via LMDB) and hashes both + /// pubkey and serialized account data using xxHash3. Returns a 64-bit hash + /// suitable for verifying state consistency across nodes. + /// + /// Acquires the write lock to ensure a consistent snapshot of the state. + pub fn checksum(&self) -> u64 { + let _locked = self.write_lock.write(); + let mut hasher = xxhash3_64::Hasher::new(); + for (pubkey, acc) in self.iter_all() { + let Some(borrowed) = acc.as_borrowed() else { + continue; + }; + hasher.write(pubkey.as_ref()); + hasher.write(borrowed.buffer()); + } + hasher.finish() + } } impl AccountsBank for AccountsDb { diff --git a/magicblock-accounts-db/src/storage.rs b/magicblock-accounts-db/src/storage.rs index 69dce7db3..9a541a9e8 100644 --- a/magicblock-accounts-db/src/storage.rs +++ b/magicblock-accounts-db/src/storage.rs @@ -371,10 +371,9 @@ impl AccountsStorage { Ok(()) } - /// Returns the total expected size of the file in bytes (Header + Data). + /// Returns the total occupied size of the storage file in bytes. pub(crate) fn size_bytes(&self) -> u64 { - (self.header().capacity_blocks as u64 * self.block_size as u64) - + METADATA_STORAGE_SIZE as u64 + self.active_segment().len() as u64 } pub(crate) fn block_size(&self) -> usize { diff --git a/magicblock-accounts-db/src/tests.rs b/magicblock-accounts-db/src/tests.rs index aba17258f..49d5e0303 100644 --- a/magicblock-accounts-db/src/tests.rs +++ b/magicblock-accounts-db/src/tests.rs @@ -464,6 +464,69 @@ fn test_database_reset() { assert_eq!(adb_reset.account_count(), 0); } +#[test] +fn test_checksum_deterministic_across_dbs() { + // Two independent DBs with identical accounts must produce identical checksums + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let config = AccountsDbConfig::default(); + + let db1 = AccountsDb::new(&config, dir1.path(), 0).unwrap(); + let db2 = AccountsDb::new(&config, dir2.path(), 0).unwrap(); + + // Insert same accounts into both DBs + for i in 0..50 { + let pubkey = Pubkey::new_unique(); + let mut account = AccountSharedData::new(LAMPORTS, SPACE, &OWNER); + account.data_as_mut_slice()[..8] + .copy_from_slice(&(i as u64).to_le_bytes()); + db1.insert_account(&pubkey, &account).unwrap(); + db2.insert_account(&pubkey, &account).unwrap(); + } + + assert_eq!( + db1.checksum(), + db2.checksum(), + "checksums must match for identical state" + ); +} + +#[test] +fn test_checksum_detects_state_change() { + let env = TestEnv::new(); + + // Create initial state + let mut accounts: Vec<_> = (0..20) + .map(|_| { + let acc = env.create_and_insert_account(); + (acc.pubkey, acc.account) + }) + .collect(); + + let original_checksum = env.checksum(); + + // Modify a single account's data + accounts[5].1.data_as_mut_slice()[0] ^= 0xFF; + env.insert_account(&accounts[5].0, &accounts[5].1).unwrap(); + + assert_ne!( + env.checksum(), + original_checksum, + "checksum must detect single account modification" + ); + + // Modify lamports on a different account + accounts[10].1.set_lamports(1_000_000); + env.insert_account(&accounts[10].0, &accounts[10].1) + .unwrap(); + + assert_ne!( + env.checksum(), + original_checksum, + "checksum must detect lamport change" + ); +} + // ============================================================== // TEST UTILITIES // ============================================================== diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 4489d9149..62e3a67b5 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -3339,6 +3339,7 @@ dependencies = [ "solana-pubkey", "thiserror 1.0.69", "tracing", + "twox-hash", ] [[package]] @@ -6250,7 +6251,7 @@ dependencies = [ [[package]] name = "solana-account" version = "2.2.1" -source = "git+https://github.com/magicblock-labs/solana-account.git?rev=6eae52b#6eae52bde25e90b3c79d4935ce2b267e35338945" +source = "git+https://github.com/magicblock-labs/solana-account.git?rev=3ff1b2ea#3ff1b2eaf16fd8b82dd8e625673df9798b07db22" dependencies = [ "bincode", "qualifier_attr", @@ -10757,6 +10758,12 @@ dependencies = [ "webpki-roots 0.24.0", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.19.0" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index 3aeaaf5ed..426f49e9a 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -78,7 +78,7 @@ rkyv = "0.7.45" schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" } solana-loader-v2-interface = "2.2" solana-loader-v3-interface = "4.0" solana-loader-v4-interface = "2.1" @@ -114,4 +114,4 @@ url = "2.5.0" solana-storage-proto = { path = "../storage-proto" } # same reason as above rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" }