diff --git a/Cargo.lock b/Cargo.lock index afa9b3121..41a1102ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2222,6 +2222,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "bonsai-trie" +version = "0.1.0" +source = "git+https://github.com/dojoengine/bonsai-trie/?branch=kariy%2Findexmap#8b509fdf2c22fb7386acc6c00f179ac1cd099243" +dependencies = [ + "bitvec", + "derive_more 0.99.19", + "hashbrown 0.14.5", + "indexmap 2.7.1", + "log", + "parity-scale-codec", + "rayon", + "serde", + "slotmap", + "smallvec", + "starknet-types-core", + "thiserror 2.0.11", +] + [[package]] name = "bonsai-trie" version = "0.1.0" @@ -2470,6 +2489,33 @@ dependencies = [ "url", ] +[[package]] +name = "cainome" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e04a357fdab01f56b676c8c41e11b154bb69eef184204cae47a4209eb4e0035" +dependencies = [ + "anyhow", + "async-trait", + "cainome-cairo-serde 0.2.1", + "cainome-cairo-serde-derive", + "cainome-parser 0.2.0", + "cainome-rs 0.2.0", + "cainome-rs-macro 0.2.0", + "camino", + "clap", + "clap_complete", + "convert_case 0.6.0", + "serde", + "serde_json", + "starknet 0.14.0", + "starknet-types-core", + "thiserror 1.0.69", + "tracing", + "tracing-subscriber", + "url", +] + [[package]] name = "cainome" version = "0.8.0" @@ -2549,6 +2595,20 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "cainome-parser" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "499219c70d1382b5f7defc6985b0266fb10b27b91dc4e8b0b0e4a10258e5d292" +dependencies = [ + "convert_case 0.6.0", + "quote", + "serde_json", + "starknet 0.14.0", + "syn 2.0.98", + "thiserror 1.0.69", +] + [[package]] name = "cainome-parser" version = "0.3.0" @@ -2582,6 +2642,25 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "cainome-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67adae2e76aeb14e514dcda34092c86170e0f84843727b1ad8e2b7398b3e0b56" +dependencies = [ + "anyhow", + "cainome-cairo-serde 0.2.1", + "cainome-parser 0.2.0", + "camino", + "prettyplease", + "proc-macro2", + "quote", + "serde_json", + "starknet 0.14.0", + "syn 2.0.98", + "thiserror 1.0.69", +] + [[package]] name = "cainome-rs" version = "0.3.1" @@ -2620,6 +2699,25 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "cainome-rs-macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c4e39d654e8e5c94f31a59ae8583771abbd784f96215dae1d1592f1b7526b2" +dependencies = [ + "anyhow", + "cainome-cairo-serde 0.2.1", + "cainome-parser 0.2.0", + "cainome-rs 0.2.0", + "proc-macro-error", + "proc-macro2", + "quote", + "serde_json", + "starknet 0.14.0", + "syn 2.0.98", + "thiserror 1.0.69", +] + [[package]] name = "cainome-rs-macro" version = "0.3.0" @@ -4024,10 +4122,10 @@ dependencies = [ "anyhow", "katana-chain-spec", "katana-cli", - "katana-db", + "katana-db 1.6.0-alpha.1", "katana-node", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "starknet 0.15.1", "tokio", ] @@ -4291,6 +4389,23 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dojo-metrics" +version = "1.2.2" +source = "git+https://github.com/dojoengine/dojo?tag=v1.2.2#aafa6669b750cec99aed145640dd6c9a9a946562" +dependencies = [ + "hyper 0.14.32", + "jemalloc-ctl", + "jemallocator", + "metrics 0.23.0", + "metrics-derive", + "metrics-exporter-prometheus", + "metrics-process", + "metrics-util", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "dojo-utils" version = "1.2.2" @@ -6215,10 +6330,10 @@ dependencies = [ "inquire", "katana-chain-spec", "katana-cli", - "katana-db", - "katana-primitives", - "katana-provider", - "katana-rpc-types", + "katana-db 1.6.0-alpha.1", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", "katana-utils", "piltover", "proptest", @@ -6237,6 +6352,22 @@ dependencies = [ "vergen-gitcl", ] +[[package]] +name = "katana-cairo" +version = "1.2.2" +source = "git+https://github.com/dojoengine/dojo?tag=v1.2.2#aafa6669b750cec99aed145640dd6c9a9a946562" +dependencies = [ + "cairo-lang-casm", + "cairo-lang-runner", + "cairo-lang-sierra", + "cairo-lang-sierra-to-casm", + "cairo-lang-starknet", + "cairo-lang-starknet-classes", + "cairo-lang-utils", + "cairo-vm 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "starknet_api 0.13.0-rc.1", +] + [[package]] name = "katana-chain-spec" version = "1.6.0-alpha.1" @@ -6245,8 +6376,8 @@ dependencies = [ "anyhow", "dirs 6.0.0", "katana-executor", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "lazy_static", "num-traits", "rstest 0.18.2", @@ -6275,7 +6406,7 @@ dependencies = [ "katana-log", "katana-messaging", "katana-node", - "katana-primitives", + "katana-primitives 1.6.0-alpha.1", "katana-rpc", "katana-slot-controller", "serde", @@ -6293,7 +6424,7 @@ name = "katana-codecs" version = "1.6.0-alpha.1" dependencies = [ "bytes", - "katana-primitives", + "katana-primitives 1.6.0-alpha.1", ] [[package]] @@ -6321,14 +6452,14 @@ dependencies = [ "futures", "hex", "katana-chain-spec", - "katana-db", + "katana-db 1.6.0-alpha.1", "katana-executor", - "katana-metrics", + "katana-metrics 1.6.0-alpha.1", "katana-pool", - "katana-primitives", - "katana-provider", - "katana-tasks", - "katana-trie", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", + "katana-tasks 1.6.0-alpha.1", + "katana-trie 1.6.0-alpha.1", "lazy_static", "metrics 0.23.0", "num-traits", @@ -6346,6 +6477,53 @@ dependencies = [ "url", ] +[[package]] +name = "katana-db" +version = "1.2.2" +source = "git+https://github.com/dojoengine/dojo?tag=v1.2.2#aafa6669b750cec99aed145640dd6c9a9a946562" +dependencies = [ + "anyhow", + "dojo-metrics", + "katana-primitives 1.2.2", + "katana-trie 1.2.2", + "metrics 0.23.0", + "page_size", + "parking_lot", + "postcard", + "reth-libmdbx", + "roaring", + "serde", + "serde_json", + "smallvec", + "tempfile", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "katana-db" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "anyhow", + "katana-metrics 1.6.0-alpha.0", + "katana-primitives 1.6.0-alpha.0", + "katana-trie 1.6.0-alpha.0", + "metrics 0.23.0", + "page_size", + "parking_lot", + "postcard", + "reth-libmdbx", + "roaring", + "serde", + "serde_json", + "smallvec", + "tempfile", + "thiserror 1.0.69", + "tracing", + "zstd 0.13.3", +] + [[package]] name = "katana-db" version = "1.6.0-alpha.1" @@ -6353,9 +6531,9 @@ dependencies = [ "anyhow", "arbitrary", "criterion", - "katana-metrics", - "katana-primitives", - "katana-trie", + "katana-metrics 1.6.0-alpha.1", + "katana-primitives 1.6.0-alpha.1", + "katana-trie 1.6.0-alpha.1", "metrics 0.23.0", "page_size", "parking_lot", @@ -6374,6 +6552,32 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "katana-db-migration" +version = "1.6.0-alpha.1" +dependencies = [ + "anyhow", + "fs_extra", + "katana-chain-spec", + "katana-core", + "katana-db 1.2.2", + "katana-db 1.6.0-alpha.0", + "katana-db 1.6.0-alpha.1", + "katana-executor", + "katana-log", + "katana-primitives 1.2.2", + "katana-primitives 1.6.0-alpha.0", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.2.2", + "katana-provider 1.6.0-alpha.0", + "katana-provider 1.6.0-alpha.1", + "similar-asserts", + "starknet 0.15.1", + "tempfile", + "tokio", + "tracing", +] + [[package]] name = "katana-executor" version = "1.6.0-alpha.1" @@ -6386,9 +6590,9 @@ dependencies = [ "cairo-vm 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "criterion", "katana-chain-spec", - "katana-primitives", - "katana-provider", - "katana-rpc-types", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", "num-traits", "oneshot", "parking_lot", @@ -6418,7 +6622,7 @@ dependencies = [ "hyper 0.14.32", "jsonrpsee", "katana-runner", - "katana-tasks", + "katana-tasks 1.6.0-alpha.1", "reqwest 0.12.15", "rust-embed", "tiny_http", @@ -6434,8 +6638,8 @@ dependencies = [ name = "katana-feeder-gateway" version = "1.6.0-alpha.1" dependencies = [ - "katana-primitives", - "katana-rpc-types", + "katana-primitives 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", "reqwest 0.12.15", "rstest 0.18.2", "serde", @@ -6448,15 +6652,32 @@ dependencies = [ "url", ] +[[package]] +name = "katana-fork" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "anyhow", + "futures", + "katana-primitives 1.6.0-alpha.0", + "katana-rpc-types 1.6.0-alpha.0", + "katana-tasks 1.6.0-alpha.0", + "parking_lot", + "starknet 0.14.0", + "thiserror 1.0.69", + "tokio", + "tracing", +] + [[package]] name = "katana-fork" version = "1.6.0-alpha.1" dependencies = [ "anyhow", "futures", - "katana-primitives", - "katana-rpc-types", - "katana-tasks", + "katana-primitives 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", + "katana-tasks 1.6.0-alpha.1", "parking_lot", "starknet 0.15.1", "thiserror 1.0.69", @@ -6514,7 +6735,7 @@ dependencies = [ "futures", "katana-chain-spec", "katana-pool", - "katana-primitives", + "katana-primitives 1.6.0-alpha.1", "reqwest 0.12.15", "serde", "serde_json", @@ -6526,6 +6747,24 @@ dependencies = [ "url", ] +[[package]] +name = "katana-metrics" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "hyper 0.14.32", + "jemalloc-ctl", + "jemallocator", + "metrics 0.23.0", + "metrics-derive", + "metrics-exporter-prometheus", + "metrics-process", + "metrics-util", + "sysinfo 0.35.1", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "katana-metrics" version = "1.6.0-alpha.1" @@ -6555,20 +6794,21 @@ dependencies = [ "jsonrpsee", "katana-chain-spec", "katana-core", - "katana-db", + "katana-db 1.6.0-alpha.1", + "katana-db-migration", "katana-executor", "katana-feeder-gateway", "katana-log", "katana-messaging", - "katana-metrics", + "katana-metrics 1.6.0-alpha.1", "katana-pipeline", "katana-pool", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "katana-rpc", "katana-rpc-api", "katana-stage", - "katana-tasks", + "katana-tasks 1.6.0-alpha.1", "serde", "serde_json", "starknet 0.15.1", @@ -6608,8 +6848,8 @@ version = "1.6.0-alpha.1" dependencies = [ "async-trait", "futures", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "katana-stage", "thiserror 1.0.69", "tokio", @@ -6623,8 +6863,8 @@ dependencies = [ "futures", "futures-util", "katana-executor", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "parking_lot", "rand 0.8.5", "starknet 0.15.1", @@ -6633,6 +6873,67 @@ dependencies = [ "tracing", ] +[[package]] +name = "katana-primitives" +version = "1.2.2" +source = "git+https://github.com/dojoengine/dojo?tag=v1.2.2#aafa6669b750cec99aed145640dd6c9a9a946562" +dependencies = [ + "alloy-primitives", + "anyhow", + "arbitrary", + "base64 0.21.7", + "derive_more 0.99.19", + "flate2", + "heapless", + "katana-cairo", + "lazy_static", + "num-bigint", + "num-traits", + "rand 0.8.5", + "serde", + "serde_json", + "serde_json_pythonic", + "serde_with 3.12.0", + "starknet 0.12.0", + "starknet-crypto 0.7.4", + "starknet-types-core", + "strum 0.25.0", + "strum_macros 0.25.3", + "thiserror 1.0.69", +] + +[[package]] +name = "katana-primitives" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "alloy-primitives", + "anyhow", + "arbitrary", + "base64 0.21.7", + "blockifier 0.0.0 (git+https://github.com/dojoengine/sequencer?rev=5d737b9c9)", + "cainome-cairo-serde 0.2.1", + "cairo-lang-starknet-classes", + "cairo-vm 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.19", + "heapless", + "lazy_static", + "num-bigint", + "num-traits", + "rand 0.8.5", + "serde", + "serde_json", + "serde_json_pythonic", + "serde_with 3.12.0", + "starknet 0.14.0", + "starknet-crypto 0.7.4", + "starknet-types-core", + "starknet_api 0.0.0 (git+https://github.com/dojoengine/sequencer?rev=5d737b9c9)", + "strum 0.25.0", + "strum_macros 0.25.3", + "thiserror 1.0.69", +] + [[package]] name = "katana-primitives" version = "1.6.0-alpha.1" @@ -6670,6 +6971,45 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "katana-provider" +version = "1.2.2" +source = "git+https://github.com/dojoengine/dojo?tag=v1.2.2#aafa6669b750cec99aed145640dd6c9a9a946562" +dependencies = [ + "anyhow", + "auto_impl", + "bitvec", + "katana-db 1.2.2", + "katana-primitives 1.2.2", + "katana-trie 1.2.2", + "parking_lot", + "serde_json", + "starknet 0.12.0", + "starknet-types-core", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "katana-provider" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "anyhow", + "auto_impl", + "bitvec", + "katana-db 1.6.0-alpha.0", + "katana-fork 1.6.0-alpha.0", + "katana-primitives 1.6.0-alpha.0", + "katana-trie 1.6.0-alpha.0", + "parking_lot", + "serde_json", + "starknet 0.14.0", + "starknet-types-core", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "katana-provider" version = "1.6.0-alpha.1" @@ -6680,11 +7020,11 @@ dependencies = [ "bitvec", "futures", "katana-chain-spec", - "katana-db", - "katana-fork", - "katana-primitives", + "katana-db 1.6.0-alpha.1", + "katana-fork 1.6.0-alpha.1", + "katana-primitives 1.6.0-alpha.1", "katana-runner", - "katana-trie", + "katana-trie 1.6.0-alpha.1", "lazy_static", "parking_lot", "rand 0.8.5", @@ -6722,16 +7062,16 @@ dependencies = [ "katana-explorer", "katana-log", "katana-messaging", - "katana-metrics", + "katana-metrics 1.6.0-alpha.1", "katana-node", "katana-pool", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "katana-rpc-api", - "katana-rpc-types", + "katana-rpc-types 1.6.0-alpha.1", "katana-rpc-types-builder", - "katana-tasks", - "katana-trie", + "katana-tasks 1.6.0-alpha.1", + "katana-trie 1.6.0-alpha.1", "katana-utils", "metrics 0.23.0", "num-bigint", @@ -6763,9 +7103,9 @@ dependencies = [ "jsonrpsee", "katana-core", "katana-pool", - "katana-primitives", - "katana-provider", - "katana-rpc-types", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", "rstest 0.18.2", "serde", "serde_json", @@ -6773,6 +7113,31 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "katana-rpc-types" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "alloy-primitives", + "anyhow", + "cainome 0.6.1", + "cainome-cairo-serde 0.2.1", + "cairo-lang-starknet-classes", + "cairo-lang-utils", + "derive_more 0.99.19", + "flate2", + "katana-primitives 1.6.0-alpha.0", + "katana-trie 1.6.0-alpha.0", + "num-traits", + "serde", + "serde_json", + "serde_json_pythonic", + "serde_with 3.12.0", + "starknet 0.14.0", + "starknet_api 0.0.0 (git+https://github.com/dojoengine/sequencer?rev=5d737b9c9)", + "thiserror 1.0.69", +] + [[package]] name = "katana-rpc-types" version = "1.6.0-alpha.1" @@ -6785,8 +7150,8 @@ dependencies = [ "cairo-lang-utils", "derive_more 0.99.19", "flate2", - "katana-primitives", - "katana-trie", + "katana-primitives 1.6.0-alpha.1", + "katana-trie 1.6.0-alpha.1", "num-traits", "rstest 0.18.2", "serde", @@ -6805,9 +7170,9 @@ version = "1.6.0-alpha.1" dependencies = [ "anyhow", "katana-executor", - "katana-primitives", - "katana-provider", - "katana-rpc-types", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", "starknet 0.15.1", ] @@ -6838,7 +7203,7 @@ name = "katana-slot-controller" version = "1.6.0-alpha.1" dependencies = [ "account_sdk", - "katana-primitives", + "katana-primitives 1.6.0-alpha.1", ] [[package]] @@ -6854,10 +7219,10 @@ dependencies = [ "katana-feeder-gateway", "katana-messaging", "katana-pool", - "katana-primitives", - "katana-provider", - "katana-rpc-types", - "katana-tasks", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", + "katana-rpc-types 1.6.0-alpha.1", + "katana-tasks 1.6.0-alpha.1", "num-traits", "starknet 0.15.1", "thiserror 1.0.69", @@ -6865,6 +7230,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "katana-tasks" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "futures", + "rayon", + "thiserror 1.0.69", + "tokio", + "tokio-metrics", + "tokio-util", + "tracing", +] + [[package]] name = "katana-tasks" version = "1.6.0-alpha.1" @@ -6878,14 +7257,46 @@ dependencies = [ "tracing", ] +[[package]] +name = "katana-trie" +version = "1.2.2" +source = "git+https://github.com/dojoengine/dojo?tag=v1.2.2#aafa6669b750cec99aed145640dd6c9a9a946562" +dependencies = [ + "anyhow", + "bitvec", + "bonsai-trie 0.1.0 (git+https://github.com/dojoengine/bonsai-trie/?branch=kariy%2Findexmap)", + "katana-primitives 1.2.2", + "serde", + "slab", + "starknet 0.12.0", + "starknet-types-core", + "thiserror 1.0.69", +] + +[[package]] +name = "katana-trie" +version = "1.6.0-alpha.0" +source = "git+https://github.com/dojoengine/katana?tag=v1.5.4#8d656bb47199db7672a669124f96191edfd2c865" +dependencies = [ + "anyhow", + "bitvec", + "bonsai-trie 0.1.0 (git+https://github.com/dojoengine/bonsai-trie/?rev=8b509fd)", + "katana-primitives 1.6.0-alpha.0", + "serde", + "slab", + "starknet 0.14.0", + "starknet-types-core", + "thiserror 1.0.69", +] + [[package]] name = "katana-trie" version = "1.6.0-alpha.1" dependencies = [ "anyhow", "bitvec", - "bonsai-trie", - "katana-primitives", + "bonsai-trie 0.1.0 (git+https://github.com/dojoengine/bonsai-trie/?rev=8b509fd)", + "katana-primitives 1.6.0-alpha.1", "serde", "slab", "starknet 0.15.1", @@ -6905,8 +7316,8 @@ dependencies = [ "katana-core", "katana-executor", "katana-node", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "katana-rpc", "rand 0.8.5", "starknet 0.15.1", @@ -10379,8 +10790,8 @@ dependencies = [ "katana-chain-spec", "katana-messaging", "katana-node", - "katana-primitives", - "katana-provider", + "katana-primitives 1.6.0-alpha.1", + "katana-provider 1.6.0-alpha.1", "prove_block", "starknet 0.15.1", "starknet-os", @@ -10808,6 +11219,26 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "starknet-crypto" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3f2175b0b3fc24ff2ec6dc07f5a720498994effca7e78b11a6e1c1bd02cad52" +dependencies = [ + "crypto-bigint", + "hex", + "hmac", + "num-bigint", + "num-integer", + "num-traits", + "rfc6979", + "sha2", + "starknet-crypto-codegen", + "starknet-curve 0.3.0", + "starknet-ff", + "zeroize", +] + [[package]] name = "starknet-crypto" version = "0.6.2" @@ -10858,6 +11289,15 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "starknet-curve" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252610baff59e4c4332ce3569f7469c5d3f9b415a2240d698fb238b2b4fc0942" +dependencies = [ + "starknet-ff", +] + [[package]] name = "starknet-curve" version = "0.4.2" @@ -11231,6 +11671,29 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "starknet_api" +version = "0.13.0-rc.1" +source = "git+https://github.com/dojoengine/sequencer?rev=d860f498#d860f498d25ad1699007286be031aef35a9692e5" +dependencies = [ + "bitvec", + "cairo-lang-starknet-classes", + "derive_more 0.99.19", + "hex", + "indexmap 2.7.1", + "itertools 0.12.1", + "once_cell", + "primitive-types", + "serde", + "serde_json", + "sha3", + "starknet-crypto 0.5.2", + "starknet-types-core", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror 1.0.69", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -11273,6 +11736,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + [[package]] name = "strum" version = "0.25.0" @@ -11291,6 +11760,19 @@ dependencies = [ "strum_macros 0.27.1", ] +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "strum_macros" version = "0.25.3" diff --git a/Cargo.toml b/Cargo.toml index 6ac1a9d07..13a9ff3db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ members = [ "crates/storage/codecs/derive", "crates/storage/db", "crates/storage/fork", + "crates/storage/migration", "crates/storage/provider", "crates/sync/pipeline", "crates/sync/stage", @@ -68,6 +69,7 @@ katana-codecs = { path = "crates/storage/codecs" } katana-codecs-derive = { path = "crates/storage/codecs/derive" } katana-core = { path = "crates/core" } katana-db = { path = "crates/storage/db" } +katana-db-migration = { path = "crates/storage/migration" } katana-executor = { path = "crates/executor" } katana-explorer = { path = "crates/explorer" } katana-feeder-gateway = { path = "crates/feeder-gateway" } diff --git a/Makefile b/Makefile index c7387c095..8447cbd31 100644 --- a/Makefile +++ b/Makefile @@ -17,12 +17,16 @@ SNOS_DB_DIR := $(DB_FIXTURES_DIR)/snos COMPATIBILITY_DB_TAR ?= $(DB_FIXTURES_DIR)/v1_2_2.tar.gz COMPATIBILITY_DB_DIR ?= $(DB_FIXTURES_DIR)/v1_2_2 +MIGRATION_FIXTURES_DIR ?= crates/storage/migration/tests/fixtures +MIGRATION_DB_TAR ?= $(DB_FIXTURES_DIR)/v1_2_2.tar.gz +MIGRATION_DB_DIR ?= $(MIGRATION_FIXTURES_DIR)/v1_2_2 + .DEFAULT_GOAL := usage .SILENT: clean .PHONY: usage help check-llvm native-deps native-deps-macos native-deps-linux native-deps-windows build-explorer clean # Virtual targets that map to actual file outputs -.PHONY: test-artifacts snos-artifacts db-compat-artifacts +.PHONY: test-artifacts snos-artifacts db-compat-artifacts migration-fixtures usage help: @echo "Usage:" @@ -30,6 +34,7 @@ usage help: @echo " test-artifacts: Prepare tests artifacts (including test database)." @echo " snos-artifacts: Prepare SNOS tests artifacts." @echo " db-compat-artifacts: Prepare database compatibility test artifacts." + @echo " migration-fixtures: Prepare migration test fixtures." @echo " native-deps-macos: Install cairo-native dependencies for macOS." @echo " native-deps-linux: Install cairo-native dependencies for Linux." @echo " native-deps-windows: Install cairo-native dependencies for Windows." @@ -41,6 +46,8 @@ snos-artifacts: $(SNOS_OUTPUT) @echo "SNOS test artifacts prepared successfully." db-compat-artifacts: $(COMPATIBILITY_DB_DIR) @echo "Database compatibility test artifacts prepared successfully." +migration-fixtures: $(MIGRATION_DB_DIR) + @echo "Migration test fixtures prepared successfully." test-artifacts: $(SNOS_DB_DIR) $(SNOS_OUTPUT) $(COMPATIBILITY_DB_DIR) @echo "All test artifacts prepared successfully." @@ -79,6 +86,13 @@ $(COMPATIBILITY_DB_DIR): $(COMPATIBILITY_DB_TAR) tar -xzf v1_2_2.tar.gz || { echo "Failed to extract backward compatibility test database\!"; exit 1; } @echo "Backward compatibility database extracted successfully." +$(MIGRATION_DB_DIR): $(MIGRATION_DB_TAR) + @echo "Extracting migration test database fixtures..." + @mkdir -p $(MIGRATION_FIXTURES_DIR) + @cd $(MIGRATION_FIXTURES_DIR) && \ + tar -xzf $(realpath $(MIGRATION_DB_TAR)) || { echo "Failed to extract migration test database\!"; exit 1; } + @echo "Migration test database fixtures extracted successfully." + check-llvm: ifndef MLIR_SYS_190_PREFIX $(error Could not find a suitable LLVM 19 toolchain (mlir), please set MLIR_SYS_190_PREFIX env pointing to the LLVM 19 dir) @@ -116,5 +130,5 @@ native-deps-windows: clean: echo "Cleaning up generated files..." - -rm -rf $(SNOS_DB_DIR) $(COMPATIBILITY_DB_DIR) $(SNOS_OUTPUT) $(EXPLORER_UI_DIST) + -rm -rf $(SNOS_DB_DIR) $(COMPATIBILITY_DB_DIR) $(MIGRATION_DB_DIR) $(SNOS_OUTPUT) $(EXPLORER_UI_DIST) echo "Clean complete." diff --git a/crates/core/src/backend/gas_oracle.rs b/crates/core/src/backend/gas_oracle.rs index 56b821ea1..36dcbad8d 100644 --- a/crates/core/src/backend/gas_oracle.rs +++ b/crates/core/src/backend/gas_oracle.rs @@ -16,13 +16,13 @@ const BUFFER_SIZE: usize = 60; const INTERVAL: Duration = Duration::from_secs(60); const ONE_GWEI: u128 = 1_000_000_000; -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum GasOracle { Fixed(FixedGasOracle), Sampled(EthereumSampledGasOracle), } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct FixedGasOracle { gas_prices: GasPrice, data_gas_prices: GasPrice, diff --git a/crates/core/src/backend/mod.rs b/crates/core/src/backend/mod.rs index 3b2abb78d..1d201240f 100644 --- a/crates/core/src/backend/mod.rs +++ b/crates/core/src/backend/mod.rs @@ -16,7 +16,6 @@ use katana_primitives::execution::TypedTransactionExecutionInfo; use katana_primitives::receipt::{Event, Receipt, ReceiptWithTxHash}; use katana_primitives::state::{compute_state_diff_hash, StateUpdates, StateUpdatesWithClasses}; use katana_primitives::transaction::{TxHash, TxWithHash}; -use katana_primitives::version::CURRENT_STARKNET_VERSION; use katana_primitives::{address, ContractAddress, Felt}; use katana_provider::providers::EmptyStateProvider; use katana_provider::traits::block::{BlockHashProvider, BlockWriter}; @@ -103,16 +102,23 @@ impl Backend { let tx_count = transactions.len(); let tx_hashes = transactions.iter().map(|tx| tx.hash).collect::>(); + let parent_hash = if block_env.number == 0 { + BlockHash::ZERO + } else { + let parent_block_num = block_env.number - 1; + self.blockchain.provider().block_hash_by_num(parent_block_num)?.unwrap() + }; + // create a new block and compute its commitment let partial_header = PartialHeader { + parent_hash, number: block_env.number, timestamp: block_env.timestamp, - protocol_version: CURRENT_STARKNET_VERSION, + protocol_version: block_env.starknet_version, l1_da_mode: L1DataAvailabilityMode::Calldata, sequencer_address: block_env.sequencer_address, l2_gas_prices: block_env.l2_gas_prices.clone(), l1_gas_prices: block_env.l1_gas_prices.clone(), - parent_hash: self.blockchain.provider().latest_hash()?, l1_data_gas_prices: block_env.l1_data_gas_prices.clone(), }; diff --git a/crates/executor/src/implementation/blockifier/mod.rs b/crates/executor/src/implementation/blockifier/mod.rs index ab54fcb90..26a3e06be 100644 --- a/crates/executor/src/implementation/blockifier/mod.rs +++ b/crates/executor/src/implementation/blockifier/mod.rs @@ -15,6 +15,7 @@ use cache::ClassCache; use katana_primitives::block::{ExecutableBlock, GasPrice as KatanaGasPrices, PartialHeader}; use katana_primitives::env::{BlockEnv, CfgEnv}; use katana_primitives::transaction::{ExecutableTx, ExecutableTxWithHash, TxWithHash}; +use katana_primitives::version::StarknetVersion; use katana_provider::traits::state::StateProvider; use starknet_api::block::{ BlockInfo, BlockNumber, BlockTimestamp, GasPriceVector, GasPrices, NonzeroGasPrice, @@ -29,7 +30,7 @@ use crate::{ pub(crate) const LOG_TARGET: &str = "katana::executor::blockifier"; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct BlockifierFactory { cfg: CfgEnv, flags: ExecutionFlags, @@ -97,6 +98,7 @@ pub struct StarknetVMProcessor<'a> { simulation_flags: ExecutionFlags, stats: ExecutionStats, bouncer: Bouncer, + starknet_version: StarknetVersion, } impl<'a> StarknetVMProcessor<'a> { @@ -109,6 +111,7 @@ impl<'a> StarknetVMProcessor<'a> { class_cache: ClassCache, ) -> Self { let transactions = Vec::new(); + let starknet_version: StarknetVersion = block_env.starknet_version; let block_context = Arc::new(utils::block_context_from_envs(&block_env, &cfg_env)); let state = state::CachedState::new(state, class_cache); @@ -138,6 +141,7 @@ impl<'a> StarknetVMProcessor<'a> { simulation_flags, stats: Default::default(), bouncer, + starknet_version, } } @@ -145,8 +149,10 @@ impl<'a> StarknetVMProcessor<'a> { let number = BlockNumber(header.number); let timestamp = BlockTimestamp(header.timestamp); - // TODO: should we enforce the gas price to not be 0, - // as there's a flag to disable gas uasge instead? + let eth_l2_gas_price = NonzeroGasPrice::new(header.l2_gas_prices.eth.get().into()) + .unwrap_or(NonzeroGasPrice::MIN); + let strk_l2_gas_price = NonzeroGasPrice::new(header.l2_gas_prices.strk.get().into()) + .unwrap_or(NonzeroGasPrice::MIN); let eth_l1_gas_price = NonzeroGasPrice::new(header.l1_gas_prices.eth.get().into()) .unwrap_or(NonzeroGasPrice::MIN); let strk_l1_gas_price = NonzeroGasPrice::new(header.l1_gas_prices.strk.get().into()) @@ -169,16 +175,14 @@ impl<'a> StarknetVMProcessor<'a> { sequencer_address: utils::to_blk_address(header.sequencer_address), gas_prices: GasPrices { eth_gas_prices: GasPriceVector { + l2_gas_price: eth_l2_gas_price, l1_gas_price: eth_l1_gas_price, l1_data_gas_price: eth_l1_data_gas_price, - // TODO: update to use the correct value - l2_gas_price: eth_l1_gas_price, }, strk_gas_prices: GasPriceVector { + l2_gas_price: strk_l2_gas_price, l1_gas_price: strk_l1_gas_price, l1_data_gas_price: strk_l1_data_gas_price, - // TODO: update to use the correct value - l2_gas_price: strk_l1_gas_price, }, }, use_kzg_da: false, @@ -220,6 +224,7 @@ impl<'a> BlockExecutor<'a> for StarknetVMProcessor<'a> { let tx = TxWithHash::from(&exec_tx); let hash = tx.hash; + let result = utils::transact( &mut state.cached_state, block_context, @@ -232,10 +237,6 @@ impl<'a> BlockExecutor<'a> for StarknetVMProcessor<'a> { Ok(exec_result) => { match &exec_result { ExecutionResult::Success { receipt, trace } => { - self.stats.l1_gas_used += receipt.resources_used().gas.l1_gas as u128; - self.stats.cairo_steps_used += - receipt.resources_used().computation_resources.n_steps as u128; - if let Some(reason) = receipt.revert_reason() { info!(target: LOG_TARGET, hash = format!("{hash:#x}"), %reason, "Transaction reverted."); } @@ -314,6 +315,7 @@ impl<'a> BlockExecutor<'a> for StarknetVMProcessor<'a> { number: self.block_context.block_info().block_number.0, timestamp: self.block_context.block_info().block_timestamp.0, sequencer_address: utils::to_address(self.block_context.block_info().sequencer_address), + starknet_version: self.starknet_version, } } } diff --git a/crates/executor/src/implementation/blockifier/utils.rs b/crates/executor/src/implementation/blockifier/utils.rs index 29f1d782f..b27fa862a 100644 --- a/crates/executor/src/implementation/blockifier/utils.rs +++ b/crates/executor/src/implementation/blockifier/utils.rs @@ -434,6 +434,10 @@ pub fn block_context_from_envs(block_env: &BlockEnv, cfg_env: &CfgEnv) -> BlockC strk_fee_token_address: to_blk_address(cfg_env.fee_token_addresses.strk), }; + let eth_l2_gas_price = NonzeroGasPrice::new(block_env.l2_gas_prices.eth.get().into()) + .unwrap_or(NonzeroGasPrice::MIN); + let strk_l2_gas_price = NonzeroGasPrice::new(block_env.l2_gas_prices.strk.get().into()) + .unwrap_or(NonzeroGasPrice::MIN); let eth_l1_gas_price = NonzeroGasPrice::new(block_env.l1_gas_prices.eth.get().into()) .unwrap_or(NonzeroGasPrice::MIN); let strk_l1_gas_price = NonzeroGasPrice::new(block_env.l1_gas_prices.strk.get().into()) @@ -446,16 +450,14 @@ pub fn block_context_from_envs(block_env: &BlockEnv, cfg_env: &CfgEnv) -> BlockC let gas_prices = GasPrices { eth_gas_prices: GasPriceVector { + l2_gas_price: eth_l2_gas_price, l1_gas_price: eth_l1_gas_price, l1_data_gas_price: eth_l1_data_gas_price, - // TODO: update to use the correct value - l2_gas_price: eth_l1_gas_price, }, strk_gas_prices: GasPriceVector { + l2_gas_price: strk_l2_gas_price, l1_gas_price: strk_l1_gas_price, l1_data_gas_price: strk_l1_data_gas_price, - // TODO: update to use the correct value - l2_gas_price: strk_l1_gas_price, }, }; @@ -475,8 +477,10 @@ pub fn block_context_from_envs(block_env: &BlockEnv, cfg_env: &CfgEnv) -> BlockC // Otherwise, there might be a mismatch between the calculated fees. // // The version of `snos` we're using is still limited up to Starknet version `0.13.3`. - const SN_VERSION: StarknetVersion = StarknetVersion::V0_13_4; - let mut versioned_constants = VersionedConstants::get(&SN_VERSION).unwrap().clone(); + let sn_version: StarknetVersion = block_env.starknet_version.try_into().expect("valid version"); + let mut versioned_constants = VersionedConstants::get(&sn_version).unwrap().clone(); + + // let mut versioned_constants = VERSIONED_CONSTANTS_V0_13_3.clone(); // NOTE: // These overrides would potentially make the `snos` run be invalid as it doesn't know about the diff --git a/crates/executor/src/utils.rs b/crates/executor/src/utils.rs index 17756cd91..5486e05b6 100644 --- a/crates/executor/src/utils.rs +++ b/crates/executor/src/utils.rs @@ -1,5 +1,7 @@ use blockifier::fee::receipt::TransactionReceipt; -use katana_primitives::execution::{CallInfo, TransactionExecutionInfo, TransactionResources}; +use katana_primitives::execution::{ + CallInfo, GasAmount, TransactionExecutionInfo, TransactionResources, +}; use katana_primitives::fee::FeeInfo; use katana_primitives::receipt::{ self, DataAvailabilityResources, DeclareTxReceipt, DeployAccountTxReceipt, Event, GasUsed, @@ -76,10 +78,15 @@ pub(crate) fn build_receipt( fn get_receipt_resources(receipt: &TransactionReceipt) -> receipt::ExecutionResources { let computation_resources = receipt.resources.computation.vm_resources.clone(); - let gas = GasUsed { - l2_gas: receipt.gas.l2_gas.0, - l1_gas: receipt.gas.l1_gas.0, - l1_data_gas: receipt.gas.l1_data_gas.0, + let gas = match receipt.gas.l2_gas { + GasAmount::ZERO => { + GasUsed::L1 { gas: receipt.gas.l1_gas.0, data_gas: receipt.gas.l1_data_gas.0 } + } + _ => GasUsed::All { + l1_gas: receipt.gas.l1_gas.0, + l2_gas: receipt.gas.l2_gas.0, + l1_data_gas: receipt.gas.l1_data_gas.0, + }, }; let da_resources = DataAvailabilityResources { diff --git a/crates/executor/tests/executor.rs b/crates/executor/tests/executor.rs index 55b4f8462..5fb0144c1 100644 --- a/crates/executor/tests/executor.rs +++ b/crates/executor/tests/executor.rs @@ -262,9 +262,9 @@ fn test_executor_with_valid_blocks_impl( "ERC_balances recepient should be set" ); - // assert the state updates after all the blocks are executed - let mut actual_total_gas: u128 = 0; - let mut actual_total_steps: u128 = 0; + // // assert the state updates after all the blocks are executed + // let mut actual_total_gas: u128 = 0; + // let mut actual_total_steps: u128 = 0; // assert the state updates let ExecutionOutput { states, transactions, stats } = executor.take_execution_output().unwrap(); @@ -273,19 +273,19 @@ fn test_executor_with_valid_blocks_impl( let actual_txs: Vec = transactions .iter() .map(|(tx, res)| { - if let Some(receipt) = res.receipt() { - let resources = receipt.resources_used(); - actual_total_gas += resources.gas.l1_gas as u128; - } - if let Some(rec) = res.receipt() { - actual_total_steps += rec.resources_used().computation_resources.n_steps as u128; - } + // if let Some(receipt) = res.receipt() { + // let resources = receipt.resources_used(); + // actual_total_gas += resources.gas.l1_gas as u128; + // } + // if let Some(rec) = res.receipt() { + // actual_total_steps += rec.resources_used().computation_resources.n_steps as u128; + // } tx.clone() }) .collect(); - assert_eq!(actual_total_gas, stats.l1_gas_used); - assert_eq!(actual_total_steps, stats.cairo_steps_used); + // assert_eq!(actual_total_gas, stats.l1_gas_used); + // assert_eq!(actual_total_steps, stats.cairo_steps_used); assert_eq!(actual_txs, expected_txs); let actual_nonce_updates = states.state_updates.nonce_updates; diff --git a/crates/feeder-gateway/src/types/mod.rs b/crates/feeder-gateway/src/types/mod.rs index a892490ea..15766aa5a 100644 --- a/crates/feeder-gateway/src/types/mod.rs +++ b/crates/feeder-gateway/src/types/mod.rs @@ -7,7 +7,7 @@ use katana_primitives::class::{ }; use katana_primitives::contract::{Nonce, StorageKey, StorageValue}; use katana_primitives::da::L1DataAvailabilityMode; -use katana_primitives::version::ProtocolVersion; +use katana_primitives::version::StarknetVersion; use katana_primitives::{ContractAddress, Felt}; use katana_rpc_types::class::ConversionError; pub use katana_rpc_types::class::RpcSierraContractClass; @@ -102,7 +102,7 @@ pub struct Block { pub transactions: Vec, pub transaction_receipts: Vec, #[serde(default)] - pub starknet_version: Option, + pub starknet_version: Option, } // -- Conversion to Katana primitive types. diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index e7dee33ee..e447b5683 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -9,6 +9,7 @@ version.workspace = true katana-chain-spec.workspace = true katana-core.workspace = true katana-db.workspace = true +katana-db-migration.workspace = true katana-executor.workspace = true katana-log.workspace = true katana-messaging.workspace = true diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index 94cbd88c5..9f6f902c4 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -338,6 +338,20 @@ impl Node { info!(%addr, "Metrics server started."); } + // Perform database migration if required + if self.db.require_migration() { + info!(target: "node", "Database migration required."); + + katana_db_migration::MigrationManager::new( + self.db.clone(), + self.config.chain.clone(), + self.backend.gas_oracle.clone(), + self.backend.executor_factory.as_ref().clone(), + )? + .migrate() + .context("Failed to migrate database")?; + } + let pool = self.pool.clone(); let backend = self.backend.clone(); let block_producer = self.block_producer.clone(); diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index 86737d1ae..a3c463f02 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -6,7 +6,7 @@ use starknet::macros::short_string; use crate::contract::ContractAddress; use crate::da::L1DataAvailabilityMode; use crate::transaction::{ExecutableTxWithHash, TxHash, TxWithHash}; -use crate::version::ProtocolVersion; +use crate::version::StarknetVersion; use crate::Felt; pub type BlockIdOrTag = starknet::core::types::BlockId; @@ -55,7 +55,7 @@ pub struct PartialHeader { pub l1_data_gas_prices: GasPrice, pub l2_gas_prices: GasPrice, pub l1_da_mode: L1DataAvailabilityMode, - pub protocol_version: ProtocolVersion, + pub protocol_version: StarknetVersion, } // TODO: Make sure the values can't be zero because in the blockifier executor, we fallback to 1 if @@ -130,7 +130,7 @@ pub struct Header { pub l1_data_gas_prices: GasPrice, pub l2_gas_prices: GasPrice, pub l1_da_mode: L1DataAvailabilityMode, - pub protocol_version: ProtocolVersion, + pub protocol_version: StarknetVersion, } impl Header { @@ -252,7 +252,7 @@ impl Default for Header { l1_data_gas_prices: GasPrice::default(), sequencer_address: ContractAddress::default(), l1_da_mode: L1DataAvailabilityMode::Calldata, - protocol_version: ProtocolVersion::default(), + protocol_version: StarknetVersion::default(), } } } diff --git a/crates/primitives/src/env.rs b/crates/primitives/src/env.rs index d6f256a34..9ea27d977 100644 --- a/crates/primitives/src/env.rs +++ b/crates/primitives/src/env.rs @@ -1,6 +1,7 @@ use crate::block::{BlockNumber, GasPrice}; use crate::chain::ChainId; use crate::contract::ContractAddress; +use crate::version::StarknetVersion; /// Block environment values. #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -17,6 +18,8 @@ pub struct BlockEnv { pub l1_data_gas_prices: GasPrice, /// The contract address of the sequencer. pub sequencer_address: ContractAddress, + + pub starknet_version: StarknetVersion, } /// The chain configuration values. diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index e01914c15..ec3c117fa 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -223,13 +223,6 @@ impl ReceiptWithTxHash { /// [docs]: https://docs.starknet.io/architecture-and-concepts/network-architecture/block-structure/#receipt_hash // pub fn compute_hash(&self) -> Felt { - let resources_used = self.resources_used(); - let gas_uasge = hash::Poseidon::hash_array(&[ - resources_used.gas.l2_gas.into(), - resources_used.gas.l1_gas.into(), - resources_used.gas.l1_data_gas.into(), - ]); - let messages_hash = self.compute_messages_to_l1_hash(); let revert_reason_hash = if let Some(reason) = self.revert_reason() { @@ -238,13 +231,33 @@ impl ReceiptWithTxHash { Felt::ZERO }; - hash::Poseidon::hash_array(&[ - self.tx_hash, - self.receipt.fee().overall_fee.into(), - messages_hash, - revert_reason_hash, - gas_uasge, - ]) + match self.resources_used().gas { + GasUsed::All { l2_gas, l1_gas, l1_data_gas } => { + println!("gas all"); + let gas_usage = + hash::Poseidon::hash_array(&[l2_gas.into(), l1_gas.into(), l1_data_gas.into()]); + + hash::Poseidon::hash_array(&[ + self.tx_hash, + self.receipt.fee().overall_fee.into(), + messages_hash, + revert_reason_hash, + gas_usage, + ]) + } + + GasUsed::L1 { gas, .. } => { + hash::Poseidon::hash_array(&[ + self.tx_hash, + self.receipt.fee().overall_fee.into(), + messages_hash, + revert_reason_hash, + Felt::ZERO, // L2 gas consumption. + gas.into(), + // data_gas.into(), + ]) + } + } } // H(n, from, to, H(payload), ...), where n, is the total number of messages, the payload is @@ -284,13 +297,18 @@ pub struct ExecutionResources { pub da_resources: DataAvailabilityResources, } -#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct GasUsed { - pub l2_gas: u64, - pub l1_gas: u64, - pub l1_data_gas: u64, +pub enum GasUsed { + All { l2_gas: u64, l1_gas: u64, l1_data_gas: u64 }, + L1 { gas: u64, data_gas: u64 }, +} + +impl Default for GasUsed { + fn default() -> Self { + GasUsed::L1 { gas: 0, data_gas: 0 } + } } #[derive(Debug, Default, Clone, PartialEq, Eq)] diff --git a/crates/primitives/src/version.rs b/crates/primitives/src/version.rs index fda4fee66..7e4f6d0ae 100644 --- a/crates/primitives/src/version.rs +++ b/crates/primitives/src/version.rs @@ -1,11 +1,11 @@ /// The currently supported version of the Starknet protocol. -pub const CURRENT_STARKNET_VERSION: ProtocolVersion = ProtocolVersion::new([0, 13, 1, 1]); // version 0.13.1.1 +pub const CURRENT_STARKNET_VERSION: StarknetVersion = StarknetVersion::new([0, 13, 1, 1]); // version 0.13.1.1 // TODO: figure out the exact format of the version string. /// Starknet protocol version. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[cfg_attr(feature = "arbitrary", derive(::arbitrary::Arbitrary))] -pub struct ProtocolVersion { +pub struct StarknetVersion { /// Each segments represents a part of the version number. segments: [u8; 4], } @@ -14,11 +14,12 @@ pub struct ProtocolVersion { pub enum ParseVersionError { #[error("invalid version format")] InvalidFormat, + #[error("failed to parse segment: {0}")] ParseSegment(#[from] std::num::ParseIntError), } -impl ProtocolVersion { +impl StarknetVersion { pub const fn new(segments: [u8; 4]) -> Self { Self { segments } } @@ -45,9 +46,9 @@ impl ProtocolVersion { } } -impl core::default::Default for ProtocolVersion { +impl core::default::Default for StarknetVersion { fn default() -> Self { - ProtocolVersion::new([0, 1, 0, 0]) + CURRENT_STARKNET_VERSION } } @@ -58,7 +59,7 @@ impl core::default::Default for ProtocolVersion { // - Version::new([1, 2, 3, 4]) will be displayed as "1.2.3.4" // - Version::new([1, 2, 3, 0]) will be displayed as "1.2.3" // - Version::new([0, 2, 3, 0]) will be displayed as "0.2.3" -impl core::fmt::Display for ProtocolVersion { +impl core::fmt::Display for StarknetVersion { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { for (idx, segment) in self.segments.iter().enumerate() { // If it's the last segment, don't print it if it's zero. @@ -77,10 +78,10 @@ impl core::fmt::Display for ProtocolVersion { } } -impl TryFrom for ProtocolVersion { +impl TryFrom for StarknetVersion { type Error = ParseVersionError; fn try_from(value: String) -> Result { - ProtocolVersion::parse(&value) + StarknetVersion::parse(&value) } } @@ -91,16 +92,55 @@ mod serde { // We de/serialize the version from/into a human-readable string format to prevent breaking the // database encoding format if ever decide to change its memory representation. - impl ::serde::Serialize for ProtocolVersion { + impl ::serde::Serialize for StarknetVersion { fn serialize(&self, serializer: S) -> Result { serializer.serialize_str(&self.to_string()) } } - impl<'de> ::serde::Deserialize<'de> for ProtocolVersion { + impl<'de> ::serde::Deserialize<'de> for StarknetVersion { fn deserialize>(deserializer: D) -> Result { let s = String::deserialize(deserializer)?; - ProtocolVersion::parse(&s).map_err(::serde::de::Error::custom) + StarknetVersion::parse(&s).map_err(::serde::de::Error::custom) + } + } +} + +/// An error when the version doesn't correspond to any of the official Starknet releases. +/// +/// List for all of the official releases can be found at +#[derive(thiserror::Error, Debug)] +#[error("invalid version: {0}")] +pub struct InvalidVersionError(StarknetVersion); + +impl TryFrom for starknet_api::block::StarknetVersion { + type Error = InvalidVersionError; + + fn try_from(version: StarknetVersion) -> Result { + match version.segments { + [0, 9, 1, 0] => Ok(Self::V0_9_1), + [0, 10, 0, 0] => Ok(Self::V0_10_0), + [0, 10, 1, 0] => Ok(Self::V0_10_1), + [0, 10, 2, 0] => Ok(Self::V0_10_2), + [0, 10, 3, 0] => Ok(Self::V0_10_3), + [0, 11, 0, 0] => Ok(Self::V0_11_0), + [0, 11, 0, 2] => Ok(Self::V0_11_0_2), + [0, 11, 1, 0] => Ok(Self::V0_11_1), + [0, 11, 2, 0] => Ok(Self::V0_11_2), + [0, 12, 0, 0] => Ok(Self::V0_12_0), + [0, 12, 1, 0] => Ok(Self::V0_12_1), + [0, 12, 2, 0] => Ok(Self::V0_12_2), + [0, 12, 3, 0] => Ok(Self::V0_12_3), + [0, 13, 0, 0] => Ok(Self::V0_13_0), + [0, 13, 1, 0] => Ok(Self::V0_13_1), + [0, 13, 1, 1] => Ok(Self::V0_13_3), + [0, 13, 2, 0] => Ok(Self::V0_13_2), + [0, 13, 2, 1] => Ok(Self::V0_13_2_1), + [0, 13, 3, 0] => Ok(Self::V0_13_3), + [0, 13, 4, 0] => Ok(Self::V0_13_4), + [0, 13, 5, 0] => Ok(Self::V0_13_5), + [0, 14, 0, 0] => Ok(Self::V0_14_0), + _ => Err(InvalidVersionError(version)), } } } @@ -112,7 +152,7 @@ mod tests { #[test] fn parse_version_valid() { let version = "1.9.0.0"; - let parsed = ProtocolVersion::parse(version).unwrap(); + let parsed = StarknetVersion::parse(version).unwrap(); assert_eq!(parsed.segments, [1, 9, 0, 0]); assert_eq!(String::from("1.9.0"), parsed.to_string()); } @@ -120,7 +160,7 @@ mod tests { #[test] fn parse_version_missing_parts() { let version = "1.9.0"; - let parsed = ProtocolVersion::parse(version).unwrap(); + let parsed = StarknetVersion::parse(version).unwrap(); assert_eq!(parsed.segments, [1, 9, 0, 0]); assert_eq!("1.9.0", parsed.to_string()); } @@ -128,13 +168,13 @@ mod tests { #[test] fn parse_version_invalid_digit_should_fail() { let version = "0.fv.1.0"; - assert!(ProtocolVersion::parse(version).is_err()); + assert!(StarknetVersion::parse(version).is_err()); } #[test] fn parse_version_missing_digit_default_zero() { let version = "1..."; - let parsed = ProtocolVersion::parse(version).unwrap(); + let parsed = StarknetVersion::parse(version).unwrap(); assert_eq!(parsed.segments, [1, 0, 0, 0]); assert_eq!("1.0.0", parsed.to_string()); } @@ -142,7 +182,7 @@ mod tests { #[test] fn parse_version_many_parts_should_succeed() { let version = "1.2.3.4"; - let parsed = ProtocolVersion::parse(version).unwrap(); + let parsed = StarknetVersion::parse(version).unwrap(); assert_eq!(parsed.segments, [1, 2, 3, 4]); assert_eq!("1.2.3.4", parsed.to_string()); } @@ -150,9 +190,9 @@ mod tests { #[test] fn parse_invalid_formats() { let version = ""; - assert!(ProtocolVersion::parse(version).is_err()); + assert!(StarknetVersion::parse(version).is_err()); let version = "1.2.3.4.5"; - assert!(ProtocolVersion::parse(version).is_err()); + assert!(StarknetVersion::parse(version).is_err()); } #[cfg(feature = "serde")] @@ -161,17 +201,17 @@ mod tests { #[test] fn rt_human_readable() { - let version = ProtocolVersion::new([1, 2, 3, 4]); + let version = StarknetVersion::new([1, 2, 3, 4]); let serialized = serde_json::to_string(&version).unwrap(); - let deserialized: ProtocolVersion = serde_json::from_str(&serialized).unwrap(); + let deserialized: StarknetVersion = serde_json::from_str(&serialized).unwrap(); assert_eq!(version, deserialized); } #[test] fn rt_non_human_readable() { - let version = ProtocolVersion::new([1, 2, 3, 4]); + let version = StarknetVersion::new([1, 2, 3, 4]); let serialized = postcard::to_stdvec(&version).unwrap(); - let deserialized: ProtocolVersion = postcard::from_bytes(&serialized).unwrap(); + let deserialized: StarknetVersion = postcard::from_bytes(&serialized).unwrap(); assert_eq!(version, deserialized); } } diff --git a/crates/rpc/rpc-types/src/receipt.rs b/crates/rpc/rpc-types/src/receipt.rs index 678b1ca60..639a5ce41 100644 --- a/crates/rpc/rpc-types/src/receipt.rs +++ b/crates/rpc/rpc-types/src/receipt.rs @@ -160,10 +160,15 @@ impl From for Event { } fn to_rpc_resources(resources: receipt::ExecutionResources) -> ExecutionResources { - ExecutionResources { - l2_gas: resources.gas.l2_gas, - l1_gas: resources.gas.l1_gas, - l1_data_gas: resources.gas.l1_data_gas, + match resources.gas { + receipt::GasUsed::All { l2_gas, l1_gas, l1_data_gas } => { + ExecutionResources { l2_gas, l1_gas, l1_data_gas } + } + + // TODO(kariy): update the rpc types to support old format + receipt::GasUsed::L1 { gas, data_gas } => { + ExecutionResources { l1_gas: gas, l1_data_gas: data_gas, l2_gas: Default::default() } + } } } diff --git a/crates/rpc/rpc-types/src/trace.rs b/crates/rpc/rpc-types/src/trace.rs index 67714180e..b5b3a4b79 100644 --- a/crates/rpc/rpc-types/src/trace.rs +++ b/crates/rpc/rpc-types/src/trace.rs @@ -94,15 +94,20 @@ pub fn to_rpc_fee_estimate(resources: &receipt::ExecutionResources, fee: &FeeInf fee::PriceUnit::Fri => PriceUnit::Fri, }; + let (l1_gas_consumed, l2_gas_consumed, l1_data_gas_consumed) = match &resources.gas { + receipt::GasUsed::All { l1_gas, l2_gas, l1_data_gas } => (*l1_gas, *l2_gas, *l1_data_gas), + receipt::GasUsed::L1 { gas, data_gas } => (*gas, 0u64, *data_gas), + }; + FeeEstimate { unit, + l1_gas_consumed, + l2_gas_consumed, + l1_data_gas_consumed, overall_fee: fee.overall_fee, l2_gas_price: fee.l2_gas_price, l1_gas_price: fee.l1_gas_price, l1_data_gas_price: fee.l1_data_gas_price, - l1_gas_consumed: resources.gas.l1_gas, - l2_gas_consumed: resources.gas.l2_gas, - l1_data_gas_consumed: resources.gas.l1_data_gas, } } diff --git a/crates/rpc/rpc/src/starknet/blockifier.rs b/crates/rpc/rpc/src/starknet/blockifier.rs index 3929af563..5b96dc47a 100644 --- a/crates/rpc/rpc/src/starknet/blockifier.rs +++ b/crates/rpc/rpc/src/starknet/blockifier.rs @@ -7,12 +7,11 @@ use katana_executor::{ EntryPointCall, ExecutionError, ExecutionFlags, ExecutionResult, ResultAndStates, }; use katana_primitives::env::{BlockEnv, CfgEnv}; -use katana_primitives::fee::{self}; use katana_primitives::transaction::ExecutableTxWithHash; use katana_primitives::Felt; use katana_provider::traits::state::StateProvider; +use katana_rpc_types::trace::to_rpc_fee_estimate; use katana_rpc_types::FeeEstimate; -use starknet::core::types::PriceUnit; #[tracing::instrument(level = "trace", target = "rpc", skip_all, fields(total_txs = transactions.len()))] pub fn simulate( @@ -74,22 +73,8 @@ pub fn estimate_fees( } else { let fee = receipt.fee(); let resources = receipt.resources_used(); - - let unit = match fee.unit { - fee::PriceUnit::Wei => PriceUnit::Wei, - fee::PriceUnit::Fri => PriceUnit::Fri, - }; - - results.push(Ok(FeeEstimate { - unit, - overall_fee: fee.overall_fee, - l2_gas_price: fee.l2_gas_price, - l1_gas_price: fee.l1_gas_price, - l2_gas_consumed: resources.gas.l2_gas, - l1_gas_consumed: resources.gas.l1_gas, - l1_data_gas_price: fee.l1_data_gas_price, - l1_data_gas_consumed: resources.gas.l1_data_gas, - })); + let estimate = to_rpc_fee_estimate(resources, fee); + results.push(Ok(estimate)); } } }; diff --git a/crates/storage/db/src/models/versioned/block/v6.rs b/crates/storage/db/src/models/versioned/block/v6.rs index 49a5b0a47..14c4c82b9 100644 --- a/crates/storage/db/src/models/versioned/block/v6.rs +++ b/crates/storage/db/src/models/versioned/block/v6.rs @@ -1,7 +1,7 @@ use katana_primitives::block::{BlockHash, BlockNumber, GasPrice}; use katana_primitives::contract::ContractAddress; use katana_primitives::da::L1DataAvailabilityMode; -use katana_primitives::version::ProtocolVersion; +use katana_primitives::version::StarknetVersion; use katana_primitives::Felt; use serde::{Deserialize, Serialize}; @@ -23,7 +23,7 @@ pub struct Header { pub l1_gas_prices: GasPrice, pub l1_data_gas_prices: GasPrice, pub l1_da_mode: L1DataAvailabilityMode, - pub protocol_version: ProtocolVersion, + pub protocol_version: StarknetVersion, } impl From
for katana_primitives::block::Header { diff --git a/crates/storage/db/src/models/versioned/transaction/mod.rs b/crates/storage/db/src/models/versioned/transaction/mod.rs index 0a51f55fc..f178fc6e4 100644 --- a/crates/storage/db/src/models/versioned/transaction/mod.rs +++ b/crates/storage/db/src/models/versioned/transaction/mod.rs @@ -34,11 +34,6 @@ impl Decompress for VersionedTx { return Ok(tx); } - // Try deserializing as V7 first, then fall back to V6 - if let Ok(transaction) = postcard::from_bytes::(bytes) { - return Ok(Self::V7(transaction)); - } - if let Ok(transaction) = postcard::from_bytes::(bytes) { return Ok(Self::V6(transaction)); } diff --git a/crates/storage/migration/Cargo.toml b/crates/storage/migration/Cargo.toml new file mode 100644 index 000000000..3afd02032 --- /dev/null +++ b/crates/storage/migration/Cargo.toml @@ -0,0 +1,36 @@ +[package] +edition.workspace = true +license.workspace = true +name = "katana-db-migration" +repository.workspace = true +version.workspace = true + +[dependencies] +katana-chain-spec.workspace = true +katana-core.workspace = true +katana-db.workspace = true +katana-executor.workspace = true +katana-primitives.workspace = true +katana-provider.workspace = true + +anyhow.workspace = true +similar-asserts.workspace = true +starknet.workspace = true +tracing.workspace = true + +# v6 types +katana-db-v1_2_2 = { git = "https://github.com/dojoengine/dojo", tag = "v1.2.2", package = "katana-db" } +katana-primitives-v1_2_2 = { git = "https://github.com/dojoengine/dojo", tag = "v1.2.2", package = "katana-primitives" } +katana-provider-v1_2_2 = { git = "https://github.com/dojoengine/dojo", tag = "v1.2.2", package = "katana-provider" } + +katana-db-v1_5_4 = { git = "https://github.com/dojoengine/katana", tag = "v1.5.4", package = "katana-db" } +katana-primitives-v1_5_4 = { git = "https://github.com/dojoengine/katana", tag = "v1.5.4", package = "katana-primitives" } +katana-provider-v1_5_4 = { git = "https://github.com/dojoengine/katana", tag = "v1.5.4", package = "katana-provider" } + +[dev-dependencies] +katana-chain-spec.workspace = true +katana-log.workspace = true +tokio = { workspace = true, features = [ "full" ] } + +fs_extra = "1.3.0" +tempfile.workspace = true diff --git a/crates/storage/migration/README.md b/crates/storage/migration/README.md new file mode 100644 index 000000000..60d895b60 --- /dev/null +++ b/crates/storage/migration/README.md @@ -0,0 +1,81 @@ +# Katana Migration + +This crate provides functionality for migrating historical blockchain data stored in Katana's database. The primary use case is re-executing historical blocks to derive new fields and data that may not be present in older database entries. + +## Overview + +When new fields are added to types like `Receipt` or `TransactionTrace`, older database entries won't have these fields. Rather than versioning all database types (which would be a huge maintenance effort), this crate allows you to re-execute historical blocks to generate the missing derived data. + +## Key Features + +- **Historical Block Re-execution**: Re-execute all blocks stored in the database +- **Database Abstraction**: Uses database abstraction traits, not concrete database implementations +- **Versioned Type Support**: Handles conversion from versioned database types (V6 → V7) +- **Standalone Implementation**: Easy to test and verify without katana-node integration + +## Usage + +```rust +use std::sync::Arc; +use katana_migration::MigrationManager; + +// Create a migration manager with your database +let migration_manager = MigrationManager::new(database); + +// Re-execute all historical blocks +migration_manager.migrate_all_blocks(executor_factory)?; + +// Or migrate a specific block +migration_manager.migrate_block(block_number, &executor_factory)?; +``` + +## Architecture + +The migration process works as follows: + +1. **Read Block Data**: Load versioned block headers and transactions from database +2. **Convert to Executable**: Convert database types to executable types required by the executor +3. **Re-execute Block**: Use katana-executor to re-execute the block +4. **Update Derived Data**: Store the newly generated receipts, traces, and other derived data + +## Type Conversions + +The migration handles conversion between different type versions: + +- **VersionedHeader**: V6 → V7 header conversion +- **VersionedTx**: V6 → V7 transaction conversion +- **Tx → ExecutableTx**: Convert stored transactions to executable format +- **Declare Transactions**: Fetch associated contract classes from storage + +## Database Tables + +The migration reads from and writes to these database tables: + +- **Read**: `Headers`, `Transactions`, `BlockBodyIndices`, `Classes`, `TxNumbers` +- **Write**: `Receipts`, `TxTraces` (and potentially other derived data tables) + +## Error Handling + +The migration gracefully handles: + +- Missing contract classes for declare transactions +- Legacy deploy transactions (not supported in ExecutableTx) +- Execution failures during re-execution +- Database transaction failures + +## Testing + +Run the tests with: + +```bash +cargo test -p katana-migration +``` + +The crate includes both unit tests and integration tests that demonstrate the migration workflow. + +## Future Enhancements + +- Historical state providers for more accurate re-execution +- Parallel block processing for better performance +- Progress tracking and resumption for large migrations +- Integration with katana-node for automated migrations \ No newline at end of file diff --git a/crates/storage/migration/src/lib.rs b/crates/storage/migration/src/lib.rs new file mode 100644 index 000000000..a225a5138 --- /dev/null +++ b/crates/storage/migration/src/lib.rs @@ -0,0 +1,359 @@ +//! Database migration crate for historical block re-execution. +//! +//! This crate provides functionality to re-execute historical blocks stored in the database +//! to derive new fields introduced in newer versions and data that may not be present in older +//! database entries. + +use std::ops::RangeInclusive; +use std::path::Path; +use std::sync::Arc; +use std::{fs, io}; + +use anyhow::{Context, Result}; +use katana_chain_spec::ChainSpec; +use katana_core::backend::gas_oracle::GasOracle; +use katana_core::backend::storage::Blockchain; +use katana_core::backend::Backend; +use katana_db::Db; +use katana_executor::implementation::blockifier::BlockifierFactory; +use katana_executor::ExecutorFactory; +use katana_primitives::block::{ + BlockHashOrNumber, BlockNumber, ExecutableBlock, PartialHeader, SealedBlockWithStatus, +}; +use katana_primitives::class::{ClassHash, ContractClass}; +use katana_primitives::env::BlockEnv; +use katana_primitives::genesis::constant::{ + DEFAULT_LEGACY_ERC20_CLASS, DEFAULT_LEGACY_ERC20_CLASS_HASH, +}; +use katana_primitives::hash::{self, StarkHash}; +use katana_primitives::transaction::{ + DeclareTxWithClass, ExecutableTx, ExecutableTxWithHash, Tx, TxWithHash, +}; +use katana_primitives::Felt; +use katana_provider::providers::db::DbProvider; +use katana_provider::traits::block::{ + BlockHashProvider, BlockNumberProvider, BlockProvider, BlockStatusProvider, BlockWriter, + HeaderProvider, +}; +use katana_provider::traits::state::StateFactoryProvider; +use katana_provider::traits::state_update::StateUpdateProvider; +use katana_provider::traits::transaction::{ReceiptProvider, TransactionProvider}; +use katana_provider::traits::trie::TrieWriter; +use starknet::macros::short_string; +use tracing::info; + +/// Migration manager for historical block re-execution. +#[derive(Debug)] +pub struct MigrationManager { + /// The new database to migrate to. + new_database: DbProvider, + old_database: DbProvider, + backend: Backend, + is_dev: bool, +} + +impl MigrationManager { + /// Create a new migration manager with the given database. + pub fn new( + old_database: katana_db::Db, + chain_spec: Arc, + gas_oracle: GasOracle, + executor: BlockifierFactory, + ) -> Result { + let old_db_path = old_database.path().to_path_buf(); + let backup_path = old_db_path.with_extension("backup"); + + // Rename the old database directory to create a backup + copy_dir_all(&old_db_path, &backup_path).context("failed to create backup")?; + drop(old_database); + + // Delete the old database directory after successful copy + fs::remove_dir_all(&old_db_path).context("failed to delete old database")?; + + // Open the backup as the old database + let old_db_env = Db::open(backup_path).context("failed to open backup (old) database")?; + let old_database = DbProvider::new(old_db_env); + + // Open a new database at the original path + let new_db_env = Db::new(old_db_path).context("failed to open new database")?; + + let new_blockchain = Blockchain::new_with_db(new_db_env.clone()); + let backend = Backend::new(chain_spec, new_blockchain, gas_oracle, executor); + + let new_database = DbProvider::new(new_db_env); + let mut new = Self { old_database, new_database, backend, is_dev: false }; + new.init_dev_db()?; + + // assert initial state are equal + { + let old_genesis_state_root = + new.old_database.historical(0.into())?.unwrap().state_root()?; + let new_genesis_state_root = + new.new_database.historical(0.into())?.unwrap().state_root()?; + + similar_asserts::assert_eq!(old_genesis_state_root, new_genesis_state_root); + } + + Ok(new) + } + + fn init_dev_db(&mut self) -> Result { + let old_db = &self.old_database; + let genesis_block_id: BlockHashOrNumber = 0.into(); + + // determine whether the genesis block is a dev or rollup chain spec + let genesis_tx_count = old_db + .transaction_count_by_block(genesis_block_id)? + .context("missing genesis block")?; + + // if the genesis block is from the DEV chain spec, follow how the dev chain spec is + // initialized in Backend::init_dev_genesis + let is_dev = if genesis_tx_count == 0 { + let mut state_updates = old_db.state_update_with_classes(genesis_block_id)?.unwrap(); + + // fix for a bug + state_updates + .classes + .insert(DEFAULT_LEGACY_ERC20_CLASS_HASH, DEFAULT_LEGACY_ERC20_CLASS.clone()); + + let old_block_hash = old_db.block_hash_by_id(genesis_block_id)?.unwrap(); + let status = old_db.block_status(genesis_block_id)?.unwrap(); + let mut block = old_db.block(genesis_block_id)?.unwrap(); + // tech debt in the initialization of dev chain spec + block.header.state_root = Felt::ZERO; + + let block_number = block.header.number; + let mut block = block.seal(); + assert_eq!(old_block_hash, block.hash); + + let class_trie_root = self + .new_database + .trie_insert_declared_classes( + block_number, + &state_updates.state_updates.declared_classes, + ) + .context("failed to update class trie")?; + + let contract_trie_root = self + .new_database + .trie_insert_contract_updates(block_number, &state_updates.state_updates) + .context("failed to update contract trie")?; + + let genesis_state_root = hash::Poseidon::hash_array(&[ + short_string!("STARKNET_STATE_V0"), + contract_trie_root, + class_trie_root, + ]); + + block.header.state_root = genesis_state_root; + self.new_database.insert_block_with_states_and_receipts( + SealedBlockWithStatus { block, status }, + state_updates, + vec![], + vec![], + )?; + + true + } else { + false + }; + + self.is_dev = is_dev; + Ok(is_dev) + } + + /// Re-execute all historical blocks in the database. + /// + /// This method reads all blocks from the database, converts them to executable format, + /// and re-executes them to derive new fields and data. + pub fn migrate(&self) -> Result<()> { + info!(target: "migration", "Starting historical block re-execution migration"); + + // Get the latest block number to determine migration range + let latest_block = + self.old_database.latest_number().context("Failed to get latest block number")?; + + if latest_block == 0 { + info!(target: "migration", "No blocks found in database, migration complete."); + return Ok(()); + } + + self.migrate_block_range(1..=latest_block)?; + + info!(target: "migration", total_blocks=%latest_block, "Database migration completed"); + + Ok(()) + } + + /// Re-execute a specific block by its number. + pub fn migrate_block_range(&self, block_range: RangeInclusive) -> Result<()> { + let block_start = *block_range.start(); + let block_end = *block_range.end(); + info!(target: "migration", %block_start, %block_end, "Migrating database"); + + for block_num in block_range { + info!(target: "migration", block=format!("{block_num}/{block_end}"), "Migrating block."); + self.migrate_block(block_num)?; + } + + Ok(()) + } + + fn migrate_block(&self, block_num: BlockNumber) -> Result<()> { + let state = self.new_database.historical(block_num.saturating_sub(1).into())?.unwrap(); + let block = self.load_executable_block(block_num)?; + + let block_env = BlockEnv { + number: block.header.number, + timestamp: block.header.timestamp, + l2_gas_prices: block.header.l2_gas_prices.clone(), + l1_gas_prices: block.header.l1_gas_prices.clone(), + l1_data_gas_prices: block.header.l1_data_gas_prices.clone(), + sequencer_address: block.header.sequencer_address, + starknet_version: block.header.protocol_version, + }; + + let mut executor = + self.backend.executor_factory.with_state_and_block_env(state, block_env.clone()); + + executor.execute_block(block)?; + let execution_output = executor.take_execution_output()?; + + self.backend.do_mine_block(&block_env, execution_output)?; + + // self.verify_block_migration(block_num)?; + + Ok(()) + } + + /// Load an executable block from the database using provider traits. + fn load_executable_block(&self, block_number: BlockNumber) -> Result { + // Load header using HeaderProvider + let old_header = self + .old_database + .header_by_number(block_number) + .with_context(|| format!("Failed to get header for block {block_number}"))? + .context("Block header not found")?; + + // Load transactions using TransactionProvider + let tx_with_hashes = self + .old_database + .transactions_by_block(block_number.into()) + .with_context(|| format!("Failed to get transactions for block {block_number}"))? + .context("Block transactions not found")?; + + // Convert to executable transactions + let mut body = Vec::with_capacity(tx_with_hashes.len()); + for tx in tx_with_hashes { + body.push(self.convert_to_executable_tx(tx)?); + } + + // Create partial header for ExecutableBlock + let partial_header = PartialHeader { + parent_hash: old_header.parent_hash, + number: old_header.number, + timestamp: old_header.timestamp, + sequencer_address: old_header.sequencer_address, + l1_gas_prices: old_header.l1_gas_prices, + l1_data_gas_prices: old_header.l1_data_gas_prices, + l2_gas_prices: old_header.l2_gas_prices, + l1_da_mode: old_header.l1_da_mode, + protocol_version: old_header.protocol_version, + }; + + Ok(ExecutableBlock { header: partial_header, body }) + } + + /// Convert a transaction to an executable transaction. + fn convert_to_executable_tx(&self, tx: TxWithHash) -> Result { + let hash = tx.hash; + let transaction = match tx.transaction { + Tx::Invoke(invoke_tx) => ExecutableTx::Invoke(invoke_tx), + Tx::L1Handler(l1_handler_tx) => ExecutableTx::L1Handler(l1_handler_tx), + Tx::DeployAccount(deploy_account_tx) => ExecutableTx::DeployAccount(deploy_account_tx), + + Tx::Declare(declare_tx) => { + // For declare transactions, we need to fetch the contract class from storage + let class_hash = declare_tx.class_hash(); + + // Retrieve the contract class from the database using provider + let contract_class = self.get_contract_class(class_hash).with_context(|| { + format!("Failed to get contract class for hash {class_hash:#x}") + })?; + + ExecutableTx::Declare(DeclareTxWithClass::new(declare_tx, contract_class)) + } + + Tx::Deploy(_deploy_tx) => { + // Legacy deploy transactions are not supported in ExecutableTx + // We'll skip these transactions for now + return Err(anyhow::anyhow!( + "Legacy deploy transactions are not supported in ExecutableTx for transaction \ + {}", + tx.hash + )); + } + }; + + Ok(ExecutableTxWithHash { transaction, hash }) + } + + fn get_contract_class(&self, class_hash: ClassHash) -> Result { + self.old_database + .latest()? + .class(class_hash) + .context("Failed to query contract class")? + .context("Contract class not found") + } + + /// Will return an Err if the block migration verification fails. + #[allow(unused)] + fn verify_block_migration(&self, block: BlockNumber) -> Result<()> { + let old_db = &self.old_database; + let new_db = &self.new_database; + let block_id: BlockHashOrNumber = block.into(); + + let old_receipts = old_db.receipts_by_block(block_id)?.unwrap_or_default().into_iter(); + let new_receipts = new_db.receipts_by_block(block_id)?.unwrap_or_default().into_iter(); + + // make sure both transactions have the same execution status (ie succeeded / reverted ) + for (idx, (new, old)) in new_receipts.zip(old_receipts).enumerate() { + assert_eq!( + new.is_reverted(), + old.is_reverted(), + "mismatch receipt status for tx {idx} in block {block}", + ) + } + + // State root + let old_state_root = old_db.historical(block_id)?.unwrap().state_root()?; + let new_state_root = new_db.historical(block_id)?.unwrap().state_root()?; + assert_eq!(old_state_root, new_state_root, "mismatch state root for block {block}"); + + // Block + let old_block_hash = old_db.block_hash_by_num(block)?.unwrap(); + let old_block = old_db.block(block.into())?.unwrap(); + + let new_block_hash = new_db.block_hash_by_num(block)?.unwrap(); + let new_block = new_db.block(block.into())?.unwrap(); + + similar_asserts::assert_eq!(old_block, new_block, "mismatch block for block {block}"); + assert_eq!(old_block_hash, new_block_hash, "mismatch block hashes for block {block}"); + + Ok(()) + } +} + +fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { + fs::create_dir_all(&dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; + } else { + fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) +} diff --git a/crates/storage/provider/src/error.rs b/crates/storage/provider/src/error.rs index 4b98e66f9..0b3481dcc 100644 --- a/crates/storage/provider/src/error.rs +++ b/crates/storage/provider/src/error.rs @@ -61,6 +61,10 @@ pub enum ProviderError { #[error("Missing compiled class hash for class hash {0:#x}")] MissingCompiledClassHash(ClassHash), + /// Error when a compiled class hash is not found but the class hash exists. + #[error("Missing contract class for class hash {0:#x}")] + MissingContractClass(ClassHash), + /// Error when a contract class change entry is not found but the block number of when the /// change happen exists in the class change list. #[error("Missing contract class change entry")] diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 8ddc73983..dc6be369c 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -273,6 +273,13 @@ where self.provider.state_update(block_id) } + fn state_update_with_classes( + &self, + block_id: BlockHashOrNumber, + ) -> ProviderResult> { + self.provider.state_update_with_classes(block_id) + } + fn declared_classes( &self, block_id: BlockHashOrNumber, diff --git a/crates/storage/provider/src/providers/db/mod.rs b/crates/storage/provider/src/providers/db/mod.rs index 7ce968f50..ea3ea7f87 100644 --- a/crates/storage/provider/src/providers/db/mod.rs +++ b/crates/storage/provider/src/providers/db/mod.rs @@ -359,6 +359,31 @@ impl StateUpdateProvider for DbProvider { } } + fn state_update_with_classes( + &self, + block_id: BlockHashOrNumber, + ) -> ProviderResult> { + let Some(state_updates) = self.state_update(block_id)? else { + return Ok(None); + }; + + let mut classes = BTreeMap::new(); + let declared_class_hashes = state_updates.declared_classes.keys(); + let deprecated_declared_class_hashes = state_updates.deprecated_declared_classes.iter(); + + for hash in declared_class_hashes.chain(deprecated_declared_class_hashes).copied() { + let class = self + .historical(block_id)? + .expect("qed; block must exist") + .class(hash)? + .expect("qed; class must exist"); + + classes.insert(hash, class); + } + + Ok(Some(StateUpdatesWithClasses { state_updates, classes })) + } + fn declared_classes( &self, block_id: BlockHashOrNumber, @@ -654,6 +679,7 @@ impl BlockEnvProvider for DbProvider { l1_gas_prices: header.l1_gas_prices, l1_data_gas_prices: header.l1_data_gas_prices, sequencer_address: header.sequencer_address, + starknet_version: header.protocol_version, })) } } diff --git a/crates/storage/provider/src/providers/fork/mod.rs b/crates/storage/provider/src/providers/fork/mod.rs index 924aab398..46fd0fb82 100644 --- a/crates/storage/provider/src/providers/fork/mod.rs +++ b/crates/storage/provider/src/providers/fork/mod.rs @@ -131,6 +131,13 @@ impl StateUpdateProvider for ForkedProvider { self.provider.state_update(block_id) } + fn state_update_with_classes( + &self, + block_id: BlockHashOrNumber, + ) -> ProviderResult> { + self.provider.state_update_with_classes(block_id) + } + fn declared_classes( &self, block_id: BlockHashOrNumber, diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 924f89a44..2cec811d7 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -14,6 +14,18 @@ use crate::ProviderResult; #[derive(Debug)] pub struct EmptyStateProvider; +impl EmptyStateProvider { + pub fn new() -> Self { + EmptyStateProvider + } +} + +impl Default for EmptyStateProvider { + fn default() -> Self { + Self::new() + } +} + impl StateProvider for EmptyStateProvider { fn class_hash_of_contract( &self, diff --git a/crates/storage/provider/src/traits/state_update.rs b/crates/storage/provider/src/traits/state_update.rs index 7d706a285..840cba060 100644 --- a/crates/storage/provider/src/traits/state_update.rs +++ b/crates/storage/provider/src/traits/state_update.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use katana_primitives::block::BlockHashOrNumber; use katana_primitives::class::{ClassHash, CompiledClassHash}; -use katana_primitives::state::StateUpdates; +use katana_primitives::state::{StateUpdates, StateUpdatesWithClasses}; use katana_primitives::ContractAddress; use crate::ProviderResult; @@ -12,6 +12,12 @@ pub trait StateUpdateProvider: Send + Sync { /// Returns the state update at the given block. fn state_update(&self, block_id: BlockHashOrNumber) -> ProviderResult>; + /// Returns the state update at the given block including the declared classes artifact. + fn state_update_with_classes( + &self, + block_id: BlockHashOrNumber, + ) -> ProviderResult>; + /// Returns all declared class hashes at the given block. fn declared_classes( &self, diff --git a/crates/storage/provider/tests/block.rs b/crates/storage/provider/tests/block.rs index 979f92843..88559db2a 100644 --- a/crates/storage/provider/tests/block.rs +++ b/crates/storage/provider/tests/block.rs @@ -93,6 +93,7 @@ where l1_gas_prices: expected_block.header.l1_gas_prices.clone(), l1_data_gas_prices: expected_block.header.l1_data_gas_prices.clone(), sequencer_address: expected_block.header.sequencer_address, + starknet_version: expected_block.header.protocol_version, }; let actual_block_hash = provider.block_hash_by_num(expected_block_num)?; @@ -201,6 +202,7 @@ where l1_gas_prices: expected_block.header.l1_gas_prices.clone(), l1_data_gas_prices: expected_block.header.l1_data_gas_prices.clone(), sequencer_address: expected_block.header.sequencer_address, + starknet_version: expected_block.header.protocol_version, }; let actual_block_hash = provider.block_hash_by_num(expected_block_num)?;