From 9c244837ec11414d5c2e4baff49ff1c7569473b2 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 15 Aug 2025 18:21:39 +0200 Subject: [PATCH 1/9] use a partial SMT proof --- crates/proto/src/generated/rpc_store.rs | 8 +++----- crates/store/src/state.rs | 13 ++++--------- proto/proto/store/rpc.proto | 6 +++--- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index f39545395..4280b1918 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -99,11 +99,9 @@ pub mod account_proofs { /// the current one. #[prost(bytes = "vec", optional, tag = "3")] pub account_code: ::core::option::Option<::prost::alloc::vec::Vec>, - /// Storage slots information for this account - #[prost(message, repeated, tag = "4")] - pub storage_maps: ::prost::alloc::vec::Vec< - account_state_header::StorageSlotMapProof, - >, + /// A sparse merkle tree, including all relevant merkle proofs for storage entries + #[prost(bytes = "vec", tag = "5")] + pub partial_smt: ::prost::alloc::vec::Vec, } /// Nested message and enum types in `AccountStateHeader`. pub mod account_state_header { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 2255b5a3b..799d56dc8 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -36,6 +36,7 @@ use miden_objects::crypto::merkle::{ MmrPeaks, MmrProof, PartialMmr, + PartialSmt, SmtProof, }; use miden_objects::note::{NoteDetails, NoteId, Nullifier}; @@ -900,8 +901,7 @@ impl State { .expect("retrieved accounts were validated against request"); if let Some(details) = &account_info.details { - let mut storage_slot_map_keys = Vec::new(); - + let mut partial = PartialSmt::new(); for StorageMapKeysProof { storage_index, storage_keys } in &request.storage_requests { @@ -910,12 +910,7 @@ impl State { { for map_key in storage_keys { let proof = storage_map.open(map_key); - - let slot_map_key = proto::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapProof { - storage_slot: u32::from(*storage_index), - smt_proof: proof.to_bytes(), - }; - storage_slot_map_keys.push(slot_map_key); + partial.add_proof(proof)?; } } else { return Err(AccountError::StorageSlotNotMap(*storage_index).into()); @@ -933,7 +928,7 @@ impl State { header: Some(AccountHeader::from(details).into()), storage_header: details.storage().to_header().to_bytes(), account_code, - storage_maps: storage_slot_map_keys, + partial_smt: partial.to_bytes(), }; headers_map.insert(account_info.summary.account_id, state_header); diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index 48d6e819b..0f44c891a 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -56,7 +56,7 @@ service Rpc { // The response includes each note's metadata and inclusion proof. // // A basic note sync can be implemented by repeatedly requesting the previous response's block until reaching the - // tip of the chain. + // tip of the chain. rpc SyncNotes(SyncNotesRequest) returns (SyncNotesResponse) {} // Returns info which can be used by the client to sync up to the latest state of the chain @@ -155,8 +155,8 @@ message AccountProofs { // the current one. optional bytes account_code = 3; - // Storage slots information for this account - repeated StorageSlotMapProof storage_maps = 4; + // A sparse merkle tree, including all relevant merkle proofs for storage entries + bytes partial_smt = 5; } // The account witness for the current state commitment of one account ID. From 75743a34d487180bdbcd80e60e27c5a7309cbef9 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Fri, 15 Aug 2025 18:34:55 +0200 Subject: [PATCH 2/9] changelog and a . --- CHANGELOG.md | 1 + proto/proto/store/rpc.proto | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca38d2add..511321edf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - Configure `NativeFaucet`, which determines the native asset used to pay fees - Configure the base verification fee - Note: fees are not yet activated, and this has no impact beyond setting these values in the block headers +- [BREAKING] `GetAccountProofs` endpoint uses a `PartialSmt` for proofs. ([#1158](https://github.com/0xMiden/miden-node/pull/1158)). ### Fixes diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index 0f44c891a..426f43e8a 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -155,7 +155,7 @@ message AccountProofs { // the current one. optional bytes account_code = 3; - // A sparse merkle tree, including all relevant merkle proofs for storage entries + // A sparse merkle tree, including all relevant merkle proofs for storage entries. bytes partial_smt = 5; } From 30a2f954728b57d4fa9c5f94b8ed88c55424eeb5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 18 Aug 2025 10:18:57 +0200 Subject: [PATCH 3/9] add lookup for storage index per leaf --- crates/proto/src/generated/rpc_store.rs | 5 ++++- crates/store/src/state.rs | 3 +++ proto/proto/store/rpc.proto | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index 4280b1918..1fe9200ce 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -99,9 +99,12 @@ pub mod account_proofs { /// the current one. #[prost(bytes = "vec", optional, tag = "3")] pub account_code: ::core::option::Option<::prost::alloc::vec::Vec>, - /// A sparse merkle tree, including all relevant merkle proofs for storage entries + /// A sparse merkle tree, including all relevant merkle proofs for storage entries. #[prost(bytes = "vec", tag = "5")] pub partial_smt: ::prost::alloc::vec::Vec, + /// Lookup table for leaf to storage index. + #[prost(bytes = "vec", tag = "6")] + pub leaf_to_storage_index: ::prost::alloc::vec::Vec, } /// Nested message and enum types in `AccountStateHeader`. pub mod account_state_header { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index 799d56dc8..b9960865a 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -902,6 +902,7 @@ impl State { if let Some(details) = &account_info.details { let mut partial = PartialSmt::new(); + let mut leaf_to_storage_index = BTreeMap::new(); for StorageMapKeysProof { storage_index, storage_keys } in &request.storage_requests { @@ -910,6 +911,7 @@ impl State { { for map_key in storage_keys { let proof = storage_map.open(map_key); + leaf_to_storage_index.insert(proof.leaf().index(), *storage_index); partial.add_proof(proof)?; } } else { @@ -929,6 +931,7 @@ impl State { storage_header: details.storage().to_header().to_bytes(), account_code, partial_smt: partial.to_bytes(), + leaf_to_storage_index: leaf_to_storage_index.to_bytes(), }; headers_map.insert(account_info.summary.account_id, state_header); diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index 426f43e8a..4ffd22745 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -157,6 +157,9 @@ message AccountProofs { // A sparse merkle tree, including all relevant merkle proofs for storage entries. bytes partial_smt = 5; + + // Lookup table for leaf to storage index. + bytes leaf_to_storage_index = 6; } // The account witness for the current state commitment of one account ID. From d58bc5e468b896e45e74b96148adbbf005ed95a5 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 18 Aug 2025 10:53:41 +0200 Subject: [PATCH 4/9] fix --- crates/proto/src/generated/rpc_store.rs | 7 ++----- crates/store/src/state.rs | 12 ++++++------ proto/proto/store/rpc.proto | 7 ++----- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index 1fe9200ce..a758c8d0d 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -99,12 +99,9 @@ pub mod account_proofs { /// the current one. #[prost(bytes = "vec", optional, tag = "3")] pub account_code: ::core::option::Option<::prost::alloc::vec::Vec>, - /// A sparse merkle tree, including all relevant merkle proofs for storage entries. + /// A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries. #[prost(bytes = "vec", tag = "5")] - pub partial_smt: ::prost::alloc::vec::Vec, - /// Lookup table for leaf to storage index. - #[prost(bytes = "vec", tag = "6")] - pub leaf_to_storage_index: ::prost::alloc::vec::Vec, + pub partial_storage_smts: ::prost::alloc::vec::Vec, } /// Nested message and enum types in `AccountStateHeader`. pub mod account_state_header { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index b9960865a..caa0d628f 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -901,8 +901,7 @@ impl State { .expect("retrieved accounts were validated against request"); if let Some(details) = &account_info.details { - let mut partial = PartialSmt::new(); - let mut leaf_to_storage_index = BTreeMap::new(); + let mut partials = BTreeMap::::default(); for StorageMapKeysProof { storage_index, storage_keys } in &request.storage_requests { @@ -911,8 +910,10 @@ impl State { { for map_key in storage_keys { let proof = storage_map.open(map_key); - leaf_to_storage_index.insert(proof.leaf().index(), *storage_index); - partial.add_proof(proof)?; + partials + .entry(*storage_index) + .or_insert_with(|| PartialSmt::new()) + .add_proof(proof)?; } } else { return Err(AccountError::StorageSlotNotMap(*storage_index).into()); @@ -930,8 +931,7 @@ impl State { header: Some(AccountHeader::from(details).into()), storage_header: details.storage().to_header().to_bytes(), account_code, - partial_smt: partial.to_bytes(), - leaf_to_storage_index: leaf_to_storage_index.to_bytes(), + partial_storage_smts: partials.to_bytes(), }; headers_map.insert(account_info.summary.account_id, state_header); diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index 4ffd22745..8e1e4fe67 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -155,11 +155,8 @@ message AccountProofs { // the current one. optional bytes account_code = 3; - // A sparse merkle tree, including all relevant merkle proofs for storage entries. - bytes partial_smt = 5; - - // Lookup table for leaf to storage index. - bytes leaf_to_storage_index = 6; + // A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries. + bytes partial_storage_smts = 5; } // The account witness for the current state commitment of one account ID. From a8c8f2d23448c14afd1ed9b4d44679f2272e1066 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 18 Aug 2025 11:41:09 +0200 Subject: [PATCH 5/9] better representation in proto --- crates/proto/src/generated/rpc_store.rs | 16 ++++++++++++++-- crates/store/src/state.rs | 5 ++++- proto/proto/store/rpc.proto | 11 ++++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index a758c8d0d..cee87a321 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -100,11 +100,23 @@ pub mod account_proofs { #[prost(bytes = "vec", optional, tag = "3")] pub account_code: ::core::option::Option<::prost::alloc::vec::Vec>, /// A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries. - #[prost(bytes = "vec", tag = "5")] - pub partial_storage_smts: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub partial_storage_smts: ::prost::alloc::vec::Vec< + account_state_header::StorageSlotMapPartialSmt, + >, } /// Nested message and enum types in `AccountStateHeader`. pub mod account_state_header { + /// Represents a single storage slot with the requested keys and their respective values. + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct StorageSlotMapPartialSmt { + /// The storage slot index (\[0..255\]). + #[prost(uint32, tag = "1")] + pub storage_slot: u32, + /// Merkle proofs of the map value as partial sparse merkle tree for compression. + #[prost(bytes = "vec", tag = "2")] + pub partial_smt: ::prost::alloc::vec::Vec, + } /// Represents a single storage slot with the requested keys and their respective values. #[derive(Clone, PartialEq, ::prost::Message)] pub struct StorageSlotMapProof { diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index caa0d628f..d82b6b339 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -931,7 +931,10 @@ impl State { header: Some(AccountHeader::from(details).into()), storage_header: details.storage().to_header().to_bytes(), account_code, - partial_storage_smts: partials.to_bytes(), + partial_storage_smts: Vec::from_iter(partials.into_iter().map(|(slot, partial_smt)| proto::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapPartialSmt { + storage_slot: slot as u32, + partial_smt: partial_smt.to_bytes(), + })), }; headers_map.insert(account_info.summary.account_id, state_header); diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index 8e1e4fe67..cd3b2b51d 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -136,6 +136,15 @@ message AccountProofs { message AccountProof { // State header for public accounts. message AccountStateHeader { + // Represents a single storage slot with the requested keys and their respective values. + message StorageSlotMapPartialSmt { + // The storage slot index ([0..255]). + uint32 storage_slot = 1; + + // Merkle proofs of the map value as partial sparse merkle tree for compression. + bytes partial_smt = 2; + } + // Represents a single storage slot with the requested keys and their respective values. message StorageSlotMapProof { // The storage slot index ([0..255]). @@ -156,7 +165,7 @@ message AccountProofs { optional bytes account_code = 3; // A sparse merkle tree per storage slot, including all relevant merkle proofs for storage entries. - bytes partial_storage_smts = 5; + repeated StorageSlotMapPartialSmt partial_storage_smts = 5; } // The account witness for the current state commitment of one account ID. From 9423fa0ba20669bcd904965e292b4865f16d7f56 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 18 Aug 2025 11:55:05 +0200 Subject: [PATCH 6/9] clippy --- crates/store/src/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index d82b6b339..f6c1b07f5 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -912,7 +912,7 @@ impl State { let proof = storage_map.open(map_key); partials .entry(*storage_index) - .or_insert_with(|| PartialSmt::new()) + .or_insert_with(PartialSmt::new) .add_proof(proof)?; } } else { @@ -932,7 +932,7 @@ impl State { storage_header: details.storage().to_header().to_bytes(), account_code, partial_storage_smts: Vec::from_iter(partials.into_iter().map(|(slot, partial_smt)| proto::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapPartialSmt { - storage_slot: slot as u32, + storage_slot: u32::from(slot), partial_smt: partial_smt.to_bytes(), })), }; From d9ea339d80b294818498a490c23fd0ba4adc050d Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 18 Aug 2025 19:48:51 +0200 Subject: [PATCH 7/9] add comment to protobuf --- proto/proto/store/rpc.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index cd3b2b51d..ab8885b1e 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -142,6 +142,8 @@ message AccountProofs { uint32 storage_slot = 1; // Merkle proofs of the map value as partial sparse merkle tree for compression. + // The respective rust types is `SparseMerkleTree` and the transformation to and from + // bytes is done via the traits `Serializable::to_bytes` and `Deserializable::from_bytes`. bytes partial_smt = 2; } From 2a666a0deaa83930c6fdae4b77fbe075d6ee9a3f Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 18 Aug 2025 20:33:22 +0200 Subject: [PATCH 8/9] protobuf comment --- crates/proto/src/generated/rpc_store.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index cee87a321..78ed4c6e2 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -114,6 +114,8 @@ pub mod account_proofs { #[prost(uint32, tag = "1")] pub storage_slot: u32, /// Merkle proofs of the map value as partial sparse merkle tree for compression. + /// The respective rust types is `SparseMerkleTree` and the transformation to and from + /// bytes is done via the traits `Serializable::to_bytes` and `Deserializable::from_bytes`. #[prost(bytes = "vec", tag = "2")] pub partial_smt: ::prost::alloc::vec::Vec, } From af22a3213227ba944b2e5ef8ccbf22c0ccfe7700 Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Tue, 19 Aug 2025 21:38:15 +0200 Subject: [PATCH 9/9] represent the internal merkle tree structure in protobuf --- Cargo.lock | 1 + crates/proto/Cargo.toml | 3 +- crates/proto/src/domain/merkle.rs | 228 +++++++++++++++++++++++ crates/proto/src/generated/primitives.rs | 35 ++++ crates/proto/src/generated/rpc_store.rs | 18 +- crates/store/src/state.rs | 10 +- proto/proto/store/rpc.proto | 13 +- proto/proto/types/primitives.proto | 28 +++ 8 files changed, 301 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b1ae6cc2..ee59b301d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2467,6 +2467,7 @@ dependencies = [ "miden-node-proto-build", "miden-node-utils", "miden-objects", + "pretty_assertions", "proptest", "prost", "prost-build", diff --git a/crates/proto/Cargo.toml b/crates/proto/Cargo.toml index ae1677656..597c135ec 100644 --- a/crates/proto/Cargo.toml +++ b/crates/proto/Cargo.toml @@ -25,7 +25,8 @@ thiserror = { workspace = true } tonic = { workspace = true } [dev-dependencies] -proptest = { version = "1.7" } +proptest = { version = "1.7" } +pretty_assertions = { workspace = true} [build-dependencies] anyhow = { workspace = true } diff --git a/crates/proto/src/domain/merkle.rs b/crates/proto/src/domain/merkle.rs index ca1ca551a..7e4ecd7f3 100644 --- a/crates/proto/src/domain/merkle.rs +++ b/crates/proto/src/domain/merkle.rs @@ -1,9 +1,14 @@ +use std::collections::{HashMap, HashSet}; + use miden_objects::Word; use miden_objects::crypto::merkle::{ Forest, + InnerNode, LeafIndex, MerklePath, MmrDelta, + NodeIndex, + PartialSmt, SmtLeaf, SmtProof, SparseMerklePath, @@ -206,3 +211,226 @@ impl From for proto::primitives::SmtOpening { } } } + +// NODE INDEX +// ------------------------------------------------------------------------------------------------ +impl From for proto::primitives::NodeIndex { + fn from(value: NodeIndex) -> Self { + proto::primitives::NodeIndex { + depth: value.depth() as u32, + value: value.value(), + } + } +} +impl TryFrom for NodeIndex { + type Error = ConversionError; + fn try_from(index: proto::primitives::NodeIndex) -> Result { + let depth = u8::try_from(index.depth)?; + let value = index.value; + Ok(NodeIndex::new(depth, value)?) + } +} + +// PARTIAL SMT +// ------------------------------------------------------------------------------------------------ + +impl TryFrom for PartialSmt { + type Error = ConversionError; + fn try_from(value: proto::primitives::PartialSmt) -> Result { + let proto::primitives::PartialSmt { root, leaves, nodes } = value; + let root = root + .as_ref() + .ok_or(proto::primitives::PartialSmt::missing_field(stringify!(root)))? + .try_into()?; + // TODO ensure `!leaves.is_empty()` + + // Convert other proto primitives to crypto types + let leaves = Result::, _>::from_iter(try_convert(leaves))?; + let mut inner = + Result::, _>::from_iter(nodes.into_iter().map(|inner| { + let node_index = NodeIndex::try_from( + inner + .index + .ok_or(proto::primitives::NodeIndex::missing_field(stringify!(index)))?, + )?; + let digest = Word::try_from( + inner + .digest + .ok_or(proto::primitives::Digest::missing_field(stringify!(digest)))?, + )?; + Ok::<_, Self::Error>((node_index, digest)) + }))?; + + let leaf_indices = + HashSet::::from_iter(leaves.iter().map(|leaf| leaf.index().into())); + + // Must contain the leaves too + inner.extend(leaves.iter().map(|leaf| (leaf.index().into(), leaf.hash()))); + + // Start constructing the partial SMT + // + // Construct a `MerklePath` per leaf by transcending from leaf digest down to depth 0. + // Then verify the merkle proof holds consistency and completeness checks and all + // required sibling nodes are present to deeduct required intermediate nodes. + let mut partial = PartialSmt::new(); + for leaf in leaves { + // Construct the merkle path: + let leaf_node_index: NodeIndex = leaf.index().into(); + let mut current = leaf_node_index.clone(); + let mut siblings = Vec::new(); + + // If we ever try to trancend beyond this depth level, something is wrong and + // we must stop decoding. + let max_depth = leaf_node_index.depth(); + // root: 00 + // / \ + // 10 11 + // / \ / \ + // 20 21 22 23 + // / \ / \ / \ / \ + // leaves ... x y + // Iterate from the leaf up to the root (exclusive) + // We start by picking the sibling of `x`, `y`, our starting point and + // then moving towards the root `0`. By definition siblings have the same parent. + loop { + let sibling_idx = current.sibling(); + // TODO FIXME for a leaf we get another leaf, we need to ensure those are part of + // the inner set or contained in the inner HashMap + let sibling_digest = if let Some(sibling_digest) = inner.get(&sibling_idx) { + // Previous round already calculated the entry or it was given explicitly + *sibling_digest + } else { + // The entry does not exist, so we need to lazily follow the missing nodes and + // calculate recursively. + + // DFS, build the subtree recursively, starting from the current sibling + let mut stack = Vec::::new(); + stack.push(sibling_idx.clone()); + loop { + let Some(idx) = stack.pop() else { + unreachable!( + "Must be an error, we must have nodes to resolve all questions, otherwise construction is borked" + ) + }; + if let Some(node_digest) = inner.get(&idx) { + if stack.is_empty() && idx == sibling_idx { + // we emptied the stack which means the current one is our desired + // starting point + break *node_digest; + } + // if the digest exists, we don't need to recurse + continue; + } + debug_assert!( + !leaf_indices.contains(&idx), + "For every relevant leaf, we must have the relevant value" + ); + let left = idx.left_child(); + let right = idx.right_child(); + if max_depth < left.depth() || max_depth < right.depth() { + // TODO might happen in case of a missing node, so we must handle this + // gracefully + unreachable!("graceful!") + } + // proceed if the inner nodes are unknown + if !inner.contains_key(&left) { + stack.push(left); + } + if !inner.contains_key(&right) { + stack.push(right); + } + // left and right exist, we can derive the digest for `idx` + if let Some(&left) = inner.get(&left) + && let Some(&right) = inner.get(&right) + { + let node = InnerNode { left, right }; + let node_digest = node.hash(); + + if stack.is_empty() && idx == sibling_idx { + // we emptied the stack which means the current one is our desired + // starting point + break node_digest; + } + inner.insert(idx, node_digest); + } + } + }; + siblings.push(sibling_digest); + + // Move up to the parent level, and repeat + current = current.parent(); + if current.depth() == 0 { + break; + } + } + + let path = MerklePath::new(siblings); + path.verify(leaf_node_index.value(), leaf.hash(), &root).expect("It's fine"); + partial.add_path(leaf, path); + } + assert_eq!(partial.root(), root); // FIXME make error + Ok(partial) + } +} + +impl From for proto::primitives::PartialSmt { + fn from(partial: PartialSmt) -> Self { + // Find all leaf digests, we need to include those, they are POIs + let mut leaves = Vec::new(); + for (key, value) in partial.entries() { + let leaf = partial.get_leaf(key).unwrap(); + leaves.push(crate::generated::primitives::SmtLeaf::from(leaf)); + } + + // Now collect the minimal set of internal nodes to be able to recalc the intermediate nodes + // forming a partial smt + let mut retained = HashMap::::new(); + for (idx, node) in partial.inner_node_indices() { + // if neither of the child keys are tracked, we cannot re-calc the inner node digest + // on-the-fly and hence need to add the node to the set to be transferred + if partial.get_value(node.left).is_err() || partial.get_value(node.left).is_err() { + retained.insert(idx, node.hash()); + continue; + } + } + let nodes = Vec::from_iter(retained.into_iter().map(|(index, digest)| { + crate::generated::primitives::InnerNode { + index: Some(crate::generated::primitives::NodeIndex::from(index)), + digest: Some(crate::generated::primitives::Digest::from(digest)), + } + })); + let root = Some(partial.root().into()); + // Remember: nodes and leaves as mutually exclusive + Self { root, nodes, leaves } + } +} + +#[cfg(test)] +mod tests { + use miden_objects::crypto::merkle::{PartialSmt, Smt}; + use pretty_assertions::assert_eq; + + use super::*; + #[test] + fn partial_smt_roundtrip() { + let mut x = Smt::new(); + + x.insert(Word::from([1_u32, 2, 3, 4]), Word::from([5_u32, 6, 7, 8])); + x.insert(Word::from([10_u32, 11, 12, 13]), Word::from([14_u32, 15, 16, 17])); + x.insert(Word::from([0x00_u32, 0xFF, 0xFF, 0xFF]), Word::from([0x00_u32; 4])); + x.insert(Word::from([0xAA_u32, 0xFF, 0xFF, 0xFF]), Word::from([0xAA_u32; 4])); + x.insert(Word::from([0xBB_u32, 0xFF, 0xFF, 0xFF]), Word::from([0xBB_u32; 4])); + x.insert(Word::from([0xCC_u32, 0xFF, 0xFF, 0xFF]), Word::from([0xCC_u32; 4])); + + let proof = x.open(&Word::from([10_u32, 11, 12, 13])); + + let mut orig = PartialSmt::new(); + orig.add_proof(proof); + let orig = orig; + + let proto = proto::primitives::PartialSmt::from(orig.clone()); + let recovered = PartialSmt::try_from(proto).unwrap(); + + assert_eq!(orig, recovered); + } +} diff --git a/crates/proto/src/generated/primitives.rs b/crates/proto/src/generated/primitives.rs index 39cb70ae2..6351ff434 100644 --- a/crates/proto/src/generated/primitives.rs +++ b/crates/proto/src/generated/primitives.rs @@ -1,4 +1,39 @@ // This file is @generated by prost-build. +/// Representation of a partial sparse merkle tree. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PartialSmt { + /// The sparse merkle tree root. + #[prost(message, optional, tag = "1")] + pub root: ::core::option::Option, + /// Set of leaves of the merkle tree. + #[prost(message, repeated, tag = "2")] + pub leaves: ::prost::alloc::vec::Vec, + /// Unique set of inner merkle tree digest, all belonging to at least one + /// merkle path of a given leave. Note that we skip all inner nodes that do + /// have two children, since we can recalculate them on-the-fly. + #[prost(message, repeated, tag = "3")] + pub nodes: ::prost::alloc::vec::Vec, +} +/// Node index representation. +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct NodeIndex { + /// The depth of the index, starting from 0 as root. + #[prost(uint32, tag = "1")] + pub depth: u32, + /// The index within a certain tree depth, left-most being zero. + #[prost(uint64, tag = "2")] + pub value: u64, +} +/// Inner node of a sparse merkle tree +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct InnerNode { + /// The position of the inner node within the tree. + #[prost(message, optional, tag = "1")] + pub index: ::core::option::Option, + /// The digest of the subtree down to the root. + #[prost(message, optional, tag = "2")] + pub digest: ::core::option::Option, +} /// Represents a single SMT leaf entry. #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct SmtLeafEntry { diff --git a/crates/proto/src/generated/rpc_store.rs b/crates/proto/src/generated/rpc_store.rs index 78ed4c6e2..d9cd260aa 100644 --- a/crates/proto/src/generated/rpc_store.rs +++ b/crates/proto/src/generated/rpc_store.rs @@ -114,20 +114,10 @@ pub mod account_proofs { #[prost(uint32, tag = "1")] pub storage_slot: u32, /// Merkle proofs of the map value as partial sparse merkle tree for compression. - /// The respective rust types is `SparseMerkleTree` and the transformation to and from - /// bytes is done via the traits `Serializable::to_bytes` and `Deserializable::from_bytes`. - #[prost(bytes = "vec", tag = "2")] - pub partial_smt: ::prost::alloc::vec::Vec, - } - /// Represents a single storage slot with the requested keys and their respective values. - #[derive(Clone, PartialEq, ::prost::Message)] - pub struct StorageSlotMapProof { - /// The storage slot index (\[0..255\]). - #[prost(uint32, tag = "1")] - pub storage_slot: u32, - /// Merkle proof of the map value - #[prost(bytes = "vec", tag = "2")] - pub smt_proof: ::prost::alloc::vec::Vec, + #[prost(message, optional, tag = "2")] + pub partial_smt: ::core::option::Option< + super::super::super::super::primitives::PartialSmt, + >, } } } diff --git a/crates/store/src/state.rs b/crates/store/src/state.rs index f6c1b07f5..7d14bf9dc 100644 --- a/crates/store/src/state.rs +++ b/crates/store/src/state.rs @@ -35,6 +35,7 @@ use miden_objects::crypto::merkle::{ MmrDelta, MmrPeaks, MmrProof, + NodeIndex, PartialMmr, PartialSmt, SmtProof, @@ -919,13 +920,6 @@ impl State { return Err(AccountError::StorageSlotNotMap(*storage_index).into()); } } - - // Only include unknown account codes - let account_code = known_code_commitments - .contains(&details.code().commitment()) - .not() - .then(|| details.code().to_bytes()); - let state_header = proto::rpc_store::account_proofs::account_proof::AccountStateHeader { header: Some(AccountHeader::from(details).into()), @@ -933,7 +927,7 @@ impl State { account_code, partial_storage_smts: Vec::from_iter(partials.into_iter().map(|(slot, partial_smt)| proto::rpc_store::account_proofs::account_proof::account_state_header::StorageSlotMapPartialSmt { storage_slot: u32::from(slot), - partial_smt: partial_smt.to_bytes(), + partial_smt: Some(proto::primitives::PartialSmt::from(partial_smt)), })), }; diff --git a/proto/proto/store/rpc.proto b/proto/proto/store/rpc.proto index ab8885b1e..d19365a3f 100644 --- a/proto/proto/store/rpc.proto +++ b/proto/proto/store/rpc.proto @@ -142,18 +142,7 @@ message AccountProofs { uint32 storage_slot = 1; // Merkle proofs of the map value as partial sparse merkle tree for compression. - // The respective rust types is `SparseMerkleTree` and the transformation to and from - // bytes is done via the traits `Serializable::to_bytes` and `Deserializable::from_bytes`. - bytes partial_smt = 2; - } - - // Represents a single storage slot with the requested keys and their respective values. - message StorageSlotMapProof { - // The storage slot index ([0..255]). - uint32 storage_slot = 1; - - // Merkle proof of the map value - bytes smt_proof = 2; + primitives.PartialSmt partial_smt = 2; } // Account header. diff --git a/proto/proto/types/primitives.proto b/proto/proto/types/primitives.proto index 28a424981..750e8fb02 100644 --- a/proto/proto/types/primitives.proto +++ b/proto/proto/types/primitives.proto @@ -4,6 +4,34 @@ package primitives; // SMT // ================================================================================================ +// Representation of a partial sparse merkle tree. +message PartialSmt { + // The sparse merkle tree root. + Digest root = 1; + // Set of leaves of the merkle tree. + repeated SmtLeaf leaves = 2; + // Unique set of inner merkle tree digest, all belonging to at least one + // merkle path of a given leave. Note that we skip all inner nodes that do + // have two children, since we can recalculate them on-the-fly. + repeated InnerNode nodes = 3; +} + +// Node index representation. +message NodeIndex { + // The depth of the index, starting from 0 as root. + uint32 depth = 1; + // The index within a certain tree depth, left-most being zero. + uint64 value = 2; +} + +// Inner node of a sparse merkle tree +message InnerNode { + // The position of the inner node within the tree. + NodeIndex index = 1; + // The digest of the subtree down to the root. + Digest digest = 2; +} + // Represents a single SMT leaf entry. message SmtLeafEntry { // The key of the entry.