diff --git a/Cargo.lock b/Cargo.lock index e3f772dbd..562f44086 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,6 +175,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.8.8" @@ -408,6 +430,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -423,6 +451,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.9.1" @@ -549,12 +583,31 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -940,6 +993,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "ecdsa" version = "0.16.9" @@ -1135,21 +1194,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1168,6 +1212,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1522,22 +1572,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.19" @@ -1814,6 +1848,28 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.34" @@ -2037,6 +2093,12 @@ version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" @@ -2077,7 +2139,7 @@ checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "miden-agglayer" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "fs-err", "miden-assembly", @@ -2098,7 +2160,7 @@ checksum = "ab2f1db9cdbd5da3eaf07fa0a8122d27b575f96b0699388c98f6c0e468cb9c1f" dependencies = [ "miden-core", "miden-utils-indexing", - "thiserror", + "thiserror 2.0.18", "winter-air", "winter-prover", ] @@ -2115,7 +2177,7 @@ dependencies = [ "miden-core", "miden-mast-package", "smallvec", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -2139,16 +2201,16 @@ dependencies = [ "rustc_version 0.4.1", "semver 1.0.27", "smallvec", - "thiserror", + "thiserror 2.0.18", ] [[package]] name = "miden-block-prover" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "miden-protocol", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -2168,7 +2230,7 @@ dependencies = [ "num-traits", "proptest", "proptest-derive", - "thiserror", + "thiserror 2.0.18", "winter-math", "winter-utils", ] @@ -2187,14 +2249,13 @@ dependencies = [ "miden-processor", "miden-utils-sync", "sha2", - "thiserror", + "thiserror 2.0.18", ] [[package]] name = "miden-crypto" -version = "0.19.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e28b6e110f339c2edc2760a8cb94863f0a055ee658a49bc90c8560eff2feef4" +version = "0.19.6" +source = "git+https://github.com/0xMiden/crypto.git?branch=bernhard-migrate-rocksdb-from-crypto#9224fe8ef0f9249d47f7904564d8f5ce6991b337" dependencies = [ "blake3", "cc", @@ -2214,11 +2275,10 @@ dependencies = [ "rand_core 0.9.5", "rand_hc", "rayon", - "rocksdb", "sha2", "sha3", "subtle", - "thiserror", + "thiserror 2.0.18", "winter-crypto", "winter-math", "winter-utils", @@ -2227,9 +2287,8 @@ dependencies = [ [[package]] name = "miden-crypto-derive" -version = "0.19.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40e95b9c7c99ed6bbf073d9e02721d812dedd2c195019c0a0e0a3dbb9cbf034" +version = "0.19.6" +source = "git+https://github.com/0xMiden/crypto.git?branch=bernhard-migrate-rocksdb-from-crypto#9224fe8ef0f9249d47f7904564d8f5ce6991b337" dependencies = [ "quote", "syn 2.0.114", @@ -2250,7 +2309,7 @@ dependencies = [ "paste", "serde", "serde_spanned 1.0.4", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -2262,6 +2321,17 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "miden-large-smt-backend-rocksdb" +version = "0.14.0" +dependencies = [ + "miden-crypto", + "miden-protocol", + "rayon", + "rocksdb", + "winter-utils", +] + [[package]] name = "miden-mast-package" version = "0.20.3" @@ -2271,7 +2341,7 @@ dependencies = [ "derive_more", "miden-assembly-syntax", "miden-core", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -2300,7 +2370,7 @@ dependencies = [ "syn 2.0.114", "terminal_size 0.3.0", "textwrap", - "thiserror", + "thiserror 2.0.18", "trybuild", "unicode-width 0.1.14", ] @@ -2392,7 +2462,7 @@ dependencies = [ "rstest", "serial_test", "tempfile", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", "tonic", @@ -2412,7 +2482,7 @@ dependencies = [ "deadpool-sync", "diesel", "miden-protocol", - "thiserror", + "thiserror 2.0.18", "tracing", ] @@ -2445,7 +2515,7 @@ dependencies = [ "rand_chacha", "rstest", "tempfile", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", "tokio-util", @@ -2472,7 +2542,7 @@ dependencies = [ "miette", "proptest", "prost", - "thiserror", + "thiserror 2.0.18", "tonic", "tonic-prost", "tonic-prost-build", @@ -2513,7 +2583,7 @@ dependencies = [ "rstest", "semver 1.0.27", "tempfile", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", "tonic", @@ -2544,6 +2614,7 @@ dependencies = [ "miden-agglayer", "miden-block-prover", "miden-crypto", + "miden-large-smt-backend-rocksdb", "miden-node-db", "miden-node-proto", "miden-node-proto-build", @@ -2560,7 +2631,7 @@ dependencies = [ "serde", "tempfile", "termtree", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", "toml 0.9.11+spec-1.1.0", @@ -2619,7 +2690,7 @@ dependencies = [ "opentelemetry_sdk", "rand", "serde", - "thiserror", + "thiserror 2.0.18", "tokio", "tonic", "tower-http", @@ -2643,7 +2714,7 @@ dependencies = [ "miden-node-utils", "miden-protocol", "miden-tx", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", "tonic", @@ -2666,7 +2737,7 @@ dependencies = [ "miden-utils-indexing", "paste", "rayon", - "thiserror", + "thiserror 2.0.18", "tokio", "tracing", "winter-prover", @@ -2675,7 +2746,7 @@ dependencies = [ [[package]] name = "miden-protocol" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "bech32", "fs-err", @@ -2696,7 +2767,7 @@ dependencies = [ "regex", "semver 1.0.27", "serde", - "thiserror", + "thiserror 2.0.18", "toml 0.9.11+spec-1.1.0", "walkdir", "winter-rand-utils", @@ -2705,7 +2776,7 @@ dependencies = [ [[package]] name = "miden-protocol-macros" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "proc-macro2", "quote", @@ -2771,7 +2842,7 @@ dependencies = [ "miden-tx", "miette", "prost", - "thiserror", + "thiserror 2.0.18", "tokio", "tonic", "tonic-prost", @@ -2783,7 +2854,7 @@ dependencies = [ [[package]] name = "miden-standards" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "fs-err", "miden-assembly", @@ -2793,14 +2864,14 @@ dependencies = [ "miden-protocol", "rand", "regex", - "thiserror", + "thiserror 2.0.18", "walkdir", ] [[package]] name = "miden-testing" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "anyhow", "itertools 0.14.0", @@ -2815,27 +2886,27 @@ dependencies = [ "miden-tx-batch-prover", "rand", "rand_chacha", - "thiserror", + "thiserror 2.0.18", "winterfell", ] [[package]] name = "miden-tx" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "miden-processor", "miden-protocol", "miden-prover", "miden-standards", "miden-verifier", - "thiserror", + "thiserror 2.0.18", ] [[package]] name = "miden-tx-batch-prover" version = "0.14.0" -source = "git+https://github.com/0xMiden/miden-base?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" +source = "git+https://github.com/0xMiden/miden-base.git?branch=next#33eabfe45ab67bdfb9f7d8907b8d50298c6b182e" dependencies = [ "miden-protocol", "miden-tx", @@ -2871,7 +2942,7 @@ version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57046b5c263b78e7fa5a6e328ca852e6319cf844faa26fbdcbb128ec555deb2a" dependencies = [ - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -2893,7 +2964,7 @@ checksum = "fe033af062937938ded511e5238db3bf8e0c1a30205850d62fb23271b3c96f85" dependencies = [ "miden-air", "miden-core", - "thiserror", + "thiserror 2.0.18", "tracing", "winter-verifier", ] @@ -2906,7 +2977,7 @@ checksum = "9d4cfab04baffdda3fb9eafa5f873604059b89a1699aa95e4f1057397a69f0b5" dependencies = [ "miden-formatting", "smallvec", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -3007,23 +3078,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe 0.1.6", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -3183,56 +3237,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "openssl" -version = "0.10.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.114", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "openssl-sys" -version = "0.9.111" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "opentelemetry" version = "0.31.0" @@ -3243,7 +3253,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror", + "thiserror 2.0.18", "tracing", ] @@ -3258,7 +3268,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "thiserror", + "thiserror 2.0.18", "tokio", "tonic", ] @@ -3288,7 +3298,7 @@ dependencies = [ "opentelemetry", "percent-encoding", "rand", - "thiserror", + "thiserror 2.0.18", "tokio", "tokio-stream", ] @@ -3667,7 +3677,7 @@ dependencies = [ "prost-reflect", "prost-types", "protox-parse", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -3679,7 +3689,7 @@ dependencies = [ "logos", "miette", "prost-types", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -3708,6 +3718,62 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.44" @@ -3854,9 +3920,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ "base64", "bytes", @@ -3868,21 +3934,21 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", + "tokio-rustls", "tower", "tower-http", "tower-service", @@ -3933,7 +3999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8a1f2315036ef6b1fbacd1972e8ee7688030b0a2121edfc2a6550febd41574d" dependencies = [ "hashbrown", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -4027,6 +4093,7 @@ version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -4042,10 +4109,10 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe 0.2.1", + "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -4054,15 +4121,44 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4151,19 +4247,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -4640,13 +4723,33 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] @@ -4720,6 +4823,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.49.0" @@ -4748,16 +4866,6 @@ dependencies = [ "syn 2.0.114", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -5016,7 +5124,7 @@ dependencies = [ "httparse", "js-sys", "pin-project", - "thiserror", + "thiserror 2.0.18", "tonic", "tower-service", "wasm-bindgen", @@ -5117,7 +5225,7 @@ checksum = "3298fe855716711a00474eceb89cc7dc254bbe67f6bc4afafdeec5f0c538771c" dependencies = [ "chrono", "smallvec", - "thiserror", + "thiserror 2.0.18", "tracing", "tracing-subscriber", ] @@ -5462,6 +5570,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi-util" version = "0.1.11" @@ -5541,6 +5658,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5586,6 +5712,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5634,6 +5775,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5652,6 +5799,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5670,6 +5823,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5700,6 +5859,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5718,6 +5883,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5736,6 +5907,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5754,6 +5931,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/Cargo.toml b/Cargo.toml index ee8ef78dd..50e8ae0ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/block-producer", "crates/db", "crates/grpc-error-macro", + "crates/large-smt-backend-rocksdb", "crates/ntx-builder", "crates/proto", "crates/remote-prover-client", @@ -45,33 +46,43 @@ debug = true [workspace.dependencies] # Workspace crates. -miden-node-block-producer = { path = "crates/block-producer", version = "0.14" } -miden-node-db = { path = "crates/db", version = "0.14" } -miden-node-grpc-error-macro = { path = "crates/grpc-error-macro", version = "0.14" } -miden-node-ntx-builder = { path = "crates/ntx-builder", version = "0.14" } -miden-node-proto = { path = "crates/proto", version = "0.14" } -miden-node-proto-build = { path = "proto", version = "0.14" } -miden-node-rpc = { path = "crates/rpc", version = "0.14" } -miden-node-store = { path = "crates/store", version = "0.14" } -miden-node-test-macro = { path = "crates/test-macro" } -miden-node-utils = { path = "crates/utils", version = "0.14" } -miden-node-validator = { path = "crates/validator", version = "0.14" } -miden-remote-prover-client = { path = "crates/remote-prover-client", version = "0.14" } +miden-large-smt-backend-rocksdb = { path = "crates/large-smt-backend-rocksdb", version = "0.14" } +miden-node-block-producer = { path = "crates/block-producer", version = "0.14" } +miden-node-db = { path = "crates/db", version = "0.14" } +miden-node-grpc-error-macro = { path = "crates/grpc-error-macro", version = "0.14" } +miden-node-ntx-builder = { path = "crates/ntx-builder", version = "0.14" } +miden-node-proto = { path = "crates/proto", version = "0.14" } +miden-node-proto-build = { path = "proto", version = "0.14" } +miden-node-rpc = { path = "crates/rpc", version = "0.14" } +miden-node-store = { path = "crates/store", version = "0.14" } +miden-node-test-macro = { path = "crates/test-macro" } +miden-node-utils = { path = "crates/utils", version = "0.14" } +miden-node-validator = { path = "crates/validator", version = "0.14" } +miden-remote-prover-client = { path = "crates/remote-prover-client", version = "0.14" } + # Temporary workaround until # is part of `rocksdb-rust` release miden-node-rocksdb-cxx-linkage-fix = { path = "crates/rocksdb-cxx-linkage-fix", version = "0.14" } # miden-base aka protocol dependencies. These should be updated in sync. -miden-block-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base" } -miden-protocol = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base" } -miden-standards = { branch = "next", git = "https://github.com/0xMiden/miden-base" } -miden-testing = { branch = "next", git = "https://github.com/0xMiden/miden-base" } -miden-tx = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base" } -miden-tx-batch-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base" } +# miden-block-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base" } +# miden-protocol = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base" } +# miden-standards = { branch = "next", git = "https://github.com/0xMiden/miden-base" } +# miden-testing = { branch = "next", git = "https://github.com/0xMiden/miden-base" } +# miden-tx = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base" } +# miden-tx-batch-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base" } # Other miden dependencies. These should align with those expected by miden-base. -miden-air = { features = ["std", "testing"], version = "0.20" } -miden-crypto = { default-features = false, version = "0.19" } +miden-air = { features = ["std", "testing"], version = "0.20" } +# miden-crypto = { default-features = false, version = "0.19.4" } +miden-block-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base.git" } +miden-protocol = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base.git" } +miden-standards = { branch = "next", git = "https://github.com/0xMiden/miden-base.git" } +miden-testing = { branch = "next", git = "https://github.com/0xMiden/miden-base.git" } +miden-tx = { branch = "next", default-features = false, git = "https://github.com/0xMiden/miden-base.git" } +miden-tx-batch-prover = { branch = "next", git = "https://github.com/0xMiden/miden-base.git" } + +miden-crypto = { version = "0.19.6" } # External dependencies anyhow = { version = "1.0" } @@ -141,3 +152,7 @@ should_panic_without_expect = "allow" # We don't care about the specific panic # Configure `cargo-typos` [workspace.metadata.typos] files.extend-exclude = ["*.svg"] # Ignore SVG files. + +[patch.crates-io] +#miden-crypto = { path = "../miden-crypto-bernhard-migrate-rocksdb-from-crypto/miden-crypto/" } +miden-crypto = { branch = "bernhard-migrate-rocksdb-from-crypto", git = "https://github.com/0xMiden/crypto.git" } diff --git a/bin/network-monitor/Cargo.toml b/bin/network-monitor/Cargo.toml index 6667a4ded..df35f8230 100644 --- a/bin/network-monitor/Cargo.toml +++ b/bin/network-monitor/Cargo.toml @@ -28,7 +28,7 @@ miden-testing = { workspace = true } miden-tx = { features = ["std"], workspace = true } rand = { version = "0.9" } rand_chacha = { version = "0.9" } -reqwest = { features = ["json"], version = "0.12" } +reqwest = { features = ["json"], version = "0.13" } serde = { features = ["derive"], version = "1.0" } serde_json = { version = "1.0" } sha2 = { version = "0.10" } diff --git a/bin/network-monitor/src/faucet.rs b/bin/network-monitor/src/faucet.rs index caeafe055..1e50a173d 100644 --- a/bin/network-monitor/src/faucet.rs +++ b/bin/network-monitor/src/faucet.rs @@ -189,14 +189,15 @@ async fn perform_faucet_test( debug!("Generated account ID: {} (length: {})", account_id, account_id.len()); // Step 1: Request PoW challenge - let pow_url = faucet_url.join("/pow")?; - let response = client - .get(pow_url) - .query(&[("account_id", &account_id), ("amount", &MINT_AMOUNT.to_string())]) - .send() - .await?; + let mut pow_url = faucet_url.join("/pow")?; + pow_url + .query_pairs_mut() + .append_pair("account_id", &account_id) + .append_pair("amount", &MINT_AMOUNT.to_string()); - let response_text = response.text().await?; + let response = client.get(pow_url).send().await?; + + let response_text: String = response.text().await?; debug!("Faucet PoW response: {}", response_text); let challenge_response: PowChallengeResponse = serde_json::from_str(&response_text) @@ -215,21 +216,18 @@ async fn perform_faucet_test( debug!("Solved PoW challenge with nonce: {}", nonce); // Step 3: Request tokens with the solution - let tokens_url = faucet_url.join("/get_tokens")?; - - let response = client - .get(tokens_url) - .query(&[ - ("account_id", account_id.as_str()), - ("is_private_note", "false"), - ("asset_amount", &MINT_AMOUNT.to_string()), - ("challenge", &challenge_response.challenge), - ("nonce", &nonce.to_string()), - ]) - .send() - .await?; - - let response_text = response.text().await?; + let mut tokens_url = faucet_url.join("/get_tokens")?; + tokens_url + .query_pairs_mut() + .append_pair("account_id", account_id.as_str()) + .append_pair("is_private_note", "false") + .append_pair("asset_amount", &MINT_AMOUNT.to_string()) + .append_pair("challenge", &challenge_response.challenge) + .append_pair("nonce", &nonce.to_string()); + + let response = client.get(tokens_url).send().await?; + + let response_text: String = response.text().await?; let tokens_response: GetTokensResponse = serde_json::from_str(&response_text) .with_context(|| format!("Failed to parse tokens response: {response_text}"))?; diff --git a/bin/remote-prover/src/server/prover.rs b/bin/remote-prover/src/server/prover.rs index 3a163a190..6ca76794e 100644 --- a/bin/remote-prover/src/server/prover.rs +++ b/bin/remote-prover/src/server/prover.rs @@ -31,8 +31,8 @@ impl Prover { } } - /// Proves a [`ProofRequest`] using the appropriate prover implementation as specified during - /// construction. + /// Proves a [`proto::ProofRequest`] using the appropriate prover implementation as specified + /// during construction. pub fn prove(&self, request: proto::ProofRequest) -> Result { match self { Prover::Transaction(prover) => prover.prove_request(request), diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 8c98e9da4..02d18e31d 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -6,7 +6,6 @@ use miden_node_store::{GenesisState, Store}; use miden_node_utils::fee::test_fee_params; use miden_node_validator::Validator; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; -use miden_protocol::testing::random_signer::RandomBlockSigner as _; use tokio::net::TcpListener; use tokio::time::sleep; use tokio::{runtime, task}; @@ -49,7 +48,7 @@ async fn block_producer_startup_is_robust_to_network_failures() { Validator { address: validator_addr, grpc_timeout, - signer: SecretKey::random(), + signer: SecretKey::new(), data_directory, } .serve() @@ -130,7 +129,7 @@ async fn start_store( store_addr: std::net::SocketAddr, data_directory: &std::path::Path, ) -> runtime::Runtime { - let genesis_state = GenesisState::new(vec![], test_fee_params(), 1, 1, SecretKey::random()); + let genesis_state = GenesisState::new(vec![], test_fee_params(), 1, 1, SecretKey::new()); Store::bootstrap(genesis_state.clone(), data_directory).expect("store should bootstrap"); let dir = data_directory.to_path_buf(); diff --git a/crates/large-smt-backend-rocksdb/Cargo.toml b/crates/large-smt-backend-rocksdb/Cargo.toml new file mode 100644 index 000000000..c7f009f92 --- /dev/null +++ b/crates/large-smt-backend-rocksdb/Cargo.toml @@ -0,0 +1,22 @@ +[package] +authors.workspace = true +description = "Large-scale Sparse Merkle Tree backed by pluggable storage - RocksDB backend" +edition.workspace = true +homepage.workspace = true +keywords = ["merkle", "miden", "node", "smt"] +license.workspace = true +name = "miden-large-smt-backend-rocksdb" +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lints] +workspace = true + +[dependencies] +miden-crypto = { features = ["concurrent", "std"], workspace = true } +miden-protocol = { features = ["std"], workspace = true } +rayon = { version = "1.10" } +rocksdb = { default-features = false, features = ["bindgen-runtime", "lz4"], version = "0.24" } +winter-utils = { version = "0.13" } diff --git a/crates/large-smt-backend-rocksdb/README.md b/crates/large-smt-backend-rocksdb/README.md new file mode 100644 index 000000000..4b612c325 --- /dev/null +++ b/crates/large-smt-backend-rocksdb/README.md @@ -0,0 +1,45 @@ +# miden-large-smt-backend-rocksdb + +Large-scale Sparse Merkle Tree backed by pluggable storage - RocksDB backend implementation. + +This crate provides `LargeSmt`, a hybrid SMT implementation that stores the top of the tree +(depths 0–23) in memory and persists the lower depths (24–64) in storage as fixed-size subtrees. +This hybrid layout scales beyond RAM while keeping common operations fast. + +## Migration Status + +This crate is the future home for `LargeSmt` and its storage backends. Currently it re-exports +types from `miden-protocol` (which re-exports from `miden-crypto`). + +The migration will be completed in phases: +1. ✅ Create this crate as a re-export layer (current state) +2. Copy the full implementation from miden-crypto to this crate +3. Update miden-crypto to remove the rocksdb feature +4. Update dependents to use this crate directly + +## Features + +- **concurrent**: Enables parallel processing with rayon (enabled by default) +- **rocksdb**: (Future) Enables RocksDB storage backend + +## Usage + +```rust +use miden_large_smt::{LargeSmt, MemoryStorage}; + +// Create an empty tree with in-memory storage +let storage = MemoryStorage::new(); +let smt = LargeSmt::new(storage).unwrap(); +``` + +## Re-exported Types + +This crate re-exports the following types from `miden-protocol`: + +- `LargeSmt` - The large-scale SMT implementation +- `LargeSmtError` - Error type for LargeSmt operations +- `MemoryStorage` - In-memory storage backend +- `SmtStorage` - Storage backend trait +- `Subtree` - Serializable subtree representation +- `StorageUpdates` / `StorageUpdateParts` - Batch update types +- Various SMT types: `Smt`, `SmtLeaf`, `SmtProof`, `LeafIndex`, etc. diff --git a/crates/large-smt-backend-rocksdb/src/lib.rs b/crates/large-smt-backend-rocksdb/src/lib.rs new file mode 100644 index 000000000..d75026a45 --- /dev/null +++ b/crates/large-smt-backend-rocksdb/src/lib.rs @@ -0,0 +1,57 @@ +//! Large-scale Sparse Merkle Tree backed by pluggable storage. +//! +//! `LargeSmt` stores the top of the tree (depths 0–23) in memory and persists the lower +//! depths (24–64) in storage as fixed-size subtrees. This hybrid layout scales beyond RAM +//! while keeping common operations fast. +//! +//! # Usage +//! +//! ```ignore +//! use miden_large_smt::{LargeSmt, MemoryStorage}; +//! +//! // Create an empty tree with in-memory storage +//! let storage = MemoryStorage::new(); +//! let smt = LargeSmt::new(storage).unwrap(); +//! ``` +//! +//! ```ignore +//! use miden_large_smt_backend_rocksdb::{LargeSmt, RocksDbConfig, RocksDbStorage}; +//! +//! let storage = RocksDbStorage::open(RocksDbConfig::new("/path/to/db")).unwrap(); +//! let smt = LargeSmt::new(storage).unwrap(); +//! ``` + +extern crate alloc; + +mod rocksdb; +// Re-export from miden-protocol. +pub use miden_protocol::crypto::merkle::smt::{ + InnerNode, + LargeSmt, + LargeSmtError, + LeafIndex, + MemoryStorage, + SMT_DEPTH, + Smt, + SmtLeaf, + SmtLeafError, + SmtProof, + SmtStorage, + StorageError, + StorageUpdateParts, + StorageUpdates, + Subtree, + SubtreeError, + SubtreeUpdate, +}; +// Also re-export commonly used types for convenience +pub use miden_protocol::{ + EMPTY_WORD, + Felt, + Word, + crypto::{ + hash::rpo::Rpo256, + merkle::{EmptySubtreeRoots, InnerNodeInfo, MerkleError, NodeIndex, SparseMerklePath}, + }, +}; +pub use rocksdb::{RocksDbConfig, RocksDbStorage}; diff --git a/crates/large-smt-backend-rocksdb/src/rocksdb.rs b/crates/large-smt-backend-rocksdb/src/rocksdb.rs new file mode 100644 index 000000000..299284f93 --- /dev/null +++ b/crates/large-smt-backend-rocksdb/src/rocksdb.rs @@ -0,0 +1,930 @@ +//! RocksDB-backed persistent storage for Sparse Merkle Trees. + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::cmp::Ordering; +use std::path::PathBuf; +use std::sync::Arc; + +use ::rocksdb::{ + BlockBasedOptions, + Cache, + ColumnFamily, + ColumnFamilyDescriptor, + DB, + DBCompactionStyle, + DBCompressionType, + DBIteratorWithThreadMode, + Error as RocksDbError, + FlushOptions, + IteratorMode, + Options, + ReadOptions, + WriteBatch, + WriteOptions, +}; +use miden_crypto::Map; +use miden_crypto::merkle::smt::MAX_LEAF_ENTRIES; +use winter_utils::{Deserializable, Serializable}; + +use crate::{ + EMPTY_WORD, + InnerNode, + NodeIndex, + SmtLeaf, + SmtLeafError, + SmtStorage, + StorageError, + StorageUpdateParts, + StorageUpdates, + Subtree, + SubtreeUpdate, + Word, +}; + +/// The name of the `RocksDB` column family used for storing SMT leaves. +const LEAVES_CF: &str = "leaves"; +/// The names of the `RocksDB` column families used for storing SMT subtrees (deep nodes). +const SUBTREE_24_CF: &str = "st24"; +const SUBTREE_32_CF: &str = "st32"; +const SUBTREE_40_CF: &str = "st40"; +const SUBTREE_48_CF: &str = "st48"; +const SUBTREE_56_CF: &str = "st56"; + +/// The name of the `RocksDB` column family used for storing metadata (e.g., root, counts). +const METADATA_CF: &str = "metadata"; +/// The name of the `RocksDB` column family used for storing level 24 hashes for fast tree +/// rebuilding. +const DEPTH_24_CF: &str = "depth24"; + +/// The key used in the `METADATA_CF` column family to store the SMT's root hash. +const ROOT_KEY: &[u8] = b"smt_root"; +/// The key used in the `METADATA_CF` column family to store the total count of non-empty leaves. +const LEAF_COUNT_KEY: &[u8] = b"leaf_count"; +/// The key used in the `METADATA_CF` column family to store the total count of key-value entries. +const ENTRY_COUNT_KEY: &[u8] = b"entry_count"; + +fn subtree_cf_opts(cache: &Cache, bloom_filter_bits: f64) -> Options { + let mut tbl = BlockBasedOptions::default(); + tbl.set_block_cache(cache); + tbl.set_bloom_filter(bloom_filter_bits, false); + tbl.set_whole_key_filtering(true); + tbl.set_pin_l0_filter_and_index_blocks_in_cache(true); + + let mut opts = Options::default(); + opts.set_block_based_table_factory(&tbl); + opts.set_write_buffer_size(128 << 20); + opts.set_max_write_buffer_number(3); + opts.set_min_write_buffer_number_to_merge(1); + opts.set_max_write_buffer_size_to_maintain(0); + opts.set_compaction_style(DBCompactionStyle::Level); + opts.set_level_zero_file_num_compaction_trigger(4); + opts.set_target_file_size_base(512 << 20); + opts.set_target_file_size_multiplier(2); + opts.set_compression_type(DBCompressionType::Lz4); + opts.set_level_zero_file_num_compaction_trigger(8); + opts +} + +trait RocksDbResultExt { + fn map_rocksdb_err(self) -> Result; +} + +impl RocksDbResultExt for Result { + fn map_rocksdb_err(self) -> Result { + self.map_err(rocksdb_error_to_storage_error) + } +} + +/// A RocksDB-backed persistent storage implementation for a Sparse Merkle Tree (SMT). +/// +/// Implements the `SmtStorage` trait, providing durable storage for SMT components +/// including leaves, subtrees (for deeper parts of the tree), and metadata like the SMT root +/// and counts. It leverages `RocksDB` column families to organize data: +/// - `LEAVES_CF` ("leaves"): Stores `SmtLeaf` data, keyed by their logical u64 index. +/// - `SUBTREE_24_CF` ("st24"): Stores serialized `Subtree` data at depth 24. +/// - `SUBTREE_32_CF` ("st32"): Stores serialized `Subtree` data at depth 32. +/// - `SUBTREE_40_CF` ("st40"): Stores serialized `Subtree` data at depth 40. +/// - `SUBTREE_48_CF` ("st48"): Stores serialized `Subtree` data at depth 48. +/// - `SUBTREE_56_CF` ("st56"): Stores serialized `Subtree` data at depth 56. +/// - `METADATA_CF` ("metadata"): Stores overall SMT metadata such as the current root hash, total +/// leaf count, and total entry count. +#[derive(Debug, Clone)] +pub struct RocksDbStorage { + db: Arc, +} + +impl RocksDbStorage { + /// Opens or creates a `RocksDB` database at the specified `path` and configures it for SMT + /// storage. + /// + /// This method sets up the necessary column families (`leaves`, `subtrees`, `metadata`) + /// and applies various `RocksDB` options for performance, such as caching, bloom filters, + /// and compaction strategies tailored for SMT workloads. + /// + /// # Errors + /// Returns `StorageError::Backend` if the database cannot be opened or configured, + /// for example, due to path issues, permissions, or `RocksDB` internal errors. + pub fn open(config: RocksDbConfig) -> Result { + // Base DB options + let mut db_opts = Options::default(); + db_opts.create_if_missing(true); + db_opts.create_missing_column_families(true); + let background_threads = i32::try_from(rayon::current_num_threads()).unwrap_or(i32::MAX); + db_opts.increase_parallelism(background_threads); + db_opts.set_max_open_files(config.max_open_files); + db_opts.set_max_background_jobs(background_threads); + db_opts.set_max_total_wal_size(512 * 1024 * 1024); + + // Shared block cache across all column families + let cache = Cache::new_lru_cache(config.cache_size); + + // Common table options for bloom filtering and cache + let mut table_opts = BlockBasedOptions::default(); + table_opts.set_block_cache(&cache); + table_opts.set_bloom_filter(10.0, false); + table_opts.set_whole_key_filtering(true); + table_opts.set_pin_l0_filter_and_index_blocks_in_cache(true); + + // Column family for leaves + let mut leaves_opts = Options::default(); + leaves_opts.set_block_based_table_factory(&table_opts); + leaves_opts.set_write_buffer_size(128 << 20); + leaves_opts.set_max_write_buffer_number(3); + leaves_opts.set_min_write_buffer_number_to_merge(1); + leaves_opts.set_max_write_buffer_size_to_maintain(0); + leaves_opts.set_compaction_style(DBCompactionStyle::Level); + leaves_opts.set_target_file_size_base(512 << 20); + leaves_opts.set_target_file_size_multiplier(2); + leaves_opts.set_compression_type(DBCompressionType::Lz4); + leaves_opts.set_level_zero_file_num_compaction_trigger(8); + + let mut depth24_opts = Options::default(); + depth24_opts.set_compression_type(DBCompressionType::Lz4); + depth24_opts.set_block_based_table_factory(&table_opts); + + let mut metadata_opts = Options::default(); + metadata_opts.set_compression_type(DBCompressionType::None); + + let cfs = vec![ + ColumnFamilyDescriptor::new(LEAVES_CF, leaves_opts), + ColumnFamilyDescriptor::new(SUBTREE_24_CF, subtree_cf_opts(&cache, 8.0)), + ColumnFamilyDescriptor::new(SUBTREE_32_CF, subtree_cf_opts(&cache, 10.0)), + ColumnFamilyDescriptor::new(SUBTREE_40_CF, subtree_cf_opts(&cache, 10.0)), + ColumnFamilyDescriptor::new(SUBTREE_48_CF, subtree_cf_opts(&cache, 12.0)), + ColumnFamilyDescriptor::new(SUBTREE_56_CF, subtree_cf_opts(&cache, 12.0)), + ColumnFamilyDescriptor::new(METADATA_CF, metadata_opts), + ColumnFamilyDescriptor::new(DEPTH_24_CF, depth24_opts), + ]; + + let db = DB::open_cf_descriptors(&db_opts, config.path, cfs).map_rocksdb_err()?; + + Ok(Self { db: Arc::new(db) }) + } + + /// Syncs the `RocksDB` database to disk. + fn sync(&self) -> Result<(), StorageError> { + let mut fopts = FlushOptions::default(); + fopts.set_wait(true); + + for name in [ + LEAVES_CF, + SUBTREE_24_CF, + SUBTREE_32_CF, + SUBTREE_40_CF, + SUBTREE_48_CF, + SUBTREE_56_CF, + METADATA_CF, + DEPTH_24_CF, + ] { + let cf = self.cf_handle(name)?; + self.db.flush_cf_opt(cf, &fopts).map_rocksdb_err()?; + } + + self.db.flush_wal(true).map_rocksdb_err()?; + Ok(()) + } + + #[inline] + fn index_db_key(index: u64) -> [u8; 8] { + index.to_be_bytes() + } + + #[inline] + fn subtree_db_key(index: NodeIndex) -> KeyBytes { + let keep = match index.depth() { + 24 => 3, + 32 => 4, + 40 => 5, + 48 => 6, + 56 => 7, + d => panic!("unsupported depth {d}"), + }; + KeyBytes::new(index.value(), keep) + } + + fn cf_handle(&self, name: &str) -> Result<&ColumnFamily, StorageError> { + self.db + .cf_handle(name) + .ok_or_else(|| StorageError::Unsupported(format!("unknown column family `{name}`"))) + } + + #[inline] + fn subtree_cf(&self, index: NodeIndex) -> &ColumnFamily { + let name = cf_for_depth(index.depth()); + self.cf_handle(name).expect("CF handle missing") + } +} + +impl SmtStorage for RocksDbStorage { + fn get_root(&self) -> Result, StorageError> { + let cf = self.cf_handle(METADATA_CF)?; + match self.db.get_cf(cf, ROOT_KEY).map_rocksdb_err()? { + Some(bytes) => { + let digest = Word::read_from_bytes(&bytes)?; + Ok(Some(digest)) + }, + None => Ok(None), + } + } + + fn set_root(&self, root: Word) -> Result<(), StorageError> { + let cf = self.cf_handle(METADATA_CF)?; + self.db.put_cf(cf, ROOT_KEY, root.to_bytes()).map_rocksdb_err()?; + Ok(()) + } + + fn leaf_count(&self) -> Result { + let cf = self.cf_handle(METADATA_CF)?; + self.db.get_cf(cf, LEAF_COUNT_KEY).map_rocksdb_err()?.map_or(Ok(0), |bytes| { + let arr: [u8; 8] = + bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { + what: "leaf count", + expected: 8, + found: bytes.len(), + })?; + Ok(usize::from_be_bytes(arr)) + }) + } + + fn entry_count(&self) -> Result { + let cf = self.cf_handle(METADATA_CF)?; + self.db.get_cf(cf, ENTRY_COUNT_KEY).map_rocksdb_err()?.map_or(Ok(0), |bytes| { + let arr: [u8; 8] = + bytes.as_slice().try_into().map_err(|_| StorageError::BadValueLen { + what: "entry count", + expected: 8, + found: bytes.len(), + })?; + Ok(usize::from_be_bytes(arr)) + }) + } + + fn insert_value( + &self, + index: u64, + key: Word, + value: Word, + ) -> Result, StorageError> { + debug_assert_ne!(value, EMPTY_WORD); + + let mut batch = WriteBatch::default(); + + let mut current_leaf_count = self.leaf_count()?; + let mut current_entry_count = self.entry_count()?; + + let leaves_cf = self.cf_handle(LEAVES_CF)?; + let db_key = Self::index_db_key(index); + + let maybe_leaf = self.get_leaf(index)?; + + let (old_value, new_leaf) = if let Some(existing_leaf) = maybe_leaf { + let (old_val, updated_leaf) = insert_leaf_entry(existing_leaf, key, value)?; + if old_val.is_none() { + current_entry_count += 1; + } + (old_val, updated_leaf) + } else { + let new_leaf = SmtLeaf::new_single(key, value); + current_leaf_count += 1; + current_entry_count += 1; + (None, new_leaf) + }; + + batch.put_cf(leaves_cf, db_key, new_leaf.to_bytes()); + + let metadata_cf = self.cf_handle(METADATA_CF)?; + batch.put_cf(metadata_cf, LEAF_COUNT_KEY, current_leaf_count.to_be_bytes()); + batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, current_entry_count.to_be_bytes()); + + self.db.write(batch).map_rocksdb_err()?; + + Ok(old_value) + } + + fn remove_value(&self, index: u64, key: Word) -> Result, StorageError> { + let maybe_leaf = self.get_leaf(index)?; + + let Some(existing_leaf) = maybe_leaf else { + return Ok(None); + }; + + let (old_value, is_empty, updated_leaf) = remove_leaf_entry(existing_leaf, key); + if old_value.is_none() { + return Ok(None); + } + + let mut batch = WriteBatch::default(); + let leaves_cf = self.cf_handle(LEAVES_CF)?; + let db_key = Self::index_db_key(index); + + let mut current_leaf_count = self.leaf_count()?; + let mut current_entry_count = self.entry_count()?; + + if is_empty { + batch.delete_cf(leaves_cf, db_key); + current_leaf_count = current_leaf_count.saturating_sub(1); + } else { + batch.put_cf(leaves_cf, db_key, updated_leaf.to_bytes()); + } + current_entry_count = current_entry_count.saturating_sub(1); + + let metadata_cf = self.cf_handle(METADATA_CF)?; + batch.put_cf(metadata_cf, LEAF_COUNT_KEY, current_leaf_count.to_be_bytes()); + batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, current_entry_count.to_be_bytes()); + + self.db.write(batch).map_rocksdb_err()?; + + Ok(old_value) + } + + fn get_leaf(&self, index: u64) -> Result, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let db_key = Self::index_db_key(index); + + match self.db.get_cf(cf, db_key).map_rocksdb_err()? { + Some(bytes) => Ok(Some(SmtLeaf::read_from_bytes(&bytes)?)), + None => Ok(None), + } + } + + fn set_leaves(&self, leaves: Map) -> Result<(), StorageError> { + let leaves_cf = self.cf_handle(LEAVES_CF)?; + let mut batch = WriteBatch::default(); + + for (index, leaf) in leaves { + let db_key = Self::index_db_key(index); + batch.put_cf(leaves_cf, db_key, leaf.to_bytes()); + } + + self.db.write(batch).map_rocksdb_err()?; + Ok(()) + } + + fn remove_leaf(&self, index: u64) -> Result, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let db_key = Self::index_db_key(index); + + let old_leaf = match self.db.get_cf(cf, db_key).map_rocksdb_err()? { + Some(bytes) => Some(SmtLeaf::read_from_bytes(&bytes)?), + None => None, + }; + + if old_leaf.is_some() { + self.db.delete_cf(cf, db_key).map_rocksdb_err()?; + } + + Ok(old_leaf) + } + + fn get_leaves(&self, indices: &[u64]) -> Result>, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let keys: Vec<[u8; 8]> = indices.iter().map(|idx| Self::index_db_key(*idx)).collect(); + let key_refs: Vec<(&ColumnFamily, &[u8])> = + keys.iter().map(|k| (cf, k.as_slice())).collect(); + + let results = self.db.multi_get_cf(key_refs); + let mut out = Vec::with_capacity(indices.len()); + for res in results { + match res.map_rocksdb_err()? { + Some(bytes) => out.push(Some(SmtLeaf::read_from_bytes(&bytes)?)), + None => out.push(None), + } + } + Ok(out) + } + + fn has_leaves(&self) -> Result { + Ok(self.leaf_count()? > 0) + } + + fn get_subtree(&self, index: NodeIndex) -> Result, StorageError> { + let cf = self.subtree_cf(index); + let key = Self::subtree_db_key(index); + + match self.db.get_cf(cf, key).map_rocksdb_err()? { + Some(bytes) => Ok(Some(Subtree::from_vec(index, &bytes)?)), + None => Ok(None), + } + } + + fn get_subtrees(&self, indices: &[NodeIndex]) -> Result>, StorageError> { + let keys: Vec<(NodeIndex, KeyBytes)> = + indices.iter().map(|&idx| (idx, Self::subtree_db_key(idx))).collect(); + + let key_refs: Vec<(&ColumnFamily, &[u8])> = + keys.iter().map(|(idx, k)| (self.subtree_cf(*idx), k.as_slice())).collect(); + + let results = self.db.multi_get_cf(key_refs); + let mut out = Vec::with_capacity(indices.len()); + + for (res, &idx) in results.into_iter().zip(indices.iter()) { + match res.map_rocksdb_err()? { + Some(bytes) => out.push(Some(Subtree::from_vec(idx, &bytes)?)), + None => out.push(None), + } + } + Ok(out) + } + + fn set_subtree(&self, subtree: &Subtree) -> Result<(), StorageError> { + let index = subtree.root_index(); + let cf = self.subtree_cf(index); + let key = Self::subtree_db_key(index); + let data = subtree.to_vec(); + + self.db.put_cf(cf, key, data).map_rocksdb_err()?; + Ok(()) + } + + fn set_subtrees(&self, subtrees: Vec) -> Result<(), StorageError> { + let mut batch = WriteBatch::default(); + + for subtree in subtrees { + let index = subtree.root_index(); + let cf = self.subtree_cf(index); + let key = Self::subtree_db_key(index); + let data = subtree.to_vec(); + batch.put_cf(cf, key, data); + } + + self.db.write(batch).map_rocksdb_err()?; + Ok(()) + } + + fn remove_subtree(&self, index: NodeIndex) -> Result<(), StorageError> { + let cf = self.subtree_cf(index); + let key = Self::subtree_db_key(index); + self.db.delete_cf(cf, key).map_rocksdb_err()?; + Ok(()) + } + + fn get_inner_node(&self, index: NodeIndex) -> Result, StorageError> { + let subtree = self.get_subtree(index)?; + Ok(subtree.and_then(|s| s.get_inner_node(index))) + } + + fn set_inner_node( + &self, + index: NodeIndex, + node: InnerNode, + ) -> Result, StorageError> { + let mut subtree = self.get_subtree(index)?.unwrap_or_else(|| Subtree::new(index)); + let old = subtree.insert_inner_node(index, node); + self.set_subtree(&subtree)?; + Ok(old) + } + + fn remove_inner_node(&self, index: NodeIndex) -> Result, StorageError> { + let Some(mut subtree) = self.get_subtree(index)? else { + return Ok(None); + }; + let old = subtree.remove_inner_node(index); + if subtree.is_empty() { + self.remove_subtree(index)?; + } else { + self.set_subtree(&subtree)?; + } + Ok(old) + } + + fn apply(&self, updates: StorageUpdates) -> Result<(), StorageError> { + let StorageUpdateParts { + new_root, + leaf_updates, + subtree_updates, + leaf_count_delta, + entry_count_delta, + } = updates.into_parts(); + + let mut batch = WriteBatch::default(); + + let leaves_cf = self.cf_handle(LEAVES_CF)?; + let metadata_cf = self.cf_handle(METADATA_CF)?; + let depth24_cf = self.cf_handle(DEPTH_24_CF)?; + + // Process leaf updates + for (index, maybe_leaf) in leaf_updates { + let key = Self::index_db_key(index); + match maybe_leaf { + Some(leaf) => batch.put_cf(leaves_cf, key, leaf.to_bytes()), + None => batch.delete_cf(leaves_cf, key), + } + } + + // Process subtree updates + let subtree_ops: Result, StorageError> = subtree_updates + .into_iter() + .map(|update| { + let (index, maybe_bytes, depth24_op) = match update { + SubtreeUpdate::Store { index, subtree } => { + let bytes = subtree.to_vec(); + let depth24_op = if index.depth() == 24 { + let hash_key = Self::index_db_key(index.value()); + let root_hash = subtree + .get_inner_node(index) + .ok_or_else(|| { + StorageError::Unsupported( + "Subtree root node not found".to_string(), + ) + })? + .hash(); + Some((hash_key, Some(root_hash.to_bytes()))) + } else { + None + }; + (index, Some(bytes), depth24_op) + }, + SubtreeUpdate::Delete { index } => { + let depth24_op = if index.depth() == 24 { + let hash_key = Self::index_db_key(index.value()); + Some((hash_key, None)) + } else { + None + }; + (index, None, depth24_op) + }, + }; + + let key = Self::subtree_db_key(index); + let subtrees_cf = self.subtree_cf(index); + + Ok((subtrees_cf, key, maybe_bytes, depth24_op)) + }) + .collect(); + + for (subtrees_cf, key, maybe_bytes, depth24_op) in subtree_ops? { + match maybe_bytes { + Some(bytes) => batch.put_cf(subtrees_cf, key, bytes), + None => batch.delete_cf(subtrees_cf, key), + } + + if let Some((hash_key, maybe_hash_bytes)) = depth24_op { + match maybe_hash_bytes { + Some(hash_bytes) => batch.put_cf(depth24_cf, hash_key, hash_bytes), + None => batch.delete_cf(depth24_cf, hash_key), + } + } + } + + if leaf_count_delta != 0 || entry_count_delta != 0 { + let current_leaf_count = self.leaf_count()?; + let current_entry_count = self.entry_count()?; + + let new_leaf_count = current_leaf_count.saturating_add_signed(leaf_count_delta); + let new_entry_count = current_entry_count.saturating_add_signed(entry_count_delta); + + batch.put_cf(metadata_cf, LEAF_COUNT_KEY, new_leaf_count.to_be_bytes()); + batch.put_cf(metadata_cf, ENTRY_COUNT_KEY, new_entry_count.to_be_bytes()); + } + + batch.put_cf(metadata_cf, ROOT_KEY, new_root.to_bytes()); + + let mut write_opts = WriteOptions::default(); + write_opts.set_sync(false); + self.db.write_opt(batch, &write_opts).map_rocksdb_err()?; + + Ok(()) + } + + fn iter_leaves(&self) -> Result + '_>, StorageError> { + let cf = self.cf_handle(LEAVES_CF)?; + let mut read_opts = ReadOptions::default(); + read_opts.set_total_order_seek(true); + let db_iter = self.db.iterator_cf_opt(cf, read_opts, IteratorMode::Start); + + Ok(Box::new(RocksDbDirectLeafIterator { iter: db_iter })) + } + + fn iter_subtrees(&self) -> Result + '_>, StorageError> { + const SUBTREE_CFS: [&str; 5] = + [SUBTREE_24_CF, SUBTREE_32_CF, SUBTREE_40_CF, SUBTREE_48_CF, SUBTREE_56_CF]; + + let mut cf_handles = Vec::new(); + for cf_name in SUBTREE_CFS { + cf_handles.push(self.cf_handle(cf_name)?); + } + + Ok(Box::new(RocksDbSubtreeIterator::new(&self.db, cf_handles))) + } + + fn get_depth24(&self) -> Result, StorageError> { + let cf = self.cf_handle(DEPTH_24_CF)?; + let iter = self.db.iterator_cf(cf, IteratorMode::Start); + let mut hashes = Vec::new(); + + for item in iter { + let (key_bytes, value_bytes) = item.map_rocksdb_err()?; + + let index = index_from_key_bytes(&key_bytes)?; + let hash = Word::read_from_bytes(&value_bytes)?; + + hashes.push((index, hash)); + } + + Ok(hashes) + } +} + +impl Drop for RocksDbStorage { + fn drop(&mut self) { + if let Err(e) = self.sync() { + panic!("failed to flush RocksDB on drop: {e}"); + } + } +} + +// ITERATORS +// -------------------------------------------------------------------------------------------- + +struct RocksDbDirectLeafIterator<'a> { + iter: DBIteratorWithThreadMode<'a, DB>, +} + +impl Iterator for RocksDbDirectLeafIterator<'_> { + type Item = (u64, SmtLeaf); + + fn next(&mut self) -> Option { + self.iter.find_map(|result| { + let (key_bytes, value_bytes) = result.ok()?; + let leaf_idx = index_from_key_bytes(&key_bytes).ok()?; + let leaf = SmtLeaf::read_from_bytes(&value_bytes).ok()?; + Some((leaf_idx, leaf)) + }) + } +} + +struct RocksDbSubtreeIterator<'a> { + db: &'a DB, + cf_handles: Vec<&'a ColumnFamily>, + current_cf_index: usize, + current_iter: Option>, +} + +impl<'a> RocksDbSubtreeIterator<'a> { + fn new(db: &'a DB, cf_handles: Vec<&'a ColumnFamily>) -> Self { + let mut iterator = Self { + db, + cf_handles, + current_cf_index: 0, + current_iter: None, + }; + iterator.advance_to_next_cf(); + iterator + } + + fn advance_to_next_cf(&mut self) { + if self.current_cf_index < self.cf_handles.len() { + let cf = self.cf_handles[self.current_cf_index]; + let mut read_opts = ReadOptions::default(); + read_opts.set_total_order_seek(true); + self.current_iter = Some(self.db.iterator_cf_opt(cf, read_opts, IteratorMode::Start)); + } else { + self.current_iter = None; + } + } + + fn try_next_from_iter( + iter: &mut DBIteratorWithThreadMode, + cf_index: usize, + ) -> Option { + iter.find_map(|result| { + let (key_bytes, value_bytes) = result.ok()?; + let depth = 24 + (cf_index * 8) as u8; + + let node_idx = subtree_root_from_key_bytes(&key_bytes, depth).ok()?; + Subtree::from_vec(node_idx, &value_bytes).ok() + }) + } +} + +impl Iterator for RocksDbSubtreeIterator<'_> { + type Item = Subtree; + + fn next(&mut self) -> Option { + loop { + let iter = self.current_iter.as_mut()?; + + if let Some(subtree) = Self::try_next_from_iter(iter, self.current_cf_index) { + return Some(subtree); + } + + self.current_cf_index += 1; + self.advance_to_next_cf(); + + self.current_iter.as_ref()?; + } + } +} + +// ROCKSDB CONFIGURATION +// -------------------------------------------------------------------------------------------- + +/// Configuration for `RocksDB` storage used by the Sparse Merkle Tree implementation. +#[derive(Debug, Clone)] +pub struct RocksDbConfig { + pub(crate) path: PathBuf, + pub(crate) cache_size: usize, + pub(crate) max_open_files: i32, +} + +impl RocksDbConfig { + /// Creates a new `RocksDbConfig` with the given database path and default settings. + pub fn new>(path: P) -> Self { + Self { + path: path.into(), + cache_size: 1 << 30, + max_open_files: 512, + } + } + + /// Sets the block cache size for `RocksDB`. + #[must_use] + pub fn with_cache_size(mut self, size: usize) -> Self { + self.cache_size = size; + self + } + + /// Sets the maximum number of files that `RocksDB` can have open simultaneously. + #[must_use] + pub fn with_max_open_files(mut self, count: i32) -> Self { + self.max_open_files = count; + self + } +} + +// SUBTREE DB KEY +// -------------------------------------------------------------------------------------------- + +#[derive(Copy, Clone, Eq, PartialEq, Debug, Hash)] +pub(crate) struct KeyBytes { + bytes: [u8; 8], + len: u8, +} + +impl KeyBytes { + #[inline] + pub fn new(value: u64, keep: usize) -> Self { + debug_assert!((3..=7).contains(&keep)); + let bytes = value.to_be_bytes(); + debug_assert!(bytes[..8 - keep].iter().all(|&b| b == 0)); + Self { bytes, len: keep as u8 } + } + + #[inline] + pub fn as_slice(&self) -> &[u8] { + &self.bytes[8 - self.len as usize..] + } +} + +impl AsRef<[u8]> for KeyBytes { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +// HELPERS +// -------------------------------------------------------------------------------------------- + +fn index_from_key_bytes(key_bytes: &[u8]) -> Result { + if key_bytes.len() != 8 { + return Err(StorageError::BadKeyLen { expected: 8, found: key_bytes.len() }); + } + let mut arr = [0u8; 8]; + arr.copy_from_slice(key_bytes); + Ok(u64::from_be_bytes(arr)) +} + +#[inline] +fn subtree_root_from_key_bytes(key_bytes: &[u8], depth: u8) -> Result { + let expected = match depth { + 24 => 3, + 32 => 4, + 40 => 5, + 48 => 6, + 56 => 7, + d => return Err(StorageError::Unsupported(format!("unsupported subtree depth {d}"))), + }; + + if key_bytes.len() != expected { + return Err(StorageError::BadSubtreeKeyLen { depth, expected, found: key_bytes.len() }); + } + let mut buf = [0u8; 8]; + buf[8 - expected..].copy_from_slice(key_bytes); + let value = u64::from_be_bytes(buf); + Ok(NodeIndex::new_unchecked(depth, value)) +} + +#[inline] +fn cf_for_depth(depth: u8) -> &'static str { + match depth { + 24 => SUBTREE_24_CF, + 32 => SUBTREE_32_CF, + 40 => SUBTREE_40_CF, + 48 => SUBTREE_48_CF, + 56 => SUBTREE_56_CF, + _ => panic!("unsupported subtree depth: {depth}"), + } +} + +fn insert_leaf_entry( + leaf: SmtLeaf, + key: Word, + value: Word, +) -> Result<(Option, SmtLeaf), StorageError> { + match leaf { + SmtLeaf::Empty(_) => Ok((None, SmtLeaf::new_single(key, value))), + SmtLeaf::Single((key_at_leaf, value_at_leaf)) => { + if key_at_leaf == key { + Ok((Some(value_at_leaf), SmtLeaf::Single((key_at_leaf, value)))) + } else { + let mut pairs = vec![(key_at_leaf, value_at_leaf), (key, value)]; + pairs.sort_by(|(key_1, _), (key_2, _)| cmp_keys(*key_1, *key_2)); + Ok((None, SmtLeaf::Multiple(pairs))) + } + }, + SmtLeaf::Multiple(mut kv_pairs) => { + match kv_pairs.binary_search_by(|kv_pair| cmp_keys(kv_pair.0, key)) { + Ok(pos) => { + let old_value = kv_pairs[pos].1; + kv_pairs[pos].1 = value; + Ok((Some(old_value), SmtLeaf::Multiple(kv_pairs))) + }, + Err(pos) => { + if kv_pairs.len() >= MAX_LEAF_ENTRIES { + return Err(SmtLeafError::TooManyLeafEntries { + actual: kv_pairs.len() + 1, + } + .into()); + } + kv_pairs.insert(pos, (key, value)); + Ok((None, SmtLeaf::Multiple(kv_pairs))) + }, + } + }, + } +} + +fn remove_leaf_entry(leaf: SmtLeaf, key: Word) -> (Option, bool, SmtLeaf) { + match leaf { + SmtLeaf::Empty(_) => (None, false, leaf), + SmtLeaf::Single((key_at_leaf, value_at_leaf)) => { + if key_at_leaf == key { + (Some(value_at_leaf), true, SmtLeaf::new_empty(key.into())) + } else { + (None, false, SmtLeaf::Single((key_at_leaf, value_at_leaf))) + } + }, + SmtLeaf::Multiple(mut kv_pairs) => { + match kv_pairs.binary_search_by(|kv_pair| cmp_keys(kv_pair.0, key)) { + Ok(pos) => { + let old_value = kv_pairs[pos].1; + kv_pairs.remove(pos); + if kv_pairs.len() == 1 { + (Some(old_value), false, SmtLeaf::Single(kv_pairs[0])) + } else { + (Some(old_value), false, SmtLeaf::Multiple(kv_pairs)) + } + }, + Err(_) => (None, false, SmtLeaf::Multiple(kv_pairs)), + } + }, + } +} + +fn cmp_keys(key_1: Word, key_2: Word) -> Ordering { + for (v1, v2) in key_1.iter().zip(key_2.iter()).rev() { + let v1 = v1.as_int(); + let v2 = v2.as_int(); + if v1 != v2 { + return v1.cmp(&v2); + } + } + + Ordering::Equal +} + +fn rocksdb_error_to_storage_error(err: RocksDbError) -> StorageError { + StorageError::Backend(Box::new(err)) +} diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index 276a4cf25..d0c11db5d 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -42,6 +42,6 @@ miden-node-store = { features = ["rocksdb"], workspace = true } miden-node-utils = { features = ["testing", "tracing-forest"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { workspace = true } -reqwest = { version = "0.12" } +reqwest = { version = "0.13" } rstest = { workspace = true } tempfile = { workspace = true } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index fd97f9195..b4dda6c95 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -34,20 +34,21 @@ miden-node-utils = { workspace = true } miden-remote-prover-client = { features = ["block-prover"], workspace = true } miden-standards = { workspace = true } # TODO remove `testing` from `miden-protocol`, required for `BlockProof::new_dummy` -miden-protocol = { features = ["std", "testing"], workspace = true } -pretty_assertions = { workspace = true } -rand = { workspace = true } -rand_chacha = { workspace = true } -serde = { features = ["derive"], version = "1" } -thiserror = { workspace = true } -tokio = { features = ["fs", "rt-multi-thread"], workspace = true } -tokio-stream = { features = ["net"], workspace = true } -toml = { version = "0.9" } -tonic = { default-features = true, workspace = true } -tonic-reflection = { workspace = true } -tower-http = { features = ["util"], workspace = true } -tracing = { workspace = true } -url = { workspace = true } +miden-large-smt-backend-rocksdb = { optional = true, workspace = true } +miden-protocol = { features = ["std", "testing"], workspace = true } +pretty_assertions = { workspace = true } +rand = { workspace = true } +rand_chacha = { workspace = true } +serde = { features = ["derive"], version = "1" } +thiserror = { workspace = true } +tokio = { features = ["fs", "rt-multi-thread"], workspace = true } +tokio-stream = { features = ["net"], workspace = true } +toml = { version = "0.9" } +tonic = { default-features = true, workspace = true } +tonic-reflection = { workspace = true } +tower-http = { features = ["util"], workspace = true } +tracing = { workspace = true } +url = { workspace = true } [build-dependencies] fs-err = { workspace = true } @@ -70,7 +71,7 @@ termtree = { version = "0.5" } [features] default = ["rocksdb"] -rocksdb = ["miden-crypto/rocksdb"] +rocksdb = ["dep:miden-large-smt-backend-rocksdb"] [[bench]] harness = false diff --git a/crates/store/benches/account_tree.rs b/crates/store/benches/account_tree.rs index 8c3f1009e..e69da7714 100644 --- a/crates/store/benches/account_tree.rs +++ b/crates/store/benches/account_tree.rs @@ -3,7 +3,7 @@ use std::path::Path; use std::sync::atomic::{AtomicUsize, Ordering}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use miden_crypto::merkle::smt::{RocksDbConfig, RocksDbStorage}; +use miden_large_smt_backend_rocksdb::{RocksDbConfig, RocksDbStorage}; use miden_node_store::AccountTreeWithHistory; use miden_protocol::Word; use miden_protocol::account::AccountId; diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 2508c9d2d..f9815190b 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -2,6 +2,8 @@ use std::collections::{BTreeMap, HashMap}; +#[cfg(feature = "rocksdb")] +use miden_large_smt_backend_rocksdb::RocksDbStorage; use miden_protocol::account::{AccountId, AccountIdPrefix}; use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::{AccountMutationSet, AccountTree, AccountWitness}; @@ -32,7 +34,7 @@ pub type InMemoryAccountTree = AccountTree>; #[cfg(feature = "rocksdb")] /// Convenience for a persistent account tree. -pub type PersistentAccountTree = AccountTree>; +pub type PersistentAccountTree = AccountTree>; // HISTORICAL ERROR TYPES // ================================================================================================ diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 2154af4c3..1499fc811 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -52,7 +52,6 @@ use miden_protocol::testing::account_id::{ ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE, ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE_2, }; -use miden_protocol::testing::random_signer::RandomBlockSigner; use miden_protocol::transaction::{ InputNoteCommitment, InputNotes, @@ -1408,8 +1407,7 @@ fn genesis_with_account_assets() { .build_existing() .unwrap(); - let genesis_state = - GenesisState::new(vec![account], test_fee_params(), 1, 0, SecretKey::random()); + let genesis_state = GenesisState::new(vec![account], test_fee_params(), 1, 0, SecretKey::new()); let genesis_block = genesis_state.into_block().unwrap(); crate::db::Db::bootstrap(":memory:".into(), &genesis_block).unwrap(); @@ -1457,8 +1455,7 @@ fn genesis_with_account_storage_map() { .build_existing() .unwrap(); - let genesis_state = - GenesisState::new(vec![account], test_fee_params(), 1, 0, SecretKey::random()); + let genesis_state = GenesisState::new(vec![account], test_fee_params(), 1, 0, SecretKey::new()); let genesis_block = genesis_state.into_block().unwrap(); crate::db::Db::bootstrap(":memory:".into(), &genesis_block).unwrap(); @@ -1504,8 +1501,7 @@ fn genesis_with_account_assets_and_storage() { .build_existing() .unwrap(); - let genesis_state = - GenesisState::new(vec![account], test_fee_params(), 1, 0, SecretKey::random()); + let genesis_state = GenesisState::new(vec![account], test_fee_params(), 1, 0, SecretKey::new()); let genesis_block = genesis_state.into_block().unwrap(); crate::db::Db::bootstrap(":memory:".into(), &genesis_block).unwrap(); @@ -1582,7 +1578,7 @@ fn genesis_with_multiple_accounts() { test_fee_params(), 1, 0, - SecretKey::random(), + SecretKey::new(), ); let genesis_block = genesis_state.into_block().unwrap(); diff --git a/crates/store/src/genesis/mod.rs b/crates/store/src/genesis/mod.rs index 5df1825d6..957df1e79 100644 --- a/crates/store/src/genesis/mod.rs +++ b/crates/store/src/genesis/mod.rs @@ -9,10 +9,10 @@ use miden_protocol::block::{ BlockNoteTree, BlockNumber, BlockProof, - BlockSigner, FeeParameters, ProvenBlock, }; +use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; use miden_protocol::crypto::merkle::mmr::{Forest, MmrPeaks}; use miden_protocol::crypto::merkle::smt::{LargeSmt, MemoryStorage, Smt}; use miden_protocol::note::Nullifier; @@ -67,7 +67,7 @@ impl GenesisState { } } -impl GenesisState { +impl GenesisState { /// Returns the block header and the account SMT pub fn into_block(self) -> Result { let accounts: Vec = self @@ -134,7 +134,7 @@ impl GenesisState { let block_proof = BlockProof::new_dummy(); - let signature = self.block_signer.sign(&header); + let signature = self.block_signer.sign(header.commitment()); // SAFETY: Header and accounts should be valid by construction. // No notes or nullifiers are created at genesis, which is consistent with the above empty // block note tree root and empty nullifier tree root. diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 3a284ceff..667cfda44 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -12,7 +12,7 @@ use miden_node_proto_build::{ }; use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; use miden_node_utils::tracing::grpc::grpc_trace_fn; -use miden_protocol::block::BlockSigner; +use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; use tokio::net::TcpListener; use tokio::task::JoinSet; use tokio_stream::wrappers::TcpListenerStream; @@ -54,8 +54,8 @@ impl Store { skip_all, err, )] - pub fn bootstrap( - genesis: GenesisState, + pub fn bootstrap( + genesis: GenesisState, data_directory: &Path, ) -> anyhow::Result<()> { let genesis = genesis diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index d237716f3..11902f4d0 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -12,6 +12,8 @@ use std::future::Future; use std::num::NonZeroUsize; use std::path::Path; +#[cfg(feature = "rocksdb")] +use miden_large_smt_backend_rocksdb::{RocksDbConfig, RocksDbStorage}; use miden_protocol::block::account_tree::{AccountTree, account_id_to_smt_key}; use miden_protocol::block::nullifier_tree::NullifierTree; use miden_protocol::block::{BlockHeader, BlockNumber, Blockchain}; @@ -22,11 +24,6 @@ use miden_protocol::{Felt, FieldElement, Word}; #[cfg(feature = "rocksdb")] use tracing::info; use tracing::instrument; -#[cfg(feature = "rocksdb")] -use { - miden_crypto::merkle::smt::RocksDbStorage, - miden_protocol::crypto::merkle::smt::RocksDbConfig, -}; use crate::COMPONENT; use crate::db::Db; diff --git a/crates/validator/src/block_validation/mod.rs b/crates/validator/src/block_validation/mod.rs index 954d043b8..69b46c4e3 100644 --- a/crates/validator/src/block_validation/mod.rs +++ b/crates/validator/src/block_validation/mod.rs @@ -1,6 +1,6 @@ use miden_node_db::{DatabaseError, Db}; -use miden_protocol::block::{BlockSigner, ProposedBlock}; -use miden_protocol::crypto::dsa::ecdsa_k256_keccak::Signature; +use miden_protocol::block::ProposedBlock; +use miden_protocol::crypto::dsa::ecdsa_k256_keccak::{SecretKey, Signature}; use miden_protocol::errors::ProposedBlockError; use miden_protocol::transaction::{TransactionHeader, TransactionId}; use tracing::{info_span, instrument}; @@ -27,9 +27,9 @@ pub enum BlockValidationError { /// Validates a block by checking that all transactions in the proposed block have been processed by /// the validator in the past. #[instrument(target = COMPONENT, skip_all, err)] -pub async fn validate_block( +pub async fn validate_block( proposed_block: ProposedBlock, - signer: &S, + signer: &SecretKey, db: &Db, ) -> Result { // Search for any proposed transactions that have not previously been validated. @@ -53,7 +53,7 @@ pub async fn validate_block( .map_err(BlockValidationError::BlockBuildingFailed)?; // Sign the header. - let signature = info_span!("sign_block").in_scope(|| signer.sign(&header)); + let signature = info_span!("sign_block").in_scope(|| signer.sign(header.commitment())); Ok(signature) } diff --git a/crates/validator/src/server/mod.rs b/crates/validator/src/server/mod.rs index 7f71161a2..02f52e1fc 100644 --- a/crates/validator/src/server/mod.rs +++ b/crates/validator/src/server/mod.rs @@ -12,7 +12,8 @@ use miden_node_utils::ErrorReport; use miden_node_utils::panic::catch_panic_layer_fn; use miden_node_utils::tracing::OpenTelemetrySpanExt; use miden_node_utils::tracing::grpc::grpc_trace_fn; -use miden_protocol::block::{BlockSigner, ProposedBlock}; +use miden_protocol::block::ProposedBlock; +use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SecretKey; use miden_protocol::transaction::{ProvenTransaction, TransactionInputs}; use miden_tx::utils::{Deserializable, Serializable}; use tokio::net::TcpListener; @@ -33,7 +34,7 @@ use crate::tx_validation::validate_transaction; /// The handle into running the gRPC validator server. /// /// Facilitates the running of the gRPC server which implements the validator API. -pub struct Validator { +pub struct Validator { /// The address of the validator component. pub address: SocketAddr, /// Server-side timeout for an individual gRPC request. @@ -42,13 +43,13 @@ pub struct Validator { pub grpc_timeout: Duration, /// The signer used to sign blocks. - pub signer: S, + pub signer: SecretKey, /// The data directory for the validator component's database files. pub data_directory: PathBuf, } -impl Validator { +impl Validator { /// Serves the validator RPC API. /// /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is @@ -99,19 +100,19 @@ impl Validator { /// The underlying implementation of the gRPC validator server. /// /// Implements the gRPC API for the validator. -struct ValidatorServer { - signer: S, +struct ValidatorServer { + signer: SecretKey, db: Arc, } -impl ValidatorServer { - fn new(signer: S, db: Db) -> Self { +impl ValidatorServer { + fn new(signer: SecretKey, db: Db) -> Self { Self { signer, db: db.into() } } } #[tonic::async_trait] -impl api_server::Api for ValidatorServer { +impl api_server::Api for ValidatorServer { /// Returns the status of the validator. async fn status( &self,