From db8554a6c7fefb90554504e4de304e66917cacec Mon Sep 17 00:00:00 2001 From: Po Date: Fri, 8 Dec 2023 17:28:34 +0100 Subject: [PATCH 01/34] chore: cont opt - dump slice to file and load --- Cargo.lock | 24 +++---- Cargo.toml | 2 +- crates/specs/Cargo.toml | 2 +- crates/specs/src/brtable.rs | 33 ++++++++- crates/specs/src/configure_table.rs | 3 +- crates/specs/src/etable.rs | 5 +- .../specs/src/external_host_call_table/mod.rs | 3 +- crates/specs/src/host_function.rs | 5 +- crates/specs/src/imtable.rs | 33 ++++++++- crates/specs/src/itable.rs | 23 ++++--- crates/specs/src/jtable.rs | 7 +- crates/specs/src/lib.rs | 54 ++++++++++----- crates/specs/src/mtable.rs | 15 ++-- crates/specs/src/state.rs | 57 +++++++++++----- crates/specs/src/step.rs | 3 +- crates/specs/src/types.rs | 3 +- crates/zkwasm/src/continuation/slice.rs | 14 ++++ crates/zkwasm/src/loader/mod.rs | 10 ++- crates/zkwasm/src/test/test_rlp_slice.rs | 68 ++++++++++++++++--- 19 files changed, 272 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c389443d7..eb42d8880 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,9 +1016,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memmap" @@ -1481,9 +1481,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -1493,9 +1493,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -1504,9 +1504,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rust-gpu-tools" @@ -1602,18 +1602,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "serde" -version = "1.0.180" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.180" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 0cb2bac1c..2af17ca4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,4 +12,4 @@ rayon = "1.8.0" wasmi = { path = "third-party/wasmi" } [profile.dev] -opt-level = 3 \ No newline at end of file +opt-level = 3 diff --git a/crates/specs/Cargo.toml b/crates/specs/Cargo.toml index d19e1db49..f07f1d826 100644 --- a/crates/specs/Cargo.toml +++ b/crates/specs/Cargo.toml @@ -20,4 +20,4 @@ rayon.workspace = true [features] default = [] cuda = ["halo2_proofs/cuda"] -continuation = [] \ No newline at end of file +continuation = [] diff --git a/crates/specs/src/brtable.rs b/crates/specs/src/brtable.rs index c6c3b3211..735f79123 100644 --- a/crates/specs/src/brtable.rs +++ b/crates/specs/src/brtable.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use serde::Deserialize; use serde::Serialize; #[derive(Serialize, Debug, Clone)] @@ -25,7 +26,7 @@ impl BrTable { } } -#[derive(Serialize, Debug, Clone)] +#[derive(Serialize, Debug, Clone, Deserialize)] pub struct ElemEntry { pub table_idx: u32, pub type_idx: u32, @@ -33,9 +34,37 @@ pub struct ElemEntry { pub func_idx: u32, } -#[derive(Debug, Default, Serialize, Clone)] +#[derive(Debug, Default, Clone)] pub struct ElemTable(BTreeMap<(u32, u32), ElemEntry>); +#[derive(Serialize, Debug, Deserialize)] +struct Entry { + key: (u32, u32), + val: ElemEntry, +} + +impl Serialize for ElemTable { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.0.iter().map(|(key, val)| Entry { + key: key.clone(), + val: val.clone(), + })) + } +} + +impl<'de> Deserialize<'de> for ElemTable { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Vec::::deserialize(deserializer) + .map(|mut v| ElemTable(v.drain(..).map(|kv| (kv.key, kv.val)).collect())) + } +} + impl ElemTable { pub fn insert(&mut self, entry: ElemEntry) { self.0.insert((entry.table_idx, entry.offset), entry); diff --git a/crates/specs/src/configure_table.rs b/crates/specs/src/configure_table.rs index b2c664fd5..ae5702f85 100644 --- a/crates/specs/src/configure_table.rs +++ b/crates/specs/src/configure_table.rs @@ -1,3 +1,4 @@ +use serde::Deserialize; use serde::Serialize; // A wasm page size is 64KB @@ -5,7 +6,7 @@ pub const WASM_BYTES_PER_PAGE: u64 = 64 * 1024 as u64; const WASM_32_MAXIMAL_PAGES_DEFAULT: u32 = 65536; -#[derive(Serialize, Debug, Clone, Copy)] +#[derive(Serialize, Debug, Clone, Copy, Deserialize)] pub struct ConfigureTable { pub init_memory_pages: u32, pub maximal_memory_pages: u32, diff --git a/crates/specs/src/etable.rs b/crates/specs/src/etable.rs index 8824e8864..1a73cdbfa 100644 --- a/crates/specs/src/etable.rs +++ b/crates/specs/src/etable.rs @@ -1,10 +1,11 @@ +use serde::Deserialize; use serde::Serialize; use super::itable::InstructionTableEntry; use crate::host_function::HostPlugin; use crate::step::StepInfo; -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct EventTableEntry { pub eid: u32, pub sp: u32, @@ -38,7 +39,7 @@ impl Iterator for RestJops { } } -#[derive(Debug, Default, Clone, Serialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct EventTable(Vec); impl EventTable { diff --git a/crates/specs/src/external_host_call_table/mod.rs b/crates/specs/src/external_host_call_table/mod.rs index a233e86dc..c92d5033a 100644 --- a/crates/specs/src/external_host_call_table/mod.rs +++ b/crates/specs/src/external_host_call_table/mod.rs @@ -1,4 +1,5 @@ use serde::ser::SerializeStruct; +use serde::Deserialize; use serde::Serialize; use crate::host_function::Signature; @@ -7,7 +8,7 @@ use crate::types::ValueType; pub mod encode; mod table; -#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, PartialOrd, Ord, Deserialize)] pub enum ExternalHostCallSignature { Argument, Return, diff --git a/crates/specs/src/host_function.rs b/crates/specs/src/host_function.rs index e5b0b8431..0f754441c 100644 --- a/crates/specs/src/host_function.rs +++ b/crates/specs/src/host_function.rs @@ -1,9 +1,10 @@ +use serde::Deserialize; use serde::Serialize; use crate::external_host_call_table::ExternalHostCallSignature; use crate::types::ValueType; -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Signature { pub params: Vec, pub return_type: Option, @@ -38,7 +39,7 @@ impl HostFunctionDesc { } } -#[derive(Clone, Debug, Serialize, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Clone, Debug, Serialize, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Deserialize)] pub enum HostPlugin { HostInput = 0, Context, diff --git a/crates/specs/src/imtable.rs b/crates/specs/src/imtable.rs index f61b00874..01d675f5c 100644 --- a/crates/specs/src/imtable.rs +++ b/crates/specs/src/imtable.rs @@ -2,9 +2,10 @@ use std::collections::BTreeMap; use crate::mtable::LocationType; use crate::mtable::VarType; +use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Debug, Clone)] +#[derive(Serialize, Debug, Clone, Deserialize)] pub struct InitMemoryTableEntry { pub ltype: LocationType, pub is_mutable: bool, @@ -15,9 +16,37 @@ pub struct InitMemoryTableEntry { pub eid: u32, } -#[derive(Serialize, Default, Debug, Clone)] +#[derive(Default, Debug, Clone)] pub struct InitMemoryTable(pub BTreeMap<(LocationType, u32), InitMemoryTableEntry>); +#[derive(Serialize, Debug, Deserialize)] +struct Entry { + key: (LocationType, u32), + val: InitMemoryTableEntry, +} + +impl Serialize for InitMemoryTable { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.0.iter().map(|(key, val)| Entry { + key: key.clone(), + val: val.clone(), + })) + } +} + +impl<'de> Deserialize<'de> for InitMemoryTable { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Vec::::deserialize(deserializer) + .map(|mut v| InitMemoryTable(v.drain(..).map(|kv| (kv.key, kv.val)).collect())) + } +} + impl InitMemoryTable { pub fn new(entries: Vec) -> Self { let mut map = BTreeMap::new(); diff --git a/crates/specs/src/itable.rs b/crates/specs/src/itable.rs index e24a2214a..93192f6e7 100644 --- a/crates/specs/src/itable.rs +++ b/crates/specs/src/itable.rs @@ -16,6 +16,7 @@ use crate::mtable::MemoryReadSize; use crate::mtable::MemoryStoreSize; use crate::types::ValueType; use num_bigint::BigUint; +use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; use strum_macros::EnumIter; @@ -106,14 +107,14 @@ impl OpcodeClassPlain { } } -#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum UnaryOp { Ctz, Clz, Popcnt, } -#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum BinOp { Add, Sub, @@ -124,7 +125,7 @@ pub enum BinOp { SignedRem, } -#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum ShiftOp { Shl, UnsignedShr, @@ -133,7 +134,7 @@ pub enum ShiftOp { Rotr, } -#[derive(Copy, Clone, Debug, Serialize, EnumIter, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, EnumIter, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum BitOp { And = 0, Or = 1, @@ -150,7 +151,7 @@ impl BitOp { } } -#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum RelOp { Eq, Ne, @@ -164,12 +165,12 @@ pub enum RelOp { UnsignedLe, } -#[derive(Copy, Clone, Debug, Serialize, EnumIter, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, EnumIter, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum TestOp { Eqz, } -#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum ConversionOp { I32WrapI64, I64ExtendI32s, @@ -181,14 +182,14 @@ pub enum ConversionOp { I64Extend32S, } -#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub struct BrTarget { pub drop: u32, pub keep: Vec, pub dst_pc: u32, } -#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum Opcode { LocalGet { vtype: VarType, @@ -585,7 +586,7 @@ impl Into for &Opcode { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct InstructionTableEntry { pub fid: u32, pub iid: u32, @@ -605,7 +606,7 @@ impl InstructionTableEntry { bn } } -#[derive(Default, Serialize, Debug, Clone)] +#[derive(Default, Serialize, Debug, Clone, Deserialize)] pub struct InstructionTable(Vec); impl InstructionTable { diff --git a/crates/specs/src/jtable.rs b/crates/specs/src/jtable.rs index f9c9179d7..0858d7f67 100644 --- a/crates/specs/src/jtable.rs +++ b/crates/specs/src/jtable.rs @@ -1,7 +1,8 @@ use super::itable::InstructionTableEntry; +use serde::Deserialize; use serde::Serialize; -#[derive(Default, Serialize, Debug, Clone)] +#[derive(Default, Serialize, Debug, Clone, Deserialize)] pub struct StaticFrameEntry { pub enable: bool, pub frame_id: u32, @@ -11,7 +12,7 @@ pub struct StaticFrameEntry { pub iid: u32, } -#[derive(Debug, Serialize, Clone)] +#[derive(Debug, Serialize, Clone, Deserialize)] pub struct JumpTableEntry { // caller eid (unique) pub eid: u32, @@ -26,7 +27,7 @@ impl JumpTableEntry { } } -#[derive(Clone, Debug, Default, Serialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct JumpTable(Vec); impl JumpTable { diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 97c65d416..796d19e7f 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -4,6 +4,8 @@ use std::collections::HashSet; use std::env; +use std::fs::File; +use std::io::BufReader; use std::io::Write; use std::path::PathBuf; use std::sync::Arc; @@ -21,6 +23,7 @@ use mtable::MTable; use mtable::MemoryTableEntry; use rayon::prelude::IntoParallelRefIterator; use rayon::prelude::ParallelIterator; +use serde::Deserialize; use serde::Serialize; use state::InitializationState; @@ -41,7 +44,7 @@ pub mod state; pub mod step; pub mod types; -#[derive(Default, Serialize, Debug, Clone)] +#[derive(Default, Serialize, Debug, Clone, Deserialize)] pub struct CompilationTable { pub itable: Arc, pub imtable: InitMemoryTable, @@ -51,7 +54,7 @@ pub struct CompilationTable { pub initialization_state: InitializationState, } -#[derive(Default, Serialize, Clone)] +#[derive(Default, Serialize, Clone, Deserialize)] pub struct ExecutionTable { pub etable: EventTable, pub jtable: Arc, @@ -128,6 +131,7 @@ impl Tables { pub fn write_json(&self, dir: Option) { fn write_file(folder: &PathBuf, filename: &str, buf: &String) { let mut folder = folder.clone(); + std::fs::create_dir_all(folder.as_path()).unwrap(); folder.push(filename); let mut fd = std::fs::File::create(folder.as_path()).unwrap(); folder.pop(); @@ -135,23 +139,37 @@ impl Tables { fd.write(buf.as_bytes()).unwrap(); } - let itable = serde_json::to_string_pretty(&self.compilation_tables.itable).unwrap(); - // let imtable = serde_json::to_string_pretty(&self.compilation_tables.imtable).unwrap(); - let etable = serde_json::to_string_pretty(&self.execution_tables.etable).unwrap(); - let external_host_call_table = serde_json::to_string_pretty( - &self - .execution_tables - .etable - .filter_external_host_call_table(), - ) - .unwrap(); - let jtable = serde_json::to_string_pretty(&self.execution_tables.jtable).unwrap(); + let compilation_table = serde_json::to_string_pretty(&self.compilation_tables).unwrap(); + let execution_table = serde_json::to_string_pretty(&self.execution_tables).unwrap(); + let post_image_table = serde_json::to_string_pretty(&self.post_image_table).unwrap(); + println!("dir path: {:?}", dir); let dir = dir.unwrap_or(env::current_dir().unwrap()); - write_file(&dir, "itable.json", &itable); - // write_file(&dir, "imtable.json", &imtable); - write_file(&dir, "etable.json", &etable); - write_file(&dir, "jtable.json", &jtable); - write_file(&dir, "external_host_table.json", &external_host_call_table); + println!("dir path unrap: {:?}", dir); + write_file(&dir, "compilation.json", &compilation_table); + write_file(&dir, "execution.json", &execution_table); + write_file(&dir, "image.json", &post_image_table); + } + + pub fn load_json(dir: PathBuf, is_last_slice: bool) -> Tables { + fn load_file(folder: &PathBuf, filename: &str) -> BufReader { + let mut folder = folder.clone(); + folder.push(filename); + let file = File::open(folder.as_path()).unwrap(); + BufReader::new(file) + } + + let compilation_tables: CompilationTable = + serde_json::from_reader(load_file(&dir, "compilation.json")).unwrap(); + let execution_tables: ExecutionTable = + serde_json::from_reader(load_file(&dir, "execution.json")).unwrap(); + let post_image_table: CompilationTable = + serde_json::from_reader(load_file(&dir, "image.json")).unwrap(); + Tables { + compilation_tables, + execution_tables, + post_image_table, + is_last_slice, + } } } diff --git a/crates/specs/src/mtable.rs b/crates/specs/src/mtable.rs index 904ade899..cd8eab88c 100644 --- a/crates/specs/src/mtable.rs +++ b/crates/specs/src/mtable.rs @@ -1,7 +1,8 @@ +use serde::Deserialize; use serde::Serialize; use strum_macros::EnumIter; -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Hash, Deserialize)] pub enum LocationType { Stack = 1, Heap = 2, @@ -21,7 +22,9 @@ impl AccessType { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, Serialize, Hash, PartialOrd, Ord)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, EnumIter, Serialize, Hash, PartialOrd, Ord, Deserialize, +)] pub enum VarType { I64 = 0, I32 = 1, @@ -36,7 +39,9 @@ impl VarType { } } -#[derive(Clone, Copy, Debug, PartialEq, EnumIter, Serialize, Hash, Eq, PartialOrd, Ord)] +#[derive( + Clone, Copy, Debug, PartialEq, EnumIter, Serialize, Hash, Eq, PartialOrd, Ord, Deserialize, +)] pub enum MemoryReadSize { U8 = 1, S8, @@ -47,7 +52,9 @@ pub enum MemoryReadSize { I64, } -#[derive(Clone, Copy, Debug, PartialEq, EnumIter, Serialize, Hash, Eq, PartialOrd, Ord)] +#[derive( + Clone, Copy, Debug, PartialEq, EnumIter, Serialize, Hash, Eq, PartialOrd, Ord, Deserialize, +)] pub enum MemoryStoreSize { Byte8 = 1, Byte16, diff --git a/crates/specs/src/state.rs b/crates/specs/src/state.rs index 11712c014..92aab9209 100644 --- a/crates/specs/src/state.rs +++ b/crates/specs/src/state.rs @@ -1,23 +1,44 @@ +use serde::Deserialize; use serde::Serialize; -#[derive(Clone, Debug, Serialize)] -pub struct InitializationState { - pub eid: T, - pub fid: T, - pub iid: T, - pub frame_id: T, - pub sp: T, - - pub host_public_inputs: T, - pub context_in_index: T, - pub context_out_index: T, - pub external_host_call_call_index: T, - - pub initial_memory_pages: T, - pub maximal_memory_pages: T, - - #[cfg(feature = "continuation")] - pub jops: T, +cfg_if::cfg_if! { + if #[cfg(feature = "continuation")] { + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct InitializationState { + pub eid: T, + pub fid: T, + pub iid: T, + pub frame_id: T, + pub sp: T, + + pub host_public_inputs: T, + pub context_in_index: T, + pub context_out_index: T, + pub external_host_call_call_index: T, + + pub initial_memory_pages: T, + pub maximal_memory_pages: T, + + pub jops: T, + } + } else { + #[derive(Clone, Debug, Serialize, Deserialize)] + pub struct InitializationState { + pub eid: T, + pub fid: T, + pub iid: T, + pub frame_id: T, + pub sp: T, + + pub host_public_inputs: T, + pub context_in_index: T, + pub context_out_index: T, + pub external_host_call_call_index: T, + + pub initial_memory_pages: T, + pub maximal_memory_pages: T + } + } } impl InitializationState { diff --git a/crates/specs/src/step.rs b/crates/specs/src/step.rs index d27f1f4e4..4b4889a8f 100644 --- a/crates/specs/src/step.rs +++ b/crates/specs/src/step.rs @@ -10,9 +10,10 @@ use crate::mtable::MemoryReadSize; use crate::mtable::MemoryStoreSize; use crate::mtable::VarType; use crate::types::ValueType; +use serde::Deserialize; use serde::Serialize; -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum StepInfo { Br { dst_pc: u32, diff --git a/crates/specs/src/types.rs b/crates/specs/src/types.rs index 826663824..a577f7c06 100644 --- a/crates/specs/src/types.rs +++ b/crates/specs/src/types.rs @@ -1,10 +1,11 @@ +use serde::Deserialize; use serde::Serialize; use crate::external_host_call_table::ExternalHostCallSignature; use crate::host_function::HostPlugin; use crate::mtable::VarType; -#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Deserialize)] pub enum ValueType { I32, I64, diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index 9ec90bd68..455ded10b 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -9,12 +9,18 @@ use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; use crate::runtime::state::UpdateCompilationTable; +use std::path::PathBuf; + pub struct Slice { table: Tables, capability: usize, } impl Slice { + pub fn new(table: Tables, capability: usize) -> Slice { + Slice {table, capability} + } + pub fn build_circuit(self) -> TestCircuit { println!( "etable entries: {}", @@ -25,6 +31,10 @@ impl Slice { builder.build_circuit(Some(self.capability)) } + + pub fn write_json(&self, dir: Option) { + self.table.write_json(dir); + } } pub struct Slices { @@ -47,6 +57,10 @@ impl Slices { } } + pub fn capability(&self) -> usize { + self.capability + } + fn pop_etable_entries(&mut self) -> Vec { self.remaining_etable_entries .drain(0..self.capability.min(self.remaining_etable_entries.len())) diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index 86778ebbe..afad86bae 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -30,6 +30,7 @@ use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; use crate::loader::err::Error; use crate::loader::err::PreCheckErr; +#[cfg(not(feature = "continuation"))] use crate::profile::Profiler; use crate::runtime::host::host_env::HostEnv; use crate::runtime::wasmi_interpreter::Execution; @@ -186,9 +187,12 @@ impl ZkWasmLoader { let compiled_module = self.compile(&env)?; let result = compiled_module.run(&mut env, wasm_runtime_io)?; - - result.tables.profile_tables(); - result.tables.write_json(None); + cfg_if::cfg_if! { + if #[cfg(not(feature = "continuation"))] { + result.tables.profile_tables(); + result.tables.write_json(None); + } + } Ok(result) } diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index 89ead8347..b30047e0e 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -3,11 +3,15 @@ use std::sync::Mutex; use crate::loader::ExecutionArg; use crate::loader::ZkWasmLoader; +use crate::runtime::ExecutionResult; +use crate::continuation::slice::Slice; +use specs::Tables; +use wasmi::RuntimeValue; use anyhow::Result; use halo2_proofs::pairing::bn256::Bn256; -fn test_slices() -> Result<()> { +fn generate_wasm_result() -> (ZkWasmLoader, ExecutionResult) { let public_inputs = vec![133]; let private_inputs: Vec = vec![ 14625441452057167097, @@ -147,15 +151,21 @@ fn test_slices() -> Result<()> { let wasm = std::fs::read("wasm/rlp.wasm").unwrap(); - let loader = ZkWasmLoader::::new(18, wasm, vec![])?; - - let execution_result = loader.run(ExecutionArg { - public_inputs, - private_inputs, - context_inputs: vec![], - context_outputs: Arc::new(Mutex::new(vec![])), - })?; + let loader = ZkWasmLoader::::new(18, wasm, vec![]).unwrap(); + + let execution_result = loader + .run(ExecutionArg { + public_inputs, + private_inputs, + context_inputs: vec![], + context_outputs: Arc::new(Mutex::new(vec![])), + }) + .unwrap(); + (loader, execution_result) +} +fn test_slices() -> Result<()> { + let (loader, execution_result) = generate_wasm_result(); let instances = execution_result .public_inputs_and_outputs .iter() @@ -179,6 +189,41 @@ fn test_slices() -> Result<()> { Ok(()) } +fn test_rpl_slice_from_file() -> Result<()> { + let (loader, execution_result) = generate_wasm_result(); + + let instances = execution_result + .public_inputs_and_outputs + .iter() + .map(|v| (*v).into()) + .collect(); + + let mut slices = loader.slice(execution_result).into_iter(); + let mut index = 0; + while let Some(slice) = slices.next() { + let mut dir = std::env::current_dir().unwrap(); + dir.push(index.to_string()); + slice.write_json(Some(dir)); + index += 1; + } + + let last_slice_index = index - 1; + while index > 0 { + index -= 1; + let mut dir = std::env::current_dir().unwrap(); + dir.push(index.to_string()); + + let table = Tables::load_json(dir.clone(), index == last_slice_index); + let slice = Slice::new(table, slices.capability()); + let circuit = slice.build_circuit(); + loader.mock_test(&circuit, &instances)?; + + std::fs::remove_dir_all(dir).unwrap(); + } + + Ok(()) +} + mod tests { use super::*; @@ -186,4 +231,9 @@ mod tests { fn test_rlp_slice_mock() { test_slices().unwrap(); } + + #[test] + fn test_rpl_slice_from_file_mock() { + test_rpl_slice_from_file().unwrap(); + } } From 2af9e83e7dd3181fb82bca5bd355ac0f78a3fbe9 Mon Sep 17 00:00:00 2001 From: Po Date: Wed, 13 Dec 2023 10:29:31 +0100 Subject: [PATCH 02/34] chore: cont-opt[2] enable dumping witness table periodically --- crates/cli/src/app_builder.rs | 36 +- crates/cli/src/command.rs | 12 + crates/cli/src/exec.rs | 32 + crates/specs/src/brtable.rs | 4 +- crates/specs/src/configure_table.rs | 2 +- crates/specs/src/etable.rs | 4 +- crates/specs/src/imtable.rs | 940 +++++++++++++++++- crates/specs/src/itable.rs | 4 +- crates/specs/src/jtable.rs | 10 +- crates/specs/src/lib.rs | 10 +- crates/specs/src/state.rs | 244 ++++- crates/specs/src/step.rs | 2 +- crates/zkwasm/Cargo.toml | 2 +- .../zkwasm/src/circuits/test_circuit/mod.rs | 2 +- .../zkwasm/src/circuits/utils/table_entry.rs | 4 +- crates/zkwasm/src/continuation/loader.rs | 7 +- crates/zkwasm/src/continuation/slice.rs | 29 +- crates/zkwasm/src/loader/mod.rs | 59 +- .../src/profile/instruction_statistic.rs | 2 +- .../runtime/host/internal_circuit_plugin.rs | 7 +- crates/zkwasm/src/runtime/mod.rs | 939 +---------------- crates/zkwasm/src/runtime/state.rs | 149 --- .../zkwasm/src/runtime/wasmi_interpreter.rs | 117 +-- crates/zkwasm/src/test/mod.rs | 9 +- crates/zkwasm/src/test/test_rlp.rs | 2 + crates/zkwasm/src/test/test_rlp_slice.rs | 72 +- .../zkwasm/src/test/test_uniform_verifier.rs | 4 +- third-party/wasmi | 2 +- 28 files changed, 1431 insertions(+), 1275 deletions(-) diff --git a/crates/cli/src/app_builder.rs b/crates/cli/src/app_builder.rs index ba042935e..e3367818b 100644 --- a/crates/cli/src/app_builder.rs +++ b/crates/cli/src/app_builder.rs @@ -11,6 +11,8 @@ use std::sync::Arc; use std::sync::Mutex; use crate::exec::exec_dry_run; +#[cfg(feature="continuation")] +use crate::exec::exec_witness_dump; use super::command::CommandBuilder; use super::exec::exec_aggregate_create_proof; @@ -77,6 +79,8 @@ pub trait AppBuilder: CommandBuilder { let app = Self::append_verify_aggregate_verify_subcommand(app); let app = Self::append_generate_solidity_verifier(app); let app = Self::append_image_checksum_subcommand(app); + #[cfg(feature="continuation")] + let app = Self::append_witness_dump_subcommand(app); app } @@ -129,13 +133,11 @@ pub trait AppBuilder: CommandBuilder { if !context_in.is_empty() || context_out_path.is_some() { warn!("All context paths are ignored when dry-run is running in service mode."); } - exec_dry_run_service(zkwasm_k, wasm_binary, phantom_functions, &listen) } else { assert!(public_inputs.len() <= Self::MAX_PUBLIC_INPUT_SIZE); let context_output = Arc::new(Mutex::new(vec![])); - exec_dry_run( zkwasm_k, wasm_binary, @@ -151,6 +153,36 @@ pub trait AppBuilder: CommandBuilder { Ok(()) } } + + #[cfg(feature="continuation")] + Some(("witness-dump", sub_matches)) => { + let public_inputs: Vec = Self::parse_single_public_arg(&sub_matches); + let private_inputs: Vec = Self::parse_single_private_arg(&sub_matches); + let context_in: Vec = Self::parse_context_in_arg(&sub_matches); + let context_out_path: Option = + Self::parse_context_out_path_arg(&sub_matches); + let context_output = Arc::new(Mutex::new(vec![])); + + let context_out = Arc::new(Mutex::new(vec![])); + + assert!(public_inputs.len() <= Self::MAX_PUBLIC_INPUT_SIZE); + + exec_witness_dump( + zkwasm_k, + wasm_binary, + phantom_functions, + &output_dir, + public_inputs, + private_inputs, + context_in, + context_output.clone(), + )?; + + write_context_output(&context_out.lock().unwrap(), context_out_path)?; + + Ok(()) + } + Some(("single-prove", sub_matches)) => { let public_inputs: Vec = Self::parse_single_public_arg(&sub_matches); let private_inputs: Vec = Self::parse_single_private_arg(&sub_matches); diff --git a/crates/cli/src/command.rs b/crates/cli/src/command.rs index 92d7466b4..f46f5c69c 100644 --- a/crates/cli/src/command.rs +++ b/crates/cli/src/command.rs @@ -27,6 +27,18 @@ pub trait CommandBuilder: ArgBuilder { app.subcommand(command) } + #[cfg(feature="continuation")] + fn append_witness_dump_subcommand(app: App) -> App { + let command = Command::new("witness-dump") + .arg(Self::single_public_arg()) + .arg(Self::single_private_arg()) + .arg(Self::dry_run_service_arg()) + .arg(Self::context_in_arg()) + .arg(Self::context_out_path_arg()); + + app.subcommand(command) + } + fn append_create_single_proof_subcommand(app: App) -> App { let command = Command::new("single-prove") .arg(Self::single_public_arg()) diff --git a/crates/cli/src/exec.rs b/crates/cli/src/exec.rs index e5d0ec7b3..eb842a6e8 100644 --- a/crates/cli/src/exec.rs +++ b/crates/cli/src/exec.rs @@ -185,6 +185,8 @@ pub fn exec_dry_run_service( private_inputs, context_inputs, context_outputs: context_outputs.clone(), + output_dir: None, + dump_table: false }) .unwrap(); println!("return value: {:?}", r); @@ -239,6 +241,32 @@ pub fn exec_dry_run( private_inputs, context_inputs, context_outputs, + output_dir: None, + dump_table: false + })?; + + Ok(()) +} + +pub fn exec_witness_dump( + zkwasm_k: u32, + wasm_binary: Vec, + phantom_functions: Vec, + output_dir: &PathBuf, + public_inputs: Vec, + private_inputs: Vec, + context_inputs: Vec, + context_outputs: Arc>>, +) -> Result<()> { + let loader = ZkWasmLoader::::new(zkwasm_k, wasm_binary, phantom_functions)?; + + loader.run(ExecutionArg { + public_inputs, + private_inputs, + context_inputs, + context_outputs, + output_dir: Some(output_dir.clone()), + dump_table: true })?; Ok(()) @@ -272,6 +300,8 @@ pub fn exec_create_proof( private_inputs, context_inputs, context_outputs, + output_dir: None, + dump_table: false })?; { @@ -370,6 +400,8 @@ pub fn exec_aggregate_create_proof( private_inputs, context_inputs, context_outputs, + output_dir: None, + dump_table: false })?; circuits.push(circuit); diff --git a/crates/specs/src/brtable.rs b/crates/specs/src/brtable.rs index 735f79123..8b447df5d 100644 --- a/crates/specs/src/brtable.rs +++ b/crates/specs/src/brtable.rs @@ -26,7 +26,7 @@ impl BrTable { } } -#[derive(Serialize, Debug, Clone, Deserialize)] +#[derive(Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct ElemEntry { pub table_idx: u32, pub type_idx: u32, @@ -34,7 +34,7 @@ pub struct ElemEntry { pub func_idx: u32, } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct ElemTable(BTreeMap<(u32, u32), ElemEntry>); #[derive(Serialize, Debug, Deserialize)] diff --git a/crates/specs/src/configure_table.rs b/crates/specs/src/configure_table.rs index ae5702f85..68e5ff689 100644 --- a/crates/specs/src/configure_table.rs +++ b/crates/specs/src/configure_table.rs @@ -6,7 +6,7 @@ pub const WASM_BYTES_PER_PAGE: u64 = 64 * 1024 as u64; const WASM_32_MAXIMAL_PAGES_DEFAULT: u32 = 65536; -#[derive(Serialize, Debug, Clone, Copy, Deserialize)] +#[derive(Serialize, Debug, Clone, Copy, Deserialize, PartialEq)] pub struct ConfigureTable { pub init_memory_pages: u32, pub maximal_memory_pages: u32, diff --git a/crates/specs/src/etable.rs b/crates/specs/src/etable.rs index 1a73cdbfa..ac9fb5538 100644 --- a/crates/specs/src/etable.rs +++ b/crates/specs/src/etable.rs @@ -5,7 +5,7 @@ use super::itable::InstructionTableEntry; use crate::host_function::HostPlugin; use crate::step::StepInfo; -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct EventTableEntry { pub eid: u32, pub sp: u32, @@ -39,7 +39,7 @@ impl Iterator for RestJops { } } -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)] pub struct EventTable(Vec); impl EventTable { diff --git a/crates/specs/src/imtable.rs b/crates/specs/src/imtable.rs index 01d675f5c..a6c718e7b 100644 --- a/crates/specs/src/imtable.rs +++ b/crates/specs/src/imtable.rs @@ -5,7 +5,14 @@ use crate::mtable::VarType; use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Debug, Clone, Deserialize)] +use crate::etable::EventTableEntry; +use crate::external_host_call_table::ExternalHostCallSignature; +use crate::mtable::AccessType; +use crate::mtable::MemoryTableEntry; +use crate::step::StepInfo; + + +#[derive(Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct InitMemoryTableEntry { pub ltype: LocationType, pub is_mutable: bool, @@ -16,7 +23,7 @@ pub struct InitMemoryTableEntry { pub eid: u32, } -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, PartialEq)] pub struct InitMemoryTable(pub BTreeMap<(LocationType, u32), InitMemoryTableEntry>); #[derive(Serialize, Debug, Deserialize)] @@ -70,3 +77,932 @@ impl InitMemoryTable { self.0.get(&(ltype, offset)) } } + + +pub fn memory_event_of_step(event: &EventTableEntry) -> Vec { + let eid = event.eid; + let sp_before_execution = event.sp; + + match &event.step_info { + StepInfo::Br { + drop, + keep, + keep_values, + .. + } => { + assert_eq!(keep.len(), keep_values.len()); + assert!(keep.len() <= 1); + + let mut sp = sp_before_execution + 1; + let mut ops = vec![]; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp + 1; + } + } + + sp += drop; + sp -= 1; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp - 1; + } + } + + ops + } + StepInfo::BrIfEqz { + condition, + drop, + keep, + keep_values, + .. + } => { + assert_eq!(keep.len(), keep_values.len()); + assert!(keep.len() <= 1); + + let mut sp = sp_before_execution + 1; + + let mut ops = vec![MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *condition as u32 as u64, + }]; + + sp = sp + 1; + + if *condition != 0 { + return ops; + } + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp + 1; + } + } + + sp += drop; + sp -= 1; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp - 1; + } + } + + ops + } + StepInfo::BrIfNez { + condition, + drop, + keep, + keep_values, + .. + } => { + assert_eq!(keep.len(), keep_values.len()); + assert!(keep.len() <= 1); + + let mut sp = sp_before_execution + 1; + + let mut ops = vec![MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *condition as u32 as u64, + }]; + + sp = sp + 1; + + if *condition == 0 { + return ops; + } + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp + 1; + } + } + + sp += drop; + sp -= 1; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp - 1; + } + } + + ops + } + StepInfo::BrTable { + index, + drop, + keep, + keep_values, + .. + } => { + assert_eq!(keep.len(), keep_values.len()); + assert!(keep.len() <= 1); + + let mut sp = sp_before_execution + 1; + + let mut ops = vec![MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *index as u32 as u64, + }]; + + sp = sp + 1; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp + 1; + } + } + + sp += drop; + sp -= 1; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp - 1; + } + } + + ops + } + StepInfo::Return { + drop, + keep, + keep_values, + } => { + assert_eq!(keep.len(), keep_values.len()); + assert!(keep.len() <= 1); + + let mut sp = sp_before_execution + 1; + let mut ops = vec![]; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp + 1; + } + } + + sp += drop; + sp -= 1; + + { + for i in 0..keep.len() { + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: keep[i].into(), + is_mutable: true, + value: keep_values[i], + }); + + sp = sp - 1; + } + } + + ops + } + StepInfo::Drop { .. } => vec![], + StepInfo::Select { + val1, + val2, + cond, + result, + vtype, + } => { + let mut sp = sp_before_execution + 1; + let mut ops = vec![]; + + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *cond, + }); + sp = sp + 1; + + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *val2, + }); + sp = sp + 1; + + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *val1, + }); + + ops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: true, + value: *result, + }); + + ops + } + StepInfo::Call { index: _ } => { + vec![] + } + StepInfo::CallIndirect { offset, .. } => { + let stack_read = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *offset as u64, + }; + + vec![stack_read] + } + StepInfo::CallHost { + args, + ret_val, + signature, + .. + } => { + let mut mops = vec![]; + let mut sp = sp_before_execution; + + for (i, (ty, val)) in signature.params.iter().zip(args.iter()).enumerate() { + mops.push(MemoryTableEntry { + eid, + offset: sp_before_execution + args.len() as u32 - i as u32, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: (*ty).into(), + is_mutable: true, + value: *val, + }); + } + + sp = sp + args.len() as u32; + + if let Some(ty) = signature.return_type { + mops.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: ty.into(), + is_mutable: true, + value: ret_val.unwrap(), + }); + } + + mops + } + StepInfo::ExternalHostCall { value, sig, .. } => match sig { + ExternalHostCallSignature::Argument => { + let stack_read = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I64, + is_mutable: true, + value: value.unwrap(), + }; + + vec![stack_read] + } + ExternalHostCallSignature::Return => { + let stack_write = MemoryTableEntry { + eid, + offset: sp_before_execution, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: VarType::I64, + is_mutable: true, + value: value.unwrap(), + }; + + vec![stack_write] + } + }, + + StepInfo::GetLocal { + vtype, + depth, + value, + } => { + let read = MemoryTableEntry { + eid, + offset: sp_before_execution + depth, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + let write = MemoryTableEntry { + eid, + offset: sp_before_execution, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + vec![read, write] + } + StepInfo::SetLocal { + vtype, + depth, + value, + } => { + let mut sp = sp_before_execution; + + let read = MemoryTableEntry { + eid, + offset: sp + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + sp += 1; + + let write = MemoryTableEntry { + eid, + offset: sp + depth, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + vec![read, write] + } + StepInfo::TeeLocal { + vtype, + depth, + value, + } => { + let read = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + let write = MemoryTableEntry { + eid, + offset: sp_before_execution + depth, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + vec![read, write] + } + + StepInfo::GetGlobal { + idx, + vtype, + is_mutable, + value, + .. + } => { + let global_get = MemoryTableEntry { + eid, + offset: *idx, + ltype: LocationType::Global, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: *is_mutable, + value: *value, + }; + + let stack_write = MemoryTableEntry { + eid, + offset: sp_before_execution, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + vec![global_get, stack_write] + } + StepInfo::SetGlobal { + idx, + vtype, + is_mutable, + value, + } => { + let stack_read = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + let global_set = MemoryTableEntry { + eid, + offset: *idx, + ltype: LocationType::Global, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: *is_mutable, + value: *value, + }; + + vec![stack_read, global_set] + } + + StepInfo::Load { + vtype, + load_size, + raw_address, + effective_address, + value, + block_value1, + block_value2, + .. + } => { + let load_address_from_stack = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *raw_address as u64, + }; + + let load_value1 = MemoryTableEntry { + eid, + offset: (*effective_address) / 8, + ltype: LocationType::Heap, + atype: AccessType::Read, + // Load u64 from address which align with 8 + vtype: VarType::I64, + is_mutable: true, + // The value will be used to lookup within imtable, hence block_value is given here + value: *block_value1, + }; + + let load_value2 = if *effective_address % 8 + load_size.byte_size() as u32 > 8 { + Some(MemoryTableEntry { + eid, + offset: effective_address / 8 + 1, + ltype: LocationType::Heap, + atype: AccessType::Read, + // Load u64 from address which align with 8 + vtype: VarType::I64, + is_mutable: true, + // The value will be used to lookup within imtable, hence block_value is given here + value: *block_value2, + }) + } else { + None + }; + + let push_value = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + vec![ + vec![load_address_from_stack, load_value1], + load_value2.map_or(vec![], |v| vec![v]), + vec![push_value], + ] + .concat() + } + StepInfo::Store { + vtype, + store_size, + raw_address, + effective_address, + value, + pre_block_value1, + updated_block_value1, + pre_block_value2, + updated_block_value2, + .. + } => { + let load_value_from_stack = MemoryTableEntry { + eid, + offset: sp_before_execution + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: *vtype, + is_mutable: true, + value: *value, + }; + + let load_address_from_stack = MemoryTableEntry { + eid, + offset: sp_before_execution + 2, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: VarType::I32, + is_mutable: true, + value: *raw_address as u64, + }; + + let load_value1 = MemoryTableEntry { + eid, + offset: effective_address / 8, + ltype: LocationType::Heap, + atype: AccessType::Read, + // Load u64 from address which align with 8 + vtype: VarType::I64, + is_mutable: true, + // The value will be used to lookup within imtable, hence block_value is given here + value: *pre_block_value1, + }; + + let write_value1 = MemoryTableEntry { + eid, + offset: effective_address / 8, + ltype: LocationType::Heap, + atype: AccessType::Write, + // Load u64 from address which align with 8 + vtype: VarType::I64, + is_mutable: true, + // The value will be used to lookup within imtable, hence block_value is given here + value: *updated_block_value1, + }; + + if *effective_address % 8 + store_size.byte_size() as u32 > 8 { + let load_value2 = MemoryTableEntry { + eid, + offset: effective_address / 8 + 1, + ltype: LocationType::Heap, + atype: AccessType::Read, + // Load u64 from address which align with 8 + vtype: VarType::I64, + is_mutable: true, + // The value will be used to lookup within imtable, hence block_value is given here + value: *pre_block_value2, + }; + + let write_value2 = MemoryTableEntry { + eid, + offset: effective_address / 8 + 1, + ltype: LocationType::Heap, + atype: AccessType::Write, + // Load u64 from address which align with 8 + vtype: VarType::I64, + is_mutable: true, + // The value will be used to lookup within imtable, hence block_value is given here + value: *updated_block_value2, + }; + vec![ + load_value_from_stack, + load_address_from_stack, + load_value1, + write_value1, + load_value2, + write_value2, + ] + } else { + vec![ + load_value_from_stack, + load_address_from_stack, + load_value1, + write_value1, + ] + } + } + + StepInfo::MemorySize => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I32, + &[], + &[event.allocated_memory_pages as u32 as u64], + ), + StepInfo::MemoryGrow { grow_size, result } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I32, + &[*grow_size as u32 as u64], + &[*result as u32 as u64], + ), + + StepInfo::I32Const { value } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I32, + &[], + &[*value as u32 as u64], + ), + StepInfo::I32BinOp { + left, right, value, .. + } + | StepInfo::I32BinShiftOp { + left, right, value, .. + } + | StepInfo::I32BinBitOp { + left, right, value, .. + } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I32, + &[*right as u32 as u64, *left as u32 as u64], + &[*value as u32 as u64], + ), + StepInfo::I32Comp { + left, right, value, .. + } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I32, + &[*right as u32 as u64, *left as u32 as u64], + &[*value as u32 as u64], + ), + + StepInfo::I64BinOp { + left, right, value, .. + } + | StepInfo::I64BinShiftOp { + left, right, value, .. + } + | StepInfo::I64BinBitOp { + left, right, value, .. + } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I64, + VarType::I64, + &[*right as u64, *left as u64], + &[*value as u64], + ), + + StepInfo::I64Const { value } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I64, + VarType::I64, + &[], + &[*value as u64], + ), + StepInfo::I64Comp { + left, right, value, .. + } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I64, + VarType::I32, + &[*right as u64, *left as u64], + &[*value as u32 as u64], + ), + StepInfo::UnaryOp { + vtype, + operand, + result, + .. + } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + *vtype, + *vtype, + &[*operand], + &[*result], + ), + + StepInfo::Test { + vtype, + value, + result, + } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + *vtype, + VarType::I32, + &[*value], + &[*result as u32 as u64], + ), + + StepInfo::I32WrapI64 { value, result } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I64, + VarType::I32, + &[*value as u64], + &[*result as u32 as u64], + ), + StepInfo::I64ExtendI32 { value, result, .. } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I64, + &[*value as u32 as u64], + &[*result as u64], + ), + StepInfo::I32SignExtendI8 { value, result } + | StepInfo::I32SignExtendI16 { value, result } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I32, + VarType::I32, + &[*value as u32 as u64], + &[*result as u32 as u64], + ), + StepInfo::I64SignExtendI8 { value, result } + | StepInfo::I64SignExtendI16 { value, result } + | StepInfo::I64SignExtendI32 { value, result } => mem_op_from_stack_only_step( + sp_before_execution, + eid, + VarType::I64, + VarType::I64, + &[*value as u64], + &[*result as u64], + ), + } +} + +pub(crate) fn mem_op_from_stack_only_step( + sp_before_execution: u32, + eid: u32, + inputs_type: VarType, + outputs_type: VarType, + pop_value: &[u64], + push_value: &[u64], +) -> Vec { + let mut mem_op = vec![]; + let mut sp = sp_before_execution; + + for i in 0..pop_value.len() { + mem_op.push(MemoryTableEntry { + eid, + offset: sp + 1, + ltype: LocationType::Stack, + atype: AccessType::Read, + vtype: inputs_type, + is_mutable: true, + value: pop_value[i], + }); + sp = sp + 1; + } + + for i in 0..push_value.len() { + mem_op.push(MemoryTableEntry { + eid, + offset: sp, + ltype: LocationType::Stack, + atype: AccessType::Write, + vtype: outputs_type, + is_mutable: true, + value: push_value[i], + }); + sp = sp - 1; + } + + mem_op +} diff --git a/crates/specs/src/itable.rs b/crates/specs/src/itable.rs index 93192f6e7..9f31134c3 100644 --- a/crates/specs/src/itable.rs +++ b/crates/specs/src/itable.rs @@ -586,7 +586,7 @@ impl Into for &Opcode { } } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct InstructionTableEntry { pub fid: u32, pub iid: u32, @@ -606,7 +606,7 @@ impl InstructionTableEntry { bn } } -#[derive(Default, Serialize, Debug, Clone, Deserialize)] +#[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct InstructionTable(Vec); impl InstructionTable { diff --git a/crates/specs/src/jtable.rs b/crates/specs/src/jtable.rs index 0858d7f67..ba4339a97 100644 --- a/crates/specs/src/jtable.rs +++ b/crates/specs/src/jtable.rs @@ -2,7 +2,7 @@ use super::itable::InstructionTableEntry; use serde::Deserialize; use serde::Serialize; -#[derive(Default, Serialize, Debug, Clone, Deserialize)] +#[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct StaticFrameEntry { pub enable: bool, pub frame_id: u32, @@ -12,7 +12,7 @@ pub struct StaticFrameEntry { pub iid: u32, } -#[derive(Debug, Serialize, Clone, Deserialize)] +#[derive(Debug, Serialize, Clone, Deserialize, PartialEq)] pub struct JumpTableEntry { // caller eid (unique) pub eid: u32, @@ -27,10 +27,14 @@ impl JumpTableEntry { } } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] pub struct JumpTable(Vec); impl JumpTable { + pub fn new(entries: Vec) -> Self{ + JumpTable(entries) + } + pub fn entries(&self) -> &Vec { &self.0 } diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 796d19e7f..48237ab85 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -44,7 +44,9 @@ pub mod state; pub mod step; pub mod types; -#[derive(Default, Serialize, Debug, Clone, Deserialize)] + + +#[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct CompilationTable { pub itable: Arc, pub imtable: InitMemoryTable, @@ -54,13 +56,13 @@ pub struct CompilationTable { pub initialization_state: InitializationState, } -#[derive(Default, Serialize, Clone, Deserialize)] +#[derive(Debug, Default, Serialize, Clone, Deserialize, PartialEq)] pub struct ExecutionTable { pub etable: EventTable, pub jtable: Arc, } -#[derive(Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct Tables { pub compilation_tables: CompilationTable, pub execution_tables: ExecutionTable, @@ -143,9 +145,7 @@ impl Tables { let execution_table = serde_json::to_string_pretty(&self.execution_tables).unwrap(); let post_image_table = serde_json::to_string_pretty(&self.post_image_table).unwrap(); - println!("dir path: {:?}", dir); let dir = dir.unwrap_or(env::current_dir().unwrap()); - println!("dir path unrap: {:?}", dir); write_file(&dir, "compilation.json", &compilation_table); write_file(&dir, "execution.json", &execution_table); write_file(&dir, "image.json", &post_image_table); diff --git a/crates/specs/src/state.rs b/crates/specs/src/state.rs index 92aab9209..2797e47bb 100644 --- a/crates/specs/src/state.rs +++ b/crates/specs/src/state.rs @@ -1,44 +1,73 @@ use serde::Deserialize; use serde::Serialize; -cfg_if::cfg_if! { - if #[cfg(feature = "continuation")] { - #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct InitializationState { - pub eid: T, - pub fid: T, - pub iid: T, - pub frame_id: T, - pub sp: T, - - pub host_public_inputs: T, - pub context_in_index: T, - pub context_out_index: T, - pub external_host_call_call_index: T, - - pub initial_memory_pages: T, - pub maximal_memory_pages: T, - - pub jops: T, - } - } else { - #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct InitializationState { - pub eid: T, - pub fid: T, - pub iid: T, - pub frame_id: T, - pub sp: T, - - pub host_public_inputs: T, - pub context_in_index: T, - pub context_out_index: T, - pub external_host_call_call_index: T, - - pub initial_memory_pages: T, - pub maximal_memory_pages: T - } - } +use crate::CompilationTable; +use crate::etable::EventTableEntry; +use crate::host_function::HostPlugin; +use crate::imtable::InitMemoryTable; +use crate::imtable::InitMemoryTableEntry; +use crate::imtable::memory_event_of_step; +use crate::mtable::AccessType; +use crate::step::StepInfo; +use crate::itable::Opcode; + +// cfg_if::cfg_if! { +// if #[cfg(feature = "continuation")] { +// #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +// pub struct InitializationState { +// pub eid: T, +// pub fid: T, +// pub iid: T, +// pub frame_id: T, +// pub sp: T, + +// pub host_public_inputs: T, +// pub context_in_index: T, +// pub context_out_index: T, +// pub external_host_call_call_index: T, + +// pub initial_memory_pages: T, +// pub maximal_memory_pages: T, + +// pub jops: T, +// } +// } else { +// #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +// pub struct InitializationState { +// pub eid: T, +// pub fid: T, +// pub iid: T, +// pub frame_id: T, +// pub sp: T, + +// pub host_public_inputs: T, +// pub context_in_index: T, +// pub context_out_index: T, +// pub external_host_call_call_index: T, + +// pub initial_memory_pages: T, +// pub maximal_memory_pages: T, +// } +// } +// } + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct InitializationState { + pub eid: T, + pub fid: T, + pub iid: T, + pub frame_id: T, + pub sp: T, + + pub host_public_inputs: T, + pub context_in_index: T, + pub context_out_index: T, + pub external_host_call_call_index: T, + + pub initial_memory_pages: T, + pub maximal_memory_pages: T, + #[cfg(feature = "continuation")] + pub jops: T, } impl InitializationState { @@ -123,3 +152,142 @@ impl InitializationState { } } } + + +pub trait UpdateCompilationTable { + fn update_init_memory_table(&self, execution_table: &Vec) -> InitMemoryTable; + + fn update_initialization_state( + &self, + execution_table: &Vec, + is_last_slice: bool, + ) -> InitializationState; +} + +impl UpdateCompilationTable for CompilationTable { + fn update_init_memory_table(&self, execution_table: &Vec) -> InitMemoryTable { + // First insert origin imtable entries which may be overwritten. + let mut map = self.imtable.entries().clone(); + + let mut it = execution_table.iter(); + while let Some(etable_entry) = it.next() { + let memory_writing_entires = memory_event_of_step(etable_entry) + .into_iter() + .filter(|entry| entry.atype == AccessType::Write); + + for mentry in memory_writing_entires { + map.insert( + (mentry.ltype, mentry.offset), + InitMemoryTableEntry { + ltype: mentry.ltype, + is_mutable: mentry.is_mutable, + offset: mentry.offset, + vtype: mentry.vtype, + value: mentry.value, + eid: etable_entry.eid, + }, + ); + } + } + + InitMemoryTable(map) + } + + fn update_initialization_state( + &self, + execution_table: &Vec, + is_last_slice: bool, + ) -> InitializationState { + let mut host_public_inputs = self.initialization_state.host_public_inputs; + let mut context_in_index = self.initialization_state.context_in_index; + let mut context_out_index = self.initialization_state.context_out_index; + let mut external_host_call_call_index = + self.initialization_state.external_host_call_call_index; + + #[cfg(feature = "continuation")] + let mut jops = self.initialization_state.jops; + for entry in execution_table { + match &entry.step_info { + // TODO: fix hard code + StepInfo::CallHost { + function_name, + args, + op_index_in_plugin, + .. + } => { + if *op_index_in_plugin == HostPlugin::HostInput as usize { + if function_name == "wasm_input" && args[0] != 0 + || function_name == "wasm_output" + { + host_public_inputs += 1; + } + } else if *op_index_in_plugin == HostPlugin::Context as usize { + if function_name == "wasm_read_context" { + context_in_index += 1; + } else if function_name == "wasm_write_context" { + context_out_index += 1; + } + } + } + StepInfo::ExternalHostCall { .. } => external_host_call_call_index += 1, + StepInfo::Call { .. } | StepInfo::CallIndirect { .. } | StepInfo::Return { .. } => { + #[cfg(feature = "continuation")] + { + jops += 1; + } + } + _ => (), + } + } + + let last_entry = execution_table.last().unwrap(); + + let post_initialization_state = if is_last_slice { + InitializationState { + eid: last_entry.eid + 1, + fid: 0, + iid: 0, + frame_id: 0, + // TODO: why not constant 4095? + sp: last_entry.sp + + if let Opcode::Return { drop, .. } = last_entry.inst.opcode { + drop + } else { + 0 + }, + + host_public_inputs, + context_in_index, + context_out_index, + external_host_call_call_index, + + initial_memory_pages: last_entry.allocated_memory_pages, + maximal_memory_pages: self.configure_table.maximal_memory_pages, + + #[cfg(feature = "continuation")] + jops, + } + } else { + InitializationState { + eid: last_entry.eid, + fid: last_entry.inst.fid, + iid: last_entry.inst.iid, + frame_id: last_entry.last_jump_eid, + sp: last_entry.sp, + + host_public_inputs, + context_in_index, + context_out_index, + external_host_call_call_index, + + initial_memory_pages: last_entry.allocated_memory_pages, + maximal_memory_pages: self.configure_table.maximal_memory_pages, + + #[cfg(feature = "continuation")] + jops, + } + }; + + post_initialization_state + } +} diff --git a/crates/specs/src/step.rs b/crates/specs/src/step.rs index 4b4889a8f..9cc9bb98e 100644 --- a/crates/specs/src/step.rs +++ b/crates/specs/src/step.rs @@ -13,7 +13,7 @@ use crate::types::ValueType; use serde::Deserialize; use serde::Serialize; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum StepInfo { Br { dst_pc: u32, diff --git a/crates/zkwasm/Cargo.toml b/crates/zkwasm/Cargo.toml index f07d0f752..8289c359d 100644 --- a/crates/zkwasm/Cargo.toml +++ b/crates/zkwasm/Cargo.toml @@ -35,4 +35,4 @@ rusty-fork = "0.3.0" [features] default = [] cuda = ["halo2_proofs/cuda", "specs/cuda"] -continuation = ["specs/continuation"] \ No newline at end of file +continuation = ["specs/continuation", "wasmi/continuation"] \ No newline at end of file diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index 192640a43..86bab146b 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -49,7 +49,7 @@ use crate::foreign::foreign_table_enable_lines; use crate::foreign::wasm_input_helper::circuits::WasmInputHelperTableConfig; use crate::foreign::wasm_input_helper::circuits::WASM_INPUT_FOREIGN_TABLE_KEY; use crate::foreign::ForeignTableConfig; -use crate::runtime::memory_event_of_step; +use specs::imtable::memory_event_of_step; use super::config::zkwasm_k; use super::image_table::ImageTableConfig; diff --git a/crates/zkwasm/src/circuits/utils/table_entry.rs b/crates/zkwasm/src/circuits/utils/table_entry.rs index c081f92e5..b19d1b445 100644 --- a/crates/zkwasm/src/circuits/utils/table_entry.rs +++ b/crates/zkwasm/src/circuits/utils/table_entry.rs @@ -5,6 +5,8 @@ use specs::mtable::AccessType; use specs::mtable::LocationType; use specs::mtable::MTable; use specs::mtable::MemoryTableEntry; +use specs::imtable::memory_event_of_step; + use std::cmp::Ordering; use std::collections::BTreeMap; use std::env; @@ -12,7 +14,7 @@ use std::io::Write; use std::path::PathBuf; use crate::circuits::config::zkwasm_k; -use crate::runtime::memory_event_of_step; + #[derive(Clone, Debug, Serialize)] pub(in crate::circuits) struct MemoryWritingEntry { diff --git a/crates/zkwasm/src/continuation/loader.rs b/crates/zkwasm/src/continuation/loader.rs index d98848813..50097cc7e 100644 --- a/crates/zkwasm/src/continuation/loader.rs +++ b/crates/zkwasm/src/continuation/loader.rs @@ -1,18 +1,13 @@ use halo2_proofs::arithmetic::MultiMillerLoop; use wasmi::RuntimeValue; -use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::loader::ZkWasmLoader; use crate::runtime::ExecutionResult; use super::slice::Slices; impl ZkWasmLoader { - fn compute_slice_capability(&self) -> usize { - ((1 << self.k) - 200) / EVENT_TABLE_ENTRY_ROWS as usize - } - pub fn slice(&self, execution_result: ExecutionResult) -> Slices { - Slices::new(execution_result.tables, self.compute_slice_capability()) + Slices::new(execution_result.tables.unwrap(), self.compute_slice_capability()) } } diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index 455ded10b..e9281da5f 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -1,16 +1,20 @@ use halo2_proofs::arithmetic::FieldExt; use specs::etable::EventTable; use specs::etable::EventTableEntry; +use specs::jtable::JumpTable; +use specs::jtable::JumpTableEntry; +use specs::state::UpdateCompilationTable; use specs::CompilationTable; use specs::ExecutionTable; use specs::Tables; use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; -use crate::runtime::state::UpdateCompilationTable; use std::path::PathBuf; +use std::sync::Arc; +#[derive(Debug, PartialEq)] pub struct Slice { table: Tables, capability: usize, @@ -18,9 +22,9 @@ pub struct Slice { impl Slice { pub fn new(table: Tables, capability: usize) -> Slice { - Slice {table, capability} + Slice { table, capability } } - + pub fn build_circuit(self) -> TestCircuit { println!( "etable entries: {}", @@ -66,6 +70,10 @@ impl Slices { .drain(0..self.capability.min(self.remaining_etable_entries.len())) .collect::>() } + + pub fn num_slices(&self) -> usize { + (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64).ceil() as usize + } } impl Iterator for Slices { @@ -109,9 +117,22 @@ impl Iterator for Slices { (updated_init_memory_table, updated_post_initialization_state) }; + let latest_eid = etable_entries.last().unwrap().eid; + + // only etable related jtable are needed + let jtable_entries: Vec = self + .origin_table + .execution_tables + .jtable + .entries() + .iter() + .filter(|e| e.eid <= latest_eid) + .map(|e| e.clone()) + .collect(); + let execution_tables = ExecutionTable { etable: EventTable::new(etable_entries), - jtable: self.origin_table.execution_tables.jtable.clone(), + jtable: Arc::new(JumpTable::new(jtable_entries)), }; let post_image_table = CompilationTable { diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index afad86bae..d369abc4a 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -1,4 +1,5 @@ use std::marker::PhantomData; +use std::path::PathBuf; use std::sync::Arc; use std::sync::Mutex; @@ -16,6 +17,7 @@ use halo2aggregator_s::circuits::utils::load_or_create_proof; use halo2aggregator_s::circuits::utils::TranscriptHash; use halo2aggregator_s::transcript::poseidon::PoseidonRead; use specs::Tables; +use wasmi::ENTRY; use wasmi::tracer::Tracer; use wasmi::ImportsBuilder; use wasmi::NotStartedModuleRef; @@ -25,12 +27,14 @@ use crate::checksum::CompilationTableWithParams; use crate::checksum::ImageCheckSum; use crate::circuits::config::init_zkwasm_runtime; use crate::circuits::config::set_zkwasm_k; +use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::circuits::image_table::IMAGE_COL_NAME; use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; +#[cfg(feature="continuation")] +use crate::continuation::slice::Slice; use crate::loader::err::Error; use crate::loader::err::PreCheckErr; -#[cfg(not(feature = "continuation"))] use crate::profile::Profiler; use crate::runtime::host::host_env::HostEnv; use crate::runtime::wasmi_interpreter::Execution; @@ -41,7 +45,7 @@ use anyhow::anyhow; mod err; -const ENTRY: &str = "zkmain"; +pub type CallbackType = Box; pub struct ExecutionArg { /// Public inputs for `wasm_input(1)` @@ -52,6 +56,9 @@ pub struct ExecutionArg { pub context_inputs: Vec, /// Context outputs for `wasm_write_context()` pub context_outputs: Arc>>, + pub output_dir: Option, + /// enable dump table for continuation + pub dump_table: bool } pub struct ExecutionReturn { @@ -66,6 +73,10 @@ pub struct ZkWasmLoader { } impl ZkWasmLoader { + pub fn compute_slice_capability(&self) -> usize { + ((1 << self.k) - 200) / EVENT_TABLE_ENTRY_ROWS as usize + } + fn precheck(&self) -> Result<()> { fn check_zkmain_exists(module: &wasmi::Module) -> Result<()> { use parity_wasm::elements::Internal; @@ -96,7 +107,7 @@ impl ZkWasmLoader { Ok(()) } - fn compile(&self, env: &HostEnv) -> Result, Tracer>> { + fn compile(&self, env: &HostEnv, callback: Option) -> Result, Tracer>> { let imports = ImportsBuilder::new().with_resolver("env", env); WasmInterpreter::compile( @@ -105,6 +116,8 @@ impl ZkWasmLoader { &env.function_description_table(), ENTRY, &self.phantom_functions, + callback, + self.compute_slice_capability() ) } @@ -151,7 +164,7 @@ impl ZkWasmLoader { vec![], Arc::new(Mutex::new(vec![])), ); - let compiled = self.compile(&env)?; + let compiled = self.compile(&env, None::)?; let table_with_params = CompilationTableWithParams { table: &compiled.tables, @@ -170,9 +183,7 @@ impl ZkWasmLoader { arg.context_inputs, arg.context_outputs, ); - - let compiled_module = self.compile(&env)?; - + let compiled_module = self.compile(&env, None::)?; compiled_module.dry_run(&mut env) } @@ -184,17 +195,33 @@ impl ZkWasmLoader { arg.context_outputs, ); - let compiled_module = self.compile(&env)?; - - let result = compiled_module.run(&mut env, wasm_runtime_io)?; - cfg_if::cfg_if! { - if #[cfg(not(feature = "continuation"))] { - result.tables.profile_tables(); - result.tables.write_json(None); + let output_dir = arg.output_dir.unwrap_or_else(|| std::env::current_dir().unwrap()); + if arg.dump_table { + let mut _index = 0; + let callback = move |_table, _capability | { + cfg_if::cfg_if! { + if #[cfg(feature = "continuation")] { + let slice = Slice::new(_table, _capability); + let mut dir = output_dir.clone(); + dir.push(_index.to_string()); + slice.write_json(Some(dir)); + _index += 1; + } + } + }; + let compiled_module = self.compile(&env, Some(callback))?; + let result = compiled_module.run(&mut env, wasm_runtime_io)?; + Ok(result) + } else { + let compiled_module = self.compile(&env, None::)?; + let result = compiled_module.run(&mut env, wasm_runtime_io)?; + if let Some(tables) = &result.tables { + tables.profile_tables(); + tables.write_json(Some(output_dir)); } + Ok(result) } - Ok(result) } pub fn circuit_with_witness( @@ -211,7 +238,7 @@ impl ZkWasmLoader { .collect(); let builder = ZkWasmCircuitBuilder { - tables: execution_result.tables, + tables: execution_result.tables.unwrap(), }; println!("output:"); diff --git a/crates/zkwasm/src/profile/instruction_statistic.rs b/crates/zkwasm/src/profile/instruction_statistic.rs index 59a1a5418..7739d21ba 100644 --- a/crates/zkwasm/src/profile/instruction_statistic.rs +++ b/crates/zkwasm/src/profile/instruction_statistic.rs @@ -6,7 +6,7 @@ use specs::mtable::MemoryTableEntry; use std::collections::BTreeMap; use std::fmt::Debug; -use crate::runtime::memory_event_of_step; +use specs::imtable::memory_event_of_step; pub trait InstructionStatistic { fn profile_instruction(&self); diff --git a/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs b/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs index 344cb2a78..08528d523 100644 --- a/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs +++ b/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs @@ -1,6 +1,7 @@ use specs::host_function::HostPlugin; use specs::host_function::Signature; use std::cell::RefCell; +use std::collections::BTreeMap; use std::collections::HashMap; use std::rc::Rc; use wasmi::FuncInstance; @@ -11,6 +12,7 @@ use wasmi::RuntimeValue; use super::ForeignContext; use super::ForeignPlugin; +#[derive(Clone)] pub(super) struct ForeignOp { pub index: Option, pub index_within_plugin: usize, @@ -21,7 +23,8 @@ pub(super) struct ForeignOp { pub struct InternalCircuitEnv { pub(super) plugins: HashMap, - pub(super) functions: HashMap, + // here we use BTreeMap to make sure op.index deterministic + pub(super) functions: BTreeMap, finalized: Rc>, } @@ -29,7 +32,7 @@ impl InternalCircuitEnv { pub(super) fn new(finalized: Rc>) -> Self { Self { plugins: HashMap::new(), - functions: HashMap::new(), + functions: BTreeMap::new(), finalized, } } diff --git a/crates/zkwasm/src/runtime/mod.rs b/crates/zkwasm/src/runtime/mod.rs index 8050b3ba4..de7306b4c 100644 --- a/crates/zkwasm/src/runtime/mod.rs +++ b/crates/zkwasm/src/runtime/mod.rs @@ -1,20 +1,13 @@ use std::cell::RefCell; use std::rc::Rc; -use specs::etable::EventTableEntry; -use specs::external_host_call_table::ExternalHostCallSignature; -use specs::mtable::AccessType; -use specs::mtable::LocationType; -use specs::mtable::MemoryTableEntry; -use specs::mtable::VarType; -use specs::step::StepInfo; use specs::CompilationTable; use specs::Tables; use self::wasmi_interpreter::WasmiRuntime; pub mod host; -pub mod state; +// pub mod state; pub mod wasmi_interpreter; pub struct CompiledImage { @@ -26,7 +19,7 @@ pub struct CompiledImage { #[derive(Clone)] pub struct ExecutionResult { - pub tables: Tables, + pub tables: Option, pub result: Option, pub public_inputs_and_outputs: Vec, pub outputs: Vec, @@ -34,931 +27,3 @@ pub struct ExecutionResult { // TODO: use feature pub type WasmInterpreter = WasmiRuntime; - -pub fn memory_event_of_step(event: &EventTableEntry) -> Vec { - let eid = event.eid; - let sp_before_execution = event.sp; - - match &event.step_info { - StepInfo::Br { - drop, - keep, - keep_values, - .. - } => { - assert_eq!(keep.len(), keep_values.len()); - assert!(keep.len() <= 1); - - let mut sp = sp_before_execution + 1; - let mut ops = vec![]; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp + 1; - } - } - - sp += drop; - sp -= 1; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp - 1; - } - } - - ops - } - StepInfo::BrIfEqz { - condition, - drop, - keep, - keep_values, - .. - } => { - assert_eq!(keep.len(), keep_values.len()); - assert!(keep.len() <= 1); - - let mut sp = sp_before_execution + 1; - - let mut ops = vec![MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *condition as u32 as u64, - }]; - - sp = sp + 1; - - if *condition != 0 { - return ops; - } - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp + 1; - } - } - - sp += drop; - sp -= 1; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp - 1; - } - } - - ops - } - StepInfo::BrIfNez { - condition, - drop, - keep, - keep_values, - .. - } => { - assert_eq!(keep.len(), keep_values.len()); - assert!(keep.len() <= 1); - - let mut sp = sp_before_execution + 1; - - let mut ops = vec![MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *condition as u32 as u64, - }]; - - sp = sp + 1; - - if *condition == 0 { - return ops; - } - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp + 1; - } - } - - sp += drop; - sp -= 1; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp - 1; - } - } - - ops - } - StepInfo::BrTable { - index, - drop, - keep, - keep_values, - .. - } => { - assert_eq!(keep.len(), keep_values.len()); - assert!(keep.len() <= 1); - - let mut sp = sp_before_execution + 1; - - let mut ops = vec![MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *index as u32 as u64, - }]; - - sp = sp + 1; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp + 1; - } - } - - sp += drop; - sp -= 1; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp - 1; - } - } - - ops - } - StepInfo::Return { - drop, - keep, - keep_values, - } => { - assert_eq!(keep.len(), keep_values.len()); - assert!(keep.len() <= 1); - - let mut sp = sp_before_execution + 1; - let mut ops = vec![]; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp + 1; - } - } - - sp += drop; - sp -= 1; - - { - for i in 0..keep.len() { - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: keep[i].into(), - is_mutable: true, - value: keep_values[i], - }); - - sp = sp - 1; - } - } - - ops - } - StepInfo::Drop { .. } => vec![], - StepInfo::Select { - val1, - val2, - cond, - result, - vtype, - } => { - let mut sp = sp_before_execution + 1; - let mut ops = vec![]; - - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *cond, - }); - sp = sp + 1; - - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *val2, - }); - sp = sp + 1; - - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *val1, - }); - - ops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: true, - value: *result, - }); - - ops - } - StepInfo::Call { index: _ } => { - vec![] - } - StepInfo::CallIndirect { offset, .. } => { - let stack_read = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *offset as u64, - }; - - vec![stack_read] - } - StepInfo::CallHost { - args, - ret_val, - signature, - .. - } => { - let mut mops = vec![]; - let mut sp = sp_before_execution; - - for (i, (ty, val)) in signature.params.iter().zip(args.iter()).enumerate() { - mops.push(MemoryTableEntry { - eid, - offset: sp_before_execution + args.len() as u32 - i as u32, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: (*ty).into(), - is_mutable: true, - value: *val, - }); - } - - sp = sp + args.len() as u32; - - if let Some(ty) = signature.return_type { - mops.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: ty.into(), - is_mutable: true, - value: ret_val.unwrap(), - }); - } - - mops - } - StepInfo::ExternalHostCall { value, sig, .. } => match sig { - ExternalHostCallSignature::Argument => { - let stack_read = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I64, - is_mutable: true, - value: value.unwrap(), - }; - - vec![stack_read] - } - ExternalHostCallSignature::Return => { - let stack_write = MemoryTableEntry { - eid, - offset: sp_before_execution, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: VarType::I64, - is_mutable: true, - value: value.unwrap(), - }; - - vec![stack_write] - } - }, - - StepInfo::GetLocal { - vtype, - depth, - value, - } => { - let read = MemoryTableEntry { - eid, - offset: sp_before_execution + depth, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - let write = MemoryTableEntry { - eid, - offset: sp_before_execution, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - vec![read, write] - } - StepInfo::SetLocal { - vtype, - depth, - value, - } => { - let mut sp = sp_before_execution; - - let read = MemoryTableEntry { - eid, - offset: sp + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - sp += 1; - - let write = MemoryTableEntry { - eid, - offset: sp + depth, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - vec![read, write] - } - StepInfo::TeeLocal { - vtype, - depth, - value, - } => { - let read = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - let write = MemoryTableEntry { - eid, - offset: sp_before_execution + depth, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - vec![read, write] - } - - StepInfo::GetGlobal { - idx, - vtype, - is_mutable, - value, - .. - } => { - let global_get = MemoryTableEntry { - eid, - offset: *idx, - ltype: LocationType::Global, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: *is_mutable, - value: *value, - }; - - let stack_write = MemoryTableEntry { - eid, - offset: sp_before_execution, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - vec![global_get, stack_write] - } - StepInfo::SetGlobal { - idx, - vtype, - is_mutable, - value, - } => { - let stack_read = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - let global_set = MemoryTableEntry { - eid, - offset: *idx, - ltype: LocationType::Global, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: *is_mutable, - value: *value, - }; - - vec![stack_read, global_set] - } - - StepInfo::Load { - vtype, - load_size, - raw_address, - effective_address, - value, - block_value1, - block_value2, - .. - } => { - let load_address_from_stack = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *raw_address as u64, - }; - - let load_value1 = MemoryTableEntry { - eid, - offset: (*effective_address) / 8, - ltype: LocationType::Heap, - atype: AccessType::Read, - // Load u64 from address which align with 8 - vtype: VarType::I64, - is_mutable: true, - // The value will be used to lookup within imtable, hence block_value is given here - value: *block_value1, - }; - - let load_value2 = if *effective_address % 8 + load_size.byte_size() as u32 > 8 { - Some(MemoryTableEntry { - eid, - offset: effective_address / 8 + 1, - ltype: LocationType::Heap, - atype: AccessType::Read, - // Load u64 from address which align with 8 - vtype: VarType::I64, - is_mutable: true, - // The value will be used to lookup within imtable, hence block_value is given here - value: *block_value2, - }) - } else { - None - }; - - let push_value = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - vec![ - vec![load_address_from_stack, load_value1], - load_value2.map_or(vec![], |v| vec![v]), - vec![push_value], - ] - .concat() - } - StepInfo::Store { - vtype, - store_size, - raw_address, - effective_address, - value, - pre_block_value1, - updated_block_value1, - pre_block_value2, - updated_block_value2, - .. - } => { - let load_value_from_stack = MemoryTableEntry { - eid, - offset: sp_before_execution + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: *vtype, - is_mutable: true, - value: *value, - }; - - let load_address_from_stack = MemoryTableEntry { - eid, - offset: sp_before_execution + 2, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: VarType::I32, - is_mutable: true, - value: *raw_address as u64, - }; - - let load_value1 = MemoryTableEntry { - eid, - offset: effective_address / 8, - ltype: LocationType::Heap, - atype: AccessType::Read, - // Load u64 from address which align with 8 - vtype: VarType::I64, - is_mutable: true, - // The value will be used to lookup within imtable, hence block_value is given here - value: *pre_block_value1, - }; - - let write_value1 = MemoryTableEntry { - eid, - offset: effective_address / 8, - ltype: LocationType::Heap, - atype: AccessType::Write, - // Load u64 from address which align with 8 - vtype: VarType::I64, - is_mutable: true, - // The value will be used to lookup within imtable, hence block_value is given here - value: *updated_block_value1, - }; - - if *effective_address % 8 + store_size.byte_size() as u32 > 8 { - let load_value2 = MemoryTableEntry { - eid, - offset: effective_address / 8 + 1, - ltype: LocationType::Heap, - atype: AccessType::Read, - // Load u64 from address which align with 8 - vtype: VarType::I64, - is_mutable: true, - // The value will be used to lookup within imtable, hence block_value is given here - value: *pre_block_value2, - }; - - let write_value2 = MemoryTableEntry { - eid, - offset: effective_address / 8 + 1, - ltype: LocationType::Heap, - atype: AccessType::Write, - // Load u64 from address which align with 8 - vtype: VarType::I64, - is_mutable: true, - // The value will be used to lookup within imtable, hence block_value is given here - value: *updated_block_value2, - }; - vec![ - load_value_from_stack, - load_address_from_stack, - load_value1, - write_value1, - load_value2, - write_value2, - ] - } else { - vec![ - load_value_from_stack, - load_address_from_stack, - load_value1, - write_value1, - ] - } - } - - StepInfo::MemorySize => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I32, - &[], - &[event.allocated_memory_pages as u32 as u64], - ), - StepInfo::MemoryGrow { grow_size, result } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I32, - &[*grow_size as u32 as u64], - &[*result as u32 as u64], - ), - - StepInfo::I32Const { value } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I32, - &[], - &[*value as u32 as u64], - ), - StepInfo::I32BinOp { - left, right, value, .. - } - | StepInfo::I32BinShiftOp { - left, right, value, .. - } - | StepInfo::I32BinBitOp { - left, right, value, .. - } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I32, - &[*right as u32 as u64, *left as u32 as u64], - &[*value as u32 as u64], - ), - StepInfo::I32Comp { - left, right, value, .. - } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I32, - &[*right as u32 as u64, *left as u32 as u64], - &[*value as u32 as u64], - ), - - StepInfo::I64BinOp { - left, right, value, .. - } - | StepInfo::I64BinShiftOp { - left, right, value, .. - } - | StepInfo::I64BinBitOp { - left, right, value, .. - } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I64, - VarType::I64, - &[*right as u64, *left as u64], - &[*value as u64], - ), - - StepInfo::I64Const { value } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I64, - VarType::I64, - &[], - &[*value as u64], - ), - StepInfo::I64Comp { - left, right, value, .. - } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I64, - VarType::I32, - &[*right as u64, *left as u64], - &[*value as u32 as u64], - ), - StepInfo::UnaryOp { - vtype, - operand, - result, - .. - } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - *vtype, - *vtype, - &[*operand], - &[*result], - ), - - StepInfo::Test { - vtype, - value, - result, - } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - *vtype, - VarType::I32, - &[*value], - &[*result as u32 as u64], - ), - - StepInfo::I32WrapI64 { value, result } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I64, - VarType::I32, - &[*value as u64], - &[*result as u32 as u64], - ), - StepInfo::I64ExtendI32 { value, result, .. } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I64, - &[*value as u32 as u64], - &[*result as u64], - ), - StepInfo::I32SignExtendI8 { value, result } - | StepInfo::I32SignExtendI16 { value, result } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I32, - VarType::I32, - &[*value as u32 as u64], - &[*result as u32 as u64], - ), - StepInfo::I64SignExtendI8 { value, result } - | StepInfo::I64SignExtendI16 { value, result } - | StepInfo::I64SignExtendI32 { value, result } => mem_op_from_stack_only_step( - sp_before_execution, - eid, - VarType::I64, - VarType::I64, - &[*value as u64], - &[*result as u64], - ), - } -} - -pub(crate) fn mem_op_from_stack_only_step( - sp_before_execution: u32, - eid: u32, - inputs_type: VarType, - outputs_type: VarType, - pop_value: &[u64], - push_value: &[u64], -) -> Vec { - let mut mem_op = vec![]; - let mut sp = sp_before_execution; - - for i in 0..pop_value.len() { - mem_op.push(MemoryTableEntry { - eid, - offset: sp + 1, - ltype: LocationType::Stack, - atype: AccessType::Read, - vtype: inputs_type, - is_mutable: true, - value: pop_value[i], - }); - sp = sp + 1; - } - - for i in 0..push_value.len() { - mem_op.push(MemoryTableEntry { - eid, - offset: sp, - ltype: LocationType::Stack, - atype: AccessType::Write, - vtype: outputs_type, - is_mutable: true, - value: push_value[i], - }); - sp = sp - 1; - } - - mem_op -} diff --git a/crates/zkwasm/src/runtime/state.rs b/crates/zkwasm/src/runtime/state.rs index d87998f73..8b1378917 100644 --- a/crates/zkwasm/src/runtime/state.rs +++ b/crates/zkwasm/src/runtime/state.rs @@ -1,150 +1 @@ -use specs::etable::EventTableEntry; -use specs::host_function::HostPlugin; -use specs::imtable::InitMemoryTable; -use specs::imtable::InitMemoryTableEntry; -use specs::itable::Opcode; -use specs::mtable::AccessType; -use specs::state::InitializationState; -use specs::step::StepInfo; -use specs::CompilationTable; -use super::memory_event_of_step; - -pub(crate) trait UpdateCompilationTable { - fn update_init_memory_table(&self, execution_table: &Vec) -> InitMemoryTable; - - fn update_initialization_state( - &self, - execution_table: &Vec, - is_last_slice: bool, - ) -> InitializationState; -} - -impl UpdateCompilationTable for CompilationTable { - fn update_init_memory_table(&self, execution_table: &Vec) -> InitMemoryTable { - // First insert origin imtable entries which may be overwritten. - let mut map = self.imtable.entries().clone(); - - let mut it = execution_table.iter(); - while let Some(etable_entry) = it.next() { - let memory_writing_entires = memory_event_of_step(etable_entry) - .into_iter() - .filter(|entry| entry.atype == AccessType::Write); - - for mentry in memory_writing_entires { - map.insert( - (mentry.ltype, mentry.offset), - InitMemoryTableEntry { - ltype: mentry.ltype, - is_mutable: mentry.is_mutable, - offset: mentry.offset, - vtype: mentry.vtype, - value: mentry.value, - eid: etable_entry.eid, - }, - ); - } - } - - InitMemoryTable(map) - } - - fn update_initialization_state( - &self, - execution_table: &Vec, - is_last_slice: bool, - ) -> InitializationState { - let mut host_public_inputs = self.initialization_state.host_public_inputs; - let mut context_in_index = self.initialization_state.context_in_index; - let mut context_out_index = self.initialization_state.context_out_index; - let mut external_host_call_call_index = - self.initialization_state.external_host_call_call_index; - - #[cfg(feature = "continuation")] - let mut jops = self.initialization_state.jops; - - for entry in execution_table { - match &entry.step_info { - // TODO: fix hard code - StepInfo::CallHost { - function_name, - args, - op_index_in_plugin, - .. - } => { - if *op_index_in_plugin == HostPlugin::HostInput as usize { - if function_name == "wasm_input" && args[0] != 0 - || function_name == "wasm_output" - { - host_public_inputs += 1; - } - } else if *op_index_in_plugin == HostPlugin::Context as usize { - if function_name == "wasm_read_context" { - context_in_index += 1; - } else if function_name == "wasm_write_context" { - context_out_index += 1; - } - } - } - StepInfo::ExternalHostCall { .. } => external_host_call_call_index += 1, - StepInfo::Call { .. } | StepInfo::CallIndirect { .. } | StepInfo::Return { .. } => { - #[cfg(feature = "continuation")] - { - jops += 1; - } - } - _ => (), - } - } - - let last_entry = execution_table.last().unwrap(); - - let post_initialization_state = if is_last_slice { - InitializationState { - eid: last_entry.eid + 1, - fid: 0, - iid: 0, - frame_id: 0, - // TODO: why not constant 4095? - sp: last_entry.sp - + if let Opcode::Return { drop, .. } = last_entry.inst.opcode { - drop - } else { - 0 - }, - - host_public_inputs, - context_in_index, - context_out_index, - external_host_call_call_index, - - initial_memory_pages: last_entry.allocated_memory_pages, - maximal_memory_pages: self.configure_table.maximal_memory_pages, - - #[cfg(feature = "continuation")] - jops, - } - } else { - InitializationState { - eid: last_entry.eid, - fid: last_entry.inst.fid, - iid: last_entry.inst.iid, - frame_id: last_entry.last_jump_eid, - sp: last_entry.sp, - - host_public_inputs, - context_in_index, - context_out_index, - external_host_call_call_index, - - initial_memory_pages: last_entry.allocated_memory_pages, - maximal_memory_pages: self.configure_table.maximal_memory_pages, - - #[cfg(feature = "continuation")] - jops, - } - }; - - post_initialization_state - } -} diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index 7153f317a..97904da7e 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -5,18 +5,18 @@ use std::sync::Arc; use anyhow::Result; use specs::host_function::HostFunctionDesc; -use specs::jtable::StaticFrameEntry; -use specs::state::InitializationState; +use specs::state::{InitializationState, UpdateCompilationTable}; use specs::CompilationTable; use specs::ExecutionTable; use specs::Tables; + use wasmi::Externals; use wasmi::ImportResolver; use wasmi::ModuleInstance; use wasmi::RuntimeValue; use wasmi::DEFAULT_VALUE_STACK_LIMIT; -use super::state::UpdateCompilationTable; + use super::CompiledImage; use super::ExecutionResult; @@ -51,7 +51,6 @@ impl Execution let instance = self.instance.run_start(externals).unwrap(); let result = instance.invoke_export(&self.entry, &[], externals)?; - Ok(result) } @@ -67,40 +66,46 @@ impl Execution let result = instance.invoke_export_trace(&self.entry, &[], externals, self.tracer.clone())?; - - let execution_tables = { - let tracer = self.tracer.borrow(); - - ExecutionTable { - etable: tracer.etable.clone(), - jtable: Arc::new(tracer.jtable.clone()), - } - }; - - let updated_init_memory_table = self - .tables - .update_init_memory_table(&execution_tables.etable.entries()); - - let post_image_table = { - CompilationTable { - itable: self.tables.itable.clone(), - imtable: updated_init_memory_table, - elem_table: self.tables.elem_table.clone(), - configure_table: self.tables.configure_table.clone(), - static_jtable: self.tables.static_jtable.clone(), - initialization_state: self - .tables - .update_initialization_state(&execution_tables.etable.entries(), true), - } - }; - - Ok(ExecutionResult { - tables: Tables { + + let mut tables: Option = None; + let tracer = self.tracer.borrow(); + if !tracer.has_dumped() { + let execution_tables = { + let tracer = self.tracer.borrow(); + + ExecutionTable { + etable: tracer.etable.clone(), + jtable: Arc::new(tracer.jtable.clone()), + } + }; + + let updated_init_memory_table = self + .tables + .update_init_memory_table(&execution_tables.etable.entries()); + + let post_image_table = { + CompilationTable { + itable: self.tables.itable.clone(), + imtable: updated_init_memory_table, + elem_table: self.tables.elem_table.clone(), + configure_table: self.tables.configure_table.clone(), + static_jtable: self.tables.static_jtable.clone(), + initialization_state: self + .tables + .update_initialization_state(&execution_tables.etable.entries(), true), + } + }; + + tables = Some(Tables { compilation_tables: self.tables, execution_tables, post_image_table, is_last_slice: true, - }, + }); + } + + Ok(ExecutionResult { + tables, result, public_inputs_and_outputs: wasm_io.public_inputs_and_outputs.borrow().clone(), outputs: wasm_io.outputs.borrow().clone(), @@ -121,51 +126,15 @@ impl WasmiRuntime { host_plugin_lookup: &HashMap, entry: &str, phantom_functions: &Vec, + callback: Option, + capability: usize ) -> Result, wasmi::tracer::Tracer>> { - let tracer = wasmi::tracer::Tracer::new(host_plugin_lookup.clone(), phantom_functions); + let tracer = wasmi::tracer::Tracer::new(host_plugin_lookup.clone(), phantom_functions, callback, capability); let tracer = Rc::new(RefCell::new(tracer)); - let instance = ModuleInstance::new(&module, imports, Some(tracer.clone())) .expect("failed to instantiate wasm module"); - let fid_of_entry = { - let idx_of_entry = instance.lookup_function_by_name(tracer.clone(), entry); - - tracer - .clone() - .borrow_mut() - .static_jtable_entries - .push(StaticFrameEntry { - enable: true, - frame_id: 0, - next_frame_id: 0, - callee_fid: idx_of_entry, - fid: 0, - iid: 0, - }); - - if instance.has_start() { - tracer - .clone() - .borrow_mut() - .static_jtable_entries - .push(StaticFrameEntry { - enable: true, - frame_id: 0, - next_frame_id: 0, - callee_fid: 0, // the fid of start function is always 0 - fid: idx_of_entry, - iid: 0, - }); - } - - if instance.has_start() { - 0 - } else { - idx_of_entry - } - }; - + let fid_of_entry = tracer.clone().borrow().get_fid_of_entry(); let itable = Arc::new(tracer.borrow().itable.clone()); let imtable = tracer.borrow().imtable.finalized(); let elem_table = Arc::new(tracer.borrow().elem_table.clone()); diff --git a/crates/zkwasm/src/test/mod.rs b/crates/zkwasm/src/test/mod.rs index 776387dc8..05f80d36c 100644 --- a/crates/zkwasm/src/test/mod.rs +++ b/crates/zkwasm/src/test/mod.rs @@ -1,5 +1,6 @@ use crate::circuits::config::zkwasm_k; use crate::circuits::TestCircuit; +use crate::loader::CallbackType; use crate::profile::Profiler; use crate::runtime::host::host_env::HostEnv; use crate::runtime::wasmi_interpreter::Execution; @@ -43,11 +44,11 @@ fn test_circuit_mock( v }; - execution_result.tables.write_json(None); + execution_result.tables.as_ref().unwrap().write_json(None); - execution_result.tables.profile_tables(); + execution_result.tables.as_ref().unwrap().profile_tables(); - let circuit = TestCircuit::new(execution_result.tables, None); + let circuit = TestCircuit::new(execution_result.tables.unwrap(), None); let prover = MockProver::run(zkwasm_k(), &circuit, vec![instance])?; assert_eq!(prover.verify(), Ok(())); @@ -71,6 +72,8 @@ fn compile_then_execute_wasm( &env.function_description_table(), function_name, &vec![], + None::, + 0 ) .unwrap(); diff --git a/crates/zkwasm/src/test/test_rlp.rs b/crates/zkwasm/src/test/test_rlp.rs index 377a0374f..c3a8342d1 100644 --- a/crates/zkwasm/src/test/test_rlp.rs +++ b/crates/zkwasm/src/test/test_rlp.rs @@ -156,6 +156,8 @@ fn build_circuit() -> Result<(ZkWasmLoader, TestCircuit, Vec)> { private_inputs, context_inputs: vec![], context_outputs: Arc::new(Mutex::new(vec![])), + output_dir: None, + dump_table: false })?; Ok((loader, circuit, instances)) diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index b30047e0e..ee36b4cdc 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -3,15 +3,15 @@ use std::sync::Mutex; use crate::loader::ExecutionArg; use crate::loader::ZkWasmLoader; -use crate::runtime::ExecutionResult; use crate::continuation::slice::Slice; +use crate::runtime::ExecutionResult; use specs::Tables; -use wasmi::RuntimeValue; use anyhow::Result; -use halo2_proofs::pairing::bn256::Bn256; +use halo2_proofs::pairing::bn256::{Bn256, Fr}; +use wasmi::RuntimeValue; -fn generate_wasm_result() -> (ZkWasmLoader, ExecutionResult) { +fn generate_wasm_result(dump_table: bool) -> Result<(ZkWasmLoader, Vec, ExecutionResult)> { let public_inputs = vec![133]; let private_inputs: Vec = vec![ 14625441452057167097, @@ -151,7 +151,7 @@ fn generate_wasm_result() -> (ZkWasmLoader, ExecutionResult let wasm = std::fs::read("wasm/rlp.wasm").unwrap(); - let loader = ZkWasmLoader::::new(18, wasm, vec![]).unwrap(); + let loader = ZkWasmLoader::::new(18, wasm, vec![])?; let execution_result = loader .run(ExecutionArg { @@ -159,24 +159,26 @@ fn generate_wasm_result() -> (ZkWasmLoader, ExecutionResult private_inputs, context_inputs: vec![], context_outputs: Arc::new(Mutex::new(vec![])), - }) - .unwrap(); - (loader, execution_result) -} + output_dir: Some(std::env::current_dir().unwrap()), + dump_table + })?; -fn test_slices() -> Result<()> { - let (loader, execution_result) = generate_wasm_result(); let instances = execution_result .public_inputs_and_outputs .iter() .map(|v| (*v).into()) .collect(); + Ok((loader, instances, execution_result)) +} +fn test_slices() -> Result<()> { + let (loader, instances, execution_result) = generate_wasm_result(false)?; let mut slices = loader.slice(execution_result).into_iter(); let mut index = 0; while let Some(slice) = slices.next() { + println!("slice {}", index); let circuit = slice.build_circuit(); @@ -190,18 +192,14 @@ fn test_slices() -> Result<()> { } fn test_rpl_slice_from_file() -> Result<()> { - let (loader, execution_result) = generate_wasm_result(); - - let instances = execution_result - .public_inputs_and_outputs - .iter() - .map(|v| (*v).into()) - .collect(); - + let (loader, instances, execution_result) = generate_wasm_result(false)?; let mut slices = loader.slice(execution_result).into_iter(); + let mut index = 0; while let Some(slice) = slices.next() { let mut dir = std::env::current_dir().unwrap(); + // push a namespace to avoid conflict with test_rpl_slice_dump when testing concurrently + dir.push("full_run_dump"); dir.push(index.to_string()); slice.write_json(Some(dir)); index += 1; @@ -211,6 +209,7 @@ fn test_rpl_slice_from_file() -> Result<()> { while index > 0 { index -= 1; let mut dir = std::env::current_dir().unwrap(); + dir.push("full_run_dump"); dir.push(index.to_string()); let table = Tables::load_json(dir.clone(), index == last_slice_index); @@ -224,6 +223,36 @@ fn test_rpl_slice_from_file() -> Result<()> { Ok(()) } + +fn test_rpl_slice_dump() -> Result<()> { + // dump slice while running + let (_, _instances, _) = generate_wasm_result(true)?; + // dump slice after a run + let (loader, _, execution_result) = generate_wasm_result(false)?; + let mut slices = loader.slice(execution_result).into_iter(); + + let last_slice_index = slices.num_slices() - 1; + let mut index = 0; + while let Some(slice) = slices.next() { + // load slice from running dump + let mut dir = std::env::current_dir().unwrap(); + dir.push(index.to_string()); + let table = Tables::load_json(dir.clone(), index == last_slice_index); + let loaded_slice = Slice::new(table, slices.capability()); + + // make sure slices generated from memory and during running is the same + assert_eq!(slice, loaded_slice); + + let circuit = loaded_slice.build_circuit(); + loader.mock_test(&circuit, &_instances)?; + + std::fs::remove_dir_all(dir).unwrap(); + index += 1; + } + + Ok(()) +} + mod tests { use super::*; @@ -236,4 +265,9 @@ mod tests { fn test_rpl_slice_from_file_mock() { test_rpl_slice_from_file().unwrap(); } + + #[test] + fn test_rpl_slice_dump_mock() { + test_rpl_slice_dump().unwrap(); + } } diff --git a/crates/zkwasm/src/test/test_uniform_verifier.rs b/crates/zkwasm/src/test/test_uniform_verifier.rs index d256b5ae3..d47bb3a23 100644 --- a/crates/zkwasm/src/test/test_uniform_verifier.rs +++ b/crates/zkwasm/src/test/test_uniform_verifier.rs @@ -32,7 +32,7 @@ fn setup_uniform_verifier() -> Result<(Params, ProvingKey)> let execution_result = test_circuit_with_env(env, WasmRuntimeIO::empty(), wasm, "zkmain")?; let builder = ZkWasmCircuitBuilder { - tables: execution_result.tables, + tables: execution_result.tables.unwrap(), }; let circuit: TestCircuit = builder.build_circuit(None); @@ -133,7 +133,7 @@ mod tests { let instances = vec![]; let builder = ZkWasmCircuitBuilder { - tables: execution_result.tables, + tables: execution_result.tables.unwrap(), }; let proof = { diff --git a/third-party/wasmi b/third-party/wasmi index 7cc3411de..d826189e8 160000 --- a/third-party/wasmi +++ b/third-party/wasmi @@ -1 +1 @@ -Subproject commit 7cc3411de4ca5ab800dec894781bd7accf668046 +Subproject commit d826189e83d30979b6e1c1c0028edd3ae0b1ccf7 From c45f18e40ae548c723b897067a4ea8b8c82d5d20 Mon Sep 17 00:00:00 2001 From: Po Date: Thu, 14 Dec 2023 16:37:52 +0100 Subject: [PATCH 03/34] chore: cont-op[3] add file type for private inputs --- crates/cli/src/args.rs | 15 ++++++- crates/specs/src/state.rs | 40 ------------------- .../src/foreign/wasm_input_helper/runtime.rs | 9 +++-- crates/zkwasm/src/loader/mod.rs | 2 +- 4 files changed, 20 insertions(+), 46 deletions(-) diff --git a/crates/cli/src/args.rs b/crates/cli/src/args.rs index fc11b7eb3..9034cc882 100644 --- a/crates/cli/src/args.rs +++ b/crates/cli/src/args.rs @@ -47,7 +47,20 @@ pub fn parse_args(values: Vec<&str>) -> Vec { u64::from_le_bytes(data) }) .collect::>() - } + }, + "file" => { + let bytes = std::fs::read(v).unwrap(); + let bytes = bytes.chunks(8); + bytes + .into_iter() + .map(|x| { + let mut data = [0u8; 8]; + data[..x.len()].copy_from_slice(x); + + u64::from_be_bytes(data) + }) + .collect() + }, _ => { panic!("Unsupported input data type: {}", t) diff --git a/crates/specs/src/state.rs b/crates/specs/src/state.rs index 2797e47bb..d495c609b 100644 --- a/crates/specs/src/state.rs +++ b/crates/specs/src/state.rs @@ -11,46 +11,6 @@ use crate::mtable::AccessType; use crate::step::StepInfo; use crate::itable::Opcode; -// cfg_if::cfg_if! { -// if #[cfg(feature = "continuation")] { -// #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -// pub struct InitializationState { -// pub eid: T, -// pub fid: T, -// pub iid: T, -// pub frame_id: T, -// pub sp: T, - -// pub host_public_inputs: T, -// pub context_in_index: T, -// pub context_out_index: T, -// pub external_host_call_call_index: T, - -// pub initial_memory_pages: T, -// pub maximal_memory_pages: T, - -// pub jops: T, -// } -// } else { -// #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] -// pub struct InitializationState { -// pub eid: T, -// pub fid: T, -// pub iid: T, -// pub frame_id: T, -// pub sp: T, - -// pub host_public_inputs: T, -// pub context_in_index: T, -// pub context_out_index: T, -// pub external_host_call_call_index: T, - -// pub initial_memory_pages: T, -// pub maximal_memory_pages: T, -// } -// } -// } - #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct InitializationState { pub eid: T, diff --git a/crates/zkwasm/src/foreign/wasm_input_helper/runtime.rs b/crates/zkwasm/src/foreign/wasm_input_helper/runtime.rs index e2fd16b57..396656836 100644 --- a/crates/zkwasm/src/foreign/wasm_input_helper/runtime.rs +++ b/crates/zkwasm/src/foreign/wasm_input_helper/runtime.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::collections::VecDeque; use std::rc::Rc; use specs::host_function::HostPlugin; @@ -12,7 +13,7 @@ use super::Op; struct Context { public_inputs: Vec, - private_inputs: Vec, + private_inputs: VecDeque, instance: Rc>>, output: Rc>>, } @@ -26,7 +27,7 @@ impl Context { ) -> Self { Context { public_inputs, - private_inputs, + private_inputs: private_inputs.into(), instance, output, } @@ -43,7 +44,7 @@ impl Context { if self.private_inputs.is_empty() { panic!("failed to read private input, please checkout your input"); } - self.private_inputs.remove(0) + self.private_inputs.pop_front().unwrap() } pub fn push_public(&mut self, value: u64) { @@ -104,7 +105,7 @@ pub fn register_wasm_input_foreign( HostPlugin::HostInput, Box::new(Context::new( public_inputs, - private_inputs, + private_inputs.into(), public_inputs_and_outputs.clone(), outputs.clone(), )), diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index d369abc4a..d0fbcc071 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -160,7 +160,7 @@ impl ZkWasmLoader { pub fn checksum(&self, params: &Params) -> Result> { let (env, _) = HostEnv::new_with_full_foreign_plugins( vec![], - vec![], + vec![].into(), vec![], Arc::new(Mutex::new(vec![])), ); From cbd3cab925d1ad038a80f4cae015edd8dc9ad7db Mon Sep 17 00:00:00 2001 From: Po Date: Mon, 18 Dec 2023 07:10:28 +0100 Subject: [PATCH 04/34] chore: cont_opt[4] caching lookup for wasmi, flexbuffer & pooling for writing witness table --- Cargo.lock | 113 ++++++++++++++++++++++- crates/specs/Cargo.toml | 1 + crates/specs/src/lib.rs | 89 ++++++++++++------ crates/zkwasm/Cargo.toml | 1 + crates/zkwasm/src/continuation/slice.rs | 4 +- crates/zkwasm/src/loader/mod.rs | 17 +++- crates/zkwasm/src/test/mod.rs | 2 +- crates/zkwasm/src/test/test_rlp_slice.rs | 6 +- third-party/wasmi | 2 +- 9 files changed, 196 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb42d8880..2ca7dff78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -177,6 +177,12 @@ version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "cc" version = "1.0.80" @@ -242,7 +248,7 @@ dependencies = [ "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "strsim", "termcolor", @@ -442,6 +448,7 @@ dependencies = [ "specs", "strum", "strum_macros", + "threadpool", "wabt", "wasmi", ] @@ -535,6 +542,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.2" @@ -634,6 +647,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "flexbuffers" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d14128f06405808ce75bfebe11e9b0f9da18719ede6d7bdb1702d6bfe0f7e8" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "num_enum", + "serde", + "serde_derive", +] + [[package]] name = "flume" version = "0.10.14" @@ -800,6 +826,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "heck" version = "0.4.1" @@ -889,7 +921,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", ] [[package]] @@ -1182,6 +1224,27 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "object" version = "0.31.1" @@ -1346,6 +1409,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1662,6 +1735,7 @@ name = "specs" version = "0.1.0" dependencies = [ "cfg-if 1.0.0", + "flexbuffers", "halo2_proofs", "lazy_static", "num-bigint", @@ -1836,6 +1910,32 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "toml_datetime" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" + +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + [[package]] name = "typenum" version = "1.16.0" @@ -2234,6 +2334,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winnow" +version = "0.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +dependencies = [ + "memchr", +] + [[package]] name = "wyz" version = "0.5.1" diff --git a/crates/specs/Cargo.toml b/crates/specs/Cargo.toml index f07f1d826..adeb9b429 100644 --- a/crates/specs/Cargo.toml +++ b/crates/specs/Cargo.toml @@ -16,6 +16,7 @@ cfg-if.workspace = true halo2_proofs.workspace = true parity-wasm.workspace = true rayon.workspace = true +flexbuffers = "2.0.0" [features] default = [] diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 48237ab85..64a209c22 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -5,7 +5,7 @@ use std::collections::HashSet; use std::env; use std::fs::File; -use std::io::BufReader; +use std::io::Read; use std::io::Write; use std::path::PathBuf; use std::sync::Arc; @@ -81,6 +81,21 @@ impl Tables { } } +fn write_file(folder: &PathBuf, filename: &str, buf: &[u8]) { + std::fs::create_dir_all(folder).unwrap(); + let mut folder = folder.clone(); + folder.push(filename); + let mut fd = std::fs::File::create(folder.as_path()).unwrap(); + folder.pop(); + + fd.write(buf).unwrap(); +} + +pub enum FileType { + JSON, + FLEXBUFFERS +} + impl Tables { pub fn create_memory_table( &self, @@ -130,41 +145,58 @@ impl Tables { MTable::new(memory_entries) } - pub fn write_json(&self, dir: Option) { - fn write_file(folder: &PathBuf, filename: &str, buf: &String) { - let mut folder = folder.clone(); - std::fs::create_dir_all(folder.as_path()).unwrap(); - folder.push(filename); - let mut fd = std::fs::File::create(folder.as_path()).unwrap(); - folder.pop(); - - fd.write(buf.as_bytes()).unwrap(); + pub fn write(&self, dir: Option, file_type: FileType) { + let dir = dir.unwrap_or(env::current_dir().unwrap()); + match file_type { + FileType::JSON => { + let compilation_table = serde_json::to_string_pretty(&self.compilation_tables).unwrap(); + let execution_table = serde_json::to_string_pretty(&self.execution_tables).unwrap(); + let post_image_table = serde_json::to_string_pretty(&self.post_image_table).unwrap(); + + write_file(&dir, "compilation.json", compilation_table.as_bytes()); + write_file(&dir, "execution.json", &execution_table.as_bytes()); + write_file(&dir, "post_image.json", &post_image_table.as_bytes()); + }, + + FileType::FLEXBUFFERS => { + let compilation_tables = flexbuffers::to_vec(&self.compilation_tables).unwrap(); + let execution_tables = flexbuffers::to_vec(&self.execution_tables).unwrap(); + let post_image_table = flexbuffers::to_vec(&self.post_image_table).unwrap(); + + write_file(&dir, "compilation.buf", &compilation_tables); + write_file(&dir, "execution.buf", &execution_tables); + write_file(&dir, "post_image.buf", &post_image_table); + } } - let compilation_table = serde_json::to_string_pretty(&self.compilation_tables).unwrap(); - let execution_table = serde_json::to_string_pretty(&self.execution_tables).unwrap(); - let post_image_table = serde_json::to_string_pretty(&self.post_image_table).unwrap(); - - let dir = dir.unwrap_or(env::current_dir().unwrap()); - write_file(&dir, "compilation.json", &compilation_table); - write_file(&dir, "execution.json", &execution_table); - write_file(&dir, "image.json", &post_image_table); } - pub fn load_json(dir: PathBuf, is_last_slice: bool) -> Tables { - fn load_file(folder: &PathBuf, filename: &str) -> BufReader { + pub fn load(dir: PathBuf, is_last_slice: bool, file_type: FileType) -> Tables { + fn load_file(folder: &PathBuf, filename: &str) -> Vec { let mut folder = folder.clone(); folder.push(filename); - let file = File::open(folder.as_path()).unwrap(); - BufReader::new(file) + let mut file = File::open(folder.as_path()).unwrap(); + let mut buf = vec![]; + file.read_to_end(&mut buf).unwrap(); + buf } + let (compilation_tables, execution_tables, post_image_table) = match file_type { + FileType::JSON => { + ( + serde_json::from_slice(load_file(&dir, "compilation.json").as_slice()).unwrap(), + serde_json::from_slice(load_file(&dir, "execution.json").as_slice()).unwrap(), + serde_json::from_slice(load_file(&dir, "post_image.json").as_slice()).unwrap() + ) + }, + FileType::FLEXBUFFERS => { + ( + flexbuffers::from_buffer(&load_file(&dir, "compilation.buf").as_slice()).unwrap(), + flexbuffers::from_buffer(&load_file(&dir, "execution.buf").as_slice()).unwrap(), + flexbuffers::from_buffer(&load_file(&dir, "post_image.buf").as_slice()).unwrap(), + ) + } + }; - let compilation_tables: CompilationTable = - serde_json::from_reader(load_file(&dir, "compilation.json")).unwrap(); - let execution_tables: ExecutionTable = - serde_json::from_reader(load_file(&dir, "execution.json")).unwrap(); - let post_image_table: CompilationTable = - serde_json::from_reader(load_file(&dir, "image.json")).unwrap(); Tables { compilation_tables, execution_tables, @@ -172,4 +204,5 @@ impl Tables { is_last_slice, } } + } diff --git a/crates/zkwasm/Cargo.toml b/crates/zkwasm/Cargo.toml index 8289c359d..b4d9c1bcf 100644 --- a/crates/zkwasm/Cargo.toml +++ b/crates/zkwasm/Cargo.toml @@ -28,6 +28,7 @@ halo2aggregator-s.workspace = true halo2_proofs.workspace = true parity-wasm.workspace = true wasmi.workspace = true +threadpool = "1.8.1" [dev-dependencies] rusty-fork = "0.3.0" diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index e9281da5f..01ff94643 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -36,8 +36,8 @@ impl Slice { builder.build_circuit(Some(self.capability)) } - pub fn write_json(&self, dir: Option) { - self.table.write_json(dir); + pub fn write_flexbuffers(&self, dir: Option) { + self.table.write(dir, specs::FileType::FLEXBUFFERS); } } diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index d0fbcc071..cef3c7954 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -197,6 +197,9 @@ impl ZkWasmLoader { let output_dir = arg.output_dir.unwrap_or_else(|| std::env::current_dir().unwrap()); if arg.dump_table { + // defaults the number of threads to the number of CPUs. + let pool = threadpool::Builder::new().build(); + let pool_cb = pool.clone(); let mut _index = 0; let callback = move |_table, _capability | { cfg_if::cfg_if! { @@ -204,20 +207,30 @@ impl ZkWasmLoader { let slice = Slice::new(_table, _capability); let mut dir = output_dir.clone(); dir.push(_index.to_string()); - slice.write_json(Some(dir)); + println!("dumping------------------>"); + pool_cb.execute(move || { + slice.write_flexbuffers(Some(dir)); + println!("Slice: {} tables has dumped!", _index); + }); _index += 1; + + while pool_cb.queued_count() > 0 { + std::thread::sleep(std::time::Duration::from_millis(10)); + } } } }; + let compiled_module = self.compile(&env, Some(callback))?; let result = compiled_module.run(&mut env, wasm_runtime_io)?; + pool.join(); Ok(result) } else { let compiled_module = self.compile(&env, None::)?; let result = compiled_module.run(&mut env, wasm_runtime_io)?; if let Some(tables) = &result.tables { tables.profile_tables(); - tables.write_json(Some(output_dir)); + tables.write(Some(output_dir), specs::FileType::FLEXBUFFERS); } Ok(result) } diff --git a/crates/zkwasm/src/test/mod.rs b/crates/zkwasm/src/test/mod.rs index 05f80d36c..23df68fdc 100644 --- a/crates/zkwasm/src/test/mod.rs +++ b/crates/zkwasm/src/test/mod.rs @@ -44,7 +44,7 @@ fn test_circuit_mock( v }; - execution_result.tables.as_ref().unwrap().write_json(None); + execution_result.tables.as_ref().unwrap().write(None, specs::FileType::JSON); execution_result.tables.as_ref().unwrap().profile_tables(); diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index ee36b4cdc..c271a3245 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -201,7 +201,7 @@ fn test_rpl_slice_from_file() -> Result<()> { // push a namespace to avoid conflict with test_rpl_slice_dump when testing concurrently dir.push("full_run_dump"); dir.push(index.to_string()); - slice.write_json(Some(dir)); + slice.write_flexbuffers(Some(dir)); index += 1; } @@ -212,7 +212,7 @@ fn test_rpl_slice_from_file() -> Result<()> { dir.push("full_run_dump"); dir.push(index.to_string()); - let table = Tables::load_json(dir.clone(), index == last_slice_index); + let table = Tables::load(dir.clone(), index == last_slice_index, specs::FileType::FLEXBUFFERS); let slice = Slice::new(table, slices.capability()); let circuit = slice.build_circuit(); loader.mock_test(&circuit, &instances)?; @@ -237,7 +237,7 @@ fn test_rpl_slice_dump() -> Result<()> { // load slice from running dump let mut dir = std::env::current_dir().unwrap(); dir.push(index.to_string()); - let table = Tables::load_json(dir.clone(), index == last_slice_index); + let table = Tables::load(dir.clone(), index == last_slice_index, specs::FileType::FLEXBUFFERS); let loaded_slice = Slice::new(table, slices.capability()); // make sure slices generated from memory and during running is the same diff --git a/third-party/wasmi b/third-party/wasmi index d826189e8..9cb31af68 160000 --- a/third-party/wasmi +++ b/third-party/wasmi @@ -1 +1 @@ -Subproject commit d826189e83d30979b6e1c1c0028edd3ae0b1ccf7 +Subproject commit 9cb31af684da4c160b7d50ad793a4c079637f3a4 From a60c749927044f86c296a9b8cda86bdce7aa116b Mon Sep 17 00:00:00 2001 From: Po Date: Mon, 18 Dec 2023 16:06:46 +0100 Subject: [PATCH 05/34] chore: cont-op -- replace callback with trait for dumping slice witness & fmt --- .gitignore | 1 + crates/cli/src/app_builder.rs | 6 +- crates/cli/src/args.rs | 6 +- crates/cli/src/command.rs | 2 +- crates/cli/src/exec.rs | 10 +-- crates/specs/src/imtable.rs | 2 - crates/specs/src/jtable.rs | 2 +- crates/specs/src/lib.rs | 40 +++++----- crates/specs/src/state.rs | 7 +- .../zkwasm/src/circuits/utils/table_entry.rs | 3 +- crates/zkwasm/src/continuation/loader.rs | 69 ++++++++++++++++- crates/zkwasm/src/continuation/slice.rs | 5 +- crates/zkwasm/src/loader/mod.rs | 76 ++++++------------- .../runtime/host/internal_circuit_plugin.rs | 2 +- .../zkwasm/src/runtime/wasmi_interpreter.rs | 23 +++--- crates/zkwasm/src/test/mod.rs | 11 ++- crates/zkwasm/src/test/test_rlp.rs | 2 +- crates/zkwasm/src/test/test_rlp_slice.rs | 40 ++++++---- third-party/wasmi | 2 +- 19 files changed, 177 insertions(+), 132 deletions(-) diff --git a/.gitignore b/.gitignore index 1480ec778..72c201188 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target *.json +*.buf output/ crates/playground/wasm crates/playground/target diff --git a/crates/cli/src/app_builder.rs b/crates/cli/src/app_builder.rs index e3367818b..d0dd437e0 100644 --- a/crates/cli/src/app_builder.rs +++ b/crates/cli/src/app_builder.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::sync::Mutex; use crate::exec::exec_dry_run; -#[cfg(feature="continuation")] +#[cfg(feature = "continuation")] use crate::exec::exec_witness_dump; use super::command::CommandBuilder; @@ -79,7 +79,7 @@ pub trait AppBuilder: CommandBuilder { let app = Self::append_verify_aggregate_verify_subcommand(app); let app = Self::append_generate_solidity_verifier(app); let app = Self::append_image_checksum_subcommand(app); - #[cfg(feature="continuation")] + #[cfg(feature = "continuation")] let app = Self::append_witness_dump_subcommand(app); app @@ -154,7 +154,7 @@ pub trait AppBuilder: CommandBuilder { } } - #[cfg(feature="continuation")] + #[cfg(feature = "continuation")] Some(("witness-dump", sub_matches)) => { let public_inputs: Vec = Self::parse_single_public_arg(&sub_matches); let private_inputs: Vec = Self::parse_single_private_arg(&sub_matches); diff --git a/crates/cli/src/args.rs b/crates/cli/src/args.rs index 9034cc882..36fa7ec74 100644 --- a/crates/cli/src/args.rs +++ b/crates/cli/src/args.rs @@ -47,7 +47,7 @@ pub fn parse_args(values: Vec<&str>) -> Vec { u64::from_le_bytes(data) }) .collect::>() - }, + } "file" => { let bytes = std::fs::read(v).unwrap(); let bytes = bytes.chunks(8); @@ -56,11 +56,11 @@ pub fn parse_args(values: Vec<&str>) -> Vec { .map(|x| { let mut data = [0u8; 8]; data[..x.len()].copy_from_slice(x); - + u64::from_be_bytes(data) }) .collect() - }, + } _ => { panic!("Unsupported input data type: {}", t) diff --git a/crates/cli/src/command.rs b/crates/cli/src/command.rs index f46f5c69c..93bec132f 100644 --- a/crates/cli/src/command.rs +++ b/crates/cli/src/command.rs @@ -27,7 +27,7 @@ pub trait CommandBuilder: ArgBuilder { app.subcommand(command) } - #[cfg(feature="continuation")] + #[cfg(feature = "continuation")] fn append_witness_dump_subcommand(app: App) -> App { let command = Command::new("witness-dump") .arg(Self::single_public_arg()) diff --git a/crates/cli/src/exec.rs b/crates/cli/src/exec.rs index eb842a6e8..d313d8bc4 100644 --- a/crates/cli/src/exec.rs +++ b/crates/cli/src/exec.rs @@ -186,7 +186,7 @@ pub fn exec_dry_run_service( context_inputs, context_outputs: context_outputs.clone(), output_dir: None, - dump_table: false + dump_table: false, }) .unwrap(); println!("return value: {:?}", r); @@ -242,7 +242,7 @@ pub fn exec_dry_run( context_inputs, context_outputs, output_dir: None, - dump_table: false + dump_table: false, })?; Ok(()) @@ -266,7 +266,7 @@ pub fn exec_witness_dump( context_inputs, context_outputs, output_dir: Some(output_dir.clone()), - dump_table: true + dump_table: true, })?; Ok(()) @@ -301,7 +301,7 @@ pub fn exec_create_proof( context_inputs, context_outputs, output_dir: None, - dump_table: false + dump_table: false, })?; { @@ -401,7 +401,7 @@ pub fn exec_aggregate_create_proof( context_inputs, context_outputs, output_dir: None, - dump_table: false + dump_table: false, })?; circuits.push(circuit); diff --git a/crates/specs/src/imtable.rs b/crates/specs/src/imtable.rs index a6c718e7b..a5971e8bb 100644 --- a/crates/specs/src/imtable.rs +++ b/crates/specs/src/imtable.rs @@ -11,7 +11,6 @@ use crate::mtable::AccessType; use crate::mtable::MemoryTableEntry; use crate::step::StepInfo; - #[derive(Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct InitMemoryTableEntry { pub ltype: LocationType, @@ -78,7 +77,6 @@ impl InitMemoryTable { } } - pub fn memory_event_of_step(event: &EventTableEntry) -> Vec { let eid = event.eid; let sp_before_execution = event.sp; diff --git a/crates/specs/src/jtable.rs b/crates/specs/src/jtable.rs index ba4339a97..40c60669b 100644 --- a/crates/specs/src/jtable.rs +++ b/crates/specs/src/jtable.rs @@ -31,7 +31,7 @@ impl JumpTableEntry { pub struct JumpTable(Vec); impl JumpTable { - pub fn new(entries: Vec) -> Self{ + pub fn new(entries: Vec) -> Self { JumpTable(entries) } diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 64a209c22..0d9c0f184 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -44,8 +44,6 @@ pub mod state; pub mod step; pub mod types; - - #[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct CompilationTable { pub itable: Arc, @@ -93,7 +91,7 @@ fn write_file(folder: &PathBuf, filename: &str, buf: &[u8]) { pub enum FileType { JSON, - FLEXBUFFERS + FLEXBUFFERS, } impl Tables { @@ -149,14 +147,16 @@ impl Tables { let dir = dir.unwrap_or(env::current_dir().unwrap()); match file_type { FileType::JSON => { - let compilation_table = serde_json::to_string_pretty(&self.compilation_tables).unwrap(); + let compilation_table = + serde_json::to_string_pretty(&self.compilation_tables).unwrap(); let execution_table = serde_json::to_string_pretty(&self.execution_tables).unwrap(); - let post_image_table = serde_json::to_string_pretty(&self.post_image_table).unwrap(); - + let post_image_table = + serde_json::to_string_pretty(&self.post_image_table).unwrap(); + write_file(&dir, "compilation.json", compilation_table.as_bytes()); write_file(&dir, "execution.json", &execution_table.as_bytes()); write_file(&dir, "post_image.json", &post_image_table.as_bytes()); - }, + } FileType::FLEXBUFFERS => { let compilation_tables = flexbuffers::to_vec(&self.compilation_tables).unwrap(); @@ -168,7 +168,6 @@ impl Tables { write_file(&dir, "post_image.buf", &post_image_table); } } - } pub fn load(dir: PathBuf, is_last_slice: bool, file_type: FileType) -> Tables { @@ -181,20 +180,16 @@ impl Tables { buf } let (compilation_tables, execution_tables, post_image_table) = match file_type { - FileType::JSON => { - ( - serde_json::from_slice(load_file(&dir, "compilation.json").as_slice()).unwrap(), - serde_json::from_slice(load_file(&dir, "execution.json").as_slice()).unwrap(), - serde_json::from_slice(load_file(&dir, "post_image.json").as_slice()).unwrap() - ) - }, - FileType::FLEXBUFFERS => { - ( - flexbuffers::from_buffer(&load_file(&dir, "compilation.buf").as_slice()).unwrap(), - flexbuffers::from_buffer(&load_file(&dir, "execution.buf").as_slice()).unwrap(), - flexbuffers::from_buffer(&load_file(&dir, "post_image.buf").as_slice()).unwrap(), - ) - } + FileType::JSON => ( + serde_json::from_slice(load_file(&dir, "compilation.json").as_slice()).unwrap(), + serde_json::from_slice(load_file(&dir, "execution.json").as_slice()).unwrap(), + serde_json::from_slice(load_file(&dir, "post_image.json").as_slice()).unwrap(), + ), + FileType::FLEXBUFFERS => ( + flexbuffers::from_buffer(&load_file(&dir, "compilation.buf").as_slice()).unwrap(), + flexbuffers::from_buffer(&load_file(&dir, "execution.buf").as_slice()).unwrap(), + flexbuffers::from_buffer(&load_file(&dir, "post_image.buf").as_slice()).unwrap(), + ), }; Tables { @@ -204,5 +199,4 @@ impl Tables { is_last_slice, } } - } diff --git a/crates/specs/src/state.rs b/crates/specs/src/state.rs index d495c609b..93b2e66a5 100644 --- a/crates/specs/src/state.rs +++ b/crates/specs/src/state.rs @@ -1,15 +1,15 @@ use serde::Deserialize; use serde::Serialize; -use crate::CompilationTable; use crate::etable::EventTableEntry; use crate::host_function::HostPlugin; +use crate::imtable::memory_event_of_step; use crate::imtable::InitMemoryTable; use crate::imtable::InitMemoryTableEntry; -use crate::imtable::memory_event_of_step; +use crate::itable::Opcode; use crate::mtable::AccessType; use crate::step::StepInfo; -use crate::itable::Opcode; +use crate::CompilationTable; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct InitializationState { @@ -113,7 +113,6 @@ impl InitializationState { } } - pub trait UpdateCompilationTable { fn update_init_memory_table(&self, execution_table: &Vec) -> InitMemoryTable; diff --git a/crates/zkwasm/src/circuits/utils/table_entry.rs b/crates/zkwasm/src/circuits/utils/table_entry.rs index b19d1b445..24310d667 100644 --- a/crates/zkwasm/src/circuits/utils/table_entry.rs +++ b/crates/zkwasm/src/circuits/utils/table_entry.rs @@ -1,11 +1,11 @@ use serde::Serialize; use specs::etable::EventTable; use specs::etable::EventTableEntry; +use specs::imtable::memory_event_of_step; use specs::mtable::AccessType; use specs::mtable::LocationType; use specs::mtable::MTable; use specs::mtable::MemoryTableEntry; -use specs::imtable::memory_event_of_step; use std::cmp::Ordering; use std::collections::BTreeMap; @@ -15,7 +15,6 @@ use std::path::PathBuf; use crate::circuits::config::zkwasm_k; - #[derive(Clone, Debug, Serialize)] pub(in crate::circuits) struct MemoryWritingEntry { index: usize, diff --git a/crates/zkwasm/src/continuation/loader.rs b/crates/zkwasm/src/continuation/loader.rs index 50097cc7e..023581c78 100644 --- a/crates/zkwasm/src/continuation/loader.rs +++ b/crates/zkwasm/src/continuation/loader.rs @@ -1,13 +1,80 @@ +use std::path::PathBuf; + use halo2_proofs::arithmetic::MultiMillerLoop; +use threadpool::ThreadPool; +use wasmi::tracer::SliceDumper; use wasmi::RuntimeValue; use crate::loader::ZkWasmLoader; use crate::runtime::ExecutionResult; +use super::slice::Slice; use super::slice::Slices; impl ZkWasmLoader { pub fn slice(&self, execution_result: ExecutionResult) -> Slices { - Slices::new(execution_result.tables.unwrap(), self.compute_slice_capability()) + Slices::new( + execution_result.tables.unwrap(), + self.compute_slice_capability(), + ) + } +} + +#[derive(Default)] +pub struct WitnessDumper { + dump_enabled: bool, + capacity: usize, + output_dir: PathBuf, + slice_index: u32, + thread_pool: ThreadPool, +} + +impl WitnessDumper { + pub(crate) fn new(dump_enabled: bool, capacity: usize, output_dir: Option) -> Self { + let thread_pool = threadpool::Builder::new().build(); + let slice_index = 0; + let output_dir = output_dir.unwrap_or_else(|| std::env::current_dir().unwrap()); + WitnessDumper { + dump_enabled, + capacity, + output_dir, + slice_index, + thread_pool, + } + } +} + +impl SliceDumper for WitnessDumper { + fn dump(&mut self, tables: specs::Tables) { + match self.dump_enabled { + true => { + cfg_if::cfg_if! { + if #[cfg(feature = "continuation")] { + let slice = Slice::new(tables, self.capacity); + let mut dir = self.output_dir.clone(); + dir.push(self.slice_index.to_string()); + + while self.thread_pool.queued_count() > 0 { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + + self.thread_pool.clone().execute(move || { + slice.write_flexbuffers(Some(dir)); + }); + + self.slice_index += 1; + } + } + } + false => {} + } + } + + fn dump_enabled(&self) -> bool { + self.dump_enabled + } + + fn get_capacity(&self) -> usize { + self.capacity } } diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index 01ff94643..2451e49a9 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -72,8 +72,9 @@ impl Slices { } pub fn num_slices(&self) -> usize { - (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64).ceil() as usize - } + (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64) + .ceil() as usize + } } impl Iterator for Slices { diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index cef3c7954..2007e5544 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -17,11 +17,12 @@ use halo2aggregator_s::circuits::utils::load_or_create_proof; use halo2aggregator_s::circuits::utils::TranscriptHash; use halo2aggregator_s::transcript::poseidon::PoseidonRead; use specs::Tables; -use wasmi::ENTRY; +use wasmi::tracer::SliceDumper; use wasmi::tracer::Tracer; use wasmi::ImportsBuilder; use wasmi::NotStartedModuleRef; use wasmi::RuntimeValue; +use wasmi::ENTRY; use crate::checksum::CompilationTableWithParams; use crate::checksum::ImageCheckSum; @@ -31,8 +32,7 @@ use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::circuits::image_table::IMAGE_COL_NAME; use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; -#[cfg(feature="continuation")] -use crate::continuation::slice::Slice; +use crate::continuation::loader::WitnessDumper; use crate::loader::err::Error; use crate::loader::err::PreCheckErr; use crate::profile::Profiler; @@ -45,8 +45,6 @@ use anyhow::anyhow; mod err; -pub type CallbackType = Box; - pub struct ExecutionArg { /// Public inputs for `wasm_input(1)` pub public_inputs: Vec, @@ -58,7 +56,7 @@ pub struct ExecutionArg { pub context_outputs: Arc>>, pub output_dir: Option, /// enable dump table for continuation - pub dump_table: bool + pub dump_table: bool, } pub struct ExecutionReturn { @@ -76,7 +74,7 @@ impl ZkWasmLoader { pub fn compute_slice_capability(&self) -> usize { ((1 << self.k) - 200) / EVENT_TABLE_ENTRY_ROWS as usize } - + fn precheck(&self) -> Result<()> { fn check_zkmain_exists(module: &wasmi::Module) -> Result<()> { use parity_wasm::elements::Internal; @@ -107,7 +105,11 @@ impl ZkWasmLoader { Ok(()) } - fn compile(&self, env: &HostEnv, callback: Option) -> Result, Tracer>> { + fn compile( + &self, + env: &HostEnv, + slice_dumper: Box, + ) -> Result, Tracer>> { let imports = ImportsBuilder::new().with_resolver("env", env); WasmInterpreter::compile( @@ -116,8 +118,7 @@ impl ZkWasmLoader { &env.function_description_table(), ENTRY, &self.phantom_functions, - callback, - self.compute_slice_capability() + slice_dumper, ) } @@ -164,7 +165,7 @@ impl ZkWasmLoader { vec![], Arc::new(Mutex::new(vec![])), ); - let compiled = self.compile(&env, None::)?; + let compiled = self.compile(&env, Box::new(WitnessDumper::default()))?; let table_with_params = CompilationTableWithParams { table: &compiled.tables, @@ -183,7 +184,7 @@ impl ZkWasmLoader { arg.context_inputs, arg.context_outputs, ); - let compiled_module = self.compile(&env, None::)?; + let compiled_module = self.compile(&env, Box::new(WitnessDumper::default()))?; compiled_module.dry_run(&mut env) } @@ -195,46 +196,19 @@ impl ZkWasmLoader { arg.context_outputs, ); - let output_dir = arg.output_dir.unwrap_or_else(|| std::env::current_dir().unwrap()); - if arg.dump_table { - // defaults the number of threads to the number of CPUs. - let pool = threadpool::Builder::new().build(); - let pool_cb = pool.clone(); - let mut _index = 0; - let callback = move |_table, _capability | { - cfg_if::cfg_if! { - if #[cfg(feature = "continuation")] { - let slice = Slice::new(_table, _capability); - let mut dir = output_dir.clone(); - dir.push(_index.to_string()); - println!("dumping------------------>"); - pool_cb.execute(move || { - slice.write_flexbuffers(Some(dir)); - println!("Slice: {} tables has dumped!", _index); - }); - _index += 1; - - while pool_cb.queued_count() > 0 { - std::thread::sleep(std::time::Duration::from_millis(10)); - } - } - } - }; - - let compiled_module = self.compile(&env, Some(callback))?; - let result = compiled_module.run(&mut env, wasm_runtime_io)?; - pool.join(); - Ok(result) - } else { - let compiled_module = self.compile(&env, None::)?; - let result = compiled_module.run(&mut env, wasm_runtime_io)?; - if let Some(tables) = &result.tables { - tables.profile_tables(); - tables.write(Some(output_dir), specs::FileType::FLEXBUFFERS); - } - Ok(result) - } + let slice_dumper = WitnessDumper::new( + arg.dump_table, + self.compute_slice_capability(), + arg.output_dir.clone(), + ); + let compiled_module = self.compile(&env, Box::new(slice_dumper))?; + let result = compiled_module.run(&mut env, wasm_runtime_io)?; + if let Some(tables) = &result.tables { + tables.profile_tables(); + tables.write(arg.output_dir, specs::FileType::FLEXBUFFERS); + } + Ok(result) } pub fn circuit_with_witness( diff --git a/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs b/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs index 08528d523..cf5b21f06 100644 --- a/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs +++ b/crates/zkwasm/src/runtime/host/internal_circuit_plugin.rs @@ -23,7 +23,7 @@ pub(super) struct ForeignOp { pub struct InternalCircuitEnv { pub(super) plugins: HashMap, - // here we use BTreeMap to make sure op.index deterministic + // here we use BTreeMap to make sure op.index deterministic pub(super) functions: BTreeMap, finalized: Rc>, } diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index 97904da7e..09bc7dc1c 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -5,18 +5,19 @@ use std::sync::Arc; use anyhow::Result; use specs::host_function::HostFunctionDesc; -use specs::state::{InitializationState, UpdateCompilationTable}; +use specs::state::InitializationState; +use specs::state::UpdateCompilationTable; use specs::CompilationTable; use specs::ExecutionTable; use specs::Tables; +use wasmi::tracer::SliceDumper; use wasmi::Externals; use wasmi::ImportResolver; use wasmi::ModuleInstance; use wasmi::RuntimeValue; use wasmi::DEFAULT_VALUE_STACK_LIMIT; - use super::CompiledImage; use super::ExecutionResult; @@ -66,23 +67,23 @@ impl Execution let result = instance.invoke_export_trace(&self.entry, &[], externals, self.tracer.clone())?; - + let mut tables: Option = None; let tracer = self.tracer.borrow(); - if !tracer.has_dumped() { + if !tracer.dump_enabled() { let execution_tables = { let tracer = self.tracer.borrow(); - + ExecutionTable { etable: tracer.etable.clone(), jtable: Arc::new(tracer.jtable.clone()), } }; - + let updated_init_memory_table = self .tables .update_init_memory_table(&execution_tables.etable.entries()); - + let post_image_table = { CompilationTable { itable: self.tables.itable.clone(), @@ -102,7 +103,7 @@ impl Execution post_image_table, is_last_slice: true, }); - } + } Ok(ExecutionResult { tables, @@ -126,10 +127,10 @@ impl WasmiRuntime { host_plugin_lookup: &HashMap, entry: &str, phantom_functions: &Vec, - callback: Option, - capability: usize + slice_dumper: Box, ) -> Result, wasmi::tracer::Tracer>> { - let tracer = wasmi::tracer::Tracer::new(host_plugin_lookup.clone(), phantom_functions, callback, capability); + let tracer = + wasmi::tracer::Tracer::new(host_plugin_lookup.clone(), phantom_functions, slice_dumper); let tracer = Rc::new(RefCell::new(tracer)); let instance = ModuleInstance::new(&module, imports, Some(tracer.clone())) .expect("failed to instantiate wasm module"); diff --git a/crates/zkwasm/src/test/mod.rs b/crates/zkwasm/src/test/mod.rs index 23df68fdc..3997ae427 100644 --- a/crates/zkwasm/src/test/mod.rs +++ b/crates/zkwasm/src/test/mod.rs @@ -1,6 +1,6 @@ use crate::circuits::config::zkwasm_k; use crate::circuits::TestCircuit; -use crate::loader::CallbackType; +use crate::continuation::loader::WitnessDumper; use crate::profile::Profiler; use crate::runtime::host::host_env::HostEnv; use crate::runtime::wasmi_interpreter::Execution; @@ -44,7 +44,11 @@ fn test_circuit_mock( v }; - execution_result.tables.as_ref().unwrap().write(None, specs::FileType::JSON); + execution_result + .tables + .as_ref() + .unwrap() + .write(None, specs::FileType::JSON); execution_result.tables.as_ref().unwrap().profile_tables(); @@ -72,8 +76,7 @@ fn compile_then_execute_wasm( &env.function_description_table(), function_name, &vec![], - None::, - 0 + Box::new(WitnessDumper::default()), ) .unwrap(); diff --git a/crates/zkwasm/src/test/test_rlp.rs b/crates/zkwasm/src/test/test_rlp.rs index c3a8342d1..71eb8f2fe 100644 --- a/crates/zkwasm/src/test/test_rlp.rs +++ b/crates/zkwasm/src/test/test_rlp.rs @@ -157,7 +157,7 @@ fn build_circuit() -> Result<(ZkWasmLoader, TestCircuit, Vec)> { context_inputs: vec![], context_outputs: Arc::new(Mutex::new(vec![])), output_dir: None, - dump_table: false + dump_table: false, })?; Ok((loader, circuit, instances)) diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index c271a3245..f987d4959 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -1,17 +1,20 @@ use std::sync::Arc; use std::sync::Mutex; +use crate::continuation::slice::Slice; use crate::loader::ExecutionArg; use crate::loader::ZkWasmLoader; -use crate::continuation::slice::Slice; use crate::runtime::ExecutionResult; use specs::Tables; use anyhow::Result; -use halo2_proofs::pairing::bn256::{Bn256, Fr}; +use halo2_proofs::pairing::bn256::Bn256; +use halo2_proofs::pairing::bn256::Fr; use wasmi::RuntimeValue; -fn generate_wasm_result(dump_table: bool) -> Result<(ZkWasmLoader, Vec, ExecutionResult)> { +fn generate_wasm_result( + dump_table: bool, +) -> Result<(ZkWasmLoader, Vec, ExecutionResult)> { let public_inputs = vec![133]; let private_inputs: Vec = vec![ 14625441452057167097, @@ -153,15 +156,14 @@ fn generate_wasm_result(dump_table: bool) -> Result<(ZkWasmLoader, Vec::new(18, wasm, vec![])?; - let execution_result = loader - .run(ExecutionArg { - public_inputs, - private_inputs, - context_inputs: vec![], - context_outputs: Arc::new(Mutex::new(vec![])), - output_dir: Some(std::env::current_dir().unwrap()), - dump_table - })?; + let execution_result = loader.run(ExecutionArg { + public_inputs, + private_inputs, + context_inputs: vec![], + context_outputs: Arc::new(Mutex::new(vec![])), + output_dir: Some(std::env::current_dir().unwrap()), + dump_table, + })?; let instances = execution_result .public_inputs_and_outputs @@ -178,7 +180,6 @@ fn test_slices() -> Result<()> { let mut index = 0; while let Some(slice) = slices.next() { - println!("slice {}", index); let circuit = slice.build_circuit(); @@ -212,7 +213,11 @@ fn test_rpl_slice_from_file() -> Result<()> { dir.push("full_run_dump"); dir.push(index.to_string()); - let table = Tables::load(dir.clone(), index == last_slice_index, specs::FileType::FLEXBUFFERS); + let table = Tables::load( + dir.clone(), + index == last_slice_index, + specs::FileType::FLEXBUFFERS, + ); let slice = Slice::new(table, slices.capability()); let circuit = slice.build_circuit(); loader.mock_test(&circuit, &instances)?; @@ -223,7 +228,6 @@ fn test_rpl_slice_from_file() -> Result<()> { Ok(()) } - fn test_rpl_slice_dump() -> Result<()> { // dump slice while running let (_, _instances, _) = generate_wasm_result(true)?; @@ -237,7 +241,11 @@ fn test_rpl_slice_dump() -> Result<()> { // load slice from running dump let mut dir = std::env::current_dir().unwrap(); dir.push(index.to_string()); - let table = Tables::load(dir.clone(), index == last_slice_index, specs::FileType::FLEXBUFFERS); + let table = Tables::load( + dir.clone(), + index == last_slice_index, + specs::FileType::FLEXBUFFERS, + ); let loaded_slice = Slice::new(table, slices.capability()); // make sure slices generated from memory and during running is the same diff --git a/third-party/wasmi b/third-party/wasmi index 9cb31af68..9be79d5b3 160000 --- a/third-party/wasmi +++ b/third-party/wasmi @@ -1 +1 @@ -Subproject commit 9cb31af684da4c160b7d50ad793a4c079637f3a4 +Subproject commit 9be79d5b3658ed693085c3d4d18cbcb607421735 From c77cd1d064ceda01594d29d54ea8c4d1772f1f20 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Mon, 11 Dec 2023 07:39:03 +0000 Subject: [PATCH 06/34] update comment --- crates/zkwasm/src/circuits/etable/assign.rs | 46 +++++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/crates/zkwasm/src/circuits/etable/assign.rs b/crates/zkwasm/src/circuits/etable/assign.rs index 85ff77263..adda194a6 100644 --- a/crates/zkwasm/src/circuits/etable/assign.rs +++ b/crates/zkwasm/src/circuits/etable/assign.rs @@ -23,26 +23,36 @@ use crate::circuits::utils::Context; * Etable Layouter with Continuation * * Not last slice - * | ---- | ------ | ---------- | --------- | - * | sel | enable | rest_mops | states ... | - * | ---- + ------ | ---------- + --------- | - * | 1 | 1 | | ... | - * | 1 | 1 | | ... | - * | 1 | 1 | | ... | - * | 0 | 0 | constant 0 | ... | permutation row(status should keep consistent with previous row) + * - `self.capability` entries with `enable = 1`. + * - one entry(only including status) with enable = 0, the status of this entry should constrain equality + * with the first entry in the next slice. * + * | -------- | ---- | ------ | ---------- | ---- | ------ | + * | | sel | enable | rest_mops | jops | states | + * | -------- | ---- + ------ | ---------- + ---- | ------ | + * | event | 1 | 1 | | | | permutation with pre image table + * | table | 1 | 1 | | | | + * | entries | 1 | 1 | | | | + * | | 1 | 1 | | | | + * | -------- | ---- | ------ | ---------- | ---- | ------ | + * | | 0 | 0 | constant 0 | | | permutation with post image table * - * Not last slice - * | ---- | ------ | ---------- | ------ | - * | sel | enable | rest_mops | states | - * | ---- + ------ | ---------- + -------| - * | 1 | 1 | | ... | - * | 1 | 1 | | ... | - * | 1 | 0 | | ... | - * | 1 | 0 | | ... | - * | 1 | 0 | | ... | - * | 1 | 0 | | ... | - * | 0 | 0 | constant 0 | ... | permutation row + * + * Last slice + * `` + * | -------- | ---- | ------ | ---------- | ---- | ------ | + * | | sel | enable | rest_mops | jops | states | + * | -------- | ---- + ------ | ---------- + ---- | -------| + * | event | 1 | 1 | | | | permutation with pre image table + * | table | 1 | 1 | | | | + * | entires | 1 | 1 | | | | + * | -------- | ---- | ------ | ---------- | ---- | ------ | + * | padding | 1 | 0 | | | | padding rows are used to copy termination status + * | | 1 | 0 | | | | to permutation row + * | | 1 | 0 | | | | + * | | 1 | 0 | | | | + * | -------- | ---- | ------ | ---------- | ---- | ------ | + * | | 0 | 0 | constant 0 | | | permutation with post image table/jops constrain with jtable */ pub(in crate::circuits) struct EventTablePermutationCells { From d46090be4329e0624fb46be45a7ebefc4da7f76b Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Mon, 11 Dec 2023 08:48:04 +0000 Subject: [PATCH 07/34] fix: use F instead of AssignedCell to avoid unwrap --- crates/zkwasm/src/circuits/mtable/assign.rs | 20 +- .../circuits/post_image_table/continuation.rs | 235 +----------------- .../src/circuits/post_image_table/mod.rs | 1 + .../src/circuits/post_image_table/trivial.rs | 1 + .../zkwasm/src/circuits/test_circuit/mod.rs | 141 +++++------ 5 files changed, 82 insertions(+), 316 deletions(-) diff --git a/crates/zkwasm/src/circuits/mtable/assign.rs b/crates/zkwasm/src/circuits/mtable/assign.rs index 0acb7ea27..437b12284 100644 --- a/crates/zkwasm/src/circuits/mtable/assign.rs +++ b/crates/zkwasm/src/circuits/mtable/assign.rs @@ -297,7 +297,7 @@ impl MemoryTableChip { etable_rest_mops_cell: &Option>, mtable: &MemoryWritingTable, imtable: &InitMemoryTable, - ) -> Result>, Error> { + ) -> Result<(Option>, F), Error> { debug!("size of memory writing table: {}", mtable.0.len()); assert!(mtable.0.len() * (MEMORY_TABLE_ENTRY_ROWS as usize) < self.maximal_available_rows); @@ -309,11 +309,11 @@ impl MemoryTableChip { self.assign_fixed(ctx)?; ctx.reset(); - let _rest_memory_updating_ops = self.compute_rest_memory_updating_ops(mtable); + let rest_memory_updating_ops = self.compute_rest_memory_updating_ops(mtable); #[cfg(feature = "continuation")] - let rest_memory_updating_ops = - self.constrain_rest_memory_updating_ops(ctx, _rest_memory_updating_ops)?; + let rest_memory_updating_ops_cell = + self.constrain_rest_memory_updating_ops(ctx, rest_memory_updating_ops)?; let rest_mops_cell = self.constrain_rest_mops_permutation(ctx, etable_rest_mops_cell, rest_mops)?; @@ -322,21 +322,15 @@ impl MemoryTableChip { * Skip subsequent advice assignment in the first pass to enhance performance. */ if rest_mops_cell.value().is_some() { - self.assign_entries( - ctx, - mtable, - rest_mops, - imtable, - self.compute_rest_memory_updating_ops(mtable), - )?; + self.assign_entries(ctx, mtable, rest_mops, imtable, rest_memory_updating_ops)?; ctx.reset(); } cfg_if::cfg_if! { if #[cfg(feature="continuation")] { - Ok(Some(rest_memory_updating_ops)) + Ok((Some(rest_memory_updating_ops_cell), F::from(rest_memory_updating_ops as u64))) } else { - Ok(None) + Ok((None, F::from(rest_memory_updating_ops as u64))) } } } diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 2cfaba863..6b9ab9de2 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -114,234 +114,8 @@ impl PostImageTableChipTrait pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, + rest_memory_writing_ops: F, ) -> Result<(), Error> { - // fn init_sel( - // region: &mut Region, - // sel: Column, - // rest_memory_finalized_count: Column, - // circuit_maximal_pages: u32, - // ) -> Result<(), Error> { - // let mut offset = INIT_MEMORY_ENTRIES_OFFSET; - - // region.assign_fixed(|| "post image table: init", sel, offset, || Ok(F::zero()))?; - - // offset += 1; - - // macro_rules! assign_address { - // ($l:expr, $o:expr) => {{ - // region.assign_fixed( - // || "post image table: init", - // sel, - // offset, - // || { - // Ok(bn_to_field(&encode_init_memory_table_address( - // BigUint::from($l as u64), - // BigUint::from($o as u64), - // ))) - // }, - // )?; - - // offset += 1; - - // Ok::<_, Error>(()) - // }}; - // } - - // for i in 0..DEFAULT_VALUE_STACK_LIMIT { - // assign_address!(LocationType::Stack, i)?; - // } - - // for i in 0..DEFAULT_VALUE_STACK_LIMIT { - // assign_address!(LocationType::Global, i)?; - // } - - // for i in 0..(circuit_maximal_pages * PAGE_ENTRIES) { - // assign_address!(LocationType::Heap, i)?; - // } - - // region.assign_advice_from_constant( - // || "post image table: init memory", - // rest_memory_finalized_count, - // offset, - // F::zero(), - // )?; - - // Ok(()) - // } - - // fn assign_static_frame_entries( - // ctx: &mut Context, - // post_image_table_col: Column, - // static_frame_entries: &Vec<(AssignedCell, AssignedCell)>, - // ) -> Result<(), Error> { - // for (enable, entry) in static_frame_entries { - // enable.copy_advice( - // || "image table: static frame entry", - // &mut ctx.region, - // post_image_table_col, - // ctx.offset, - // )?; - // ctx.next(); - - // entry.copy_advice( - // || "image table: static frame entry", - // &mut ctx.region, - // post_image_table_col, - // ctx.offset, - // )?; - // ctx.next(); - // } - - // Ok(()) - // } - - // fn assign_instructions( - // ctx: &mut Context, - // post_image_table_col: Column, - // instructions: &Vec>, - // ) -> Result<(), Error> { - // for cell in instructions { - // cell.copy_advice( - // || "post image table: instruction and br table", - // &mut ctx.region, - // post_image_table_col, - // ctx.offset, - // )?; - - // ctx.next(); - // } - - // Ok(()) - // } - - // fn assign_br_table( - // ctx: &mut Context, - // post_image_table_col: Column, - // br_table: &Vec>, - // ) -> Result<(), Error> { - // for cell in br_table { - // cell.copy_advice( - // || "post image table: instruction and br table", - // &mut ctx.region, - // post_image_table_col, - // ctx.offset, - // )?; - - // ctx.next(); - // } - - // Ok(()) - // } - - // fn assign_padding( - // ctx: &mut Context, - // post_image_table_col: Column, - // padding: &Vec>, - // ) -> Result<(), Error> { - // for cell in padding { - // cell.copy_advice( - // || "post image table: instruction and br table", - // &mut ctx.region, - // post_image_table_col, - // ctx.offset, - // )?; - - // ctx.next(); - // } - - // Ok(()) - // } - - // fn assign_init_memory_entries( - // ctx: &mut Context, - // sel: Column, - // post_image_table_col: Column, - // update_col: Column, - // rest_memory_finalized_ops_col: Column, - // pre_image_table: &ImageTableLayouter, - // post_image_table: &ImageTableLayouter, - // permutation_cells: &ImageTableLayouter>, - // circuit_maximal_pages: u32, - // ) -> Result<(), Error> { - // assert!(ctx.offset < INIT_MEMORY_ENTRIES_OFFSET); - // ctx.offset = INIT_MEMORY_ENTRIES_OFFSET; - - // assert_eq!( - // pre_image_table.init_memory_entries.as_ref().unwrap().len(), - // post_image_table.init_memory_entries.as_ref().unwrap().len() - // ); - - // init_sel( - // &mut ctx.region, - // sel, - // rest_memory_finalized_ops_col, - // circuit_maximal_pages, - // )?; - - // permutation_cells - // .rest_memory_writing_ops - // .as_ref() - // .unwrap() - // .copy_advice( - // || "post image table: init memory", - // &mut ctx.region, - // rest_memory_finalized_ops_col, - // ctx.offset, - // )?; - - // let mut rest_memory_writing_ops = *permutation_cells - // .rest_memory_writing_ops - // .as_ref() - // .unwrap() - // .value() - // .unwrap(); - - // for (pre, post) in pre_image_table - // .init_memory_entries - // .as_ref() - // .unwrap() - // .iter() - // .zip( - // post_image_table - // .init_memory_entries - // .as_ref() - // .unwrap() - // .iter(), - // ) - // { - // ctx.region.assign_advice( - // || "post image table: init memory", - // post_image_table_col, - // ctx.offset, - // || Ok(*post), - // )?; - - // ctx.region.assign_advice( - // || "post image table: init memory", - // rest_memory_finalized_ops_col, - // ctx.offset, - // || Ok(rest_memory_writing_ops), - // )?; - - // if pre != post { - // ctx.region.assign_advice( - // || "post image table: init memory", - // update_col, - // ctx.offset, - // || Ok(F::one()), - // )?; - - // rest_memory_writing_ops = rest_memory_writing_ops - F::one(); - // } - - // ctx.next(); - // } - - // assert_eq!(rest_memory_writing_ops, F::zero()); - - // Ok::<_, Error>(()) - // } - layouter.assign_region( || "post image table", |region| { @@ -560,12 +334,7 @@ impl PostImageTableChipTrait let entries = { let mut offset = base_offset; - let mut rest_memory_writing_ops = *permutation_cells - .rest_memory_writing_ops - .as_ref() - .unwrap() - .value() - .unwrap(); + let mut rest_memory_writing_ops = rest_memory_writing_ops; pre_image_table .init_memory_entries diff --git a/crates/zkwasm/src/circuits/post_image_table/mod.rs b/crates/zkwasm/src/circuits/post_image_table/mod.rs index d84f60fef..7e99f1091 100644 --- a/crates/zkwasm/src/circuits/post_image_table/mod.rs +++ b/crates/zkwasm/src/circuits/post_image_table/mod.rs @@ -42,6 +42,7 @@ pub(in crate::circuits) trait PostImageTableChipTrait< pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, + rest_memory_writing_ops: F, ) -> Result<(), Error>; } diff --git a/crates/zkwasm/src/circuits/post_image_table/trivial.rs b/crates/zkwasm/src/circuits/post_image_table/trivial.rs index 2ff070cc8..cac7219e9 100644 --- a/crates/zkwasm/src/circuits/post_image_table/trivial.rs +++ b/crates/zkwasm/src/circuits/post_image_table/trivial.rs @@ -56,6 +56,7 @@ impl PostImageTableChipTrait> _pre_image_table: ImageTableLayouter, _post_image_table: ImageTableLayouter, _permutation_cells: ImageTableLayouter>, + _rest_memory_writing_ops: F, ) -> Result<(), Error> { Ok(()) } diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index 86bab146b..a6ed9e8b2 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -229,77 +229,77 @@ impl Circuit for TestCircuit { )? ); - let (etable_permutation_cells, rest_memory_writing_ops, static_frame_entries) = layouter - .assign_region( - || "jtable mtable etable", - |region| { - let mut ctx = Context::new(region); - - let memory_writing_table: MemoryWritingTable = - self.tables.create_memory_table(memory_event_of_step).into(); - - let etable = exec_with_profile!( - || "Prepare memory info for etable", - EventTableWithMemoryInfo::new( - &self.tables.execution_tables.etable, + let ( + etable_permutation_cells, + (rest_memory_writing_ops_cell, rest_memory_writing_ops), + static_frame_entries, + ) = layouter.assign_region( + || "jtable mtable etable", + |region| { + let mut ctx = Context::new(region); + + let memory_writing_table: MemoryWritingTable = + self.tables.create_memory_table(memory_event_of_step).into(); + + let etable = exec_with_profile!( + || "Prepare memory info for etable", + EventTableWithMemoryInfo::new( + &self.tables.execution_tables.etable, + &memory_writing_table, + ) + ); + + let etable_permutation_cells = exec_with_profile!( + || "Assign etable", + echip.assign( + &mut ctx, + &etable, + &self.tables.compilation_tables.configure_table, + &self.tables.compilation_tables.initialization_state, + &self.tables.post_image_table.initialization_state, + self.tables.is_last_slice, + )? + ); + + let rest_memory_writing_ops = { + ctx.reset(); + + exec_with_profile!( + || "Assign mtable", + mchip.assign( + &mut ctx, + &etable_permutation_cells.rest_mops, &memory_writing_table, - ) - ); - - let etable_permutation_cells = exec_with_profile!( - || "Assign etable", - echip.assign( + &self.tables.compilation_tables.imtable + )? + ) + }; + + let jtable_info = { + ctx.reset(); + exec_with_profile!( + || "Assign frame table", + jchip.assign( &mut ctx, - &etable, - &self.tables.compilation_tables.configure_table, - &self.tables.compilation_tables.initialization_state, - &self.tables.post_image_table.initialization_state, - self.tables.is_last_slice, + &self.tables.execution_tables.jtable, + &etable_permutation_cells.rest_jops, + &self.tables.compilation_tables.static_jtable, )? - ); - - let rest_memory_writing_ops = { - ctx.reset(); - - exec_with_profile!( - || "Assign mtable", - mchip.assign( - &mut ctx, - &etable_permutation_cells.rest_mops, - &memory_writing_table, - &self.tables.compilation_tables.imtable - )? - ) - }; - - let jtable_info = { - ctx.reset(); - exec_with_profile!( - || "Assign frame table", - jchip.assign( - &mut ctx, - &self.tables.execution_tables.jtable, - &etable_permutation_cells.rest_jops, - &self.tables.compilation_tables.static_jtable, - )? - ) - }; - - { - ctx.reset(); - exec_with_profile!( - || "Assign bit table", - bit_chip.assign(&mut ctx, &etable)? - ); - } - - Ok(( - etable_permutation_cells, - rest_memory_writing_ops, - jtable_info, - )) - }, - )?; + ) + }; + + { + ctx.reset(); + exec_with_profile!(|| "Assign bit table", bit_chip.assign(&mut ctx, &etable)?); + } + + Ok(( + etable_permutation_cells, + rest_memory_writing_ops, + jtable_info, + )) + }, + )?; exec_with_profile!( || "Assign context cont chip", @@ -367,8 +367,9 @@ impl Circuit for TestCircuit { br_table: pre_image_table_cells.br_table, padding: pre_image_table_cells.padding, init_memory_entries: pre_image_table_cells.init_memory_entries, - rest_memory_writing_ops, - } + rest_memory_writing_ops: rest_memory_writing_ops_cell, + }, + rest_memory_writing_ops )? ); From bd1c40df2f4d430a69cdcc5aae983086578b3a08 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Mon, 11 Dec 2023 14:17:45 +0000 Subject: [PATCH 08/34] fix: fix termination state and imtable --- crates/specs/src/lib.rs | 55 ++++++++++--------- crates/zkwasm/src/circuits/image_table/mod.rs | 15 +++++ .../zkwasm/src/circuits/test_circuit/mod.rs | 8 ++- crates/zkwasm/src/loader/mod.rs | 17 +++++- 4 files changed, 68 insertions(+), 27 deletions(-) diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 0d9c0f184..3bfa81236 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -19,6 +19,7 @@ use itable::InstructionTable; use jtable::JumpTable; use jtable::StaticFrameEntry; use mtable::AccessType; +use mtable::LocationType; use mtable::MTable; use mtable::MemoryTableEntry; use rayon::prelude::IntoParallelRefIterator; @@ -108,33 +109,37 @@ impl Tables { .collect::>>() .concat(); - let init_value = memory_entries - .par_iter() - .map(|entry| { - self.compilation_tables - .imtable - .try_find(entry.ltype, entry.offset) - }) - .collect::>(); - let mut set = HashSet::::default(); - memory_entries - .iter() - .zip(init_value.into_iter()) - .for_each(|(entry, init_memory_entry)| { - if let Some(init_memory_entry) = init_memory_entry { - set.insert(MemoryTableEntry { - eid: init_memory_entry.eid, - offset: entry.offset, - ltype: entry.ltype, - atype: AccessType::Init, - vtype: entry.vtype, - is_mutable: entry.is_mutable, - value: init_memory_entry.value, - }); - } - }); + memory_entries.iter().for_each(|entry| { + let init_memory_entry = self + .compilation_tables + .imtable + .try_find(entry.ltype, entry.offset); + + if let Some(init_memory_entry) = init_memory_entry { + set.insert(MemoryTableEntry { + eid: init_memory_entry.eid, + offset: entry.offset, + ltype: entry.ltype, + atype: AccessType::Init, + vtype: entry.vtype, + is_mutable: entry.is_mutable, + value: init_memory_entry.value, + }); + } else if entry.ltype == LocationType::Heap { + // Heap value without init memory entry should equal 0 + set.insert(MemoryTableEntry { + eid: 0, + offset: entry.offset, + ltype: entry.ltype, + atype: AccessType::Init, + vtype: entry.vtype, + is_mutable: entry.is_mutable, + value: 0, + }); + } + }); memory_entries.append(&mut set.into_iter().collect()); diff --git a/crates/zkwasm/src/circuits/image_table/mod.rs b/crates/zkwasm/src/circuits/image_table/mod.rs index 2eae9ceee..c26e7452b 100644 --- a/crates/zkwasm/src/circuits/image_table/mod.rs +++ b/crates/zkwasm/src/circuits/image_table/mod.rs @@ -9,9 +9,11 @@ use specs::brtable::BrTable; use specs::brtable::ElemTable; use specs::encode::image_table::ImageTableEncoder; use specs::imtable::InitMemoryTable; +use specs::imtable::InitMemoryTableEntry; use specs::itable::InstructionTable; use specs::jtable::StaticFrameEntry; use specs::mtable::LocationType; +use specs::mtable::VarType; use specs::state::InitializationState; use specs::CompilationTable; use std::marker::PhantomData; @@ -165,6 +167,19 @@ impl EncodeCompilationTableValues for CompilationTable { layouter.for_each(|(ltype, offset)| { if let Some(entry) = init_memory_table.try_find(ltype, offset) { + cells.push(bn_to_field::( + &ImageTableEncoder::InitMemory.encode(entry.encode()), + )); + } else if ltype == LocationType::Heap { + let entry = InitMemoryTableEntry { + ltype, + is_mutable: true, + offset, + vtype: VarType::I64, + value: 0, + eid: 0, + }; + cells.push(bn_to_field::( &ImageTableEncoder::InitMemory.encode(entry.encode()), )); diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index a6ed9e8b2..890d089bc 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -12,6 +12,7 @@ use halo2_proofs::plonk::Error; use halo2_proofs::plonk::Fixed; use log::debug; use log::info; +use specs::ExecutionTable; use specs::Tables; use wasmi::DEFAULT_VALUE_STACK_LIMIT; @@ -90,7 +91,12 @@ impl Circuit for TestCircuit { fn without_witnesses(&self) -> Self { TestCircuit::new( - Tables::default(self.tables.is_last_slice), + Tables { + compilation_tables: self.tables.compilation_tables.clone(), + execution_tables: ExecutionTable::default(), + post_image_table: self.tables.post_image_table.clone(), + is_last_slice: self.tables.is_last_slice, + }, self.slice_capability, ) } diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index 2007e5544..f7b9850e4 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -16,6 +16,7 @@ use halo2_proofs::poly::commitment::ParamsVerifier; use halo2aggregator_s::circuits::utils::load_or_create_proof; use halo2aggregator_s::circuits::utils::TranscriptHash; use halo2aggregator_s::transcript::poseidon::PoseidonRead; +use specs::ExecutionTable; use specs::Tables; use wasmi::tracer::SliceDumper; use wasmi::tracer::Tracer; @@ -123,8 +124,22 @@ impl ZkWasmLoader { } fn circuit_without_witness(&self, last_slice_circuit: bool) -> Result> { + let (env, _) = HostEnv::new_with_full_foreign_plugins( + vec![], + vec![], + vec![], + Arc::new(Mutex::new(vec![])), + ); + + let compiled_module = self.compile(&env)?; + let builder = ZkWasmCircuitBuilder { - tables: Tables::default(last_slice_circuit), + tables: Tables { + compilation_tables: compiled_module.tables.clone(), + execution_tables: ExecutionTable::default(), + post_image_table: compiled_module.tables, // FIXME: odd + is_last_slice: last_slice_circuit, + }, }; Ok(builder.build_circuit::(None)) From c095762f981a8929adc4ebd4ca9638d67631ff8d Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Tue, 12 Dec 2023 03:48:22 +0000 Subject: [PATCH 09/34] fix etable assignment --- crates/zkwasm/src/circuits/etable/assign.rs | 24 +++++++------------ .../zkwasm/src/circuits/utils/step_status.rs | 4 ++-- third-party/wasmi | 2 +- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/crates/zkwasm/src/circuits/etable/assign.rs b/crates/zkwasm/src/circuits/etable/assign.rs index adda194a6..1685b1350 100644 --- a/crates/zkwasm/src/circuits/etable/assign.rs +++ b/crates/zkwasm/src/circuits/etable/assign.rs @@ -3,7 +3,6 @@ use halo2_proofs::circuit::AssignedCell; use halo2_proofs::plonk::Error; use log::debug; use specs::configure_table::ConfigureTable; -use specs::itable::Opcode; use specs::itable::OpcodeClassPlain; use specs::state::InitializationState; use std::collections::BTreeMap; @@ -326,6 +325,7 @@ impl EventTableChip { event_table: &EventTableWithMemoryInfo, configure_table: &ConfigureTable, initialization_state: &InitializationState, + post_initialization_state: &InitializationState, mut rest_mops: u32, mut jops: u32, ) -> Result<(), Error> { @@ -394,19 +394,12 @@ impl EventTableChip { .collect::>(); let terminate_status = Status { - eid: status.last().unwrap().eid + 1, - fid: 0, - iid: 0, - sp: status.last().unwrap().sp - + if let Opcode::Return { drop, .. } = - &event_table.0.last().unwrap().eentry.inst.opcode - { - *drop - } else { - 0 - }, - last_jump_eid: 0, - allocated_memory_pages: status.last().unwrap().allocated_memory_pages, + eid: post_initialization_state.eid, + fid: post_initialization_state.fid, + iid: post_initialization_state.iid, + sp: post_initialization_state.sp, + last_jump_eid: post_initialization_state.frame_id, + allocated_memory_pages: post_initialization_state.initial_memory_pages, }; status.push(terminate_status); @@ -419,7 +412,7 @@ impl EventTableChip { current: &status[index], next: &status[index + 1], current_external_host_call_index: external_host_call_call_index, - configure_table: *configure_table, + configure_table, host_public_inputs, context_in_index, context_out_index, @@ -521,6 +514,7 @@ impl EventTableChip { event_table, configure_table, &initialization_state, + post_initialization_state, rest_mops, jops, )?; diff --git a/crates/zkwasm/src/circuits/utils/step_status.rs b/crates/zkwasm/src/circuits/utils/step_status.rs index 9238a9ad9..acc2e2863 100644 --- a/crates/zkwasm/src/circuits/utils/step_status.rs +++ b/crates/zkwasm/src/circuits/utils/step_status.rs @@ -10,12 +10,12 @@ pub struct Status { pub allocated_memory_pages: u32, } -pub struct StepStatus<'a> { +pub struct StepStatus<'a, 'b> { pub current: &'a Status, pub next: &'a Status, pub current_external_host_call_index: u32, pub host_public_inputs: u32, pub context_in_index: u32, pub context_out_index: u32, - pub configure_table: ConfigureTable, + pub configure_table: &'b ConfigureTable, } diff --git a/third-party/wasmi b/third-party/wasmi index 9be79d5b3..cc61c1b84 160000 --- a/third-party/wasmi +++ b/third-party/wasmi @@ -1 +1 @@ -Subproject commit 9be79d5b3658ed693085c3d4d18cbcb607421735 +Subproject commit cc61c1b840306d0e51962a65d9205a9f7b3f74c0 From f99839adf632ce154a13125b026b2cda3fc441b5 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Tue, 12 Dec 2023 07:27:59 +0000 Subject: [PATCH 10/34] fix: fix constraints of etable --- crates/zkwasm/src/circuits/etable/mod.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/crates/zkwasm/src/circuits/etable/mod.rs b/crates/zkwasm/src/circuits/etable/mod.rs index fb68ba3cc..817d72c1e 100644 --- a/crates/zkwasm/src/circuits/etable/mod.rs +++ b/crates/zkwasm/src/circuits/etable/mod.rs @@ -390,10 +390,10 @@ impl EventTableConfig { .into_iter() .reduce(|acc, x| acc + x) .unwrap() - - constant_from!(1), + - enabled_cell.curr_expr(meta), ] .into_iter() - .map(|expr| expr * enabled_cell.curr_expr(meta) * fixed_curr!(meta, step_sel)) + .map(|expr| expr * fixed_curr!(meta, step_sel)) .collect::>() }); @@ -466,7 +466,6 @@ impl EventTableConfig { &|meta, config: &Rc>>| { config.input_index_increase(meta, &common_config) }, - // Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -479,7 +478,6 @@ impl EventTableConfig { &|meta, config: &Rc>>| { config.external_host_call_index_increase(meta, &common_config) }, - //Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -489,7 +487,6 @@ impl EventTableConfig { sp_cell.curr_expr(meta) - sp_cell.next_expr(meta), meta, &|meta, config: &Rc>>| config.sp_diff(meta), - //Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -501,7 +498,6 @@ impl EventTableConfig { &|meta, config: &Rc>>| { config.allocated_memory_pages_diff(meta) }, - // Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -513,7 +509,6 @@ impl EventTableConfig { &|meta, config: &Rc>>| { config.context_input_index_increase(meta, &common_config) }, - //Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -526,7 +521,6 @@ impl EventTableConfig { &|meta, config: &Rc>>| { config.context_output_index_increase(meta, &common_config) }, - // Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -549,7 +543,6 @@ impl EventTableConfig { .next_fid(meta, &common_config) .map(|x| x - fid_cell.curr_expr(meta)) }, - // Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -563,7 +556,6 @@ impl EventTableConfig { .next_iid(meta, &common_config) .map(|x| iid_cell.curr_expr(meta) + enabled_cell.curr_expr(meta) - x) }, - // Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -577,7 +569,6 @@ impl EventTableConfig { .next_frame_id(meta, &common_config) .map(|x| x - frame_id_cell.curr_expr(meta)) }, - // Some(&|meta| enabled_cell.curr_expr(meta)), None, )] }); @@ -634,7 +625,6 @@ impl EventTableConfig { vec![ (maximal_memory_pages_cell.next_expr(meta) - maximal_memory_pages_cell.curr_expr(meta)) -// * enabled_cell.expr(meta) * fixed_curr!(meta, step_sel), ] }); From a6053b2aa80364215d15ef819324174dc8b2a0be Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Tue, 12 Dec 2023 08:50:23 +0000 Subject: [PATCH 11/34] refine image table --- .../zkwasm/src/circuits/image_table/assign.rs | 15 +++++++++------ crates/zkwasm/src/circuits/image_table/mod.rs | 17 +++++++++++++---- crates/zkwasm/src/circuits/jtable/assign.rs | 3 ++- crates/zkwasm/src/circuits/jtable/mod.rs | 7 +++++++ .../circuits/post_image_table/continuation.rs | 17 +++++++---------- .../zkwasm/src/circuits/post_image_table/mod.rs | 1 + .../src/circuits/post_image_table/trivial.rs | 1 + crates/zkwasm/src/circuits/test_circuit/mod.rs | 3 +-- crates/zkwasm/src/circuits/utils/image_table.rs | 15 +++++++++------ 9 files changed, 50 insertions(+), 29 deletions(-) diff --git a/crates/zkwasm/src/circuits/image_table/assign.rs b/crates/zkwasm/src/circuits/image_table/assign.rs index 4fb7035c7..70c245076 100644 --- a/crates/zkwasm/src/circuits/image_table/assign.rs +++ b/crates/zkwasm/src/circuits/image_table/assign.rs @@ -5,22 +5,26 @@ use halo2_proofs::arithmetic::FieldExt; use halo2_proofs::circuit::AssignedCell; use halo2_proofs::circuit::Layouter; use halo2_proofs::plonk::Error; -use wasmi::DEFAULT_VALUE_STACK_LIMIT; use super::ImageTableChip; use super::ImageTableLayouter; -use crate::circuits::image_table::INIT_MEMORY_ENTRIES_OFFSET; use crate::circuits::utils::image_table::ImageTableAssigner; use crate::circuits::utils::Context; -impl ImageTableChip { +impl< + const INIT_MEMORY_ENTRIES_OFFSET: usize, + const STACK_LIMIT: usize, + const GLOBAL_LIMIT: usize, + F: FieldExt, + > ImageTableChip +{ pub(crate) fn assign( self, layouter: &mut impl Layouter, image_table_assigner: &mut ImageTableAssigner< INIT_MEMORY_ENTRIES_OFFSET, - DEFAULT_VALUE_STACK_LIMIT, - DEFAULT_VALUE_STACK_LIMIT, + STACK_LIMIT, + GLOBAL_LIMIT, >, image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, @@ -199,7 +203,6 @@ impl ImageTableChip { br_table: Some(result.br_table_entires), padding: Some(result.padding_entires), init_memory_entries: None, - rest_memory_writing_ops: None, }) }, ) diff --git a/crates/zkwasm/src/circuits/image_table/mod.rs b/crates/zkwasm/src/circuits/image_table/mod.rs index c26e7452b..3a40534e8 100644 --- a/crates/zkwasm/src/circuits/image_table/mod.rs +++ b/crates/zkwasm/src/circuits/image_table/mod.rs @@ -79,7 +79,6 @@ pub struct ImageTableLayouter { pub br_table: Option>, pub padding: Option>, pub init_memory_entries: Option>, - pub rest_memory_writing_ops: Option, } impl ImageTableLayouter { @@ -239,7 +238,6 @@ impl EncodeCompilationTableValues for CompilationTable { br_table, padding: None, init_memory_entries, - rest_memory_writing_ops: None, } } } @@ -258,11 +256,22 @@ impl ImageTableConfig { } #[derive(Clone)] -pub struct ImageTableChip { +pub struct ImageTableChip< + const INIT_MEMORY_ENTRIES_OFFSET: usize, + const STACK_LIMIT: usize, + const GLOBAL_LIMIT: usize, + F: FieldExt, +> { config: ImageTableConfig, } -impl ImageTableChip { +impl< + const INIT_MEMORY_ENTRIES_OFFSET: usize, + const STACK_LIMIT: usize, + const GLOBAL_LIMIT: usize, + F: FieldExt, + > ImageTableChip +{ pub fn new(config: ImageTableConfig) -> Self { ImageTableChip { config } } diff --git a/crates/zkwasm/src/circuits/jtable/assign.rs b/crates/zkwasm/src/circuits/jtable/assign.rs index 8f6afd195..8510d52de 100644 --- a/crates/zkwasm/src/circuits/jtable/assign.rs +++ b/crates/zkwasm/src/circuits/jtable/assign.rs @@ -6,6 +6,7 @@ use specs::jtable::StaticFrameEntry; use super::JtableOffset; use super::JumpTableChip; +use super::STATIC_FRAME_ENTRY_NUMBER; use crate::circuits::utils::bn_to_field; use crate::circuits::utils::Context; @@ -72,7 +73,7 @@ impl JumpTableChip { let mut cells = vec![]; static_entries.resize( - 2, + STATIC_FRAME_ENTRY_NUMBER, StaticFrameEntry { enable: false, frame_id: 0, diff --git a/crates/zkwasm/src/circuits/jtable/mod.rs b/crates/zkwasm/src/circuits/jtable/mod.rs index a4cd595e2..786fb62a4 100644 --- a/crates/zkwasm/src/circuits/jtable/mod.rs +++ b/crates/zkwasm/src/circuits/jtable/mod.rs @@ -10,6 +10,13 @@ mod assign; mod configure; pub(crate) mod expression; +// 1. jumps to zkmain +// 2. jumps to start(if exists) +pub(crate) const STATIC_FRAME_ENTRY_NUMBER: usize = 2; + +// enable and data should encode in image table +pub(crate) const STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY: usize = STATIC_FRAME_ENTRY_NUMBER * 2; + pub enum JtableOffset { JtableOffsetEnable = 0, JtableOffsetRest = 1, diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 6b9ab9de2..d2a78c2cf 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -114,6 +114,7 @@ impl PostImageTableChipTrait pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, + rest_memory_writing_ops_cell: Option>, rest_memory_writing_ops: F, ) -> Result<(), Error> { layouter.assign_region( @@ -319,16 +320,12 @@ impl PostImageTableChipTrait // First line is placeholder for default lookup let offset = base_offset + 1; - permutation_cells - .rest_memory_writing_ops - .as_ref() - .unwrap() - .copy_advice( - || "post image table: init memory", - &mut ctx.borrow_mut().region, - self.config.rest_memory_finalized_count, - offset, - )?; + rest_memory_writing_ops_cell.as_ref().unwrap().copy_advice( + || "post image table: init memory", + &mut ctx.borrow_mut().region, + self.config.rest_memory_finalized_count, + offset, + )?; } let entries = { diff --git a/crates/zkwasm/src/circuits/post_image_table/mod.rs b/crates/zkwasm/src/circuits/post_image_table/mod.rs index 7e99f1091..1f1ca6192 100644 --- a/crates/zkwasm/src/circuits/post_image_table/mod.rs +++ b/crates/zkwasm/src/circuits/post_image_table/mod.rs @@ -42,6 +42,7 @@ pub(in crate::circuits) trait PostImageTableChipTrait< pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, + rest_memory_writing_ops_cell: Option>, rest_memory_writing_ops: F, ) -> Result<(), Error>; } diff --git a/crates/zkwasm/src/circuits/post_image_table/trivial.rs b/crates/zkwasm/src/circuits/post_image_table/trivial.rs index cac7219e9..a146b0c0e 100644 --- a/crates/zkwasm/src/circuits/post_image_table/trivial.rs +++ b/crates/zkwasm/src/circuits/post_image_table/trivial.rs @@ -56,6 +56,7 @@ impl PostImageTableChipTrait> _pre_image_table: ImageTableLayouter, _post_image_table: ImageTableLayouter, _permutation_cells: ImageTableLayouter>, + _rest_memory_writing_ops_cell: Option>, _rest_memory_writing_ops: F, ) -> Result<(), Error> { Ok(()) diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index 890d089bc..f30a9cfe9 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -350,7 +350,6 @@ impl Circuit for TestCircuit { br_table: None, padding: None, init_memory_entries: None, - rest_memory_writing_ops: None, } )? ); @@ -373,8 +372,8 @@ impl Circuit for TestCircuit { br_table: pre_image_table_cells.br_table, padding: pre_image_table_cells.padding, init_memory_entries: pre_image_table_cells.init_memory_entries, - rest_memory_writing_ops: rest_memory_writing_ops_cell, }, + rest_memory_writing_ops_cell, rest_memory_writing_ops )? ); diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index ec00409bf..f483c8339 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -1,6 +1,7 @@ use specs::state::InitializationState; use crate::circuits::image_table::PAGE_ENTRIES; +use crate::circuits::jtable::STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; /* * -------------------- @@ -21,14 +22,15 @@ use crate::circuits::image_table::PAGE_ENTRIES; * Heap * -------------------- */ +#[allow(dead_code)] pub(crate) struct Layouter { pub(crate) initialization_state: InitializationState, pub(crate) static_frame_entries: Vec<(T, T)>, pub(crate) instructions: Vec, pub(crate) br_table_entires: Vec, - // NOTE: padding entries also need constain_equal for other image + // NOTE: unused instructions and br_table entries. pub(crate) padding_entires: Vec, - pub(crate) _init_memory_entires: Vec, + pub(crate) init_memory_entires: Vec, } pub(crate) struct ImageTableAssigner< @@ -37,6 +39,7 @@ pub(crate) struct ImageTableAssigner< const GLOBAL_CAPABILITY: usize, > { pub(crate) heap_capability: u32, + initialization_state_offset: usize, static_frame_entries_offset: usize, instruction_offset: usize, @@ -55,8 +58,7 @@ impl< let initialization_state_offset = 0; let static_frame_entries_offset = initialization_state_offset + InitializationState::::field_count(); - // FIXME: magic number - let instruction_offset = static_frame_entries_offset + 4; + let instruction_offset = static_frame_entries_offset + STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; let br_table_offset = instruction_offset + instruction_number; let padding_offset = br_table_offset + br_table_number; let init_memory_offset = INIT_MEMORY_OFFSET; @@ -65,6 +67,7 @@ impl< Self { heap_capability: pages_capability * PAGE_ENTRIES, + initialization_state_offset, static_frame_entries_offset, instruction_offset, @@ -130,7 +133,7 @@ impl< let instructions = self.exec_instruction(instruction_handler)?; let br_table_entires = self.exec_br_table_entires(br_table_handler)?; let padding_entires = self.exec_padding_entires(padding_handler)?; - let _init_memory_entires = self.exec_init_memory_entires(init_memory_entries_handler)?; + let init_memory_entires = self.exec_init_memory_entires(init_memory_entries_handler)?; Ok(Layouter { initialization_state, @@ -138,7 +141,7 @@ impl< instructions, br_table_entires, padding_entires, - _init_memory_entires, + init_memory_entires, }) } } From dd9b8121d1a2b314915980fb486ba342dd0da238 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Tue, 12 Dec 2023 10:09:17 +0000 Subject: [PATCH 12/34] refine image table --- crates/specs/src/brtable.rs | 2 +- crates/specs/src/lib.rs | 2 + crates/zkwasm/src/checksum/mod.rs | 2 +- .../zkwasm/src/circuits/image_table/assign.rs | 32 +-- crates/zkwasm/src/circuits/image_table/mod.rs | 223 +----------------- crates/zkwasm/src/circuits/jtable/assign.rs | 8 +- .../circuits/post_image_table/continuation.rs | 29 +-- .../src/circuits/post_image_table/mod.rs | 10 +- .../src/circuits/post_image_table/trivial.rs | 10 +- .../zkwasm/src/circuits/test_circuit/mod.rs | 24 +- .../zkwasm/src/circuits/utils/image_table.rs | 209 ++++++++++++++-- crates/zkwasm/src/continuation/slice.rs | 1 + .../zkwasm/src/runtime/wasmi_interpreter.rs | 27 ++- 13 files changed, 246 insertions(+), 333 deletions(-) diff --git a/crates/specs/src/brtable.rs b/crates/specs/src/brtable.rs index 8b447df5d..2c333402e 100644 --- a/crates/specs/src/brtable.rs +++ b/crates/specs/src/brtable.rs @@ -13,7 +13,7 @@ pub struct BrTableEntry { pub dst_pc: u32, } -#[derive(Debug)] +#[derive(Default, Serialize, Debug, Clone)] pub struct BrTable(Vec); impl BrTable { diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 3bfa81236..278a613b6 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -10,6 +10,7 @@ use std::io::Write; use std::path::PathBuf; use std::sync::Arc; +use brtable::BrTable; use brtable::ElemTable; use configure_table::ConfigureTable; use etable::EventTable; @@ -49,6 +50,7 @@ pub mod types; pub struct CompilationTable { pub itable: Arc, pub imtable: InitMemoryTable, + pub br_table: Arc, pub elem_table: Arc, pub configure_table: Arc, pub static_jtable: Arc>, diff --git a/crates/zkwasm/src/checksum/mod.rs b/crates/zkwasm/src/checksum/mod.rs index 5d03b8743..3ec68d439 100644 --- a/crates/zkwasm/src/checksum/mod.rs +++ b/crates/zkwasm/src/checksum/mod.rs @@ -3,7 +3,7 @@ use halo2_proofs::arithmetic::CurveAffine; use halo2_proofs::poly::commitment::Params; use specs::CompilationTable; -use crate::circuits::image_table::EncodeCompilationTableValues; +use crate::circuits::utils::image_table::EncodeCompilationTableValues; pub trait ImageCheckSum { fn checksum(&self) -> Output; diff --git a/crates/zkwasm/src/circuits/image_table/assign.rs b/crates/zkwasm/src/circuits/image_table/assign.rs index 70c245076..b0eb61b6f 100644 --- a/crates/zkwasm/src/circuits/image_table/assign.rs +++ b/crates/zkwasm/src/circuits/image_table/assign.rs @@ -7,25 +7,15 @@ use halo2_proofs::circuit::Layouter; use halo2_proofs::plonk::Error; use super::ImageTableChip; -use super::ImageTableLayouter; use crate::circuits::utils::image_table::ImageTableAssigner; +use crate::circuits::utils::image_table::ImageTableLayouter; use crate::circuits::utils::Context; -impl< - const INIT_MEMORY_ENTRIES_OFFSET: usize, - const STACK_LIMIT: usize, - const GLOBAL_LIMIT: usize, - F: FieldExt, - > ImageTableChip -{ +impl ImageTableChip { pub(crate) fn assign( self, layouter: &mut impl Layouter, - image_table_assigner: &mut ImageTableAssigner< - INIT_MEMORY_ENTRIES_OFFSET, - STACK_LIMIT, - GLOBAL_LIMIT, - >, + image_table_assigner: &mut ImageTableAssigner, image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, ) -> Result>, Error> { @@ -97,8 +87,6 @@ impl< image_table .instructions - .as_ref() - .unwrap() .iter() .map(|entry| { let offset = ctx.borrow().offset; @@ -123,9 +111,7 @@ impl< ctx.borrow_mut().offset = base_offset; image_table - .br_table - .as_ref() - .unwrap() + .br_table_entires .iter() .map(|entry| { let offset = ctx.borrow().offset; @@ -166,8 +152,6 @@ impl< image_table .init_memory_entries - .as_ref() - .unwrap() .iter() .map(|entry| { let offset = ctx.borrow().offset; @@ -199,10 +183,10 @@ impl< Ok(ImageTableLayouter { initialization_state: result.initialization_state, static_frame_entries: result.static_frame_entries, - instructions: Some(result.instructions), - br_table: Some(result.br_table_entires), - padding: Some(result.padding_entires), - init_memory_entries: None, + instructions: result.instructions, + br_table_entires: result.br_table_entires, + padding_entires: result.padding_entires, + init_memory_entries: result.init_memory_entries, }) }, ) diff --git a/crates/zkwasm/src/circuits/image_table/mod.rs b/crates/zkwasm/src/circuits/image_table/mod.rs index 3a40534e8..b54234233 100644 --- a/crates/zkwasm/src/circuits/image_table/mod.rs +++ b/crates/zkwasm/src/circuits/image_table/mod.rs @@ -4,32 +4,18 @@ use halo2_proofs::plonk::Column; use halo2_proofs::plonk::Expression; use halo2_proofs::plonk::Fixed; use halo2_proofs::plonk::VirtualCells; -use num_bigint::BigUint; -use specs::brtable::BrTable; -use specs::brtable::ElemTable; -use specs::encode::image_table::ImageTableEncoder; -use specs::imtable::InitMemoryTable; -use specs::imtable::InitMemoryTableEntry; -use specs::itable::InstructionTable; -use specs::jtable::StaticFrameEntry; -use specs::mtable::LocationType; -use specs::mtable::VarType; -use specs::state::InitializationState; -use specs::CompilationTable; use std::marker::PhantomData; use wasmi::DEFAULT_VALUE_STACK_LIMIT; -use crate::circuits::config::zkwasm_k; -use crate::circuits::utils::bn_to_field; use crate::curr; use super::test_circuit::RESERVE_ROWS; +use super::utils::image_table::INIT_MEMORY_ENTRIES_OFFSET; mod assign; mod configure; pub const IMAGE_COL_NAME: &str = "img_col"; -pub const INIT_MEMORY_ENTRIES_OFFSET: usize = 40960; /* * 8192: 64 * 1024 / 8 * A page is 64KB, an entry is 8B @@ -50,198 +36,6 @@ pub fn compute_maximal_pages(k: u32) -> u32 { pages } -pub(crate) struct InitMemoryLayouter { - pub(crate) stack: u32, - pub(crate) global: u32, - pub(crate) pages: u32, -} - -impl InitMemoryLayouter { - fn for_each(self, mut f: impl FnMut((LocationType, u32))) { - for offset in 0..self.stack { - f((LocationType::Stack, offset)) - } - - for offset in 0..self.global { - f((LocationType::Global, offset)) - } - - for offset in 0..(self.pages * PAGE_ENTRIES) { - f((LocationType::Heap, offset)) - } - } -} - -pub struct ImageTableLayouter { - pub initialization_state: InitializationState, - pub static_frame_entries: Vec<(T, T)>, - pub instructions: Option>, - pub br_table: Option>, - pub padding: Option>, - pub init_memory_entries: Option>, -} - -impl ImageTableLayouter { - pub fn plain(&self) -> Vec { - let mut buf = vec![]; - - buf.append(&mut self.initialization_state.plain()); - buf.append( - &mut self - .static_frame_entries - .clone() - .to_vec() - .into_iter() - .map(|(enable, fid)| vec![enable, fid]) - .collect::>>() - .concat(), - ); - buf.append(&mut self.instructions.clone().unwrap()); - buf.append(&mut self.br_table.clone().unwrap()); - buf.append(&mut vec![F::zero(); INIT_MEMORY_ENTRIES_OFFSET - buf.len()]); - buf.append(&mut self.init_memory_entries.clone().unwrap()); - - buf - } -} - -pub trait EncodeCompilationTableValues { - fn encode_compilation_table_values(&self) -> ImageTableLayouter; -} - -impl EncodeCompilationTableValues for CompilationTable { - fn encode_compilation_table_values(&self) -> ImageTableLayouter { - fn msg_of_initialization_state( - initialization_state: &InitializationState, - ) -> InitializationState { - initialization_state.map(|field| F::from(*field as u64)) - } - - fn msg_of_instruction_table(instruction_table: &InstructionTable) -> Vec { - let mut cells = vec![]; - - cells.push(bn_to_field( - &ImageTableEncoder::Instruction.encode(BigUint::from(0u64)), - )); - - for e in instruction_table.entries() { - cells.push(bn_to_field( - &ImageTableEncoder::Instruction.encode(e.encode()), - )); - } - - cells - } - - fn msg_of_br_table(br_table: &BrTable, elem_table: &ElemTable) -> Vec { - let mut cells = vec![]; - - cells.push(bn_to_field( - &ImageTableEncoder::BrTable.encode(BigUint::from(0u64)), - )); - - for e in br_table.entries() { - cells.push(bn_to_field(&ImageTableEncoder::BrTable.encode(e.encode()))); - } - - for e in elem_table.entries() { - cells.push(bn_to_field(&ImageTableEncoder::BrTable.encode(e.encode()))); - } - - cells - } - - fn msg_of_init_memory_table(init_memory_table: &InitMemoryTable) -> Vec { - let mut cells = vec![]; - - cells.push(bn_to_field( - &ImageTableEncoder::InitMemory.encode(BigUint::from(0u64)), - )); - - let layouter = InitMemoryLayouter { - stack: DEFAULT_VALUE_STACK_LIMIT as u32, - global: DEFAULT_VALUE_STACK_LIMIT as u32, - pages: compute_maximal_pages(zkwasm_k()), - }; - - layouter.for_each(|(ltype, offset)| { - if let Some(entry) = init_memory_table.try_find(ltype, offset) { - cells.push(bn_to_field::( - &ImageTableEncoder::InitMemory.encode(entry.encode()), - )); - } else if ltype == LocationType::Heap { - let entry = InitMemoryTableEntry { - ltype, - is_mutable: true, - offset, - vtype: VarType::I64, - value: 0, - eid: 0, - }; - - cells.push(bn_to_field::( - &ImageTableEncoder::InitMemory.encode(entry.encode()), - )); - } else { - cells.push(bn_to_field::( - &ImageTableEncoder::InitMemory.encode(BigUint::from(0u64)), - )); - } - }); - - cells - } - - fn msg_of_static_frame_table( - static_frame_table: &Vec, - ) -> Vec<(F, F)> { - let mut cells = static_frame_table - .into_iter() - .map(|entry| (F::one(), bn_to_field(&entry.encode()))) - .collect::>(); - - cells.resize( - 2, - ( - F::zero(), - bn_to_field( - &StaticFrameEntry { - enable: false, - frame_id: 0, - next_frame_id: 0, - callee_fid: 0, - fid: 0, - iid: 0, - } - .encode(), - ), - ), - ); - - cells - } - - let initialization_state = msg_of_initialization_state(&self.initialization_state); - let static_frame_entries = msg_of_static_frame_table(&self.static_jtable); - - let instructions = Some(msg_of_instruction_table(&self.itable)); - let br_table = Some(msg_of_br_table( - &self.itable.create_brtable(), - &self.elem_table, - )); - let init_memory_entries = Some(msg_of_init_memory_table(&self.imtable)); - - ImageTableLayouter { - initialization_state, - static_frame_entries, - instructions, - br_table, - padding: None, - init_memory_entries, - } - } -} - #[derive(Clone)] pub struct ImageTableConfig { _memory_addr_sel: Column, @@ -256,22 +50,11 @@ impl ImageTableConfig { } #[derive(Clone)] -pub struct ImageTableChip< - const INIT_MEMORY_ENTRIES_OFFSET: usize, - const STACK_LIMIT: usize, - const GLOBAL_LIMIT: usize, - F: FieldExt, -> { +pub struct ImageTableChip { config: ImageTableConfig, } -impl< - const INIT_MEMORY_ENTRIES_OFFSET: usize, - const STACK_LIMIT: usize, - const GLOBAL_LIMIT: usize, - F: FieldExt, - > ImageTableChip -{ +impl ImageTableChip { pub fn new(config: ImageTableConfig) -> Self { ImageTableChip { config } } diff --git a/crates/zkwasm/src/circuits/jtable/assign.rs b/crates/zkwasm/src/circuits/jtable/assign.rs index 8510d52de..07a7d143b 100644 --- a/crates/zkwasm/src/circuits/jtable/assign.rs +++ b/crates/zkwasm/src/circuits/jtable/assign.rs @@ -206,7 +206,13 @@ impl JumpTableChip { self.init(ctx)?; ctx.reset(); - let mut rest_jops = jtable.entries().len() as u64 * 2 + static_entries.len() as u64; + let mut rest_jops = jtable.entries().len() as u64 * 2; + + for entry in static_entries { + if entry.enable { + rest_jops += 1; + } + } let frame_table_start_jump_cells = self.assign_static_entries(ctx, &mut rest_jops, static_entries)?; diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index d2a78c2cf..7d7b3699c 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -16,11 +16,10 @@ use specs::mtable::LocationType; use wasmi::DEFAULT_VALUE_STACK_LIMIT; use crate::circuits::image_table::ImageTableConfig; -use crate::circuits::image_table::ImageTableLayouter; -use crate::circuits::image_table::INIT_MEMORY_ENTRIES_OFFSET; use crate::circuits::mtable::MemoryTableConfig; use crate::circuits::utils::bn_to_field; use crate::circuits::utils::image_table::ImageTableAssigner; +use crate::circuits::utils::image_table::ImageTableLayouter; use crate::circuits::utils::Context; use crate::constant_from; use crate::curr; @@ -106,11 +105,7 @@ impl PostImageTableChipTrait fn assign( self, layouter: &mut impl Layouter, - image_table_assigner: &mut ImageTableAssigner< - INIT_MEMORY_ENTRIES_OFFSET, - DEFAULT_VALUE_STACK_LIMIT, - DEFAULT_VALUE_STACK_LIMIT, - >, + image_table_assigner: &mut ImageTableAssigner, pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, @@ -185,8 +180,6 @@ impl PostImageTableChipTrait permutation_cells .instructions - .as_ref() - .unwrap() .iter() .map(|entry| { let offset = ctx.borrow().offset; @@ -211,9 +204,7 @@ impl PostImageTableChipTrait ctx.borrow_mut().offset = base_offset; permutation_cells - .br_table - .as_ref() - .unwrap() + .br_table_entires .iter() .map(|entry| { let offset = ctx.borrow().offset; @@ -238,9 +229,7 @@ impl PostImageTableChipTrait ctx.borrow_mut().offset = start_offset; permutation_cells - .padding - .as_ref() - .unwrap() + .padding_entires .iter() .map(|entry| { let offset = ctx.borrow().offset; @@ -335,16 +324,8 @@ impl PostImageTableChipTrait pre_image_table .init_memory_entries - .as_ref() - .unwrap() .iter() - .zip( - post_image_table - .init_memory_entries - .as_ref() - .unwrap() - .iter(), - ) + .zip(post_image_table.init_memory_entries.iter()) .map(|(pre, post)| { let entry = ctx.borrow_mut().region.assign_advice( || "post image table: init memory", diff --git a/crates/zkwasm/src/circuits/post_image_table/mod.rs b/crates/zkwasm/src/circuits/post_image_table/mod.rs index 1f1ca6192..e444451c9 100644 --- a/crates/zkwasm/src/circuits/post_image_table/mod.rs +++ b/crates/zkwasm/src/circuits/post_image_table/mod.rs @@ -5,13 +5,11 @@ use halo2_proofs::plonk::Column; use halo2_proofs::plonk::ConstraintSystem; use halo2_proofs::plonk::Error; use halo2_proofs::plonk::Fixed; -use wasmi::DEFAULT_VALUE_STACK_LIMIT; use super::image_table::ImageTableConfig; -use super::image_table::ImageTableLayouter; -use super::image_table::INIT_MEMORY_ENTRIES_OFFSET; use super::mtable::MemoryTableConfig; use super::utils::image_table::ImageTableAssigner; +use super::utils::image_table::ImageTableLayouter; pub(self) mod continuation; pub(self) mod trivial; @@ -34,11 +32,7 @@ pub(in crate::circuits) trait PostImageTableChipTrait< fn assign( self, layouter: &mut impl Layouter, - image_table_assigner: &mut ImageTableAssigner< - INIT_MEMORY_ENTRIES_OFFSET, - DEFAULT_VALUE_STACK_LIMIT, - DEFAULT_VALUE_STACK_LIMIT, - >, + image_table_assigner: &mut ImageTableAssigner, pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, diff --git a/crates/zkwasm/src/circuits/post_image_table/trivial.rs b/crates/zkwasm/src/circuits/post_image_table/trivial.rs index a146b0c0e..d73416d45 100644 --- a/crates/zkwasm/src/circuits/post_image_table/trivial.rs +++ b/crates/zkwasm/src/circuits/post_image_table/trivial.rs @@ -7,13 +7,11 @@ use halo2_proofs::plonk::Column; use halo2_proofs::plonk::ConstraintSystem; use halo2_proofs::plonk::Error; use halo2_proofs::plonk::Fixed; -use wasmi::DEFAULT_VALUE_STACK_LIMIT; use crate::circuits::image_table::ImageTableConfig; -use crate::circuits::image_table::ImageTableLayouter; -use crate::circuits::image_table::INIT_MEMORY_ENTRIES_OFFSET; use crate::circuits::mtable::MemoryTableConfig; use crate::circuits::utils::image_table::ImageTableAssigner; +use crate::circuits::utils::image_table::ImageTableLayouter; use super::PostImageTableChipTrait; use super::PostImageTableConfigTrait; @@ -48,11 +46,7 @@ impl PostImageTableChipTrait> fn assign( self, _layouter: &mut impl Layouter, - _image_table_assigner: &mut ImageTableAssigner< - INIT_MEMORY_ENTRIES_OFFSET, - DEFAULT_VALUE_STACK_LIMIT, - DEFAULT_VALUE_STACK_LIMIT, - >, + _image_table_assigner: &mut ImageTableAssigner, _pre_image_table: ImageTableLayouter, _post_image_table: ImageTableLayouter, _permutation_cells: ImageTableLayouter>, diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index f30a9cfe9..cc5369e78 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -14,7 +14,6 @@ use log::debug; use log::info; use specs::ExecutionTable; use specs::Tables; -use wasmi::DEFAULT_VALUE_STACK_LIMIT; use crate::circuits::bit_table::BitTableChip; use crate::circuits::bit_table::BitTableConfig; @@ -23,10 +22,7 @@ use crate::circuits::etable::EventTableConfig; use crate::circuits::external_host_call_table::ExternalHostCallChip; use crate::circuits::external_host_call_table::ExternalHostCallTableConfig; use crate::circuits::image_table::compute_maximal_pages; -use crate::circuits::image_table::EncodeCompilationTableValues; use crate::circuits::image_table::ImageTableChip; -use crate::circuits::image_table::ImageTableLayouter; -use crate::circuits::image_table::INIT_MEMORY_ENTRIES_OFFSET; use crate::circuits::jtable::JumpTableChip; use crate::circuits::jtable::JumpTableConfig; use crate::circuits::mtable::MemoryTableChip; @@ -36,7 +32,9 @@ use crate::circuits::post_image_table::PostImageTableChipTrait; use crate::circuits::post_image_table::PostImageTableConfigTrait; use crate::circuits::rtable::RangeTableChip; use crate::circuits::rtable::RangeTableConfig; +use crate::circuits::utils::image_table::EncodeCompilationTableValues; use crate::circuits::utils::image_table::ImageTableAssigner; +use crate::circuits::utils::image_table::ImageTableLayouter; use crate::circuits::utils::table_entry::EventTableWithMemoryInfo; use crate::circuits::utils::table_entry::MemoryWritingTable; use crate::circuits::utils::Context; @@ -316,11 +314,7 @@ impl Circuit for TestCircuit { )? ); - let mut image_table_assigner = ImageTableAssigner::< - INIT_MEMORY_ENTRIES_OFFSET, - DEFAULT_VALUE_STACK_LIMIT, - DEFAULT_VALUE_STACK_LIMIT, - >::new( + let mut image_table_assigner = ImageTableAssigner::new( // Add one for default lookup value self.tables.compilation_tables.itable.entries().len() + 1, // FIXME: avoid compute @@ -346,10 +340,10 @@ impl Circuit for TestCircuit { ImageTableLayouter { initialization_state: etable_permutation_cells.pre_initialization_state, static_frame_entries, - instructions: None, - br_table: None, - padding: None, - init_memory_entries: None, + instructions: vec![], + br_table_entires: vec![], + padding_entires: vec![], + init_memory_entries: vec![], } )? ); @@ -369,8 +363,8 @@ impl Circuit for TestCircuit { initialization_state: etable_permutation_cells.post_initialization_state, static_frame_entries: pre_image_table_cells.static_frame_entries, instructions: pre_image_table_cells.instructions, - br_table: pre_image_table_cells.br_table, - padding: pre_image_table_cells.padding, + br_table_entires: pre_image_table_cells.br_table_entires, + padding_entires: pre_image_table_cells.padding_entires, init_memory_entries: pre_image_table_cells.init_memory_entries, }, rest_memory_writing_ops_cell, diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index f483c8339..5689e01e0 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -1,7 +1,44 @@ +use anyhow::Error; +use halo2_proofs::arithmetic::FieldExt; +use num_bigint::BigUint; +use specs::encode::image_table::ImageTableEncoder; +use specs::imtable::InitMemoryTableEntry; +use specs::mtable::LocationType; +use specs::mtable::VarType; use specs::state::InitializationState; +use specs::CompilationTable; +use wasmi::DEFAULT_VALUE_STACK_LIMIT; +use crate::circuits::config::zkwasm_k; +use crate::circuits::image_table::compute_maximal_pages; use crate::circuits::image_table::PAGE_ENTRIES; use crate::circuits::jtable::STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; +use crate::circuits::jtable::STATIC_FRAME_ENTRY_NUMBER; +use crate::circuits::utils::bn_to_field; + +pub const STACK_CAPABILITY: usize = DEFAULT_VALUE_STACK_LIMIT; +pub const GLOBAL_CAPABILITY: usize = DEFAULT_VALUE_STACK_LIMIT; +pub const INIT_MEMORY_ENTRIES_OFFSET: usize = 40960; + +pub(crate) struct InitMemoryLayouter { + pub(crate) pages: u32, +} + +impl InitMemoryLayouter { + fn for_each(self, mut f: impl FnMut((LocationType, u32))) { + for offset in 0..STACK_CAPABILITY { + f((LocationType::Stack, offset as u32)) + } + + for offset in 0..GLOBAL_CAPABILITY { + f((LocationType::Global, offset as u32)) + } + + for offset in 0..(self.pages * PAGE_ENTRIES) { + f((LocationType::Heap, offset)) + } + } +} /* * -------------------- @@ -23,21 +60,17 @@ use crate::circuits::jtable::STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; * -------------------- */ #[allow(dead_code)] -pub(crate) struct Layouter { +pub(crate) struct ImageTableLayouter { pub(crate) initialization_state: InitializationState, pub(crate) static_frame_entries: Vec<(T, T)>, pub(crate) instructions: Vec, pub(crate) br_table_entires: Vec, // NOTE: unused instructions and br_table entries. pub(crate) padding_entires: Vec, - pub(crate) init_memory_entires: Vec, + pub(crate) init_memory_entries: Vec, } -pub(crate) struct ImageTableAssigner< - const INIT_MEMORY_OFFSET: usize, - const STACK_CAPABILITY: usize, - const GLOBAL_CAPABILITY: usize, -> { +pub(crate) struct ImageTableAssigner { pub(crate) heap_capability: u32, initialization_state_offset: usize, @@ -48,12 +81,7 @@ pub(crate) struct ImageTableAssigner< init_memory_offset: usize, } -impl< - const INIT_MEMORY_OFFSET: usize, - const STACK_CAPABILITY: usize, - const GLOBAL_CAPABILITY: usize, - > ImageTableAssigner -{ +impl ImageTableAssigner { pub fn new(instruction_number: usize, br_table_number: usize, pages_capability: u32) -> Self { let initialization_state_offset = 0; let static_frame_entries_offset = @@ -61,7 +89,7 @@ impl< let instruction_offset = static_frame_entries_offset + STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; let br_table_offset = instruction_offset + instruction_number; let padding_offset = br_table_offset + br_table_number; - let init_memory_offset = INIT_MEMORY_OFFSET; + let init_memory_offset = INIT_MEMORY_ENTRIES_OFFSET; assert!(padding_offset <= init_memory_offset); @@ -112,7 +140,7 @@ impl< padding_handler(self.padding_offset, self.padding_offset) } - pub fn exec_init_memory_entires( + pub fn exec_init_memory_entries( &mut self, mut init_memory_entries_handler: impl FnMut(usize) -> Result, Error>, ) -> Result, Error> { @@ -127,21 +155,164 @@ impl< br_table_handler: impl FnMut(usize) -> Result, Error>, padding_handler: impl FnMut(usize, usize) -> Result, Error>, init_memory_entries_handler: impl FnMut(usize) -> Result, Error>, - ) -> Result, Error> { + ) -> Result, Error> { let initialization_state = self.exec_initialization_state(initialization_state_handler)?; let static_frame_entries = self.exec_static_frame_entries(static_frame_entries_handler)?; let instructions = self.exec_instruction(instruction_handler)?; let br_table_entires = self.exec_br_table_entires(br_table_handler)?; let padding_entires = self.exec_padding_entires(padding_handler)?; - let init_memory_entires = self.exec_init_memory_entires(init_memory_entries_handler)?; + let init_memory_entries = self.exec_init_memory_entries(init_memory_entries_handler)?; - Ok(Layouter { + Ok(ImageTableLayouter { initialization_state, static_frame_entries, instructions, br_table_entires, padding_entires, - init_memory_entires, + init_memory_entries, }) } } + +pub(crate) trait EncodeCompilationTableValues { + fn encode_compilation_table_values(&self) -> ImageTableLayouter; +} + +impl EncodeCompilationTableValues for CompilationTable { + fn encode_compilation_table_values(&self) -> ImageTableLayouter { + // FIXME: ugly + let pages_capability = compute_maximal_pages(zkwasm_k()); + + let initialization_state_handler = + |_| Ok(self.initialization_state.map(|v| F::from((*v) as u64))); + + let static_frame_entries_handler = |_| { + // Encode disabled static frame entry in image table + assert_eq!(self.static_jtable.len(), STATIC_FRAME_ENTRY_NUMBER); + + Ok(self + .static_jtable + .iter() + .map(|entry| (F::from(entry.enable as u64), bn_to_field(&entry.encode()))) + .collect()) + }; + + let instruction_handler = |_| { + let mut cells = vec![]; + + cells.push(bn_to_field( + &ImageTableEncoder::Instruction.encode(BigUint::from(0u64)), + )); + + for e in self.itable.entries() { + cells.push(bn_to_field( + &ImageTableEncoder::Instruction.encode(e.encode()), + )); + } + + Ok(cells) + }; + + let br_table_handler = |_| { + let mut cells = vec![]; + + cells.push(bn_to_field( + &ImageTableEncoder::BrTable.encode(BigUint::from(0u64)), + )); + + for e in self.br_table.entries() { + cells.push(bn_to_field(&ImageTableEncoder::BrTable.encode(e.encode()))); + } + + for e in self.elem_table.entries() { + cells.push(bn_to_field(&ImageTableEncoder::BrTable.encode(e.encode()))); + } + + Ok(cells) + }; + + let padding_handler = |start, end| Ok(vec![F::zero(); end - start]); + + let init_memory_entries_handler = |_| { + let mut cells = vec![]; + + cells.push(bn_to_field( + &ImageTableEncoder::InitMemory.encode(BigUint::from(0u64)), + )); + + let layouter = InitMemoryLayouter { + pages: pages_capability, + }; + + layouter.for_each(|(ltype, offset)| { + if let Some(entry) = self.imtable.try_find(ltype, offset) { + cells.push(bn_to_field::( + &ImageTableEncoder::InitMemory.encode(entry.encode()), + )); + } else if ltype == LocationType::Heap { + let entry = InitMemoryTableEntry { + ltype, + is_mutable: true, + offset, + vtype: VarType::I64, + value: 0, + eid: 0, + }; + + cells.push(bn_to_field::( + &ImageTableEncoder::InitMemory.encode(entry.encode()), + )); + } else { + cells.push(bn_to_field::( + &ImageTableEncoder::InitMemory.encode(BigUint::from(0u64)), + )); + } + }); + + Ok(cells) + }; + + let mut assigner = ImageTableAssigner::new( + self.itable.entries().len(), + self.br_table.entries().len() + self.elem_table.entries().len(), + pages_capability, + ); + + let layouter = assigner + .exec::<_, Error>( + initialization_state_handler, + static_frame_entries_handler, + instruction_handler, + br_table_handler, + padding_handler, + init_memory_entries_handler, + ) + .unwrap(); + + layouter + } +} + +impl ImageTableLayouter { + pub fn plain(&self) -> Vec { + let mut buf = vec![]; + + buf.append(&mut self.initialization_state.plain()); + buf.append( + &mut self + .static_frame_entries + .clone() + .to_vec() + .into_iter() + .map(|(enable, fid)| vec![enable, fid]) + .collect::>>() + .concat(), + ); + buf.append(&mut self.instructions.clone()); + buf.append(&mut self.br_table_entires.clone()); + buf.append(&mut self.padding_entires.clone()); + buf.append(&mut self.init_memory_entries.clone()); + + buf + } +} diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index 2451e49a9..7e1e2c0cd 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -139,6 +139,7 @@ impl Iterator for Slices { let post_image_table = CompilationTable { itable: self.origin_table.compilation_tables.itable.clone(), imtable: updated_init_memory_table, + br_table: self.origin_table.compilation_tables.br_table.clone(), elem_table: self.origin_table.compilation_tables.elem_table.clone(), configure_table: self.origin_table.compilation_tables.configure_table.clone(), static_jtable: self.origin_table.compilation_tables.static_jtable.clone(), diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index 09bc7dc1c..a6baa3cf9 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -84,18 +84,19 @@ impl Execution .tables .update_init_memory_table(&execution_tables.etable.entries()); - let post_image_table = { - CompilationTable { - itable: self.tables.itable.clone(), - imtable: updated_init_memory_table, - elem_table: self.tables.elem_table.clone(), - configure_table: self.tables.configure_table.clone(), - static_jtable: self.tables.static_jtable.clone(), - initialization_state: self - .tables - .update_initialization_state(&execution_tables.etable.entries(), true), - } - }; + let post_image_table = { + CompilationTable { + itable: self.tables.itable.clone(), + imtable: updated_init_memory_table, + br_table: self.tables.br_table.clone(), + elem_table: self.tables.elem_table.clone(), + configure_table: self.tables.configure_table.clone(), + static_jtable: self.tables.static_jtable.clone(), + initialization_state: self + .tables + .update_initialization_state(&execution_tables.etable.entries(), true), + } + }; tables = Some(Tables { compilation_tables: self.tables, @@ -138,6 +139,7 @@ impl WasmiRuntime { let fid_of_entry = tracer.clone().borrow().get_fid_of_entry(); let itable = Arc::new(tracer.borrow().itable.clone()); let imtable = tracer.borrow().imtable.finalized(); + let br_table = Arc::new(itable.create_brtable()); let elem_table = Arc::new(tracer.borrow().elem_table.clone()); let configure_table = Arc::new(tracer.borrow().configure_table.clone()); let static_jtable = Arc::new(tracer.borrow().static_jtable_entries.clone()); @@ -165,6 +167,7 @@ impl WasmiRuntime { tables: CompilationTable { itable, imtable, + br_table, elem_table, configure_table, static_jtable, From 5205f318818c9f66e4d0533771492ef456ee97ba Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Wed, 13 Dec 2023 03:38:11 +0000 Subject: [PATCH 13/34] fix hardcode --- .../zkwasm/src/circuits/post_image_table/continuation.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 7d7b3699c..44af5ef26 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -13,13 +13,14 @@ use halo2_proofs::plonk::Fixed; use num_bigint::BigUint; use specs::encode::init_memory_table::encode_init_memory_table_address; use specs::mtable::LocationType; -use wasmi::DEFAULT_VALUE_STACK_LIMIT; use crate::circuits::image_table::ImageTableConfig; use crate::circuits::mtable::MemoryTableConfig; use crate::circuits::utils::bn_to_field; use crate::circuits::utils::image_table::ImageTableAssigner; use crate::circuits::utils::image_table::ImageTableLayouter; +use crate::circuits::utils::image_table::GLOBAL_CAPABILITY; +use crate::circuits::utils::image_table::STACK_CAPABILITY; use crate::circuits::utils::Context; use crate::constant_from; use crate::curr; @@ -285,11 +286,11 @@ impl PostImageTableChipTrait }}; } - for i in 0..DEFAULT_VALUE_STACK_LIMIT { + for i in 0..STACK_CAPABILITY { assign_address!(LocationType::Stack, i)?; } - for i in 0..DEFAULT_VALUE_STACK_LIMIT { + for i in 0..GLOBAL_CAPABILITY { assign_address!(LocationType::Global, i)?; } From ae734edcf194c621eae8b4e04bdf0d0a294218e0 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Wed, 13 Dec 2023 04:05:34 +0000 Subject: [PATCH 14/34] add transpose for InitializationState --- crates/specs/src/state.rs | 23 ++++++++++++++++++- .../zkwasm/src/circuits/image_table/assign.rs | 18 +++++++-------- .../circuits/post_image_table/continuation.rs | 18 +++++++-------- 3 files changed, 38 insertions(+), 21 deletions(-) diff --git a/crates/specs/src/state.rs b/crates/specs/src/state.rs index 93b2e66a5..b76122ea3 100644 --- a/crates/specs/src/state.rs +++ b/crates/specs/src/state.rs @@ -246,7 +246,28 @@ impl UpdateCompilationTable for CompilationTable { jops, } }; - post_initialization_state } } + + +impl InitializationState> { + pub fn transpose(self) -> Result, E> { + Ok(InitializationState { + eid: self.eid?, + fid: self.fid?, + iid: self.iid?, + frame_id: self.frame_id?, + sp: self.sp?, + host_public_inputs: self.host_public_inputs?, + context_in_index: self.context_in_index?, + context_out_index: self.context_out_index?, + external_host_call_call_index: self.external_host_call_call_index?, + initial_memory_pages: self.initial_memory_pages?, + maximal_memory_pages: self.maximal_memory_pages?, + + #[cfg(feature = "continuation")] + jops: self.jops?, + }) + } +} diff --git a/crates/zkwasm/src/circuits/image_table/assign.rs b/crates/zkwasm/src/circuits/image_table/assign.rs index b0eb61b6f..46b7c5a06 100644 --- a/crates/zkwasm/src/circuits/image_table/assign.rs +++ b/crates/zkwasm/src/circuits/image_table/assign.rs @@ -31,21 +31,19 @@ impl ImageTableChip { permutation_cells.initialization_state.map(|field| { let offset = ctx.borrow().offset; - field - .copy_advice( - || "image table: initialization state", - &mut ctx.borrow_mut().region, - self.config.col, - offset, - ) - .unwrap(); + field.copy_advice( + || "image table: initialization state", + &mut ctx.borrow_mut().region, + self.config.col, + offset, + )?; ctx.borrow_mut().next(); - field.clone() + Ok(field.clone()) }); - Ok::<_, Error>(initialization_state) + initialization_state.transpose() }; let static_frame_entries_handler = |base_offset| { diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 44af5ef26..23840cebf 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -125,21 +125,19 @@ impl PostImageTableChipTrait permutation_cells.initialization_state.map(|field| { let offset = ctx.borrow().offset; - field - .copy_advice( - || "image table: initialization state", - &mut ctx.borrow_mut().region, - self.config.post_image_table, - offset, - ) - .unwrap(); + field.copy_advice( + || "image table: initialization state", + &mut ctx.borrow_mut().region, + self.config.post_image_table, + offset, + )?; ctx.borrow_mut().next(); - field.clone() + Ok(field.clone()) }); - Ok::<_, Error>(initialization_state) + initialization_state.transpose() }; let static_frame_entries_handler = |base_offset| { From 3168fb0e0861a6bce2278278ed1a174d53ad3931 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Wed, 13 Dec 2023 09:46:35 +0000 Subject: [PATCH 15/34] chore: refine image table assignment --- .../zkwasm/src/circuits/image_table/assign.rs | 80 ++++++------------- .../circuits/post_image_table/continuation.rs | 12 +-- 2 files changed, 25 insertions(+), 67 deletions(-) diff --git a/crates/zkwasm/src/circuits/image_table/assign.rs b/crates/zkwasm/src/circuits/image_table/assign.rs index 46b7c5a06..f451d881e 100644 --- a/crates/zkwasm/src/circuits/image_table/assign.rs +++ b/crates/zkwasm/src/circuits/image_table/assign.rs @@ -24,6 +24,23 @@ impl ImageTableChip { |region| { let ctx = Rc::new(RefCell::new(Context::new(region))); + macro_rules! assign { + ($v:expr) => {{ + let offset = ctx.borrow().offset; + + let cell = ctx.borrow_mut().region.assign_advice( + || "pre image table", + self.config.col, + offset, + || Ok($v), + ); + + ctx.borrow_mut().next(); + + cell + }}; + } + let initialization_state_handler = |base_offset| { ctx.borrow_mut().offset = base_offset; @@ -73,10 +90,8 @@ impl ImageTableChip { )?; ctx.borrow_mut().next(); - Ok::<_, Error>((enable.clone(), entry.clone())) + Ok((enable.clone(), entry.clone())) }) - .collect::>>() - .into_iter() .collect::, Error>>() }; @@ -86,22 +101,7 @@ impl ImageTableChip { image_table .instructions .iter() - .map(|entry| { - let offset = ctx.borrow().offset; - - let cell = ctx.borrow_mut().region.assign_advice( - || "image table", - self.config.col, - offset, - || Ok(*entry), - ); - - ctx.borrow_mut().next(); - - cell - }) - .collect::>>() - .into_iter() + .map(|entry| assign!(*entry)) .collect::, Error>>() }; @@ -111,37 +111,15 @@ impl ImageTableChip { image_table .br_table_entires .iter() - .map(|entry| { - let offset = ctx.borrow().offset; - - let cell = ctx.borrow_mut().region.assign_advice( - || "image table", - self.config.col, - offset, - || Ok(*entry), - ); - - ctx.borrow_mut().next(); - - cell - }) - .collect::>>() - .into_iter() + .map(|entry| assign!(*entry)) .collect::, Error>>() }; let padding_handler = |start_offset, end_offset| { + ctx.borrow_mut().offset = start_offset; + (start_offset..end_offset) - .map(|offset| { - ctx.borrow_mut().region.assign_advice( - || "image table: padding", - self.config.col, - offset, - || Ok(F::zero()), - ) - }) - .collect::>>() - .into_iter() + .map(|_| assign!(F::zero())) .collect::, Error>>() }; @@ -152,20 +130,10 @@ impl ImageTableChip { .init_memory_entries .iter() .map(|entry| { - let offset = ctx.borrow().offset; - let cell = ctx.borrow_mut().region.assign_advice( - || "image table", - self.config.col, - offset, - || Ok(*entry), - ); - ctx.borrow_mut().next(); + assign!(*entry) - cell }) - .collect::>>() - .into_iter() .collect::, Error>>() }; diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 23840cebf..4c5404589 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -167,10 +167,8 @@ impl PostImageTableChipTrait )?; ctx.borrow_mut().next(); - Ok::<_, Error>((enable.clone(), entry.clone())) + Ok((enable.clone(), entry.clone())) }) - .collect::>>() - .into_iter() .collect::, Error>>() }; @@ -194,8 +192,6 @@ impl PostImageTableChipTrait Ok(entry) }) - .collect::>>() - .into_iter() .collect::, Error>>() }; @@ -219,8 +215,6 @@ impl PostImageTableChipTrait Ok(entry) }) - .collect::>>() - .into_iter() .collect::, Error>>() }; @@ -244,8 +238,6 @@ impl PostImageTableChipTrait Ok(entry) }) - .collect::>>() - .into_iter() .collect::, Error>>() }; @@ -355,8 +347,6 @@ impl PostImageTableChipTrait Ok(entry) }) - .collect::>>() - .into_iter() .collect::, Error>>() }?; From 66b16d7115db6bb9619bad4c88b566f3d0f53c9f Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Wed, 13 Dec 2023 09:49:44 +0000 Subject: [PATCH 16/34] fmt code --- crates/zkwasm/src/circuits/image_table/assign.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/zkwasm/src/circuits/image_table/assign.rs b/crates/zkwasm/src/circuits/image_table/assign.rs index f451d881e..194a77d86 100644 --- a/crates/zkwasm/src/circuits/image_table/assign.rs +++ b/crates/zkwasm/src/circuits/image_table/assign.rs @@ -129,11 +129,7 @@ impl ImageTableChip { image_table .init_memory_entries .iter() - .map(|entry| { - - assign!(*entry) - - }) + .map(|entry| assign!(*entry)) .collect::, Error>>() }; From 5970057172f80ed4127533641a12988b2d1f7fa3 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Thu, 14 Dec 2023 06:46:55 +0000 Subject: [PATCH 17/34] set the number of frame table entry always 2 --- crates/specs/src/jtable.rs | 4 ++ crates/specs/src/lib.rs | 3 +- .../zkwasm/src/circuits/image_table/assign.rs | 50 ++++++++++--------- crates/zkwasm/src/circuits/image_table/mod.rs | 30 ++++++----- crates/zkwasm/src/circuits/jtable/assign.rs | 29 ++++------- crates/zkwasm/src/circuits/jtable/mod.rs | 5 +- .../circuits/post_image_table/continuation.rs | 50 ++++++++++--------- .../zkwasm/src/circuits/test_circuit/mod.rs | 8 +-- .../zkwasm/src/circuits/utils/image_table.rs | 33 ++++++++---- .../zkwasm/src/runtime/wasmi_interpreter.rs | 14 +++++- 10 files changed, 124 insertions(+), 102 deletions(-) diff --git a/crates/specs/src/jtable.rs b/crates/specs/src/jtable.rs index 40c60669b..92ed84621 100644 --- a/crates/specs/src/jtable.rs +++ b/crates/specs/src/jtable.rs @@ -2,6 +2,10 @@ use super::itable::InstructionTableEntry; use serde::Deserialize; use serde::Serialize; +// 1. jumps to zkmain +// 2. jumps to start(if exists) +pub const STATIC_FRAME_ENTRY_NUMBER: usize = 2; + #[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct StaticFrameEntry { pub enable: bool, diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 278a613b6..ec67340d2 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -19,6 +19,7 @@ use imtable::InitMemoryTable; use itable::InstructionTable; use jtable::JumpTable; use jtable::StaticFrameEntry; +use jtable::STATIC_FRAME_ENTRY_NUMBER; use mtable::AccessType; use mtable::LocationType; use mtable::MTable; @@ -53,7 +54,7 @@ pub struct CompilationTable { pub br_table: Arc, pub elem_table: Arc, pub configure_table: Arc, - pub static_jtable: Arc>, + pub static_jtable: Arc<[StaticFrameEntry; STATIC_FRAME_ENTRY_NUMBER]>, pub initialization_state: InitializationState, } diff --git a/crates/zkwasm/src/circuits/image_table/assign.rs b/crates/zkwasm/src/circuits/image_table/assign.rs index 194a77d86..f8fb5e51e 100644 --- a/crates/zkwasm/src/circuits/image_table/assign.rs +++ b/crates/zkwasm/src/circuits/image_table/assign.rs @@ -5,6 +5,7 @@ use halo2_proofs::arithmetic::FieldExt; use halo2_proofs::circuit::AssignedCell; use halo2_proofs::circuit::Layouter; use halo2_proofs::plonk::Error; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use super::ImageTableChip; use crate::circuits::utils::image_table::ImageTableAssigner; @@ -66,33 +67,36 @@ impl ImageTableChip { let static_frame_entries_handler = |base_offset| { ctx.borrow_mut().offset = base_offset; - permutation_cells - .static_frame_entries - .iter() - .map(|(enable, entry)| { - let offset = ctx.borrow().offset; + let mut cells = vec![]; - enable.copy_advice( - || "image table: static frame entry", - &mut ctx.borrow_mut().region, - self.config.col, - offset, - )?; - ctx.borrow_mut().next(); + for (enable, entry) in &permutation_cells.static_frame_entries { + let offset = ctx.borrow().offset; - let offset = ctx.borrow().offset; + enable.copy_advice( + || "image table: static frame entry", + &mut ctx.borrow_mut().region, + self.config.col, + offset, + )?; + ctx.borrow_mut().next(); - entry.copy_advice( - || "image table: static frame entry", - &mut ctx.borrow_mut().region, - self.config.col, - offset, - )?; - ctx.borrow_mut().next(); + let offset = ctx.borrow().offset; - Ok((enable.clone(), entry.clone())) - }) - .collect::, Error>>() + entry.copy_advice( + || "image table: static frame entry", + &mut ctx.borrow_mut().region, + self.config.col, + offset, + )?; + ctx.borrow_mut().next(); + + cells.push((enable.clone(), entry.clone())); + } + + Ok(cells.try_into().expect(&format!( + "The number of static frame entries should be {}", + STATIC_FRAME_ENTRY_NUMBER + ))) }; let instruction_handler = |base_offset| { diff --git a/crates/zkwasm/src/circuits/image_table/mod.rs b/crates/zkwasm/src/circuits/image_table/mod.rs index b54234233..d3c8fdb2d 100644 --- a/crates/zkwasm/src/circuits/image_table/mod.rs +++ b/crates/zkwasm/src/circuits/image_table/mod.rs @@ -5,35 +5,39 @@ use halo2_proofs::plonk::Expression; use halo2_proofs::plonk::Fixed; use halo2_proofs::plonk::VirtualCells; use std::marker::PhantomData; -use wasmi::DEFAULT_VALUE_STACK_LIMIT; use crate::curr; use super::test_circuit::RESERVE_ROWS; +use super::utils::image_table::GLOBAL_CAPABILITY; use super::utils::image_table::INIT_MEMORY_ENTRIES_OFFSET; +use super::utils::image_table::STACK_CAPABILITY; mod assign; mod configure; pub const IMAGE_COL_NAME: &str = "img_col"; -/* - * 8192: 64 * 1024 / 8 - * A page is 64KB, an entry is 8B - */ -pub const PAGE_ENTRIES: u32 = 8192; + +pub const PAGE_SIZE: u32 = 64 * 1024; +// A block is 8 bytes +pub const PAGE_ENTRIES: u32 = PAGE_SIZE / 8; /// Compute maximal number of pages supported by the circuit. -/// circuit size - reserved rows for blind - initialization_state/static frame entries/instructions/br_table +/// circuit size - reserved rows for blind - init memory entries base offset /// - stack entries - global entries pub fn compute_maximal_pages(k: u32) -> u32 { - let bytes: u32 = - ((1usize << k) - RESERVE_ROWS - INIT_MEMORY_ENTRIES_OFFSET - DEFAULT_VALUE_STACK_LIMIT * 2) - .try_into() - .unwrap(); + let rows: u32 = ((1usize << k) + - RESERVE_ROWS + - INIT_MEMORY_ENTRIES_OFFSET + - STACK_CAPABILITY + - GLOBAL_CAPABILITY) + .try_into() + .unwrap(); - let pages = bytes / PAGE_ENTRIES; + // A block is 8 bytes. + let bytes = rows * 8; - pages + bytes / PAGE_SIZE } #[derive(Clone)] diff --git a/crates/zkwasm/src/circuits/jtable/assign.rs b/crates/zkwasm/src/circuits/jtable/assign.rs index 07a7d143b..4dfda5e4a 100644 --- a/crates/zkwasm/src/circuits/jtable/assign.rs +++ b/crates/zkwasm/src/circuits/jtable/assign.rs @@ -3,10 +3,10 @@ use halo2_proofs::circuit::AssignedCell; use halo2_proofs::plonk::Error; use specs::jtable::JumpTable; use specs::jtable::StaticFrameEntry; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use super::JtableOffset; use super::JumpTableChip; -use super::STATIC_FRAME_ENTRY_NUMBER; use crate::circuits::utils::bn_to_field; use crate::circuits::utils::Context; @@ -66,24 +66,10 @@ impl JumpTableChip { &self, ctx: &mut Context<'_, F>, rest_jops: &mut u64, - static_entries: &Vec, - ) -> Result, AssignedCell)>, Error> { - let mut static_entries = static_entries.clone(); - + static_entries: &[StaticFrameEntry; STATIC_FRAME_ENTRY_NUMBER], + ) -> Result<[(AssignedCell, AssignedCell); STATIC_FRAME_ENTRY_NUMBER], Error> { let mut cells = vec![]; - static_entries.resize( - STATIC_FRAME_ENTRY_NUMBER, - StaticFrameEntry { - enable: false, - frame_id: 0, - next_frame_id: 0, - callee_fid: 0, - fid: 0, - iid: 0, - }, - ); - for entry in static_entries { ctx.region.assign_fixed( || "jtable start entries", @@ -123,7 +109,10 @@ impl JumpTableChip { } } - Ok(cells) + Ok(cells.try_into().expect(&format!( + "The number of static frame entries should be {}", + STATIC_FRAME_ENTRY_NUMBER + ))) } fn assign_jtable_entries( @@ -197,8 +186,8 @@ impl JumpTableChip { ctx: &mut Context<'_, F>, jtable: &JumpTable, etable_jops_cell: &Option>, - static_entries: &Vec, - ) -> Result, AssignedCell)>, Error> { + static_entries: &[StaticFrameEntry; STATIC_FRAME_ENTRY_NUMBER], + ) -> Result<[(AssignedCell, AssignedCell); STATIC_FRAME_ENTRY_NUMBER], Error> { if etable_jops_cell.is_some() { self.constraint_to_etable_jops(ctx, etable_jops_cell.as_ref().unwrap())?; } diff --git a/crates/zkwasm/src/circuits/jtable/mod.rs b/crates/zkwasm/src/circuits/jtable/mod.rs index 786fb62a4..34132e87a 100644 --- a/crates/zkwasm/src/circuits/jtable/mod.rs +++ b/crates/zkwasm/src/circuits/jtable/mod.rs @@ -4,16 +4,13 @@ use halo2_proofs::plonk::Advice; use halo2_proofs::plonk::Column; use halo2_proofs::plonk::ConstraintSystem; use halo2_proofs::plonk::Fixed; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use std::marker::PhantomData; mod assign; mod configure; pub(crate) mod expression; -// 1. jumps to zkmain -// 2. jumps to start(if exists) -pub(crate) const STATIC_FRAME_ENTRY_NUMBER: usize = 2; - // enable and data should encode in image table pub(crate) const STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY: usize = STATIC_FRAME_ENTRY_NUMBER * 2; diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 4c5404589..e79a2534c 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -12,6 +12,7 @@ use halo2_proofs::plonk::Error; use halo2_proofs::plonk::Fixed; use num_bigint::BigUint; use specs::encode::init_memory_table::encode_init_memory_table_address; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use specs::mtable::LocationType; use crate::circuits::image_table::ImageTableConfig; @@ -143,33 +144,36 @@ impl PostImageTableChipTrait let static_frame_entries_handler = |base_offset| { ctx.borrow_mut().offset = base_offset; - permutation_cells - .static_frame_entries - .iter() - .map(|(enable, entry)| { - let offset = ctx.borrow().offset; + let mut cells = vec![]; - enable.copy_advice( - || "image table: static frame entry", - &mut ctx.borrow_mut().region, - self.config.post_image_table, - offset, - )?; - ctx.borrow_mut().next(); + for (enable, entry) in &permutation_cells.static_frame_entries { + let offset = ctx.borrow().offset; - let offset = ctx.borrow().offset; + enable.copy_advice( + || "image table: static frame entry", + &mut ctx.borrow_mut().region, + self.config.post_image_table, + offset, + )?; + ctx.borrow_mut().next(); - entry.copy_advice( - || "image table: static frame entry", - &mut ctx.borrow_mut().region, - self.config.post_image_table, - offset, - )?; - ctx.borrow_mut().next(); + let offset = ctx.borrow().offset; - Ok((enable.clone(), entry.clone())) - }) - .collect::, Error>>() + entry.copy_advice( + || "image table: static frame entry", + &mut ctx.borrow_mut().region, + self.config.post_image_table, + offset, + )?; + ctx.borrow_mut().next(); + + cells.push((enable.clone(), entry.clone())); + } + + Ok(cells.try_into().expect(&format!( + "The number of static frame entries should be {}", + STATIC_FRAME_ENTRY_NUMBER + ))) }; let instruction_handler = |base_offset| { diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index cc5369e78..4bdbc0e3a 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -317,13 +317,7 @@ impl Circuit for TestCircuit { let mut image_table_assigner = ImageTableAssigner::new( // Add one for default lookup value self.tables.compilation_tables.itable.entries().len() + 1, - // FIXME: avoid compute - self.tables - .compilation_tables - .itable - .create_brtable() - .entries() - .len() + self.tables.compilation_tables.br_table.entries().len() + self.tables.compilation_tables.elem_table.entries().len() + 1, config.circuit_maximal_pages, diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index 5689e01e0..f909df3f2 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -3,6 +3,7 @@ use halo2_proofs::arithmetic::FieldExt; use num_bigint::BigUint; use specs::encode::image_table::ImageTableEncoder; use specs::imtable::InitMemoryTableEntry; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use specs::mtable::LocationType; use specs::mtable::VarType; use specs::state::InitializationState; @@ -13,7 +14,6 @@ use crate::circuits::config::zkwasm_k; use crate::circuits::image_table::compute_maximal_pages; use crate::circuits::image_table::PAGE_ENTRIES; use crate::circuits::jtable::STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; -use crate::circuits::jtable::STATIC_FRAME_ENTRY_NUMBER; use crate::circuits::utils::bn_to_field; pub const STACK_CAPABILITY: usize = DEFAULT_VALUE_STACK_LIMIT; @@ -62,7 +62,7 @@ impl InitMemoryLayouter { #[allow(dead_code)] pub(crate) struct ImageTableLayouter { pub(crate) initialization_state: InitializationState, - pub(crate) static_frame_entries: Vec<(T, T)>, + pub(crate) static_frame_entries: [(T, T); STATIC_FRAME_ENTRY_NUMBER], pub(crate) instructions: Vec, pub(crate) br_table_entires: Vec, // NOTE: unused instructions and br_table entries. @@ -114,8 +114,13 @@ impl ImageTableAssigner { pub fn exec_static_frame_entries( &mut self, - mut static_frame_entries_handler: impl FnMut(usize) -> Result, Error>, - ) -> Result, Error> { + mut static_frame_entries_handler: impl FnMut( + usize, + ) -> Result< + [(T, T); STATIC_FRAME_ENTRY_NUMBER], + Error, + >, + ) -> Result<[(T, T); STATIC_FRAME_ENTRY_NUMBER], Error> { static_frame_entries_handler(self.static_frame_entries_offset) } @@ -150,7 +155,10 @@ impl ImageTableAssigner { pub fn exec( &mut self, initialization_state_handler: impl FnMut(usize) -> Result, Error>, - static_frame_entries_handler: impl FnMut(usize) -> Result, Error>, + static_frame_entries_handler: impl FnMut( + usize, + ) + -> Result<[(T, T); STATIC_FRAME_ENTRY_NUMBER], Error>, instruction_handler: impl FnMut(usize) -> Result, Error>, br_table_handler: impl FnMut(usize) -> Result, Error>, padding_handler: impl FnMut(usize, usize) -> Result, Error>, @@ -190,11 +198,16 @@ impl EncodeCompilationTableValues for CompilationTable { // Encode disabled static frame entry in image table assert_eq!(self.static_jtable.len(), STATIC_FRAME_ENTRY_NUMBER); - Ok(self - .static_jtable - .iter() - .map(|entry| (F::from(entry.enable as u64), bn_to_field(&entry.encode()))) - .collect()) + let mut cells = vec![]; + + for entry in self.static_jtable.as_ref() { + cells.push((F::from(entry.enable as u64), bn_to_field(&entry.encode()))); + } + + Ok(cells.try_into().expect(&format!( + "The number of static frame entries should be {}", + STATIC_FRAME_ENTRY_NUMBER + ))) }; let instruction_handler = |_| { diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index a6baa3cf9..d6b903e81 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -5,6 +5,8 @@ use std::sync::Arc; use anyhow::Result; use specs::host_function::HostFunctionDesc; +use specs::jtable::StaticFrameEntry; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use specs::state::InitializationState; use specs::state::UpdateCompilationTable; use specs::CompilationTable; @@ -142,7 +144,17 @@ impl WasmiRuntime { let br_table = Arc::new(itable.create_brtable()); let elem_table = Arc::new(tracer.borrow().elem_table.clone()); let configure_table = Arc::new(tracer.borrow().configure_table.clone()); - let static_jtable = Arc::new(tracer.borrow().static_jtable_entries.clone()); + let static_jtable = Arc::new( + tracer + .borrow() + .static_jtable_entries + .clone() + .try_into() + .expect(&format!( + "The number of static frame entries should be {}", + STATIC_FRAME_ENTRY_NUMBER + )), + ); let initialization_state = InitializationState { eid: 1, fid: fid_of_entry, From 945c29d4a636f021d5e49790bd970077430987f7 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Thu, 14 Dec 2023 07:22:02 +0000 Subject: [PATCH 18/34] fix code style --- crates/zkwasm/src/circuits/jtable/assign.rs | 3 +- crates/zkwasm/src/circuits/jtable/mod.rs | 2 +- crates/zkwasm/src/circuits/mtable/assign.rs | 38 ++++++++++----------- crates/zkwasm/src/circuits/mtable/mod.rs | 14 +++----- 4 files changed, 27 insertions(+), 30 deletions(-) diff --git a/crates/zkwasm/src/circuits/jtable/assign.rs b/crates/zkwasm/src/circuits/jtable/assign.rs index 4dfda5e4a..49ddb26f1 100644 --- a/crates/zkwasm/src/circuits/jtable/assign.rs +++ b/crates/zkwasm/src/circuits/jtable/assign.rs @@ -195,8 +195,9 @@ impl JumpTableChip { self.init(ctx)?; ctx.reset(); + // non-static entry includes `call`` and `return`` op let mut rest_jops = jtable.entries().len() as u64 * 2; - + // static entry only includes `return` op for entry in static_entries { if entry.enable { rest_jops += 1; diff --git a/crates/zkwasm/src/circuits/jtable/mod.rs b/crates/zkwasm/src/circuits/jtable/mod.rs index 34132e87a..5450888f1 100644 --- a/crates/zkwasm/src/circuits/jtable/mod.rs +++ b/crates/zkwasm/src/circuits/jtable/mod.rs @@ -11,7 +11,7 @@ mod assign; mod configure; pub(crate) mod expression; -// enable and data should encode in image table +// enable and data should be encoded in image table pub(crate) const STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY: usize = STATIC_FRAME_ENTRY_NUMBER * 2; pub enum JtableOffset { diff --git a/crates/zkwasm/src/circuits/mtable/assign.rs b/crates/zkwasm/src/circuits/mtable/assign.rs index 437b12284..0be113dc0 100644 --- a/crates/zkwasm/src/circuits/mtable/assign.rs +++ b/crates/zkwasm/src/circuits/mtable/assign.rs @@ -41,9 +41,9 @@ impl MemoryTableChip { #[cfg(feature = "continuation")] ctx.region.assign_advice_from_constant( - || "rest_memory_updating_ops terminate", - self.config.rest_memory_updating_ops.0.col, - ctx.offset + self.config.rest_memory_updating_ops.0.rot as usize, + || "rest_memory_finalize_ops terminate", + self.config.rest_memory_finalize_ops.0.col, + ctx.offset + self.config.rest_memory_finalize_ops.0.rot as usize, F::zero(), )?; } @@ -55,16 +55,16 @@ impl MemoryTableChip { } #[cfg(feature = "continuation")] - fn constrain_rest_memory_updating_ops( + fn constrain_rest_memory_finalize_ops( &self, ctx: &mut Context<'_, F>, - rest_memory_updating_ops: u32, + rest_memory_finalize_ops: u32, ) -> Result, Error> { // Overwrite in assign_entries let cell = self .config - .rest_memory_updating_ops - .assign(ctx, F::from(rest_memory_updating_ops as u64))?; + .rest_memory_finalize_ops + .assign(ctx, F::from(rest_memory_finalize_ops as u64))?; Ok(cell) } @@ -94,7 +94,7 @@ impl MemoryTableChip { mtable: &MemoryWritingTable, init_rest_mops: u64, _imtable: &InitMemoryTable, - mut _rest_memory_updating_ops: u32, + mut _rest_memory_finalize_ops: u32, ) -> Result<(), Error> { macro_rules! assign_advice { ($cell:ident, $value:expr) => { @@ -195,8 +195,8 @@ impl MemoryTableChip { #[cfg(feature = "continuation")] assign_advice!( - rest_memory_updating_ops, - F::from(_rest_memory_updating_ops as u64) + rest_memory_finalize_ops, + F::from(_rest_memory_finalize_ops as u64) ); assign_advice!( @@ -220,13 +220,13 @@ impl MemoryTableChip { if entry.entry.atype == AccessType::Write && !entry.entry.is_same_location(&next_entry.entry) { - _rest_memory_updating_ops -= 1; + _rest_memory_finalize_ops -= 1; } } else { // It's last entry if entry.entry.atype == AccessType::Write { - _rest_memory_updating_ops -= 1; + _rest_memory_finalize_ops -= 1; } } @@ -269,7 +269,7 @@ impl MemoryTableChip { Ok(()) } - fn compute_rest_memory_updating_ops(&self, mtable: &MemoryWritingTable) -> u32 { + fn compute_rest_memory_finalize_ops(&self, mtable: &MemoryWritingTable) -> u32 { let mut ops = 0u32; let mut iter = mtable.0.iter().peekable(); @@ -309,11 +309,11 @@ impl MemoryTableChip { self.assign_fixed(ctx)?; ctx.reset(); - let rest_memory_updating_ops = self.compute_rest_memory_updating_ops(mtable); + let rest_memory_finalize_ops = self.compute_rest_memory_finalize_ops(mtable); #[cfg(feature = "continuation")] - let rest_memory_updating_ops_cell = - self.constrain_rest_memory_updating_ops(ctx, rest_memory_updating_ops)?; + let rest_memory_finalize_ops_cell = + self.constrain_rest_memory_finalize_ops(ctx, rest_memory_finalize_ops)?; let rest_mops_cell = self.constrain_rest_mops_permutation(ctx, etable_rest_mops_cell, rest_mops)?; @@ -322,15 +322,15 @@ impl MemoryTableChip { * Skip subsequent advice assignment in the first pass to enhance performance. */ if rest_mops_cell.value().is_some() { - self.assign_entries(ctx, mtable, rest_mops, imtable, rest_memory_updating_ops)?; + self.assign_entries(ctx, mtable, rest_mops, imtable, rest_memory_finalize_ops)?; ctx.reset(); } cfg_if::cfg_if! { if #[cfg(feature="continuation")] { - Ok((Some(rest_memory_updating_ops_cell), F::from(rest_memory_updating_ops as u64))) + Ok((Some(rest_memory_finalize_ops_cell), F::from(rest_memory_finalize_ops as u64))) } else { - Ok((None, F::from(rest_memory_updating_ops as u64))) + Ok((None, F::from(rest_memory_finalize_ops as u64))) } } } diff --git a/crates/zkwasm/src/circuits/mtable/mod.rs b/crates/zkwasm/src/circuits/mtable/mod.rs index 2eace1f46..59f4b0df9 100644 --- a/crates/zkwasm/src/circuits/mtable/mod.rs +++ b/crates/zkwasm/src/circuits/mtable/mod.rs @@ -46,10 +46,6 @@ pub struct MemoryTableConfig { end_eid_cell: AllocatedU32StateCell, eid_diff_cell: AllocatedU32StateCell, rest_mops_cell: AllocatedCommonRangeCell, - // offset_align_left: AllocatedU32Cell, - // offset_align_right: AllocatedU32Cell, - // offset_align_left_diff_cell: AllocatedU32Cell, - // offset_align_right_diff_cell: AllocatedU32Cell, offset_cell: AllocatedU32Cell, offset_diff_cell: AllocatedU32Cell, @@ -59,7 +55,7 @@ pub struct MemoryTableConfig { init_encode_cell: AllocatedUnlimitedCell, #[cfg(feature = "continuation")] - rest_memory_updating_ops: AllocatedUnlimitedCell, + rest_memory_finalize_ops: AllocatedUnlimitedCell, value: AllocatedU64Cell, } @@ -103,7 +99,7 @@ impl MemoryTableConfig { let init_encode_cell = allocator.alloc_unlimited_cell(); #[cfg(feature = "continuation")] - let rest_memory_updating_ops = { + let rest_memory_finalize_ops = { let cell = allocator.alloc_unlimited_cell(); // FIXME: try to avoid this? meta.enable_equality(cell.0.col); @@ -347,8 +343,8 @@ impl MemoryTableConfig { { meta.create_gate("mc13. rest memory updating ops", |meta| { vec![ - rest_memory_updating_ops.curr_expr(meta) - - rest_memory_updating_ops.next_expr(meta) + rest_memory_finalize_ops.curr_expr(meta) + - rest_memory_finalize_ops.next_expr(meta) - (constant_from!(1) - is_next_same_offset_cell.curr_expr(meta)) * (constant_from!(1) - is_init_cell.curr_expr(meta)), ] @@ -387,7 +383,7 @@ impl MemoryTableConfig { encode_cell, #[cfg(feature = "continuation")] - rest_memory_updating_ops, + rest_memory_finalize_ops, } } } From 664ab04480c30cd062b719d23c1f30dbed0984aa Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Thu, 14 Dec 2023 07:46:06 +0000 Subject: [PATCH 19/34] fix comment --- crates/zkwasm/src/checksum/mod.rs | 9 +++-- crates/zkwasm/src/circuits/mtable/assign.rs | 29 +--------------- crates/zkwasm/src/circuits/mtable/mod.rs | 33 ++----------------- .../zkwasm/src/circuits/test_circuit/mod.rs | 7 ++-- .../zkwasm/src/circuits/utils/image_table.rs | 17 +++------- crates/zkwasm/src/loader/mod.rs | 3 +- 6 files changed, 19 insertions(+), 79 deletions(-) diff --git a/crates/zkwasm/src/checksum/mod.rs b/crates/zkwasm/src/checksum/mod.rs index 3ec68d439..bf04d7133 100644 --- a/crates/zkwasm/src/checksum/mod.rs +++ b/crates/zkwasm/src/checksum/mod.rs @@ -6,7 +6,7 @@ use specs::CompilationTable; use crate::circuits::utils::image_table::EncodeCompilationTableValues; pub trait ImageCheckSum { - fn checksum(&self) -> Output; + fn checksum(&self, page_capability: u32) -> Output; } pub(crate) struct CompilationTableWithParams<'a, 'b, C: CurveAffine> { @@ -15,8 +15,11 @@ pub(crate) struct CompilationTableWithParams<'a, 'b, C: CurveAffine> { } impl<'a, 'b, C: CurveAffine> ImageCheckSum> for CompilationTableWithParams<'a, 'b, C> { - fn checksum(&self) -> Vec { - let cells = self.table.encode_compilation_table_values().plain(); + fn checksum(&self, page_capability: u32) -> Vec { + let cells = self + .table + .encode_compilation_table_values(page_capability) + .plain(); let c = best_multiexp_gpu_cond(&cells[..], &self.params.get_g_lagrange()[0..cells.len()]); vec![c.into()] diff --git a/crates/zkwasm/src/circuits/mtable/assign.rs b/crates/zkwasm/src/circuits/mtable/assign.rs index 0be113dc0..045e060c0 100644 --- a/crates/zkwasm/src/circuits/mtable/assign.rs +++ b/crates/zkwasm/src/circuits/mtable/assign.rs @@ -6,7 +6,6 @@ use halo2_proofs::plonk::Error; use log::debug; use specs::encode::init_memory_table::encode_init_memory_table_entry; use specs::encode::memory_table::encode_memory_table_entry; -use specs::imtable::InitMemoryTable; use specs::mtable::AccessType; use specs::mtable::LocationType; use specs::mtable::VarType; @@ -60,7 +59,6 @@ impl MemoryTableChip { ctx: &mut Context<'_, F>, rest_memory_finalize_ops: u32, ) -> Result, Error> { - // Overwrite in assign_entries let cell = self .config .rest_memory_finalize_ops @@ -93,7 +91,6 @@ impl MemoryTableChip { ctx: &mut Context<'_, F>, mtable: &MemoryWritingTable, init_rest_mops: u64, - _imtable: &InitMemoryTable, mut _rest_memory_finalize_ops: u32, ) -> Result<(), Error> { macro_rules! assign_advice { @@ -154,26 +151,6 @@ impl MemoryTableChip { assign_bit_if!(entry.entry.atype.is_init(), is_init_cell); - if ctx.offset == 207328 { - println!("is init: {}", entry.entry.atype.is_init()); - } - - //if entry.entry.atype.is_init() { - // let init_memory_entry = imtable - // .try_find(entry.entry.ltype, entry.entry.offset) - // .unwrap(); - - // assign_advice!(offset_align_left, left_offset); - // assign_advice!(offset_align_right, right_offset); - // assign_advice!( - // offset_align_left_diff_cell, - // entry.entry.offset - left_offset - // ); - // assign_advice!( - // offset_align_right_diff_cell, - // right_offset - entry.entry.offset - // ); - assign_advice!( init_encode_cell, bn_to_field(&encode_init_memory_table_entry( @@ -184,7 +161,6 @@ impl MemoryTableChip { entry.entry.value.into() )) ); - //} assign_u32_state!(start_eid_cell, entry.entry.eid); assign_u32_state!(end_eid_cell, entry.end_eid); @@ -264,8 +240,6 @@ impl MemoryTableChip { ctx.step(MEMORY_TABLE_ENTRY_ROWS as usize); } - println!("mtable end: {}", ctx.offset); - Ok(()) } @@ -296,7 +270,6 @@ impl MemoryTableChip { ctx: &mut Context<'_, F>, etable_rest_mops_cell: &Option>, mtable: &MemoryWritingTable, - imtable: &InitMemoryTable, ) -> Result<(Option>, F), Error> { debug!("size of memory writing table: {}", mtable.0.len()); assert!(mtable.0.len() * (MEMORY_TABLE_ENTRY_ROWS as usize) < self.maximal_available_rows); @@ -322,7 +295,7 @@ impl MemoryTableChip { * Skip subsequent advice assignment in the first pass to enhance performance. */ if rest_mops_cell.value().is_some() { - self.assign_entries(ctx, mtable, rest_mops, imtable, rest_memory_finalize_ops)?; + self.assign_entries(ctx, mtable, rest_mops, rest_memory_finalize_ops)?; ctx.reset(); } diff --git a/crates/zkwasm/src/circuits/mtable/mod.rs b/crates/zkwasm/src/circuits/mtable/mod.rs index 59f4b0df9..92d46adeb 100644 --- a/crates/zkwasm/src/circuits/mtable/mod.rs +++ b/crates/zkwasm/src/circuits/mtable/mod.rs @@ -89,7 +89,6 @@ impl MemoryTableConfig { let eid_diff_cell = allocator.alloc_u32_state_cell(); let rest_mops_cell = allocator.alloc_common_range_cell(); - // TODO: cut allocated u32 cell let offset_cell = allocator.alloc_u32_cell(); let offset_diff_cell = allocator.alloc_u32_cell(); @@ -193,32 +192,8 @@ impl MemoryTableConfig { .collect::>() }); - // meta.create_gate("mc7a. init", |meta| { - // vec![ - // // offset_left_align <= offset && offset <= offset_right_align - // is_init_cell.curr_expr(meta) - // * (offset_align_left.curr_expr(meta) - // + offset_align_left_diff_cell.curr_expr(meta) - // - offset_cell.curr_expr(meta)), - // is_init_cell.curr_expr(meta) - // * (offset_cell.curr_expr(meta) + offset_align_right_diff_cell.curr_expr(meta) - // - offset_align_right.curr_expr(meta)), - // ] - // .into_iter() - // .map(|x| x * fixed_curr!(meta, entry_sel)) - // .collect::>() - // }); - - #[cfg(not(feature = "continuation"))] - meta.create_gate("mc7a. init start_eid equals 0", |meta| { - vec![is_init_cell.curr_expr(meta) * start_eid_cell.curr_expr(meta)] - .into_iter() - .map(|x| x * fixed_curr!(meta, entry_sel)) - .collect::>() - }); - meta.create_gate( - "mc7b. global must has init (because of mutability check).", + "mc7a. global must has init (because of mutability check).", |meta| { vec![ (is_next_same_offset_cell.expr(meta) - constant_from!(1)) @@ -231,7 +206,7 @@ impl MemoryTableConfig { }, ); - meta.create_gate("mc7c. init encode.", |meta| { + meta.create_gate("mc7b. init encode.", |meta| { vec![ encode_init_memory_table_entry( is_stack_cell.curr_expr(meta) * constant_from!(LocationType::Stack as u64) @@ -374,10 +349,6 @@ impl MemoryTableConfig { offset_diff_cell, offset_diff_inv_cell, offset_diff_inv_helper_cell, - // offset_align_left, - // offset_align_right, - // offset_align_left_diff_cell, - // offset_align_right_diff_cell, value, init_encode_cell, encode_cell, diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index 4bdbc0e3a..41e4565ce 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -274,7 +274,6 @@ impl Circuit for TestCircuit { &mut ctx, &etable_permutation_cells.rest_mops, &memory_writing_table, - &self.tables.compilation_tables.imtable )? ) }; @@ -330,7 +329,7 @@ impl Circuit for TestCircuit { &mut image_table_assigner, self.tables .compilation_tables - .encode_compilation_table_values(), + .encode_compilation_table_values(config.circuit_maximal_pages), ImageTableLayouter { initialization_state: etable_permutation_cells.pre_initialization_state, static_frame_entries, @@ -349,10 +348,10 @@ impl Circuit for TestCircuit { &mut image_table_assigner, self.tables .compilation_tables - .encode_compilation_table_values(), + .encode_compilation_table_values(config.circuit_maximal_pages), self.tables .post_image_table - .encode_compilation_table_values(), + .encode_compilation_table_values(config.circuit_maximal_pages), ImageTableLayouter { initialization_state: etable_permutation_cells.post_initialization_state, static_frame_entries: pre_image_table_cells.static_frame_entries, diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index f909df3f2..d526cfa2c 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -10,8 +10,6 @@ use specs::state::InitializationState; use specs::CompilationTable; use wasmi::DEFAULT_VALUE_STACK_LIMIT; -use crate::circuits::config::zkwasm_k; -use crate::circuits::image_table::compute_maximal_pages; use crate::circuits::image_table::PAGE_ENTRIES; use crate::circuits::jtable::STATIC_FRAME_ENTRY_IMAGE_TABLE_ENTRY; use crate::circuits::utils::bn_to_field; @@ -183,14 +181,11 @@ impl ImageTableAssigner { } pub(crate) trait EncodeCompilationTableValues { - fn encode_compilation_table_values(&self) -> ImageTableLayouter; + fn encode_compilation_table_values(&self, page_capability: u32) -> ImageTableLayouter; } impl EncodeCompilationTableValues for CompilationTable { - fn encode_compilation_table_values(&self) -> ImageTableLayouter { - // FIXME: ugly - let pages_capability = compute_maximal_pages(zkwasm_k()); - + fn encode_compilation_table_values(&self, page_capability: u32) -> ImageTableLayouter { let initialization_state_handler = |_| Ok(self.initialization_state.map(|v| F::from((*v) as u64))); @@ -254,7 +249,7 @@ impl EncodeCompilationTableValues for CompilationTable { )); let layouter = InitMemoryLayouter { - pages: pages_capability, + pages: page_capability, }; layouter.for_each(|(ltype, offset)| { @@ -288,7 +283,7 @@ impl EncodeCompilationTableValues for CompilationTable { let mut assigner = ImageTableAssigner::new( self.itable.entries().len(), self.br_table.entries().len() + self.elem_table.entries().len(), - pages_capability, + page_capability, ); let layouter = assigner @@ -314,10 +309,8 @@ impl ImageTableLayouter { buf.append( &mut self .static_frame_entries - .clone() - .to_vec() - .into_iter() .map(|(enable, fid)| vec![enable, fid]) + .into_iter() .collect::>>() .concat(), ); diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index f7b9850e4..4a4a3954b 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -30,6 +30,7 @@ use crate::checksum::ImageCheckSum; use crate::circuits::config::init_zkwasm_runtime; use crate::circuits::config::set_zkwasm_k; use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; +use crate::circuits::image_table::compute_maximal_pages; use crate::circuits::image_table::IMAGE_COL_NAME; use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; @@ -187,7 +188,7 @@ impl ZkWasmLoader { params, }; - Ok(table_with_params.checksum()) + Ok(table_with_params.checksum(compute_maximal_pages(self.k))) } } From 4f8abe6c0e897e640eb00b5fdc03a4d02a80c522 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Thu, 14 Dec 2023 13:30:05 +0000 Subject: [PATCH 20/34] add name for post image table col --- crates/zkwasm/src/circuits/post_image_table/continuation.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index e79a2534c..3aaeaa4f4 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -31,6 +31,8 @@ use crate::next; use super::PostImageTableChipTrait; use super::PostImageTableConfigTrait; +pub const POST_IMAGE_TABLE: &str = "post_img_col"; + #[derive(Clone)] pub(in crate::circuits) struct ContinuationPostImageTableConfig { memory_addr_sel: Column, @@ -49,7 +51,7 @@ impl PostImageTableConfigTrait for ContinuationPostImageTableCon ) -> Self { let update = meta.advice_column(); let rest_memory_finalized_count = meta.advice_column(); - let post_image_table = meta.advice_column(); + let post_image_table = meta.named_advice_column(POST_IMAGE_TABLE.to_owned()); meta.enable_equality(rest_memory_finalized_count); meta.enable_equality(post_image_table); From cbc68c5a6a10faf08b734d6207744d3282ba2ae9 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Thu, 14 Dec 2023 14:06:05 +0000 Subject: [PATCH 21/34] fix image table assigner --- crates/zkwasm/src/circuits/utils/image_table.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index d526cfa2c..e72fe1221 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -140,7 +140,7 @@ impl ImageTableAssigner { &mut self, mut padding_handler: impl FnMut(usize, usize) -> Result, Error>, ) -> Result, Error> { - padding_handler(self.padding_offset, self.padding_offset) + padding_handler(self.padding_offset, self.init_memory_offset) } pub fn exec_init_memory_entries( @@ -281,8 +281,8 @@ impl EncodeCompilationTableValues for CompilationTable { }; let mut assigner = ImageTableAssigner::new( - self.itable.entries().len(), - self.br_table.entries().len() + self.elem_table.entries().len(), + self.itable.entries().len() + 1, + self.br_table.entries().len() + self.elem_table.entries().len() + 1, page_capability, ); From 37beae0f5272c8765bcaa65eb28684b3578e1f78 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Fri, 15 Dec 2023 05:37:43 +0000 Subject: [PATCH 22/34] fix circuit_without_witness --- crates/cli/src/exec.rs | 17 +++++++- crates/zkwasm/src/circuits/mod.rs | 6 +++ crates/zkwasm/src/continuation/loader.rs | 7 +++ crates/zkwasm/src/continuation/slice.rs | 8 ---- crates/zkwasm/src/loader/mod.rs | 55 ++++++++++++++++-------- crates/zkwasm/src/test/test_rlp.rs | 2 +- crates/zkwasm/src/test/test_rlp_slice.rs | 7 ++- 7 files changed, 70 insertions(+), 32 deletions(-) diff --git a/crates/cli/src/exec.rs b/crates/cli/src/exec.rs index d313d8bc4..ac8c25d15 100644 --- a/crates/cli/src/exec.rs +++ b/crates/cli/src/exec.rs @@ -2,6 +2,7 @@ use anyhow::Result; use delphinus_zkwasm::circuits::TestCircuit; use delphinus_zkwasm::loader::ExecutionArg; use delphinus_zkwasm::loader::ZkWasmLoader; +use delphinus_zkwasm::runtime::host::host_env::HostEnv; use halo2_proofs::arithmetic::BaseExt; use halo2_proofs::pairing::bn256::Bn256; use halo2_proofs::pairing::bn256::Fr; @@ -96,12 +97,18 @@ pub fn exec_image_checksum( ) -> Result<()> { let loader = ZkWasmLoader::::new(zkwasm_k, wasm_binary, phantom_functions)?; + let image = if cfg!(feature = "continuation") { + todo!("read slice image from file?"); + } else { + loader.compile_without_env()?.tables + }; + let params = load_or_build_unsafe_params::( zkwasm_k, Some(&output_dir.join(format!("K{}.params", zkwasm_k))), ); - let checksum = loader.checksum(¶ms)?; + let checksum = loader.checksum(&image, ¶ms)?; assert_eq!(checksum.len(), 1); let checksum = checksum[0]; @@ -363,7 +370,13 @@ pub fn exec_verify_proof( let proof = load_proof(proof_path); - loader.verify_proof(¶ms, vkey, instances, proof)?; + let image = if cfg!(feature = "continuation") { + todo!("read slice image from file?") + } else { + loader.compile_without_env()?.tables + }; + + loader.verify_proof(&image, ¶ms, vkey, &instances, proof)?; info!("Verifing proof passed"); diff --git a/crates/zkwasm/src/circuits/mod.rs b/crates/zkwasm/src/circuits/mod.rs index 672f3b1c3..dca24ab37 100644 --- a/crates/zkwasm/src/circuits/mod.rs +++ b/crates/zkwasm/src/circuits/mod.rs @@ -64,6 +64,12 @@ pub struct ZkWasmCircuitBuilder { impl ZkWasmCircuitBuilder { pub fn build_circuit(self, slice_capability: Option) -> TestCircuit { + #[cfg(feature = "continuation")] + assert!(slice_capability.is_some()); + + #[cfg(not(feature = "continuation"))] + assert!(slice_capability.is_none()); + TestCircuit::new(self.tables, slice_capability) } } diff --git a/crates/zkwasm/src/continuation/loader.rs b/crates/zkwasm/src/continuation/loader.rs index 023581c78..bc86e0cd1 100644 --- a/crates/zkwasm/src/continuation/loader.rs +++ b/crates/zkwasm/src/continuation/loader.rs @@ -12,6 +12,13 @@ use super::slice::Slice; use super::slice::Slices; impl ZkWasmLoader { +<<<<<<< HEAD +======= + pub(crate) fn compute_slice_capability(&self) -> usize { + ((1 << self.k) - 200) / EVENT_TABLE_ENTRY_ROWS as usize + } + +>>>>>>> 1fd2066 (fix circuit_without_witness) pub fn slice(&self, execution_result: ExecutionResult) -> Slices { Slices::new( execution_result.tables.unwrap(), diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index 7e1e2c0cd..add768960 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -86,14 +86,6 @@ impl Iterator for Slices { } let mut etable_entries = self.pop_etable_entries(); - // let etable = EventTable::new(etable_entries); - - // let is_last_slice = self.remaining_etable_entries.is_empty(); - - // if !is_last_slice { - // self.remaining_etable_entries - // .insert(0, etable.entries().last().unwrap().clone()); - // } let (updated_init_memory_table, updated_post_initialization_state) = { let updated_init_memory_table = self diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index 4a4a3954b..ad19efdd7 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -16,6 +16,7 @@ use halo2_proofs::poly::commitment::ParamsVerifier; use halo2aggregator_s::circuits::utils::load_or_create_proof; use halo2aggregator_s::circuits::utils::TranscriptHash; use halo2aggregator_s::transcript::poseidon::PoseidonRead; +use specs::CompilationTable; use specs::ExecutionTable; use specs::Tables; use wasmi::tracer::SliceDumper; @@ -124,7 +125,7 @@ impl ZkWasmLoader { ) } - fn circuit_without_witness(&self, last_slice_circuit: bool) -> Result> { + pub fn compile_without_env(&self) -> Result, Tracer>> { let (env, _) = HostEnv::new_with_full_foreign_plugins( vec![], vec![], @@ -132,7 +133,11 @@ impl ZkWasmLoader { Arc::new(Mutex::new(vec![])), ); - let compiled_module = self.compile(&env)?; + self.compile(&env) + } + + fn circuit_without_witness(&self, last_slice_circuit: bool) -> Result> { + let compiled_module = self.compile_without_env()?; let builder = ZkWasmCircuitBuilder { tables: Tables { @@ -143,7 +148,13 @@ impl ZkWasmLoader { }, }; - Ok(builder.build_circuit::(None)) + Ok( + builder.build_circuit::(if cfg!(feature = "continuation") { + Some(self.compute_slice_capability()) + } else { + None + }), + ) } pub fn new(k: u32, image: Vec, phantom_functions: Vec) -> Result { @@ -174,17 +185,13 @@ impl ZkWasmLoader { Ok(keygen_vk(¶ms, &circuit).unwrap()) } - pub fn checksum(&self, params: &Params) -> Result> { - let (env, _) = HostEnv::new_with_full_foreign_plugins( - vec![], - vec![].into(), - vec![], - Arc::new(Mutex::new(vec![])), - ); - let compiled = self.compile(&env, Box::new(WitnessDumper::default()))?; - + pub fn checksum<'a, 'b>( + &self, + image: &'b CompilationTable, + params: &'a Params, + ) -> Result> { let table_with_params = CompilationTableWithParams { - table: &compiled.tables, + table: &image, params, }; @@ -287,9 +294,10 @@ impl ZkWasmLoader { pub fn verify_proof( &self, + image: &CompilationTable, params: &Params, vkey: VerifyingKey, - instances: Vec, + instances: &Vec, proof: Vec, ) -> Result<()> { let params_verifier: ParamsVerifier = params.verifier(instances.len()).unwrap(); @@ -318,7 +326,7 @@ impl ZkWasmLoader { &mut PoseidonRead::init(&proof[..]), ) .unwrap(); - let checksum = self.checksum(params)?; + let checksum = self.checksum(image, params)?; assert!(vec![img_col_commitment[img_col_idx as usize]] == checksum) } @@ -345,7 +353,7 @@ mod tests { use super::ZkWasmLoader; impl ZkWasmLoader { - pub(crate) fn bench_test(&self, circuit: TestCircuit, instances: Vec) { + pub(crate) fn bench_test(&self, circuit: TestCircuit, instances: &Vec) { fn prepare_param(k: u32) -> Params { let path = PathBuf::from(format!("test_param.{}.data", k)); @@ -369,12 +377,21 @@ mod tests { } let params = prepare_param(self.k); - let vkey = self.create_vkey(¶ms, true).unwrap(); + let vkey = self + .create_vkey(¶ms, circuit.tables.is_last_slice) + .unwrap(); let proof = self - .create_proof(¶ms, vkey.clone(), circuit, &instances) + .create_proof(¶ms, vkey.clone(), circuit.clone(), &instances) .unwrap(); - self.verify_proof(¶ms, vkey, instances, proof).unwrap(); + self.verify_proof( + &circuit.tables.compilation_tables, + ¶ms, + vkey, + instances, + proof, + ) + .unwrap(); } } } diff --git a/crates/zkwasm/src/test/test_rlp.rs b/crates/zkwasm/src/test/test_rlp.rs index 71eb8f2fe..6aac963fa 100644 --- a/crates/zkwasm/src/test/test_rlp.rs +++ b/crates/zkwasm/src/test/test_rlp.rs @@ -181,7 +181,7 @@ mod tests { fn test_rlp_bench() { let (loader, circuit, instances) = build_circuit().unwrap(); - loader.bench_test(circuit, instances) + loader.bench_test(circuit, &instances) } } } diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index f987d4959..c5cc2df4c 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -182,9 +182,12 @@ fn test_slices() -> Result<()> { while let Some(slice) = slices.next() { println!("slice {}", index); - let circuit = slice.build_circuit(); + if index != 0 { + let circuit = slice.build_circuit(); - loader.mock_test(&circuit, &instances)?; + loader.mock_test(&circuit, &instances)?; + loader.bench_test(circuit, &instances); + } index += 1; } From 829a66b30ca6b9d7dc605373b480cdabcdd9bc9a Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Fri, 15 Dec 2023 05:39:09 +0000 Subject: [PATCH 23/34] fix rlp_slice test --- crates/cli/src/exec.rs | 1 - crates/zkwasm/src/test/test_rlp_slice.rs | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/cli/src/exec.rs b/crates/cli/src/exec.rs index ac8c25d15..19febe4fe 100644 --- a/crates/cli/src/exec.rs +++ b/crates/cli/src/exec.rs @@ -2,7 +2,6 @@ use anyhow::Result; use delphinus_zkwasm::circuits::TestCircuit; use delphinus_zkwasm::loader::ExecutionArg; use delphinus_zkwasm::loader::ZkWasmLoader; -use delphinus_zkwasm::runtime::host::host_env::HostEnv; use halo2_proofs::arithmetic::BaseExt; use halo2_proofs::pairing::bn256::Bn256; use halo2_proofs::pairing::bn256::Fr; diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index c5cc2df4c..97c7f2e8e 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -182,12 +182,10 @@ fn test_slices() -> Result<()> { while let Some(slice) = slices.next() { println!("slice {}", index); - if index != 0 { - let circuit = slice.build_circuit(); + let circuit = slice.build_circuit(); - loader.mock_test(&circuit, &instances)?; - loader.bench_test(circuit, &instances); - } + loader.mock_test(&circuit, &instances)?; + loader.bench_test(circuit, &instances); index += 1; } From 57adf94677d278ff5b909ed3850a1856668cacde Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Fri, 15 Dec 2023 05:55:39 +0000 Subject: [PATCH 24/34] fix compiling error --- crates/zkwasm/src/loader/mod.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index ad19efdd7..7eb7c581f 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -148,13 +148,12 @@ impl ZkWasmLoader { }, }; - Ok( - builder.build_circuit::(if cfg!(feature = "continuation") { - Some(self.compute_slice_capability()) - } else { - None - }), - ) + #[cfg(feature = "continuation")] + let slice_capabitlity = Some(self.compute_slice_capability()); + #[cfg(not(feature = "continuation"))] + let slice_capabitlity = None; + + Ok(builder.build_circuit::(slice_capabitlity)) } pub fn new(k: u32, image: Vec, phantom_functions: Vec) -> Result { From 0748504b5a58ad7943e9d573d29aa3b7ad3c304c Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Mon, 18 Dec 2023 08:48:44 +0000 Subject: [PATCH 25/34] fix code style --- crates/zkwasm/src/circuits/etable/assign.rs | 4 +- crates/zkwasm/src/circuits/jtable/assign.rs | 11 ++-- crates/zkwasm/src/circuits/mtable/assign.rs | 53 ++++++++----------- crates/zkwasm/src/circuits/mtable/mod.rs | 10 ++-- .../zkwasm/src/circuits/utils/image_table.rs | 2 +- crates/zkwasm/src/loader/mod.rs | 2 +- 6 files changed, 34 insertions(+), 48 deletions(-) diff --git a/crates/zkwasm/src/circuits/etable/assign.rs b/crates/zkwasm/src/circuits/etable/assign.rs index 1685b1350..08f7237a8 100644 --- a/crates/zkwasm/src/circuits/etable/assign.rs +++ b/crates/zkwasm/src/circuits/etable/assign.rs @@ -219,7 +219,7 @@ impl EventTableChip { }) } - fn assign_post_initialization_state( + fn assign_padding_and_post_initialization_state( &self, ctx: &mut Context<'_, F>, initialization_state: &InitializationState, @@ -520,7 +520,7 @@ impl EventTableChip { )?; let post_initialization_state_cells = - self.assign_post_initialization_state(ctx, &post_initialization_state)?; + self.assign_padding_and_post_initialization_state(ctx, &post_initialization_state)?; cfg_if::cfg_if! { if #[cfg(feature = "continuation")] { diff --git a/crates/zkwasm/src/circuits/jtable/assign.rs b/crates/zkwasm/src/circuits/jtable/assign.rs index 49ddb26f1..3d007ff60 100644 --- a/crates/zkwasm/src/circuits/jtable/assign.rs +++ b/crates/zkwasm/src/circuits/jtable/assign.rs @@ -195,14 +195,9 @@ impl JumpTableChip { self.init(ctx)?; ctx.reset(); - // non-static entry includes `call`` and `return`` op - let mut rest_jops = jtable.entries().len() as u64 * 2; - // static entry only includes `return` op - for entry in static_entries { - if entry.enable { - rest_jops += 1; - } - } + // non-static entry includes `call`` and `return`` op, static entry only includes `return` op + let mut rest_jops = jtable.entries().len() as u64 * 2 + + static_entries.iter().filter(|entry| entry.enable).count() as u64; let frame_table_start_jump_cells = self.assign_static_entries(ctx, &mut rest_jops, static_entries)?; diff --git a/crates/zkwasm/src/circuits/mtable/assign.rs b/crates/zkwasm/src/circuits/mtable/assign.rs index 045e060c0..dc5e37d3a 100644 --- a/crates/zkwasm/src/circuits/mtable/assign.rs +++ b/crates/zkwasm/src/circuits/mtable/assign.rs @@ -41,8 +41,8 @@ impl MemoryTableChip { #[cfg(feature = "continuation")] ctx.region.assign_advice_from_constant( || "rest_memory_finalize_ops terminate", - self.config.rest_memory_finalize_ops.0.col, - ctx.offset + self.config.rest_memory_finalize_ops.0.rot as usize, + self.config.rest_memory_finalize_ops_cell.0.col, + ctx.offset + self.config.rest_memory_finalize_ops_cell.0.rot as usize, F::zero(), )?; } @@ -61,7 +61,7 @@ impl MemoryTableChip { ) -> Result, Error> { let cell = self .config - .rest_memory_finalize_ops + .rest_memory_finalize_ops_cell .assign(ctx, F::from(rest_memory_finalize_ops as u64))?; Ok(cell) @@ -171,7 +171,7 @@ impl MemoryTableChip { #[cfg(feature = "continuation")] assign_advice!( - rest_memory_finalize_ops, + rest_memory_finalize_ops_cell, F::from(_rest_memory_finalize_ops as u64) ); @@ -192,18 +192,12 @@ impl MemoryTableChip { rest_mops -= 1; } - if let Some(next_entry) = iter.peek() { - if entry.entry.atype == AccessType::Write - && !entry.entry.is_same_location(&next_entry.entry) - { - _rest_memory_finalize_ops -= 1; - } - } else { - // It's last entry - - if entry.entry.atype == AccessType::Write { - _rest_memory_finalize_ops -= 1; - } + if entry.entry.atype == AccessType::Write + && iter.peek().map_or(true, |next_entry| { + !next_entry.entry.is_same_location(&entry.entry) + }) + { + _rest_memory_finalize_ops -= 1; } ctx.step(MEMORY_TABLE_ENTRY_ROWS as usize); @@ -243,26 +237,22 @@ impl MemoryTableChip { Ok(()) } - fn compute_rest_memory_finalize_ops(&self, mtable: &MemoryWritingTable) -> u32 { - let mut ops = 0u32; + fn count_rest_memory_finalize_ops(&self, mtable: &MemoryWritingTable) -> u32 { + let mut count = 0u32; let mut iter = mtable.0.iter().peekable(); while let Some(entry) = iter.next() { - if let Some(next_entry) = iter.peek() { - if entry.entry.atype == AccessType::Write - && !entry.entry.is_same_location(&next_entry.entry) - { - ops += 1; - } - } else { - if entry.entry.atype == AccessType::Write { - ops += 1; - } + if entry.entry.atype == AccessType::Write + && iter.peek().map_or(true, |next_entry| { + !next_entry.entry.is_same_location(&entry.entry) + }) + { + count += 1; } } - ops + count } pub(crate) fn assign( @@ -282,7 +272,7 @@ impl MemoryTableChip { self.assign_fixed(ctx)?; ctx.reset(); - let rest_memory_finalize_ops = self.compute_rest_memory_finalize_ops(mtable); + let rest_memory_finalize_ops = self.count_rest_memory_finalize_ops(mtable); #[cfg(feature = "continuation")] let rest_memory_finalize_ops_cell = @@ -303,7 +293,8 @@ impl MemoryTableChip { if #[cfg(feature="continuation")] { Ok((Some(rest_memory_finalize_ops_cell), F::from(rest_memory_finalize_ops as u64))) } else { - Ok((None, F::from(rest_memory_finalize_ops as u64))) + // Useless rest_memory_finalize_ops if continuation is disabled + Ok((None, F::zero())) } } } diff --git a/crates/zkwasm/src/circuits/mtable/mod.rs b/crates/zkwasm/src/circuits/mtable/mod.rs index 92d46adeb..d3fc02683 100644 --- a/crates/zkwasm/src/circuits/mtable/mod.rs +++ b/crates/zkwasm/src/circuits/mtable/mod.rs @@ -55,7 +55,7 @@ pub struct MemoryTableConfig { init_encode_cell: AllocatedUnlimitedCell, #[cfg(feature = "continuation")] - rest_memory_finalize_ops: AllocatedUnlimitedCell, + rest_memory_finalize_ops_cell: AllocatedUnlimitedCell, value: AllocatedU64Cell, } @@ -98,7 +98,7 @@ impl MemoryTableConfig { let init_encode_cell = allocator.alloc_unlimited_cell(); #[cfg(feature = "continuation")] - let rest_memory_finalize_ops = { + let rest_memory_finalize_ops_cell = { let cell = allocator.alloc_unlimited_cell(); // FIXME: try to avoid this? meta.enable_equality(cell.0.col); @@ -318,8 +318,8 @@ impl MemoryTableConfig { { meta.create_gate("mc13. rest memory updating ops", |meta| { vec![ - rest_memory_finalize_ops.curr_expr(meta) - - rest_memory_finalize_ops.next_expr(meta) + rest_memory_finalize_ops_cell.curr_expr(meta) + - rest_memory_finalize_ops_cell.next_expr(meta) - (constant_from!(1) - is_next_same_offset_cell.curr_expr(meta)) * (constant_from!(1) - is_init_cell.curr_expr(meta)), ] @@ -354,7 +354,7 @@ impl MemoryTableConfig { encode_cell, #[cfg(feature = "continuation")] - rest_memory_finalize_ops, + rest_memory_finalize_ops_cell, } } } diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index e72fe1221..5f437cdda 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -49,7 +49,7 @@ impl InitMemoryLayouter { * Br Table * -------------------- * Padding - * -------------------- Init Memory Offset + * -------------------- Init Memory Offset(Constant) * Stack * -------------------- * Global diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index 7eb7c581f..aac16038a 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -143,7 +143,7 @@ impl ZkWasmLoader { tables: Tables { compilation_tables: compiled_module.tables.clone(), execution_tables: ExecutionTable::default(), - post_image_table: compiled_module.tables, // FIXME: odd + post_image_table: compiled_module.tables, is_last_slice: last_slice_circuit, }, }; From 231db4b764414118fe98ccf2fa7b8ad9657db137 Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Mon, 18 Dec 2023 13:31:31 +0000 Subject: [PATCH 26/34] fix: fix a bug of post image table --- .../src/circuits/image_table/configure.rs | 6 +- crates/zkwasm/src/circuits/image_table/mod.rs | 3 +- crates/zkwasm/src/circuits/mod.rs | 9 ++- crates/zkwasm/src/circuits/mtable/assign.rs | 24 +++++--- crates/zkwasm/src/circuits/mtable/mod.rs | 21 +++++-- .../circuits/post_image_table/continuation.rs | 37 ++++++------ .../src/circuits/post_image_table/mod.rs | 57 ------------------- .../src/circuits/post_image_table/trivial.rs | 25 ++++---- .../zkwasm/src/circuits/test_circuit/mod.rs | 17 +++--- .../zkwasm/src/circuits/utils/image_table.rs | 29 ++++++++-- 10 files changed, 107 insertions(+), 121 deletions(-) delete mode 100644 crates/zkwasm/src/circuits/post_image_table/mod.rs diff --git a/crates/zkwasm/src/circuits/image_table/configure.rs b/crates/zkwasm/src/circuits/image_table/configure.rs index 14d5a880d..e0dcc107f 100644 --- a/crates/zkwasm/src/circuits/image_table/configure.rs +++ b/crates/zkwasm/src/circuits/image_table/configure.rs @@ -15,12 +15,12 @@ use crate::curr; impl ImageTableConfig { pub(in crate::circuits) fn configure( meta: &mut ConstraintSystem, - _memory_addr_sel: Column, + memory_addr_sel: Option>, ) -> Self { let col = meta.named_advice_column(IMAGE_COL_NAME.to_owned()); meta.enable_equality(col); Self { - _memory_addr_sel, + memory_addr_sel, col, _mark: PhantomData, } @@ -53,7 +53,7 @@ impl ImageTableConfig { let (addr, encode) = expr(meta); vec![ - (addr, fixed_curr!(meta, self._memory_addr_sel)), + (addr, fixed_curr!(meta, self.memory_addr_sel.unwrap())), ( ImageTableEncoder::InitMemory.encode(encode), curr!(meta, self.col), diff --git a/crates/zkwasm/src/circuits/image_table/mod.rs b/crates/zkwasm/src/circuits/image_table/mod.rs index d3c8fdb2d..5a793aca3 100644 --- a/crates/zkwasm/src/circuits/image_table/mod.rs +++ b/crates/zkwasm/src/circuits/image_table/mod.rs @@ -40,9 +40,10 @@ pub fn compute_maximal_pages(k: u32) -> u32 { bytes / PAGE_SIZE } +#[allow(dead_code)] #[derive(Clone)] pub struct ImageTableConfig { - _memory_addr_sel: Column, + memory_addr_sel: Option>, col: Column, _mark: PhantomData, } diff --git a/crates/zkwasm/src/circuits/mod.rs b/crates/zkwasm/src/circuits/mod.rs index dca24ab37..4a32f3932 100644 --- a/crates/zkwasm/src/circuits/mod.rs +++ b/crates/zkwasm/src/circuits/mod.rs @@ -16,10 +16,17 @@ mod external_host_call_table; mod mtable; mod traits; +#[cfg(feature = "continuation")] +#[path = "./post_image_table/continuation.rs"] +pub mod post_image_table; + +#[cfg(not(feature = "continuation"))] +#[path = "./post_image_table/trivial.rs"] +pub mod post_image_table; + pub mod config; pub mod image_table; pub mod jtable; -pub mod post_image_table; pub mod rtable; pub mod test_circuit; pub mod utils; diff --git a/crates/zkwasm/src/circuits/mtable/assign.rs b/crates/zkwasm/src/circuits/mtable/assign.rs index dc5e37d3a..c01d9e188 100644 --- a/crates/zkwasm/src/circuits/mtable/assign.rs +++ b/crates/zkwasm/src/circuits/mtable/assign.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::collections::HashSet; use halo2_proofs::arithmetic::FieldExt; use halo2_proofs::circuit::AssignedCell; @@ -92,7 +93,7 @@ impl MemoryTableChip { mtable: &MemoryWritingTable, init_rest_mops: u64, mut _rest_memory_finalize_ops: u32, - ) -> Result<(), Error> { + ) -> Result, Error> { macro_rules! assign_advice { ($cell:ident, $value:expr) => { self.config.$cell.assign(ctx, $value)? @@ -129,6 +130,7 @@ impl MemoryTableChip { }; } + let mut memory_finalized_table = HashSet::new(); let mut rest_mops = init_rest_mops; let mut iter = mtable.0.iter().peekable(); @@ -198,6 +200,8 @@ impl MemoryTableChip { }) { _rest_memory_finalize_ops -= 1; + + memory_finalized_table.insert((entry.entry.ltype, entry.entry.offset)); } ctx.step(MEMORY_TABLE_ENTRY_ROWS as usize); @@ -234,7 +238,7 @@ impl MemoryTableChip { ctx.step(MEMORY_TABLE_ENTRY_ROWS as usize); } - Ok(()) + Ok(memory_finalized_table) } fn count_rest_memory_finalize_ops(&self, mtable: &MemoryWritingTable) -> u32 { @@ -260,7 +264,7 @@ impl MemoryTableChip { ctx: &mut Context<'_, F>, etable_rest_mops_cell: &Option>, mtable: &MemoryWritingTable, - ) -> Result<(Option>, F), Error> { + ) -> Result<(Option>, F, HashSet<(LocationType, u32)>), Error> { debug!("size of memory writing table: {}", mtable.0.len()); assert!(mtable.0.len() * (MEMORY_TABLE_ENTRY_ROWS as usize) < self.maximal_available_rows); @@ -284,17 +288,21 @@ impl MemoryTableChip { /* * Skip subsequent advice assignment in the first pass to enhance performance. */ - if rest_mops_cell.value().is_some() { - self.assign_entries(ctx, mtable, rest_mops, rest_memory_finalize_ops)?; + let _memory_finalized_set = if rest_mops_cell.value().is_some() { + let set = self.assign_entries(ctx, mtable, rest_mops, rest_memory_finalize_ops)?; ctx.reset(); - } + + set + } else { + HashSet::new() + }; cfg_if::cfg_if! { if #[cfg(feature="continuation")] { - Ok((Some(rest_memory_finalize_ops_cell), F::from(rest_memory_finalize_ops as u64))) + Ok((Some(rest_memory_finalize_ops_cell), F::from(rest_memory_finalize_ops as u64), _memory_finalized_set)) } else { // Useless rest_memory_finalize_ops if continuation is disabled - Ok((None, F::zero())) + Ok((None, F::zero(), HashSet::new())) } } } diff --git a/crates/zkwasm/src/circuits/mtable/mod.rs b/crates/zkwasm/src/circuits/mtable/mod.rs index d3fc02683..21ef2783a 100644 --- a/crates/zkwasm/src/circuits/mtable/mod.rs +++ b/crates/zkwasm/src/circuits/mtable/mod.rs @@ -13,8 +13,6 @@ use halo2_proofs::plonk::ConstraintSystem; use halo2_proofs::plonk::Expression; use halo2_proofs::plonk::Fixed; use halo2_proofs::plonk::VirtualCells; -use specs::encode::image_table::ImageTableEncoder; -use specs::encode::init_memory_table::encode_init_memory_table_address; use specs::encode::init_memory_table::encode_init_memory_table_entry; use specs::encode::memory_table::encode_memory_table_entry; use specs::mtable::LocationType; @@ -227,6 +225,8 @@ impl MemoryTableConfig { image_table.init_memory_lookup(meta, "mc7c. imtable init", |meta| { cfg_if::cfg_if! { if #[cfg(feature = "continuation")] { + use specs::encode::init_memory_table::encode_init_memory_table_address; + ( encode_init_memory_table_address( is_stack_cell.curr_expr(meta) * constant_from!(LocationType::Stack as u64) @@ -317,14 +317,21 @@ impl MemoryTableConfig { #[cfg(feature = "continuation")] { meta.create_gate("mc13. rest memory updating ops", |meta| { + let is_write = constant_from!(1) - is_init_cell.curr_expr(meta); + let next_entry_at_different_position = + constant_from!(1) - is_next_same_offset_cell.curr_expr(meta); + + let is_memory_finalized_position_bit = is_write * next_entry_at_different_position; + vec![ + // `* enabled_cell`: If disabled, rest_memory_finalize_ops_cell should keep the same, + // The termination rest_memory_finalize_ops_cell is constant 0 at the last selected(sel=1) step. rest_memory_finalize_ops_cell.curr_expr(meta) - rest_memory_finalize_ops_cell.next_expr(meta) - - (constant_from!(1) - is_next_same_offset_cell.curr_expr(meta)) - * (constant_from!(1) - is_init_cell.curr_expr(meta)), + - is_memory_finalized_position_bit * enabled_cell.curr_expr(meta), ] .into_iter() - .map(|x| x * enabled_cell.curr_expr(meta) * fixed_curr!(meta, entry_sel)) + .map(|x| x * fixed_curr!(meta, entry_sel)) .collect::>() }); } @@ -391,6 +398,7 @@ impl ConfigureLookupTable for MemoryTableConfig { } } +#[cfg(feature = "continuation")] impl MemoryTableConfig { pub(in crate::circuits) fn configure_in_post_init_memory_table( &self, @@ -398,6 +406,9 @@ impl MemoryTableConfig { name: &'static str, expr: impl FnOnce(&mut VirtualCells<'_, F>) -> (Expression, Expression), ) { + use specs::encode::image_table::ImageTableEncoder; + use specs::encode::init_memory_table::encode_init_memory_table_address; + meta.lookup_any(name, |meta| { let (address, encode) = expr(meta); vec![ diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 3aaeaa4f4..2a0ff8df8 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::collections::HashSet; use std::marker::PhantomData; use std::rc::Rc; @@ -18,6 +19,7 @@ use specs::mtable::LocationType; use crate::circuits::image_table::ImageTableConfig; use crate::circuits::mtable::MemoryTableConfig; use crate::circuits::utils::bn_to_field; +use crate::circuits::utils::image_table::image_table_offset_to_memory_location; use crate::circuits::utils::image_table::ImageTableAssigner; use crate::circuits::utils::image_table::ImageTableLayouter; use crate::circuits::utils::image_table::GLOBAL_CAPABILITY; @@ -28,13 +30,10 @@ use crate::curr; use crate::fixed_curr; use crate::next; -use super::PostImageTableChipTrait; -use super::PostImageTableConfigTrait; - pub const POST_IMAGE_TABLE: &str = "post_img_col"; #[derive(Clone)] -pub(in crate::circuits) struct ContinuationPostImageTableConfig { +pub(in crate::circuits) struct PostImageTableConfig { memory_addr_sel: Column, post_image_table: Column, update: Column, @@ -42,13 +41,14 @@ pub(in crate::circuits) struct ContinuationPostImageTableConfig { _mark: PhantomData, } -impl PostImageTableConfigTrait for ContinuationPostImageTableConfig { - fn configure( +impl PostImageTableConfig { + pub(in crate::circuits) fn configure( meta: &mut ConstraintSystem, - memory_addr_sel: Column, + memory_addr_sel: Option>, memory_table: &MemoryTableConfig, pre_image_table: &ImageTableConfig, ) -> Self { + let memory_addr_sel = memory_addr_sel.unwrap(); let update = meta.advice_column(); let rest_memory_finalized_count = meta.advice_column(); let post_image_table = meta.named_advice_column(POST_IMAGE_TABLE.to_owned()); @@ -95,26 +95,24 @@ impl PostImageTableConfigTrait for ContinuationPostImageTableCon } } -pub(in crate::circuits) struct ContinuationPostImageTableChip { - config: ContinuationPostImageTableConfig, +pub(in crate::circuits) struct PostImageTableChip { + config: PostImageTableConfig, } -impl PostImageTableChipTrait> - for ContinuationPostImageTableChip -{ - fn new(config: ContinuationPostImageTableConfig) -> Self { +impl PostImageTableChip { + pub(in crate::circuits) fn new(config: PostImageTableConfig) -> Self { Self { config } } - fn assign( + pub(in crate::circuits) fn assign( self, layouter: &mut impl Layouter, image_table_assigner: &mut ImageTableAssigner, - pre_image_table: ImageTableLayouter, post_image_table: ImageTableLayouter, permutation_cells: ImageTableLayouter>, rest_memory_writing_ops_cell: Option>, rest_memory_writing_ops: F, + memory_finalized_set: HashSet<(LocationType, u32)>, ) -> Result<(), Error> { layouter.assign_region( || "post image table", @@ -319,11 +317,10 @@ impl PostImageTableChipTrait let mut rest_memory_writing_ops = rest_memory_writing_ops; - pre_image_table + post_image_table .init_memory_entries .iter() - .zip(post_image_table.init_memory_entries.iter()) - .map(|(pre, post)| { + .map(|post| { let entry = ctx.borrow_mut().region.assign_advice( || "post image table: init memory", self.config.post_image_table, @@ -338,7 +335,9 @@ impl PostImageTableChipTrait || Ok(rest_memory_writing_ops), )?; - if pre != post { + let position = image_table_offset_to_memory_location(offset); + + if memory_finalized_set.contains(&position) { ctx.borrow_mut().region.assign_advice( || "post image table: init memory", self.config.update, diff --git a/crates/zkwasm/src/circuits/post_image_table/mod.rs b/crates/zkwasm/src/circuits/post_image_table/mod.rs deleted file mode 100644 index e444451c9..000000000 --- a/crates/zkwasm/src/circuits/post_image_table/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -use halo2_proofs::arithmetic::FieldExt; -use halo2_proofs::circuit::AssignedCell; -use halo2_proofs::circuit::Layouter; -use halo2_proofs::plonk::Column; -use halo2_proofs::plonk::ConstraintSystem; -use halo2_proofs::plonk::Error; -use halo2_proofs::plonk::Fixed; - -use super::image_table::ImageTableConfig; -use super::mtable::MemoryTableConfig; -use super::utils::image_table::ImageTableAssigner; -use super::utils::image_table::ImageTableLayouter; - -pub(self) mod continuation; -pub(self) mod trivial; - -pub(in crate::circuits) trait PostImageTableConfigTrait { - fn configure( - _meta: &mut ConstraintSystem, - _memory_addr_sel: Column, - _memory_table: &MemoryTableConfig, - _pre_image_table: &ImageTableConfig, - ) -> Self; -} - -pub(in crate::circuits) trait PostImageTableChipTrait< - F: FieldExt, - Config: PostImageTableConfigTrait, -> -{ - fn new(config: Config) -> Self; - fn assign( - self, - layouter: &mut impl Layouter, - image_table_assigner: &mut ImageTableAssigner, - pre_image_table: ImageTableLayouter, - post_image_table: ImageTableLayouter, - permutation_cells: ImageTableLayouter>, - rest_memory_writing_ops_cell: Option>, - rest_memory_writing_ops: F, - ) -> Result<(), Error>; -} - -cfg_if::cfg_if! { - if #[cfg(feature = "continuation")] { - use self::continuation::*; - - pub(in crate::circuits) type PostImageTableConfig = ContinuationPostImageTableConfig; - pub(in crate::circuits) type PostImageTableChip = ContinuationPostImageTableChip; - - } else { - use self::trivial::*; - - pub(in crate::circuits) type PostImageTableConfig = TrivialPostImageTableConfig; - pub(in crate::circuits) type PostImageTableChip = TrivialPostImageTableChip; - } -} diff --git a/crates/zkwasm/src/circuits/post_image_table/trivial.rs b/crates/zkwasm/src/circuits/post_image_table/trivial.rs index d73416d45..7be976129 100644 --- a/crates/zkwasm/src/circuits/post_image_table/trivial.rs +++ b/crates/zkwasm/src/circuits/post_image_table/trivial.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::marker::PhantomData; use halo2_proofs::arithmetic::FieldExt; @@ -7,24 +8,22 @@ use halo2_proofs::plonk::Column; use halo2_proofs::plonk::ConstraintSystem; use halo2_proofs::plonk::Error; use halo2_proofs::plonk::Fixed; +use specs::mtable::LocationType; use crate::circuits::image_table::ImageTableConfig; use crate::circuits::mtable::MemoryTableConfig; use crate::circuits::utils::image_table::ImageTableAssigner; use crate::circuits::utils::image_table::ImageTableLayouter; -use super::PostImageTableChipTrait; -use super::PostImageTableConfigTrait; - #[derive(Clone)] -pub(in crate::circuits) struct TrivialPostImageTableConfig { +pub(in crate::circuits) struct PostImageTableConfig { _mark: PhantomData, } -impl PostImageTableConfigTrait for TrivialPostImageTableConfig { - fn configure( +impl PostImageTableConfig { + pub(in crate::circuits) fn configure( _meta: &mut ConstraintSystem, - _memory_addr_sel: Column, + _memory_addr_sel: Option>, _memory_table: &MemoryTableConfig, _pre_image_table: &ImageTableConfig, ) -> Self { @@ -32,26 +31,24 @@ impl PostImageTableConfigTrait for TrivialPostImageTableConfig { +pub(in crate::circuits) struct PostImageTableChip { _mark: PhantomData, } -impl PostImageTableChipTrait> - for TrivialPostImageTableChip -{ - fn new(_config: TrivialPostImageTableConfig) -> Self { +impl PostImageTableChip { + pub(in crate::circuits) fn new(_config: PostImageTableConfig) -> Self { Self { _mark: PhantomData } } - fn assign( + pub(in crate::circuits) fn assign( self, _layouter: &mut impl Layouter, _image_table_assigner: &mut ImageTableAssigner, - _pre_image_table: ImageTableLayouter, _post_image_table: ImageTableLayouter, _permutation_cells: ImageTableLayouter>, _rest_memory_writing_ops_cell: Option>, _rest_memory_writing_ops: F, + _memory_finalized_set: HashSet<(LocationType, u32)>, ) -> Result<(), Error> { Ok(()) } diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index 41e4565ce..9027bcf50 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -28,8 +28,6 @@ use crate::circuits::jtable::JumpTableConfig; use crate::circuits::mtable::MemoryTableChip; use crate::circuits::mtable::MemoryTableConfig; use crate::circuits::post_image_table::PostImageTableChip; -use crate::circuits::post_image_table::PostImageTableChipTrait; -use crate::circuits::post_image_table::PostImageTableConfigTrait; use crate::circuits::rtable::RangeTableChip; use crate::circuits::rtable::RangeTableConfig; use crate::circuits::utils::image_table::EncodeCompilationTableValues; @@ -108,7 +106,12 @@ impl Circuit for TestCircuit { meta.enable_constant(constants); meta.enable_equality(constants); } - let memory_addr_sel = meta.fixed_column(); + + let memory_addr_sel = if cfg!(feature = "continuation") { + Some(meta.fixed_column()) + } else { + None + }; let foreign_table_from_zero_index = meta.fixed_column(); @@ -235,7 +238,7 @@ impl Circuit for TestCircuit { let ( etable_permutation_cells, - (rest_memory_writing_ops_cell, rest_memory_writing_ops), + (rest_memory_writing_ops_cell, rest_memory_writing_ops, memory_finalized_set), static_frame_entries, ) = layouter.assign_region( || "jtable mtable etable", @@ -346,9 +349,6 @@ impl Circuit for TestCircuit { post_image_chip.assign( &mut layouter, &mut image_table_assigner, - self.tables - .compilation_tables - .encode_compilation_table_values(config.circuit_maximal_pages), self.tables .post_image_table .encode_compilation_table_values(config.circuit_maximal_pages), @@ -361,7 +361,8 @@ impl Circuit for TestCircuit { init_memory_entries: pre_image_table_cells.init_memory_entries, }, rest_memory_writing_ops_cell, - rest_memory_writing_ops + rest_memory_writing_ops, + memory_finalized_set )? ); diff --git a/crates/zkwasm/src/circuits/utils/image_table.rs b/crates/zkwasm/src/circuits/utils/image_table.rs index 5f437cdda..c6f63a97c 100644 --- a/crates/zkwasm/src/circuits/utils/image_table.rs +++ b/crates/zkwasm/src/circuits/utils/image_table.rs @@ -38,6 +38,24 @@ impl InitMemoryLayouter { } } +pub fn image_table_offset_to_memory_location(offset: usize) -> (LocationType, u32) { + // Minus one for default lookup entry. + let mut offset = offset - INIT_MEMORY_ENTRIES_OFFSET - 1; + + if offset < STACK_CAPABILITY { + return (LocationType::Stack, offset as u32); + } + + offset -= STACK_CAPABILITY; + + if offset < GLOBAL_CAPABILITY { + return (LocationType::Global, offset as u32); + } + + offset -= GLOBAL_CAPABILITY; + return (LocationType::Heap, offset as u32); +} + /* * -------------------- * Initialization State @@ -49,7 +67,7 @@ impl InitMemoryLayouter { * Br Table * -------------------- * Padding - * -------------------- Init Memory Offset(Constant) + * -------------------- Init Memory Offset(Constant INIT_MEMORY_ENTRIES_OFFSET) * Stack * -------------------- * Global @@ -57,8 +75,7 @@ impl InitMemoryLayouter { * Heap * -------------------- */ -#[allow(dead_code)] -pub(crate) struct ImageTableLayouter { +pub struct ImageTableLayouter { pub(crate) initialization_state: InitializationState, pub(crate) static_frame_entries: [(T, T); STATIC_FRAME_ENTRY_NUMBER], pub(crate) instructions: Vec, @@ -68,8 +85,8 @@ pub(crate) struct ImageTableLayouter { pub(crate) init_memory_entries: Vec, } -pub(crate) struct ImageTableAssigner { - pub(crate) heap_capability: u32, +pub struct ImageTableAssigner { + pub heap_capability: u32, initialization_state_offset: usize, static_frame_entries_offset: usize, @@ -80,6 +97,8 @@ pub(crate) struct ImageTableAssigner { } impl ImageTableAssigner { + /// `instruction_number` and `br_table_number` came from wasm image. Instructions, br table entries and paddings + /// are compacted within a fixed range. `page_capability` is computed based on K. pub fn new(instruction_number: usize, br_table_number: usize, pages_capability: u32) -> Self { let initialization_state_offset = 0; let static_frame_entries_offset = From aaedeceeb39643961d843cd74802b258ce3593ea Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Thu, 21 Dec 2023 07:40:41 +0000 Subject: [PATCH 27/34] refine etable assignment --- crates/zkwasm/src/circuits/etable/assign.rs | 317 +++++++------------- 1 file changed, 106 insertions(+), 211 deletions(-) diff --git a/crates/zkwasm/src/circuits/etable/assign.rs b/crates/zkwasm/src/circuits/etable/assign.rs index 08f7237a8..2a428bf2a 100644 --- a/crates/zkwasm/src/circuits/etable/assign.rs +++ b/crates/zkwasm/src/circuits/etable/assign.rs @@ -62,6 +62,84 @@ pub(in crate::circuits) struct EventTablePermutationCells { } impl EventTableChip { + fn assign_step_state( + &self, + ctx: &mut Context<'_, F>, + state: &InitializationState, + ) -> Result>, Error> { + cfg_if::cfg_if! { + if #[cfg(feature="continuation")] { + macro_rules! assign_u32_state { + ($cell:ident, $value:expr) => { + self.config.common_config.$cell.assign(ctx, $value)? + }; + } + } else { + macro_rules! assign_u32_state { + ($cell:ident, $value:expr) => { + self.config.common_config.$cell.assign_u32(ctx, $value)? + }; + } + } + } + + macro_rules! assign_common_range_advice { + ($cell:ident, $value:expr) => { + self.config + .common_config + .$cell + .assign(ctx, F::from($value as u64))? + }; + } + + let eid = assign_u32_state!(eid_cell, state.eid); + let fid = assign_common_range_advice!(fid_cell, state.fid); + let iid = assign_common_range_advice!(iid_cell, state.iid); + let sp = assign_common_range_advice!(sp_cell, state.sp); + let frame_id = assign_u32_state!(frame_id_cell, state.frame_id); + + let host_public_inputs = + assign_common_range_advice!(input_index_cell, state.host_public_inputs); + let context_in_index = + assign_common_range_advice!(context_input_index_cell, state.context_in_index); + let context_out_index = + assign_common_range_advice!(context_output_index_cell, state.context_out_index); + let external_host_call_call_index = assign_common_range_advice!( + external_host_call_index_cell, + state.external_host_call_call_index + ); + + let initial_memory_pages = + assign_common_range_advice!(mpages_cell, state.initial_memory_pages); + let maximal_memory_pages = + assign_common_range_advice!(maximal_memory_pages_cell, state.maximal_memory_pages); + + #[cfg(feature = "continuation")] + let jops = assign_common_range_advice!(jops_cell, state.jops); + + // The context will be stepped by EVENT_TABLE_ENTRY_ROWS. + ctx.step(EVENT_TABLE_ENTRY_ROWS as usize); + + Ok(InitializationState { + eid, + fid, + iid, + frame_id, + sp, + + host_public_inputs, + context_in_index, + context_out_index, + external_host_call_call_index, + + initial_memory_pages, + maximal_memory_pages, + + #[cfg(feature = "continuation")] + jops, + }) + } + fn compute_rest_mops_and_jops( &self, op_configs: &BTreeMap>>>, @@ -138,184 +216,16 @@ impl EventTableChip { Ok((rest_mops_cell, rest_jops_cell)) } - fn assign_initialization_state( - &self, - ctx: &mut Context<'_, F>, - initialization_state: &InitializationState, - ) -> Result>, Error> { - cfg_if::cfg_if! { - if #[cfg(feature = "continuation")] { - macro_rules! assign_u32_state { - ($cell:ident, $value:expr) => { - self.config.common_config.$cell.assign(ctx, $value)? - } - } - } else { - macro_rules! assign_u32_state { - ($cell:ident, $value:expr) => { - assign_common_range_advice!($cell, $value) - } - } - } - } - - macro_rules! assign_common_range_advice { - ($cell:ident, $value:expr) => { - self.config - .common_config - .$cell - .assign(ctx, F::from($value as u64))? - }; - } - - let eid = assign_u32_state!(eid_cell, initialization_state.eid); - let fid = assign_common_range_advice!(fid_cell, initialization_state.fid); - let iid = assign_common_range_advice!(iid_cell, initialization_state.iid); - let sp = assign_common_range_advice!(sp_cell, initialization_state.sp); - let frame_id = assign_u32_state!(frame_id_cell, initialization_state.frame_id); - - let host_public_inputs = - assign_common_range_advice!(input_index_cell, initialization_state.host_public_inputs); - let context_in_index = assign_common_range_advice!( - context_input_index_cell, - initialization_state.context_in_index - ); - let context_out_index = assign_common_range_advice!( - context_output_index_cell, - initialization_state.context_out_index - ); - let external_host_call_call_index = assign_common_range_advice!( - external_host_call_index_cell, - initialization_state.external_host_call_call_index - ); - - let initial_memory_pages = - assign_common_range_advice!(mpages_cell, initialization_state.initial_memory_pages); - let maximal_memory_pages = assign_common_range_advice!( - maximal_memory_pages_cell, - initialization_state.maximal_memory_pages - ); - - #[cfg(feature = "continuation")] - let jops = assign_common_range_advice!(jops_cell, initialization_state.jops); - - Ok(InitializationState { - eid, - fid, - iid, - frame_id, - sp, - - host_public_inputs, - context_in_index, - context_out_index, - external_host_call_call_index, - - initial_memory_pages, - maximal_memory_pages, - - #[cfg(feature = "continuation")] - jops, - }) - } - fn assign_padding_and_post_initialization_state( &self, ctx: &mut Context<'_, F>, initialization_state: &InitializationState, ) -> Result>, Error> { - cfg_if::cfg_if! { - if #[cfg(feature="continuation")] { - macro_rules! assign_u32_state { - ($cell:ident, $value:expr) => { - self.config.common_config.$cell.assign(ctx, $value)? - }; - } - } else { - macro_rules! assign_u32_state { - ($cell:ident, $value:expr) => { - self.config.common_config.$cell.assign_u32(ctx, $value)? - }; - } - } - } - - macro_rules! assign_common_range_advice { - ($cell:ident, $value:expr) => { - self.config - .common_config - .$cell - .assign(ctx, F::from($value as u64))? - }; - } - - macro_rules! assign_one_step { - () => {{ - let eid = assign_u32_state!(eid_cell, initialization_state.eid); - let fid = assign_common_range_advice!(fid_cell, initialization_state.fid); - let iid = assign_common_range_advice!(iid_cell, initialization_state.iid); - let sp = assign_common_range_advice!(sp_cell, initialization_state.sp); - let frame_id = assign_u32_state!(frame_id_cell, initialization_state.frame_id); - - let host_public_inputs = assign_common_range_advice!( - input_index_cell, - initialization_state.host_public_inputs - ); - let context_in_index = assign_common_range_advice!( - context_input_index_cell, - initialization_state.context_in_index - ); - let context_out_index = assign_common_range_advice!( - context_output_index_cell, - initialization_state.context_out_index - ); - let external_host_call_call_index = assign_common_range_advice!( - external_host_call_index_cell, - initialization_state.external_host_call_call_index - ); - - let initial_memory_pages = assign_common_range_advice!( - mpages_cell, - initialization_state.initial_memory_pages - ); - let maximal_memory_pages = assign_common_range_advice!( - maximal_memory_pages_cell, - initialization_state.maximal_memory_pages - ); - - #[cfg(feature = "continuation")] - let jops = assign_common_range_advice!(jops_cell, initialization_state.jops); - - InitializationState { - eid, - fid, - iid, - frame_id, - sp, - - host_public_inputs, - context_in_index, - context_out_index, - external_host_call_call_index, - - initial_memory_pages, - maximal_memory_pages, - - #[cfg(feature = "continuation")] - jops, - } - }}; - } - while ctx.offset < self.capability * EVENT_TABLE_ENTRY_ROWS as usize { - assign_one_step!(); - - ctx.step(EVENT_TABLE_ENTRY_ROWS as usize); + self.assign_step_state(ctx, initialization_state)?; } - let post_initialization_state = assign_one_step!(); - - Ok(post_initialization_state) + self.assign_step_state(ctx, initialization_state) } fn assign_entries( @@ -341,22 +251,6 @@ impl EventTableChip { }; } - cfg_if::cfg_if!( - if #[cfg(feature = "continuation")] { - macro_rules! assign_u32_state { - ($cell:ident, $value:expr) => { - self.config.common_config.$cell.assign(ctx, $value)? - }; - } - } else { - macro_rules! assign_u32_state { - ($cell:ident, $value:expr) => { - assign_advice!($cell, F::from($value as u64)) - }; - } - } - ); - let mut host_public_inputs = initialization_state.host_public_inputs; let mut context_in_index = initialization_state.context_in_index; let mut context_out_index = initialization_state.context_out_index; @@ -427,34 +321,37 @@ impl EventTableChip { assign_advice!(enabled_cell, F::one()); assign_advice!(rest_mops_cell, F::from(rest_mops as u64)); - assign_advice!(jops_cell, F::from(jops as u64)); - assign_advice!(input_index_cell, F::from(host_public_inputs as u64)); - assign_advice!(context_input_index_cell, F::from(context_in_index as u64)); - assign_advice!(context_output_index_cell, F::from(context_out_index as u64)); - assign_advice!( - external_host_call_index_cell, - F::from(external_host_call_call_index as u64) - ); - assign_advice!(sp_cell, F::from(entry.eentry.sp as u64)); - assign_advice!( - mpages_cell, - F::from(entry.eentry.allocated_memory_pages as u64) - ); - assign_advice!( - maximal_memory_pages_cell, - F::from(configure_table.maximal_memory_pages as u64) - ); - assign_u32_state!(frame_id_cell, entry.eentry.last_jump_eid); - assign_u32_state!(eid_cell, entry.eentry.eid); - assign_advice!(fid_cell, F::from(entry.eentry.inst.fid as u64)); - assign_advice!(iid_cell, F::from(entry.eentry.inst.iid as u64)); assign_advice!(itable_lookup_cell, bn_to_field(&entry.eentry.inst.encode())); + assign_advice!(jops_cell, F::from(jops as u64)); let op_config = op_configs .get(&((&entry.eentry.inst.opcode).into())) .unwrap(); op_config.assign(ctx, &step_status, &entry)?; + // Be careful, the function will step context. + self.assign_step_state( + ctx, + &InitializationState { + eid: entry.eentry.eid, + fid: entry.eentry.inst.fid, + iid: entry.eentry.inst.iid, + sp: entry.eentry.sp, + frame_id: entry.eentry.last_jump_eid, + + host_public_inputs, + context_in_index, + context_out_index, + external_host_call_call_index, + + initial_memory_pages: entry.eentry.allocated_memory_pages, + maximal_memory_pages: configure_table.maximal_memory_pages, + + #[cfg(feature = "continuation")] + jops, + }, + )?; + if op_config.is_host_public_input(&entry.eentry) { host_public_inputs += 1; } @@ -474,8 +371,6 @@ impl EventTableChip { } else { jops -= op_config.jops() } - - ctx.step(EVENT_TABLE_ENTRY_ROWS as usize); } Ok(()) @@ -505,8 +400,8 @@ impl EventTableChip { initialization_state, ); - let pre_initialization_state_cells = - self.assign_initialization_state(ctx, &initialization_state)?; + let pre_initialization_state_cells = self.assign_step_state(ctx, &initialization_state)?; + ctx.reset(); self.assign_entries( ctx, From 31c8ed3462ec3eb309f3f9f935cdc626b930e46f Mon Sep 17 00:00:00 2001 From: Zhang Junyu Date: Fri, 22 Dec 2023 09:10:09 +0000 Subject: [PATCH 28/34] decrease the degree of post init memory lookup --- crates/specs/src/encode/init_memory_table.rs | 2 + .../zkwasm/src/circuits/mtable/allocator.rs | 2 +- crates/zkwasm/src/circuits/mtable/assign.rs | 64 ++++++++-- crates/zkwasm/src/circuits/mtable/mod.rs | 115 ++++++++++-------- .../circuits/post_image_table/continuation.rs | 34 +++++- .../zkwasm/src/circuits/test_circuit/mod.rs | 2 +- 6 files changed, 151 insertions(+), 68 deletions(-) diff --git a/crates/specs/src/encode/init_memory_table.rs b/crates/specs/src/encode/init_memory_table.rs index 56a8e8e25..8430e599e 100644 --- a/crates/specs/src/encode/init_memory_table.rs +++ b/crates/specs/src/encode/init_memory_table.rs @@ -5,6 +5,7 @@ use super::FromBn; use crate::imtable::InitMemoryTableEntry; pub(crate) const INIT_MEMORY_ENCODE_BOUNDARY: u32 = 224; +pub const MEMORY_ADDRESS_OFFSET: u32 = 97; pub fn encode_init_memory_table_address(location_type: T, offset: T) -> T { location_type * T::from_bn(&(1u64.to_biguint().unwrap() << 32)) + offset @@ -23,6 +24,7 @@ pub fn encode_init_memory_table_entry( const EID_OFFSET_SHIFT: u32 = VALUE_SHIFT + u64::BITS; const VALUE_SHIFT: u32 = 0; + assert_eq!(OFFSET_SHIFT, MEMORY_ADDRESS_OFFSET); assert!(LTYPE_SHIFT + 8 <= INIT_MEMORY_ENCODE_BOUNDARY); let encode = is_mutable * T::from_bn(&(1u64.to_biguint().unwrap() << IS_MUTABLE_SHIFT)) diff --git a/crates/zkwasm/src/circuits/mtable/allocator.rs b/crates/zkwasm/src/circuits/mtable/allocator.rs index d643f75e9..c03c47991 100644 --- a/crates/zkwasm/src/circuits/mtable/allocator.rs +++ b/crates/zkwasm/src/circuits/mtable/allocator.rs @@ -84,7 +84,7 @@ pub(super) enum MemoryTableCellType { const BIT_COLUMNS: usize = 3; const U16_COLUMNS: usize = U32_CELLS.next_multiple_of(2) / 2 + U64_CELLS; const COMMON_RANGE_COLUMNS: usize = if cfg!(feature = "continuation") { 1 } else { 2 }; -const UNLIMITED_COLUMNS: usize = if cfg!(feature = "continuation") { 3 } else { 2 }; +const UNLIMITED_COLUMNS: usize = if cfg!(feature = "continuation") { 4 } else { 2 }; const U32_CELLS: usize = if cfg!(feature = "continuation") { 5 } else { 2 }; const U64_CELLS: usize = 1; diff --git a/crates/zkwasm/src/circuits/mtable/assign.rs b/crates/zkwasm/src/circuits/mtable/assign.rs index c01d9e188..daa6a79a4 100644 --- a/crates/zkwasm/src/circuits/mtable/assign.rs +++ b/crates/zkwasm/src/circuits/mtable/assign.rs @@ -134,6 +134,7 @@ impl MemoryTableChip { let mut rest_mops = init_rest_mops; let mut iter = mtable.0.iter().peekable(); + let mut current_address_init_encode = None; while let Some(entry) = iter.next() { assign_bit!(enabled_cell); @@ -153,17 +154,26 @@ impl MemoryTableChip { assign_bit_if!(entry.entry.atype.is_init(), is_init_cell); - assign_advice!( - init_encode_cell, - bn_to_field(&encode_init_memory_table_entry( + if entry.entry.atype.is_init() { + current_address_init_encode = Some(bn_to_field(&encode_init_memory_table_entry( (entry.entry.ltype as u64).into(), entry.entry.offset.into(), (entry.entry.is_mutable as u64).into(), entry.entry.eid.into(), - entry.entry.value.into() - )) + entry.entry.value.into(), + ))); + } + assign_advice!( + init_encode_cell, + current_address_init_encode.unwrap_or(F::zero()) ); + if let Some(next_entry) = iter.peek() { + if !next_entry.entry.is_same_location(&entry.entry) { + current_address_init_encode = None; + } + } + assign_u32_state!(start_eid_cell, entry.entry.eid); assign_u32_state!(end_eid_cell, entry.end_eid); assign_u32_state!(eid_diff_cell, entry.end_eid - entry.entry.eid - 1); @@ -172,10 +182,22 @@ impl MemoryTableChip { assign_advice!(value, entry.entry.value); #[cfg(feature = "continuation")] - assign_advice!( - rest_memory_finalize_ops_cell, - F::from(_rest_memory_finalize_ops as u64) - ); + { + use specs::encode::init_memory_table::encode_init_memory_table_address; + + assign_advice!( + rest_memory_finalize_ops_cell, + F::from(_rest_memory_finalize_ops as u64) + ); + + assign_advice!( + address_encode_cell, + bn_to_field(&encode_init_memory_table_address( + (entry.entry.ltype as u64).into(), + entry.entry.offset.into() + )) + ); + } assign_advice!( encode_cell, @@ -202,6 +224,30 @@ impl MemoryTableChip { _rest_memory_finalize_ops -= 1; memory_finalized_table.insert((entry.entry.ltype, entry.entry.offset)); + + #[cfg(feature = "continuation")] + { + use num_bigint::BigUint; + use specs::encode::init_memory_table::encode_init_memory_table_address; + use specs::encode::init_memory_table::MEMORY_ADDRESS_OFFSET; + + assign_advice!( + post_init_encode_cell, + bn_to_field( + &((encode_init_memory_table_address::( + (entry.entry.ltype as u64).into(), + entry.entry.offset.into() + )) * MEMORY_ADDRESS_OFFSET + + (encode_init_memory_table_entry::( + (entry.entry.ltype as u64).into(), + entry.entry.offset.into(), + (entry.entry.is_mutable as u64).into(), + entry.entry.eid.into(), + entry.entry.value.into() + ))) + ) + ); + } } ctx.step(MEMORY_TABLE_ENTRY_ROWS as usize); diff --git a/crates/zkwasm/src/circuits/mtable/mod.rs b/crates/zkwasm/src/circuits/mtable/mod.rs index 21ef2783a..3509d1541 100644 --- a/crates/zkwasm/src/circuits/mtable/mod.rs +++ b/crates/zkwasm/src/circuits/mtable/mod.rs @@ -52,6 +52,10 @@ pub struct MemoryTableConfig { encode_cell: AllocatedUnlimitedCell, init_encode_cell: AllocatedUnlimitedCell, + #[cfg(feature = "continuation")] + address_encode_cell: AllocatedUnlimitedCell, + #[cfg(feature = "continuation")] + post_init_encode_cell: AllocatedUnlimitedCell, #[cfg(feature = "continuation")] rest_memory_finalize_ops_cell: AllocatedUnlimitedCell, @@ -95,6 +99,11 @@ impl MemoryTableConfig { let encode_cell = allocator.alloc_unlimited_cell(); let init_encode_cell = allocator.alloc_unlimited_cell(); + #[cfg(feature = "continuation")] + let post_init_encode_cell = allocator.alloc_unlimited_cell(); + #[cfg(feature = "continuation")] + let address_encode_cell = allocator.alloc_unlimited_cell(); + #[cfg(feature = "continuation")] let rest_memory_finalize_ops_cell = { let cell = allocator.alloc_unlimited_cell(); @@ -105,6 +114,13 @@ impl MemoryTableConfig { let value = allocator.alloc_u64_cell(); + macro_rules! location { + ($meta:expr) => { + is_stack_cell.curr_expr($meta) * constant_from!(LocationType::Stack as u64) + + is_global_cell.curr_expr($meta) * constant_from!(LocationType::Global as u64) + + is_heap_cell.curr_expr($meta) * constant_from!(LocationType::Heap) + }; + } meta.create_gate("mc1. enable seq", |meta| { vec![ (enabled_cell.curr_expr(meta) - constant_from!(1)) @@ -206,16 +222,14 @@ impl MemoryTableConfig { meta.create_gate("mc7b. init encode.", |meta| { vec![ - encode_init_memory_table_entry( - is_stack_cell.curr_expr(meta) * constant_from!(LocationType::Stack as u64) - + is_heap_cell.curr_expr(meta) * constant_from!(LocationType::Heap as u64) - + is_global_cell.curr_expr(meta) - * constant_from!(LocationType::Global as u64), + (encode_init_memory_table_entry( + location!(meta), offset_cell.curr_expr(meta), is_mutable.curr_expr(meta), start_eid_cell.curr_expr(meta), value.u64_cell.curr_expr(meta), - ) - init_encode_cell.curr_expr(meta), + ) - init_encode_cell.curr_expr(meta)) + * is_init_cell.curr_expr(meta), ] .into_iter() .map(|x| x * fixed_curr!(meta, entry_sel)) @@ -225,21 +239,12 @@ impl MemoryTableConfig { image_table.init_memory_lookup(meta, "mc7c. imtable init", |meta| { cfg_if::cfg_if! { if #[cfg(feature = "continuation")] { - use specs::encode::init_memory_table::encode_init_memory_table_address; - ( - encode_init_memory_table_address( - is_stack_cell.curr_expr(meta) * constant_from!(LocationType::Stack as u64) - + is_heap_cell.curr_expr(meta) - * constant_from!(LocationType::Heap as u64) - + is_global_cell.curr_expr(meta) - * constant_from!(LocationType::Global as u64), - offset_cell.curr_expr(meta), - ) * fixed_curr!(meta, entry_sel) * is_init_cell.curr_expr(meta), - init_encode_cell.curr_expr(meta) * fixed_curr!(meta, entry_sel) * is_init_cell.curr_expr(meta), + address_encode_cell.curr_expr(meta) * fixed_curr!(meta, entry_sel), + init_encode_cell.curr_expr(meta) * fixed_curr!(meta, entry_sel), ) } else { - init_encode_cell.curr_expr(meta) * fixed_curr!(meta, entry_sel) * is_init_cell.curr_expr(meta) + init_encode_cell.curr_expr(meta) * fixed_curr!(meta, entry_sel) } } }); @@ -302,10 +307,7 @@ impl MemoryTableConfig { (constant_from!(1) - enabled_cell.curr_expr(meta)) * encode_cell.curr_expr(meta), encode_memory_table_entry( offset_cell.curr_expr(meta), - is_stack_cell.curr_expr(meta) * constant_from!(LocationType::Stack as u64) - + is_global_cell.curr_expr(meta) - * constant_from!(LocationType::Global as u64) - + is_heap_cell.curr_expr(meta) * constant_from!(LocationType::Heap), + location!(meta), is_i32_cell.curr_expr(meta), ) - encode_cell.curr_expr(meta), ] @@ -316,19 +318,44 @@ impl MemoryTableConfig { #[cfg(feature = "continuation")] { - meta.create_gate("mc13. rest memory updating ops", |meta| { - let is_write = constant_from!(1) - is_init_cell.curr_expr(meta); + use specs::encode::init_memory_table::encode_init_memory_table_address; + use specs::encode::init_memory_table::MEMORY_ADDRESS_OFFSET; + + meta.create_gate("mc13. post init memory entry", |meta| { + let is_writing = constant_from!(1) - is_init_cell.curr_expr(meta); let next_entry_at_different_position = constant_from!(1) - is_next_same_offset_cell.curr_expr(meta); - let is_memory_finalized_position_bit = is_write * next_entry_at_different_position; + let is_memory_finalized_position_bit = + is_writing * next_entry_at_different_position; vec![ + // rest_memory_finalize_ops_cell decreases. // `* enabled_cell`: If disabled, rest_memory_finalize_ops_cell should keep the same, // The termination rest_memory_finalize_ops_cell is constant 0 at the last selected(sel=1) step. rest_memory_finalize_ops_cell.curr_expr(meta) - rest_memory_finalize_ops_cell.next_expr(meta) - - is_memory_finalized_position_bit * enabled_cell.curr_expr(meta), + - is_memory_finalized_position_bit.clone() * enabled_cell.curr_expr(meta), + // encode address_encode_cell. + (encode_init_memory_table_address( + location!(meta), + offset_cell.curr_expr(meta), + ) - address_encode_cell.curr_expr(meta)), + // post_init_encode_cell assigned iff at memory finalized position. + post_init_encode_cell.curr_expr(meta) + * (constant_from!(1) - is_memory_finalized_position_bit.clone()), + // encode post_init_encode_cell. + (post_init_encode_cell.curr_expr(meta) + - address_encode_cell.curr_expr(meta) + * constant_from!(MEMORY_ADDRESS_OFFSET) + - encode_init_memory_table_entry( + location!(meta), + offset_cell.curr_expr(meta), + is_mutable.curr_expr(meta), + start_eid_cell.curr_expr(meta), + value.u64_cell.curr_expr(meta), + )) + * is_memory_finalized_position_bit, ] .into_iter() .map(|x| x * fixed_curr!(meta, entry_sel)) @@ -360,6 +387,10 @@ impl MemoryTableConfig { init_encode_cell, encode_cell, + #[cfg(feature = "continuation")] + post_init_encode_cell, + #[cfg(feature = "continuation")] + address_encode_cell, #[cfg(feature = "continuation")] rest_memory_finalize_ops_cell, } @@ -404,35 +435,17 @@ impl MemoryTableConfig { &self, meta: &mut ConstraintSystem, name: &'static str, - expr: impl FnOnce(&mut VirtualCells<'_, F>) -> (Expression, Expression), + expr: impl FnOnce(&mut VirtualCells<'_, F>) -> Expression, ) { use specs::encode::image_table::ImageTableEncoder; - use specs::encode::init_memory_table::encode_init_memory_table_address; meta.lookup_any(name, |meta| { - let (address, encode) = expr(meta); - vec![ - ( - address, - // FIXME: Add a bit cell to indicate finalized state - encode_init_memory_table_address( - self.is_stack_cell.expr(meta) - + self.is_heap_cell.expr(meta) * constant_from!(LocationType::Heap) - + self.is_global_cell.expr(meta) * constant_from!(LocationType::Global), - self.offset_cell.expr(meta), - ) * fixed_curr!(meta, self.entry_sel) - * (constant_from!(1) - self.is_init_cell.expr(meta)) - * (constant_from!(1) - self.is_next_same_offset_cell.expr(meta)), - ), - ( - encode, - // FIXME: Add a bit cell to indicate finalized state - ImageTableEncoder::InitMemory.encode(self.init_encode_cell.expr(meta)) - * fixed_curr!(meta, self.entry_sel) - * (constant_from!(1) - self.is_init_cell.expr(meta)) - * (constant_from!(1) - self.is_next_same_offset_cell.expr(meta)), - ), - ] + let encode = expr(meta); + vec![( + encode, + ImageTableEncoder::InitMemory.encode(self.post_init_encode_cell.expr(meta)) + * fixed_curr!(meta, self.entry_sel), + )] }); } } diff --git a/crates/zkwasm/src/circuits/post_image_table/continuation.rs b/crates/zkwasm/src/circuits/post_image_table/continuation.rs index 2a0ff8df8..2ad4dc9b0 100644 --- a/crates/zkwasm/src/circuits/post_image_table/continuation.rs +++ b/crates/zkwasm/src/circuits/post_image_table/continuation.rs @@ -13,6 +13,7 @@ use halo2_proofs::plonk::Error; use halo2_proofs::plonk::Fixed; use num_bigint::BigUint; use specs::encode::init_memory_table::encode_init_memory_table_address; +use specs::encode::init_memory_table::MEMORY_ADDRESS_OFFSET; use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use specs::mtable::LocationType; @@ -38,6 +39,7 @@ pub(in crate::circuits) struct PostImageTableConfig { post_image_table: Column, update: Column, rest_memory_finalized_count: Column, + memory_finalized_lookup_encode: Column, _mark: PhantomData, } @@ -52,6 +54,7 @@ impl PostImageTableConfig { let update = meta.advice_column(); let rest_memory_finalized_count = meta.advice_column(); let post_image_table = meta.named_advice_column(POST_IMAGE_TABLE.to_owned()); + let memory_finalized_lookup_encode = meta.advice_column(); meta.enable_equality(rest_memory_finalized_count); meta.enable_equality(post_image_table); @@ -74,15 +77,20 @@ impl PostImageTableConfig { ] }); + meta.create_gate("post image table: memory_finalized_lookup_encode", |meta| { + vec![ + fixed_curr!(meta, memory_addr_sel) + * curr!(meta, update) + * (fixed_curr!(meta, memory_addr_sel) * constant_from!(MEMORY_ADDRESS_OFFSET) + + curr!(meta, post_image_table) + - curr!(meta, memory_finalized_lookup_encode)), + ] + }); + memory_table.configure_in_post_init_memory_table( meta, "post image table: lookup updating value", - |meta| { - ( - fixed_curr!(meta, memory_addr_sel) * curr!(meta, update), - curr!(meta, post_image_table) * curr!(meta, update), - ) - }, + |meta| curr!(meta, memory_finalized_lookup_encode), ); Self { @@ -90,6 +98,7 @@ impl PostImageTableConfig { post_image_table, update, rest_memory_finalized_count, + memory_finalized_lookup_encode, _mark: PhantomData, } } @@ -345,6 +354,19 @@ impl PostImageTableChip { || Ok(F::one()), )?; + let address: BigUint = + encode_init_memory_table_address::( + (position.0 as u64).into(), + position.1.into(), + ) * MEMORY_ADDRESS_OFFSET; + + ctx.borrow_mut().region.assign_advice( + || "post image table: init memory lookup", + self.config.memory_finalized_lookup_encode, + offset, + || Ok(bn_to_field::(&address) + *post), + )?; + rest_memory_writing_ops = rest_memory_writing_ops - F::one(); } diff --git a/crates/zkwasm/src/circuits/test_circuit/mod.rs b/crates/zkwasm/src/circuits/test_circuit/mod.rs index 9027bcf50..62f0aa05a 100644 --- a/crates/zkwasm/src/circuits/test_circuit/mod.rs +++ b/crates/zkwasm/src/circuits/test_circuit/mod.rs @@ -53,7 +53,7 @@ use super::image_table::ImageTableConfig; use super::post_image_table::PostImageTableConfig; pub const VAR_COLUMNS: usize = if cfg!(feature = "continuation") { - 59 + 60 } else { 51 }; From a22fffbb61e48d8b5b06e3545e7927129c314c2f Mon Sep 17 00:00:00 2001 From: Po Date: Fri, 8 Dec 2023 17:28:34 +0100 Subject: [PATCH 29/34] chore: cont opt - dump slice to file and load --- crates/specs/src/brtable.rs | 4 ++-- crates/specs/src/lib.rs | 1 + crates/zkwasm/src/loader/mod.rs | 1 + crates/zkwasm/src/test/test_rlp_slice.rs | 9 +-------- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/crates/specs/src/brtable.rs b/crates/specs/src/brtable.rs index 2c333402e..916b57c31 100644 --- a/crates/specs/src/brtable.rs +++ b/crates/specs/src/brtable.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Debug, Clone)] +#[derive(Serialize, Debug, Clone, Deserialize)] pub struct BrTableEntry { pub fid: u32, pub iid: u32, @@ -13,7 +13,7 @@ pub struct BrTableEntry { pub dst_pc: u32, } -#[derive(Default, Serialize, Debug, Clone)] +#[derive(Default, Serialize, Debug, Clone, Deserialize)] pub struct BrTable(Vec); impl BrTable { diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index ec67340d2..6b13500d9 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -181,6 +181,7 @@ impl Tables { pub fn load(dir: PathBuf, is_last_slice: bool, file_type: FileType) -> Tables { fn load_file(folder: &PathBuf, filename: &str) -> Vec { let mut folder = folder.clone(); + std::fs::create_dir_all(folder.as_path()).unwrap(); folder.push(filename); let mut file = File::open(folder.as_path()).unwrap(); let mut buf = vec![]; diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index aac16038a..ff371ceff 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -38,6 +38,7 @@ use crate::circuits::ZkWasmCircuitBuilder; use crate::continuation::loader::WitnessDumper; use crate::loader::err::Error; use crate::loader::err::PreCheckErr; +#[cfg(not(feature = "continuation"))] use crate::profile::Profiler; use crate::runtime::host::host_env::HostEnv; use crate::runtime::wasmi_interpreter::Execution; diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index 97c7f2e8e..6601fc94f 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -154,7 +154,7 @@ fn generate_wasm_result( let wasm = std::fs::read("wasm/rlp.wasm").unwrap(); - let loader = ZkWasmLoader::::new(18, wasm, vec![])?; + let loader = ZkWasmLoader::::new(18, wasm, vec![]).unwrap(); let execution_result = loader.run(ExecutionArg { public_inputs, @@ -164,13 +164,6 @@ fn generate_wasm_result( output_dir: Some(std::env::current_dir().unwrap()), dump_table, })?; - - let instances = execution_result - .public_inputs_and_outputs - .iter() - .map(|v| (*v).into()) - .collect(); - Ok((loader, instances, execution_result)) } fn test_slices() -> Result<()> { From fdafec02fab740ef128494d5ef8b3a755fbc624f Mon Sep 17 00:00:00 2001 From: Po Date: Wed, 13 Dec 2023 10:29:31 +0100 Subject: [PATCH 30/34] chore: cont-opt[2] enable dumping witness table periodically --- crates/cli/src/app_builder.rs | 2 +- crates/specs/src/lib.rs | 1 + crates/specs/src/state.rs | 2 +- crates/zkwasm/src/continuation/loader.rs | 7 ------- crates/zkwasm/src/continuation/slice.rs | 5 ++--- crates/zkwasm/src/loader/mod.rs | 4 ++-- crates/zkwasm/src/test/test_rlp_slice.rs | 3 ++- 7 files changed, 9 insertions(+), 15 deletions(-) diff --git a/crates/cli/src/app_builder.rs b/crates/cli/src/app_builder.rs index d0dd437e0..1b2424fcb 100644 --- a/crates/cli/src/app_builder.rs +++ b/crates/cli/src/app_builder.rs @@ -154,7 +154,7 @@ pub trait AppBuilder: CommandBuilder { } } - #[cfg(feature = "continuation")] + #[cfg(feature="continuation")] Some(("witness-dump", sub_matches)) => { let public_inputs: Vec = Self::parse_single_public_arg(&sub_matches); let private_inputs: Vec = Self::parse_single_private_arg(&sub_matches); diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 6b13500d9..b8b069584 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -47,6 +47,7 @@ pub mod state; pub mod step; pub mod types; + #[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct CompilationTable { pub itable: Arc, diff --git a/crates/specs/src/state.rs b/crates/specs/src/state.rs index b76122ea3..2787121fe 100644 --- a/crates/specs/src/state.rs +++ b/crates/specs/src/state.rs @@ -6,9 +6,9 @@ use crate::host_function::HostPlugin; use crate::imtable::memory_event_of_step; use crate::imtable::InitMemoryTable; use crate::imtable::InitMemoryTableEntry; -use crate::itable::Opcode; use crate::mtable::AccessType; use crate::step::StepInfo; +use crate::itable::Opcode; use crate::CompilationTable; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] diff --git a/crates/zkwasm/src/continuation/loader.rs b/crates/zkwasm/src/continuation/loader.rs index bc86e0cd1..023581c78 100644 --- a/crates/zkwasm/src/continuation/loader.rs +++ b/crates/zkwasm/src/continuation/loader.rs @@ -12,13 +12,6 @@ use super::slice::Slice; use super::slice::Slices; impl ZkWasmLoader { -<<<<<<< HEAD -======= - pub(crate) fn compute_slice_capability(&self) -> usize { - ((1 << self.k) - 200) / EVENT_TABLE_ENTRY_ROWS as usize - } - ->>>>>>> 1fd2066 (fix circuit_without_witness) pub fn slice(&self, execution_result: ExecutionResult) -> Slices { Slices::new( execution_result.tables.unwrap(), diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index add768960..3178a446d 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -72,9 +72,8 @@ impl Slices { } pub fn num_slices(&self) -> usize { - (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64) - .ceil() as usize - } + (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64).ceil() as usize + } } impl Iterator for Slices { diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index ff371ceff..64322712d 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -20,6 +20,7 @@ use specs::CompilationTable; use specs::ExecutionTable; use specs::Tables; use wasmi::tracer::SliceDumper; +use wasmi::ENTRY; use wasmi::tracer::Tracer; use wasmi::ImportsBuilder; use wasmi::NotStartedModuleRef; @@ -32,13 +33,13 @@ use crate::circuits::config::init_zkwasm_runtime; use crate::circuits::config::set_zkwasm_k; use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::circuits::image_table::compute_maximal_pages; +use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::circuits::image_table::IMAGE_COL_NAME; use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; use crate::continuation::loader::WitnessDumper; use crate::loader::err::Error; use crate::loader::err::PreCheckErr; -#[cfg(not(feature = "continuation"))] use crate::profile::Profiler; use crate::runtime::host::host_env::HostEnv; use crate::runtime::wasmi_interpreter::Execution; @@ -78,7 +79,6 @@ impl ZkWasmLoader { pub fn compute_slice_capability(&self) -> usize { ((1 << self.k) - 200) / EVENT_TABLE_ENTRY_ROWS as usize } - fn precheck(&self) -> Result<()> { fn check_zkmain_exists(module: &wasmi::Module) -> Result<()> { use parity_wasm::elements::Internal; diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index 6601fc94f..db04e249a 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -154,7 +154,7 @@ fn generate_wasm_result( let wasm = std::fs::read("wasm/rlp.wasm").unwrap(); - let loader = ZkWasmLoader::::new(18, wasm, vec![]).unwrap(); + let loader = ZkWasmLoader::::new(18, wasm, vec![])?; let execution_result = loader.run(ExecutionArg { public_inputs, @@ -173,6 +173,7 @@ fn test_slices() -> Result<()> { let mut index = 0; while let Some(slice) = slices.next() { + println!("slice {}", index); let circuit = slice.build_circuit(); From 5434ced99783e1ca3f0d8cdb4b6a11ddab0f6435 Mon Sep 17 00:00:00 2001 From: Po Date: Mon, 18 Dec 2023 07:10:28 +0100 Subject: [PATCH 31/34] chore: cont_opt[4] caching lookup for wasmi, flexbuffer & pooling for writing witness table --- crates/specs/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index b8b069584..118352836 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -209,4 +209,5 @@ impl Tables { is_last_slice, } } + } From 6addb1c3b739fca22cf5bed1b14f12bef88df3df Mon Sep 17 00:00:00 2001 From: Po Date: Mon, 18 Dec 2023 16:06:46 +0100 Subject: [PATCH 32/34] chore: cont-op -- replace callback with trait for dumping slice witness & fmt --- crates/cli/src/app_builder.rs | 2 +- crates/specs/src/lib.rs | 2 -- crates/zkwasm/src/continuation/slice.rs | 5 +++-- crates/zkwasm/src/loader/mod.rs | 9 +++++++++ crates/zkwasm/src/runtime/wasmi_interpreter.rs | 2 -- crates/zkwasm/src/test/test_rlp_slice.rs | 1 - 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/cli/src/app_builder.rs b/crates/cli/src/app_builder.rs index 1b2424fcb..d0dd437e0 100644 --- a/crates/cli/src/app_builder.rs +++ b/crates/cli/src/app_builder.rs @@ -154,7 +154,7 @@ pub trait AppBuilder: CommandBuilder { } } - #[cfg(feature="continuation")] + #[cfg(feature = "continuation")] Some(("witness-dump", sub_matches)) => { let public_inputs: Vec = Self::parse_single_public_arg(&sub_matches); let private_inputs: Vec = Self::parse_single_private_arg(&sub_matches); diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 118352836..6b13500d9 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -47,7 +47,6 @@ pub mod state; pub mod step; pub mod types; - #[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct CompilationTable { pub itable: Arc, @@ -209,5 +208,4 @@ impl Tables { is_last_slice, } } - } diff --git a/crates/zkwasm/src/continuation/slice.rs b/crates/zkwasm/src/continuation/slice.rs index 3178a446d..add768960 100644 --- a/crates/zkwasm/src/continuation/slice.rs +++ b/crates/zkwasm/src/continuation/slice.rs @@ -72,8 +72,9 @@ impl Slices { } pub fn num_slices(&self) -> usize { - (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64).ceil() as usize - } + (self.origin_table.execution_tables.etable.entries().len() as f64 / self.capability as f64) + .ceil() as usize + } } impl Iterator for Slices { diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index 64322712d..a7552d938 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -185,6 +185,15 @@ impl ZkWasmLoader { Ok(keygen_vk(¶ms, &circuit).unwrap()) } + pub fn checksum(&self, params: &Params) -> Result> { + let (env, _) = HostEnv::new_with_full_foreign_plugins( + vec![], + vec![].into(), + vec![], + Arc::new(Mutex::new(vec![])), + ); + let compiled = self.compile(&env, Box::new(WitnessDumper::default()))?; + pub fn checksum<'a, 'b>( &self, image: &'b CompilationTable, diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index d6b903e81..baedf4a5d 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -5,8 +5,6 @@ use std::sync::Arc; use anyhow::Result; use specs::host_function::HostFunctionDesc; -use specs::jtable::StaticFrameEntry; -use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use specs::state::InitializationState; use specs::state::UpdateCompilationTable; use specs::CompilationTable; diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index db04e249a..b2ed2cda0 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -173,7 +173,6 @@ fn test_slices() -> Result<()> { let mut index = 0; while let Some(slice) = slices.next() { - println!("slice {}", index); let circuit = slice.build_circuit(); From 6fbdd47752e2ec9f4927e3714f22cc8057fa8d5b Mon Sep 17 00:00:00 2001 From: Po Date: Tue, 9 Jan 2024 01:04:05 -0800 Subject: [PATCH 33/34] rebase cont_dev --- crates/specs/src/brtable.rs | 4 ++-- crates/zkwasm/src/loader/mod.rs | 13 +------------ crates/zkwasm/src/runtime/wasmi_interpreter.rs | 1 + crates/zkwasm/src/test/test_rlp_slice.rs | 9 ++++++++- third-party/wasmi | 2 +- 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/crates/specs/src/brtable.rs b/crates/specs/src/brtable.rs index 916b57c31..5128572e9 100644 --- a/crates/specs/src/brtable.rs +++ b/crates/specs/src/brtable.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use serde::Deserialize; use serde::Serialize; -#[derive(Serialize, Debug, Clone, Deserialize)] +#[derive(Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct BrTableEntry { pub fid: u32, pub iid: u32, @@ -13,7 +13,7 @@ pub struct BrTableEntry { pub dst_pc: u32, } -#[derive(Default, Serialize, Debug, Clone, Deserialize)] +#[derive(Default, Serialize, Debug, Clone, Deserialize, PartialEq)] pub struct BrTable(Vec); impl BrTable { diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index a7552d938..683c4b2af 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -25,7 +25,6 @@ use wasmi::tracer::Tracer; use wasmi::ImportsBuilder; use wasmi::NotStartedModuleRef; use wasmi::RuntimeValue; -use wasmi::ENTRY; use crate::checksum::CompilationTableWithParams; use crate::checksum::ImageCheckSum; @@ -33,7 +32,6 @@ use crate::circuits::config::init_zkwasm_runtime; use crate::circuits::config::set_zkwasm_k; use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::circuits::image_table::compute_maximal_pages; -use crate::circuits::etable::EVENT_TABLE_ENTRY_ROWS; use crate::circuits::image_table::IMAGE_COL_NAME; use crate::circuits::TestCircuit; use crate::circuits::ZkWasmCircuitBuilder; @@ -134,7 +132,7 @@ impl ZkWasmLoader { Arc::new(Mutex::new(vec![])), ); - self.compile(&env) + self.compile(&env, Box::new(WitnessDumper::default())) } fn circuit_without_witness(&self, last_slice_circuit: bool) -> Result> { @@ -185,15 +183,6 @@ impl ZkWasmLoader { Ok(keygen_vk(¶ms, &circuit).unwrap()) } - pub fn checksum(&self, params: &Params) -> Result> { - let (env, _) = HostEnv::new_with_full_foreign_plugins( - vec![], - vec![].into(), - vec![], - Arc::new(Mutex::new(vec![])), - ); - let compiled = self.compile(&env, Box::new(WitnessDumper::default()))?; - pub fn checksum<'a, 'b>( &self, image: &'b CompilationTable, diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index baedf4a5d..45ecc0213 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use anyhow::Result; use specs::host_function::HostFunctionDesc; +use specs::jtable::STATIC_FRAME_ENTRY_NUMBER; use specs::state::InitializationState; use specs::state::UpdateCompilationTable; use specs::CompilationTable; diff --git a/crates/zkwasm/src/test/test_rlp_slice.rs b/crates/zkwasm/src/test/test_rlp_slice.rs index b2ed2cda0..cff3df517 100644 --- a/crates/zkwasm/src/test/test_rlp_slice.rs +++ b/crates/zkwasm/src/test/test_rlp_slice.rs @@ -164,6 +164,13 @@ fn generate_wasm_result( output_dir: Some(std::env::current_dir().unwrap()), dump_table, })?; + + let instances = execution_result + .public_inputs_and_outputs + .iter() + .map(|v| (*v).into()) + .collect(); + Ok((loader, instances, execution_result)) } fn test_slices() -> Result<()> { @@ -178,7 +185,7 @@ fn test_slices() -> Result<()> { let circuit = slice.build_circuit(); loader.mock_test(&circuit, &instances)?; - loader.bench_test(circuit, &instances); + // loader.bench_test(circuit, &instances); index += 1; } diff --git a/third-party/wasmi b/third-party/wasmi index cc61c1b84..61fc9f38d 160000 --- a/third-party/wasmi +++ b/third-party/wasmi @@ -1 +1 @@ -Subproject commit cc61c1b840306d0e51962a65d9205a9f7b3f74c0 +Subproject commit 61fc9f38d71ba265e24ab7a8bae3fdd4e3398be2 From b0cacad49a7d819bc593bf38ba5bde8e70802f53 Mon Sep 17 00:00:00 2001 From: Po Date: Tue, 9 Jan 2024 20:04:31 -0800 Subject: [PATCH 34/34] feat: add proof-from-trace cli --- crates/cli/src/app_builder.rs | 25 +++++++ crates/cli/src/args.rs | 30 ++++++++- crates/cli/src/command.rs | 9 +++ crates/cli/src/exec.rs | 66 ++++++++++++++++++- crates/specs/src/lib.rs | 12 ++++ crates/zkwasm/src/continuation/loader.rs | 4 ++ crates/zkwasm/src/loader/mod.rs | 17 +++-- .../zkwasm/src/runtime/wasmi_interpreter.rs | 2 +- crates/zkwasm/src/test/mod.rs | 5 +- third-party/wasmi | 2 +- 10 files changed, 159 insertions(+), 13 deletions(-) diff --git a/crates/cli/src/app_builder.rs b/crates/cli/src/app_builder.rs index d0dd437e0..af065bbbb 100644 --- a/crates/cli/src/app_builder.rs +++ b/crates/cli/src/app_builder.rs @@ -17,6 +17,7 @@ use crate::exec::exec_witness_dump; use super::command::CommandBuilder; use super::exec::exec_aggregate_create_proof; use super::exec::exec_create_proof; +use super::exec::exec_create_proof_from_trace; use super::exec::exec_dry_run_service; use super::exec::exec_image_checksum; use super::exec::exec_setup; @@ -74,6 +75,7 @@ pub trait AppBuilder: CommandBuilder { let app = Self::append_setup_subcommand(app); let app = Self::append_dry_run_subcommand(app); let app = Self::append_create_single_proof_subcommand(app); + let app = Self::append_create_proof_from_trace_subcommand(app); let app = Self::append_verify_single_proof_subcommand(app); let app = Self::append_create_aggregate_proof_subcommand(app); let app = Self::append_verify_aggregate_verify_subcommand(app); @@ -168,6 +170,7 @@ pub trait AppBuilder: CommandBuilder { assert!(public_inputs.len() <= Self::MAX_PUBLIC_INPUT_SIZE); exec_witness_dump( + Self::NAME, zkwasm_k, wasm_binary, phantom_functions, @@ -210,6 +213,28 @@ pub trait AppBuilder: CommandBuilder { Ok(()) } + Some(("proof-from-trace", sub_matches)) => { + let tables_dir = Self::parse_tables_path_arg(&sub_matches); + let param_dir = Self::parse_params_path_arg(&sub_matches); + let context_out_path: Option = + Self::parse_context_out_path_arg(&sub_matches); + + let context_out = Arc::new(Mutex::new(vec![])); + + exec_create_proof_from_trace( + Self::NAME, + zkwasm_k, + wasm_binary, + phantom_functions, + &output_dir, + &tables_dir, + ¶m_dir, + )?; + + write_context_output(&context_out.lock().unwrap(), context_out_path)?; + + Ok(()) + } Some(("single-verify", sub_matches)) => { let proof_path: PathBuf = Self::parse_proof_path_arg(&sub_matches); let instance_path: PathBuf = Self::parse_single_instance_arg(&sub_matches); diff --git a/crates/cli/src/args.rs b/crates/cli/src/args.rs index 36fa7ec74..3f80cc697 100644 --- a/crates/cli/src/args.rs +++ b/crates/cli/src/args.rs @@ -57,7 +57,7 @@ pub fn parse_args(values: Vec<&str>) -> Vec { let mut data = [0u8; 8]; data[..x.len()].copy_from_slice(x); - u64::from_be_bytes(data) + u64::from_le_bytes(data) }) .collect() } @@ -144,6 +144,34 @@ pub trait ArgBuilder { .clone() } + fn tables_path_arg<'a>() -> Arg<'a> { + arg!( + -t --tables "Path of tables." + ) + .value_parser(value_parser!(PathBuf)) + } + + fn parse_tables_path_arg(matches: &ArgMatches) -> PathBuf { + matches + .get_one::("tables") + .expect("tables path is required.") + .clone() + } + + fn params_path_arg<'a>() -> Arg<'a> { + arg!( + -p --param "Path of params." + ) + .value_parser(value_parser!(PathBuf)) + } + + fn parse_params_path_arg(matches: &ArgMatches) -> PathBuf { + matches + .get_one::("param") + .expect("params path is required.") + .clone() + } + fn sol_dir_arg<'a>() -> Arg<'a> { arg!( -s --sol_dir [SOL_DIRECTORY] "Path of solidity directory." diff --git a/crates/cli/src/command.rs b/crates/cli/src/command.rs index 93bec132f..52ee77aaa 100644 --- a/crates/cli/src/command.rs +++ b/crates/cli/src/command.rs @@ -49,6 +49,15 @@ pub trait CommandBuilder: ArgBuilder { app.subcommand(command) } + fn append_create_proof_from_trace_subcommand(app: App) -> App { + let command = Command::new("proof-from-trace") + .arg(Self::tables_path_arg()) + .arg(Self::params_path_arg()) + .arg(Self::context_out_path_arg()); + + app.subcommand(command) + } + fn append_verify_single_proof_subcommand(app: App) -> App { let command = Command::new("single-verify") .arg(Self::proof_path_arg()) diff --git a/crates/cli/src/exec.rs b/crates/cli/src/exec.rs index 19febe4fe..6d6b7ee6c 100644 --- a/crates/cli/src/exec.rs +++ b/crates/cli/src/exec.rs @@ -1,5 +1,6 @@ use anyhow::Result; use delphinus_zkwasm::circuits::TestCircuit; +use delphinus_zkwasm::continuation::slice::Slice; use delphinus_zkwasm::loader::ExecutionArg; use delphinus_zkwasm::loader::ZkWasmLoader; use halo2_proofs::arithmetic::BaseExt; @@ -28,6 +29,7 @@ use notify::RecursiveMode; use notify::Watcher; use serde::Deserialize; use serde::Serialize; +use specs::Tables; use std::fs; use std::io::Write; use std::path::Path; @@ -255,6 +257,7 @@ pub fn exec_dry_run( } pub fn exec_witness_dump( + prefix: &'static str, zkwasm_k: u32, wasm_binary: Vec, phantom_functions: Vec, @@ -266,7 +269,7 @@ pub fn exec_witness_dump( ) -> Result<()> { let loader = ZkWasmLoader::::new(zkwasm_k, wasm_binary, phantom_functions)?; - loader.run(ExecutionArg { + let result = loader.run(ExecutionArg { public_inputs, private_inputs, context_inputs, @@ -275,6 +278,18 @@ pub fn exec_witness_dump( dump_table: true, })?; + let instances: Vec = result + .public_inputs_and_outputs + .clone() + .iter() + .map(|v| (*v).into()) + .collect(); + + store_instance( + &vec![instances.clone()], + &output_dir.join(format!("{}.{}.instance.data", prefix, 0)), + ); + Ok(()) } @@ -337,6 +352,49 @@ pub fn exec_create_proof( Ok(()) } +pub fn exec_create_proof_from_trace( + prefix: &'static str, + zkwasm_k: u32, + wasm_binary: Vec, + phantom_functions: Vec, + output_dir: &PathBuf, + tables_dir: &PathBuf, + param_dir: &PathBuf, +) -> Result<()> { + let loader = ZkWasmLoader::::new(zkwasm_k, wasm_binary, phantom_functions)?; + + let table = Tables::load(tables_dir.clone(), false, specs::FileType::FLEXBUFFERS); + let capacity = loader.compute_slice_capability(); + let slice = Slice::new(table, capacity); + let circuit = slice.build_circuit(); + + let instances = Tables::load_instances(output_dir); + + + let params = load_or_build_unsafe_params::( + zkwasm_k, + Some(¶m_dir.join(format!("K{}.params", zkwasm_k))), + ); + + let vkey = load_vkey::>( + ¶ms, + &output_dir.join(format!("{}.{}.vkey.data", prefix, 0)), + ); + + let proof = loader.create_proof(¶ms, vkey, circuit, &instances)?; + + { + let proof_path = tables_dir.join(format!("{}.{}.transcript.data", prefix, 0)); + println!("write transcript to {:?}", proof_path); + let mut fd = std::fs::File::create(&proof_path)?; + fd.write_all(&proof)?; + } + + info!("Proof has been created."); + + Ok(()) +} + pub fn exec_verify_proof( prefix: &'static str, zkwasm_k: u32, @@ -367,10 +425,12 @@ pub fn exec_verify_proof( &output_dir.join(format!("{}.{}.vkey.data", prefix, 0)), ); - let proof = load_proof(proof_path); + let proof = load_proof(&proof_path.join(format!("{}.{}.transcript.data", prefix, 0))); let image = if cfg!(feature = "continuation") { - todo!("read slice image from file?") + let table = Tables::load(proof_path.clone(), false, specs::FileType::FLEXBUFFERS); + table.compilation_tables + // todo!("read slice image from file?") } else { loader.compile_without_env()?.tables }; diff --git a/crates/specs/src/lib.rs b/crates/specs/src/lib.rs index 6b13500d9..1754eb00c 100644 --- a/crates/specs/src/lib.rs +++ b/crates/specs/src/lib.rs @@ -15,6 +15,7 @@ use brtable::ElemTable; use configure_table::ConfigureTable; use etable::EventTable; use etable::EventTableEntry; +use halo2_proofs::pairing::bn256::Fr; use imtable::InitMemoryTable; use itable::InstructionTable; use jtable::JumpTable; @@ -29,6 +30,7 @@ use rayon::prelude::ParallelIterator; use serde::Deserialize; use serde::Serialize; use state::InitializationState; +use halo2_proofs::arithmetic::BaseExt; #[macro_use] extern crate lazy_static; @@ -208,4 +210,14 @@ impl Tables { is_last_slice, } } + + pub fn load_instances(instance_path: &PathBuf) -> Vec { + let mut instances = vec![]; + let mut fd = std::fs::File::open(&instance_path).unwrap(); + while let Ok(f) = Fr::read(&mut fd) { + instances.push(f); + } + + instances + } } diff --git a/crates/zkwasm/src/continuation/loader.rs b/crates/zkwasm/src/continuation/loader.rs index 023581c78..70f8e93c8 100644 --- a/crates/zkwasm/src/continuation/loader.rs +++ b/crates/zkwasm/src/continuation/loader.rs @@ -42,6 +42,10 @@ impl WitnessDumper { thread_pool, } } + + pub(crate) fn join(&self) { + self.thread_pool.join(); + } } impl SliceDumper for WitnessDumper { diff --git a/crates/zkwasm/src/loader/mod.rs b/crates/zkwasm/src/loader/mod.rs index 683c4b2af..029205da5 100644 --- a/crates/zkwasm/src/loader/mod.rs +++ b/crates/zkwasm/src/loader/mod.rs @@ -1,5 +1,7 @@ +use std::cell::RefCell; use std::marker::PhantomData; use std::path::PathBuf; +use std::rc::Rc; use std::sync::Arc; use std::sync::Mutex; @@ -110,7 +112,7 @@ impl ZkWasmLoader { fn compile( &self, env: &HostEnv, - slice_dumper: Box, + slice_dumper: Rc>, ) -> Result, Tracer>> { let imports = ImportsBuilder::new().with_resolver("env", env); @@ -131,8 +133,7 @@ impl ZkWasmLoader { vec![], Arc::new(Mutex::new(vec![])), ); - - self.compile(&env, Box::new(WitnessDumper::default())) + self.compile(&env, Rc::new(RefCell::new(WitnessDumper::default()))) } fn circuit_without_witness(&self, last_slice_circuit: bool) -> Result> { @@ -205,7 +206,7 @@ impl ZkWasmLoader { arg.context_inputs, arg.context_outputs, ); - let compiled_module = self.compile(&env, Box::new(WitnessDumper::default()))?; + let compiled_module = self.compile(&env, Rc::new(RefCell::new(WitnessDumper::default())))?; compiled_module.dry_run(&mut env) } @@ -222,13 +223,17 @@ impl ZkWasmLoader { self.compute_slice_capability(), arg.output_dir.clone(), ); - let compiled_module = self.compile(&env, Box::new(slice_dumper))?; + let slice_dumper = Rc::new(RefCell::new(slice_dumper)); + let compiled_module = self.compile(&env, slice_dumper.clone())?; let result = compiled_module.run(&mut env, wasm_runtime_io)?; + slice_dumper.borrow_mut().join(); + if let Some(tables) = &result.tables { tables.profile_tables(); - tables.write(arg.output_dir, specs::FileType::FLEXBUFFERS); + tables.write(arg.output_dir.clone(), specs::FileType::FLEXBUFFERS); } + Ok(result) } diff --git a/crates/zkwasm/src/runtime/wasmi_interpreter.rs b/crates/zkwasm/src/runtime/wasmi_interpreter.rs index 45ecc0213..d74e1d6bf 100644 --- a/crates/zkwasm/src/runtime/wasmi_interpreter.rs +++ b/crates/zkwasm/src/runtime/wasmi_interpreter.rs @@ -129,7 +129,7 @@ impl WasmiRuntime { host_plugin_lookup: &HashMap, entry: &str, phantom_functions: &Vec, - slice_dumper: Box, + slice_dumper: Rc>, ) -> Result, wasmi::tracer::Tracer>> { let tracer = wasmi::tracer::Tracer::new(host_plugin_lookup.clone(), phantom_functions, slice_dumper); diff --git a/crates/zkwasm/src/test/mod.rs b/crates/zkwasm/src/test/mod.rs index 3997ae427..81d532d9c 100644 --- a/crates/zkwasm/src/test/mod.rs +++ b/crates/zkwasm/src/test/mod.rs @@ -1,3 +1,6 @@ +use std::cell::RefCell; +use std::rc::Rc; + use crate::circuits::config::zkwasm_k; use crate::circuits::TestCircuit; use crate::continuation::loader::WitnessDumper; @@ -76,7 +79,7 @@ fn compile_then_execute_wasm( &env.function_description_table(), function_name, &vec![], - Box::new(WitnessDumper::default()), + Rc::new(RefCell::new(WitnessDumper::default())), ) .unwrap(); diff --git a/third-party/wasmi b/third-party/wasmi index 61fc9f38d..978eb5df6 160000 --- a/third-party/wasmi +++ b/third-party/wasmi @@ -1 +1 @@ -Subproject commit 61fc9f38d71ba265e24ab7a8bae3fdd4e3398be2 +Subproject commit 978eb5df6b2fe8b7a060fbb8c9ca32e63ac716e1