diff --git a/Cargo.toml b/Cargo.toml index ab5bdd2ba7..637aca99dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ std = [ "parity-wasm/std", "validation/std", ] +continuation = ["specs/continuation"] # Enables OS supported virtual memory. # # Note diff --git a/src/lib.rs b/src/lib.rs index b4183e2025..e5ef408fdc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -279,7 +279,7 @@ pub use self::{ host::{Externals, NopExternals, RuntimeArgs}, imports::{ImportResolver, ImportsBuilder, ModuleImportResolver}, memory::{MemoryInstance, MemoryRef, LINEAR_MEMORY_PAGE_SIZE}, - module::{ExternVal, ModuleInstance, ModuleRef, NotStartedModuleRef}, + module::{ExternVal, ModuleInstance, ModuleRef, NotStartedModuleRef, ENTRY}, runner::{StackRecycler, DEFAULT_CALL_STACK_LIMIT, DEFAULT_VALUE_STACK_LIMIT}, table::{TableInstance, TableRef}, types::{GlobalDescriptor, MemoryDescriptor, Signature, TableDescriptor}, diff --git a/src/module.rs b/src/module.rs index f600611824..090ea2535d 100644 --- a/src/module.rs +++ b/src/module.rs @@ -30,7 +30,7 @@ use core::{ fmt, }; use parity_wasm::elements::{External, InitExpr, Instruction, Internal, ResizableLimits, Type}; -use specs::configure_table::ConfigureTable; +use specs::{configure_table::ConfigureTable, jtable::StaticFrameEntry}; use validation::{DEFAULT_MEMORY_INDEX, DEFAULT_TABLE_INDEX}; /// Reference to a [`ModuleInstance`]. @@ -56,6 +56,8 @@ impl ::core::ops::Deref for ModuleRef { } } +pub const ENTRY: &str = "zkmain"; + /// An external value is the runtime representation of an entity /// that can be imported or exported. #[derive(PartialEq)] @@ -607,7 +609,6 @@ impl ModuleInstance { tracer: Option>>, ) -> Result, Error> { let module = loaded_module.module(); - let mut extern_vals = Vec::new(); for import_entry in module.import_section().map(|s| s.entries()).unwrap_or(&[]) { let module_name = import_entry.module(); @@ -646,6 +647,58 @@ impl ModuleInstance { let module_ref = Self::with_externvals(loaded_module, extern_vals.iter(), tracer.clone()); + let instance = module_ref + .as_ref() + .expect("failed to instantiate wasm module"); + // set tracer's initial fid_of_entry + let tracer = tracer.expect("failed to initialize tracer"); + let fid_of_entry = { + let idx_of_entry = instance.lookup_function_by_name(tracer.clone(), ENTRY); + tracer + .clone() + .borrow_mut() + .static_jtable_entries + .push(StaticFrameEntry { + enable: true, + frame_id: 0, + next_frame_id: 0, + callee_fid: idx_of_entry, + fid: 0, + iid: 0, + }); + + tracer + .clone() + .borrow_mut() + .static_jtable_entries + .push(if instance.has_start() { + StaticFrameEntry { + enable: true, + frame_id: 0, + next_frame_id: 0, + callee_fid: 0, // the fid of start function is always 0 + fid: idx_of_entry, + iid: 0, + } + } else { + StaticFrameEntry { + enable: false, + frame_id: 0, + next_frame_id: 0, + callee_fid: 0, + fid: 0, + iid: 0, + } + }); + + if instance.has_start() { + 0 + } else { + idx_of_entry + } + }; + tracer.clone().borrow_mut().set_fid_of_entry(fid_of_entry); + module_ref } diff --git a/src/prepare/tests.rs b/src/prepare/tests.rs index 5ba1c3d5ea..7e3c5b0c50 100644 --- a/src/prepare/tests.rs +++ b/src/prepare/tests.rs @@ -282,7 +282,7 @@ fn if_without_else() { }), isa::Instruction::I32Const(2), isa::Instruction::Return(isa::DropKeep { - drop: 1, // 1 param + drop: 1, // 1 param keep: isa::Keep::Single(ValueType::I32), // 1 result }), isa::Instruction::I32Const(3), diff --git a/src/runner.rs b/src/runner.rs index ca11f1e654..297e08e4d6 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -2013,7 +2013,7 @@ impl Interpreter { }; macro_rules! trace_post { - () => {{ + ($is_return: ident) => {{ if let Some(tracer) = self.get_tracer_if_active() { let post_status = self.run_instruction_post(pre_status, function_context, &instruction); @@ -2039,28 +2039,39 @@ impl Interpreter { last_jump_eid, post_status, ); + + // invoke callback to dump continuation slice tables + if tracer.dump_enabled() { + // println!("capacity: {}, tracer eid: {}, {}", tracer.slice_capability(), tracer.eid(), tracer.get_prev_eid()); + let is_last_slice = self.call_stack.is_empty() && $is_return; + // println!("is last slice: {}", is_last_slice); + assert!(tracer.eid() > tracer.get_prev_eid(), "eid: {}, prev_edi: {}", tracer.eid(), tracer.get_prev_eid()); + if (tracer.eid() - tracer.get_prev_eid() > tracer.slice_capability()) || is_last_slice { + tracer.dump_witness(is_last_slice); + } + } } }}; } match self.run_instruction(function_context, &instruction)? { InstructionOutcome::RunNextInstruction => { - trace_post!(); + trace_post!(false); } InstructionOutcome::Branch(target) => { - trace_post!(); + trace_post!(false); iter = instructions.iterate_from(target.dst_pc); self.value_stack.drop_keep(target.drop_keep); } InstructionOutcome::ExecuteCall(func_ref) => { // We don't record updated pc, the value should be recorded in the next trace log. - trace_post!(); + trace_post!(false); function_context.position = iter.position(); return Ok(RunResult::NestedCall(func_ref)); } InstructionOutcome::Return(drop_keep) => { - trace_post!(); + trace_post!(true); if let Some(tracer) = self.tracer.clone() { if tracer @@ -2996,7 +3007,7 @@ impl Interpreter { } /// Function execution context. -struct FunctionContext { +pub(crate) struct FunctionContext { /// Is context initialized. pub is_initialized: bool, /// Internal function reference. @@ -3027,7 +3038,7 @@ impl FunctionContext { self.is_initialized } - pub fn initialize( + fn initialize( &mut self, _locals: &[Local], _value_stack: &mut ValueStack, diff --git a/src/tracer/etable.rs b/src/tracer/etable.rs index 58e94a8525..7b8894f947 100644 --- a/src/tracer/etable.rs +++ b/src/tracer/etable.rs @@ -131,12 +131,15 @@ pub(crate) trait ETable { allocated_memory_pages: u32, last_jump_eid: u32, step_info: StepInfo, - ) -> EventTableEntry; + ); } impl ETable for EventTable { fn get_latest_eid(&self) -> u32 { - self.entries().last().unwrap().eid + match self.entries().last() { + Some(e) => e.eid, + None => 0, + } } fn get_last_entry_mut(&mut self) -> Option<&mut EventTableEntry> { @@ -150,7 +153,7 @@ impl ETable for EventTable { allocated_memory_pages: u32, last_jump_eid: u32, step_info: StepInfo, - ) -> EventTableEntry { + ) { let sp = (DEFAULT_VALUE_STACK_LIMIT as u32) .checked_sub(sp) .unwrap() @@ -158,7 +161,7 @@ impl ETable for EventTable { .unwrap(); let eentry = EventTableEntry { - eid: (self.entries().len() + 1).try_into().unwrap(), + eid: (self.get_latest_eid() + 1).try_into().unwrap(), sp, allocated_memory_pages, last_jump_eid, @@ -166,8 +169,6 @@ impl ETable for EventTable { step_info, }; - self.entries_mut().push(eentry.clone()); - - eentry + self.entries_mut().push(eentry); } } diff --git a/src/tracer/imtable.rs b/src/tracer/imtable.rs index 7fd44c890c..65fd564538 100644 --- a/src/tracer/imtable.rs +++ b/src/tracer/imtable.rs @@ -3,7 +3,7 @@ use specs::{ mtable::{LocationType, VarType}, }; -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct IMTable(Vec); impl IMTable { diff --git a/src/tracer/mod.rs b/src/tracer/mod.rs index f8cded51d8..33c1eef561 100644 --- a/src/tracer/mod.rs +++ b/src/tracer/mod.rs @@ -1,17 +1,24 @@ -use std::collections::HashMap; +use core::{cell::RefCell, fmt::Debug}; +use std::{collections::HashMap, rc::Rc, sync::Arc}; use specs::{ - brtable::{ElemEntry, ElemTable}, + brtable::{ElemEntry, ElemTable, BrTable}, configure_table::ConfigureTable, etable::EventTable, host_function::HostFunctionDesc, + imtable::InitMemoryTable, itable::{InstructionTable, InstructionTableEntry}, - jtable::{JumpTable, StaticFrameEntry}, + jtable::{JumpTable, StaticFrameEntry, STATIC_FRAME_ENTRY_NUMBER}, mtable::VarType, + state::{InitializationState, UpdateCompilationTable}, types::FunctionType, + CompilationTable, + ExecutionTable, + Tables, }; use crate::{ + func::FuncInstanceInternal, runner::{from_value_internal_to_u64_with_typ, ValueInternal}, FuncRef, GlobalRef, @@ -19,6 +26,7 @@ use crate::{ Module, ModuleRef, Signature, + DEFAULT_VALUE_STACK_LIMIT, }; use self::{etable::ETable, imtable::IMTable, phantom::PhantomFunction}; @@ -34,10 +42,23 @@ pub struct FuncDesc { pub signature: Signature, } +pub trait SliceDumper { + fn dump(&mut self, tables: Tables); + fn get_capacity(&self) -> usize; + fn dump_enabled(&self) -> bool; +} + +impl Debug for dyn SliceDumper { + fn fmt(&self, _: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + Ok(()) + } +} + #[derive(Debug)] pub struct Tracer { pub itable: InstructionTable, pub imtable: IMTable, + pub br_table: BrTable, pub etable: EventTable, pub jtable: JumpTable, pub elem_table: ElemTable, @@ -54,6 +75,16 @@ pub struct Tracer { // Wasm Image Function Idx pub wasm_input_func_idx: Option, pub wasm_input_func_ref: Option, + // wasm perf opt by caching + itable_entries: HashMap, + function_map: HashMap, + host_function_map: HashMap, + // continuation + witness_dumper: Rc>, + fid_of_entry: u32, + prev_eid: u32, + cur_imtable: InitMemoryTable, + cur_state: InitializationState, } impl Tracer { @@ -61,10 +92,12 @@ impl Tracer { pub fn new( host_plugin_lookup: HashMap, phantom_functions: &Vec, + witness_dumper: Rc>, ) -> Self { Tracer { itable: InstructionTable::default(), imtable: IMTable::default(), + br_table: BrTable::default(), etable: EventTable::default(), last_jump_eid: vec![], jtable: JumpTable::default(), @@ -80,6 +113,15 @@ impl Tracer { phantom_functions_ref: vec![], wasm_input_func_ref: None, wasm_input_func_idx: None, + itable_entries: HashMap::new(), + function_map: HashMap::new(), + host_function_map: HashMap::new(), + witness_dumper, + // #[cfg(feature="continuation")] + fid_of_entry: 0, // change when initializing module + prev_eid: 0, + cur_imtable: InitMemoryTable::default(), + cur_state: InitializationState::default(), // change when setting fid_of_entry } } @@ -113,9 +155,117 @@ impl Tracer { } } +impl Tracer { + pub(crate) fn dump_witness(&mut self, is_last_slice: bool) { + // keep etable eid + self.prev_eid = self.eid() - 1; + let mut etable = std::mem::take(&mut self.etable); + let etable_entires = etable.entries_mut(); + // If it is not the last slice, push a step to keep eid correct. + if !is_last_slice { + let last_entry = etable_entires.last().unwrap().clone(); + self.etable = EventTable::new(vec![last_entry]) + } + + let static_jtable = Arc::new( + self + .static_jtable_entries + .clone() + .try_into() + .expect(&format!( + "The number of static frame entries should be {}", + STATIC_FRAME_ENTRY_NUMBER + )), + ); + + let br_table = Arc::new(self.br_table.clone()); + + let compilation_tables = CompilationTable { + itable: Arc::new(self.itable.clone()), + imtable: self.cur_imtable.clone(), + br_table: br_table.clone(), + elem_table: Arc::new(self.elem_table.clone()), + configure_table: Arc::new(self.configure_table), + static_jtable: Arc::clone(&static_jtable), + initialization_state: self.cur_state.clone(), + }; + + // update current state + self.cur_state = + compilation_tables.update_initialization_state(etable_entires, is_last_slice); + + // update current memory table + // If it is not the last slice, push a helper step to get the post initialization state. + if !is_last_slice { + etable_entires.pop(); + } + self.cur_imtable = compilation_tables.update_init_memory_table(etable_entires); + + let post_image_table = CompilationTable { + itable: Arc::new(self.itable.clone()), + imtable: self.cur_imtable.clone(), + br_table, + elem_table: Arc::new(self.elem_table.clone()), + configure_table: Arc::new(self.configure_table), + static_jtable: static_jtable, + initialization_state: self.cur_state.clone(), + }; + + let execution_tables = ExecutionTable { + etable, + jtable: Arc::new(self.jtable.clone()), + }; + + self.witness_dumper.borrow_mut().dump(Tables { + compilation_tables, + execution_tables, + post_image_table, + is_last_slice, + }); + } + + pub(crate) fn get_prev_eid(&self) -> u32 { + self.prev_eid + } + + pub(crate) fn slice_capability(&self) -> u32 { + self.witness_dumper.borrow().get_capacity() as u32 + } + + pub fn dump_enabled(&self) -> bool { + self.witness_dumper.borrow().dump_enabled() + } + + pub(crate) fn set_fid_of_entry(&mut self, fid_of_entry: u32) { + self.fid_of_entry = fid_of_entry; + + self.cur_state = InitializationState { + eid: 1, + fid: fid_of_entry, + iid: 0, + frame_id: 0, + sp: DEFAULT_VALUE_STACK_LIMIT as u32 - 1, + + host_public_inputs: 1, + context_in_index: 1, + context_out_index: 1, + external_host_call_call_index: 1, + + initial_memory_pages: self.configure_table.init_memory_pages, + maximal_memory_pages: self.configure_table.maximal_memory_pages, + #[cfg(feature = "continuation")] + jops: 0, + }; + } + + pub fn get_fid_of_entry(&self) -> u32 { + self.fid_of_entry + } +} + impl Tracer { pub(crate) fn push_init_memory(&mut self, memref: MemoryRef) { - // one page contains 64KB*1024/8=8192 u64 entries + // one page contains 64KB/8 = 64*1024/8=8192 u64 entries const ENTRIES: u32 = 8192; let pages = (*memref).limits().initial(); @@ -130,6 +280,10 @@ impl Tracer { .push(false, true, i, VarType::I64, u64::from_le_bytes(buf)); } } + + // update current memory table + self.cur_imtable = self.imtable.finalized(); + self.br_table = self.itable.create_brtable(); } pub(crate) fn push_global(&mut self, globalidx: u32, globalref: &GlobalRef) { @@ -248,6 +402,22 @@ impl Tracer { self.function_lookup .push((func.clone(), func_index_in_itable)); + + match *func.as_internal() { + FuncInstanceInternal::Internal { + image_func_index, .. + } => { + self.function_map + .insert(image_func_index, func_index_in_itable); + } + FuncInstanceInternal::Host { + host_func_index, .. + } => { + self.host_function_map + .insert(host_func_index, func_index_in_itable); + } + } + self.function_index_translation.insert( func_index, FuncDesc { @@ -305,6 +475,10 @@ impl Tracer { pc, instruction.into(&self.function_index_translation), ); + self.itable_entries.insert( + ((funcdesc.index_within_jtable as u64) << 32) + pc as u64, + self.itable.entries().last().unwrap().clone(), + ); } else { break; } @@ -321,24 +495,23 @@ impl Tracer { } pub fn lookup_function(&self, function: &FuncRef) -> u32 { - let pos = self - .function_lookup - .iter() - .position(|m| m.0 == *function) - .unwrap(); - self.function_lookup.get(pos).unwrap().1 + match *function.as_internal() { + FuncInstanceInternal::Internal { + image_func_index, .. + } => *self.function_map.get(&image_func_index).unwrap(), + FuncInstanceInternal::Host { + host_func_index, .. + } => *self.host_function_map.get(&host_func_index).unwrap(), + } } pub fn lookup_ientry(&self, function: &FuncRef, pos: u32) -> InstructionTableEntry { let function_idx = self.lookup_function(function); - for ientry in self.itable.entries() { - if ientry.fid == function_idx && ientry.iid as u32 == pos { - return ientry.clone(); - } - } + let key = ((function_idx as u64) << 32) + pos as u64; + self.itable_entries.get(&key).unwrap().clone() - unreachable!() + // unreachable!() } pub fn lookup_first_inst(&self, function: &FuncRef) -> InstructionTableEntry { diff --git a/validation/src/lib.rs b/validation/src/lib.rs index 2b5cd8bd8c..038a4e8752 100644 --- a/validation/src/lib.rs +++ b/validation/src/lib.rs @@ -16,8 +16,9 @@ pub const DEFAULT_MEMORY_INDEX: u32 = 0; /// Index of default table. pub const DEFAULT_TABLE_INDEX: u32 = 0; -/// Maximal number of pages that a wasm instance supports. -pub const LINEAR_MEMORY_MAX_PAGES: u32 = 65536; +/// Maximal number of pages that a wasm instance supports, +/// if it's 65536, tracer would be out of memory +pub const LINEAR_MEMORY_MAX_PAGES: u32 = 1024; use alloc::{string::String, vec::Vec}; use core::fmt;