From 2ab6356a9b9af11483638fcd2c4bb8276123e19a Mon Sep 17 00:00:00 2001 From: Frank Emrich Date: Mon, 21 Apr 2025 22:24:00 +0100 Subject: [PATCH 01/34] cranelift: stack-switching support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This initial commit represents the "pr2" base commit with minimal merge conflicts resolved. Due to OOB conflicts, this commit is not functional as-is, but using it as a base in order to allow for easier reviewing of the delta from this commit to what will be used for the PR against upstream. Co-authored-by: Daniel Hillerström Co-authored-by: Paul Osborne --- cranelift/codegen/src/ir/function.rs | 7 + cranelift/codegen/src/isa/x64/abi.rs | 7 + crates/cranelift/src/func_environ.rs | 297 +- crates/cranelift/src/lib.rs | 7 +- .../src/stack_switching/control_effect.rs | 84 + .../src/stack_switching/fatpointer.rs | 71 + .../src/stack_switching/instructions.rs | 2543 +++++++++++++++++ crates/cranelift/src/stack_switching/mod.rs | 3 + .../src/translate/code_translator.rs | 168 +- 9 files changed, 3092 insertions(+), 95 deletions(-) create mode 100644 crates/cranelift/src/stack_switching/control_effect.rs create mode 100644 crates/cranelift/src/stack_switching/fatpointer.rs create mode 100644 crates/cranelift/src/stack_switching/instructions.rs create mode 100644 crates/cranelift/src/stack_switching/mod.rs diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 99ca7d39f70a..b078b8935303 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -255,6 +255,13 @@ impl FunctionStencil { self.get_dyn_scale(dyn_ty) } + /// Find the data for the given stack slot + pub fn get_stack_slot_data(&self, stack_slot: StackSlot) -> &StackSlotData { + self.sized_stack_slots + .get(stack_slot) + .expect("undeclared stack slot: {stack_slot}") + } + /// Get a concrete `Type` from a user defined `DynamicType`. pub fn get_concrete_dynamic_ty(&self, ty: DynamicType) -> Option { self.dfg diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index 41c24eb49d25..56b15489ee7e 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -101,6 +101,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { mut args: ArgsAccumulator, ) -> CodegenResult<(u32, Option)> { let is_fastcall = call_conv == CallConv::WindowsFastcall; + let is_tail = call_conv == CallConv::Tail; let mut next_gpr = 0; let mut next_vreg = 0; @@ -181,6 +182,11 @@ impl ABIMachineSpec for X64ABIMachineSpec { // This is consistent with LLVM's behavior, and is needed for // some uses of Cranelift (e.g., the rustc backend). // + // - Otherwise, if the calling convention is Tail, we behave as in + // the previous case, even if `enable_llvm_abi_extensions` is not + // set in the flags: This is a custom calling convention defined + // by Cranelift, LLVM doesn't know about it. + // // - Otherwise, both SysV and Fastcall specify behavior (use of // vector register, a register pair, or passing by reference // depending on the case), but for simplicity, we will just panic if @@ -194,6 +200,7 @@ impl ABIMachineSpec for X64ABIMachineSpec { if param.value_type.bits() > 64 && !(param.value_type.is_vector() || param.value_type.is_float()) && !flags.enable_llvm_abi_extensions() + && !is_tail { panic!( "i128 args/return values not supported unless LLVM ABI extensions are enabled" diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 8741b15a2afa..6b67fe3da2fe 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -5,7 +5,7 @@ use crate::translate::{ FuncTranslationState, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize, TargetEnvironment, }; -use crate::{BuiltinFunctionSignatures, TRAP_INTERNAL_ASSERT}; +use crate::{BuiltinFunctionSignatures, TRAP_INTERNAL_ASSERT, stack_switching}; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::{Imm64, Offset32, V128Imm}; @@ -24,9 +24,9 @@ use wasmparser::{Operator, WasmFeatures}; use wasmtime_environ::{ BuiltinFunctionIndex, DataIndex, ElemIndex, EngineOrModuleTypeIndex, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex, ModuleTranslation, - ModuleTypesBuilder, PtrSize, Table, TableIndex, TripleExt, Tunables, TypeConvert, TypeIndex, - VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, - WasmResult, WasmValType, + ModuleTypesBuilder, PtrSize, Table, TableIndex, TagIndex, TripleExt, Tunables, TypeConvert, + TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, + WasmRefType, WasmResult, WasmValType, }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; use wasmtime_math::f64_cvt_to_int_bounds; @@ -94,9 +94,9 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`. pub struct FuncEnvironment<'module_environment> { compiler: &'module_environment Compiler, - isa: &'module_environment (dyn TargetIsa + 'module_environment), - module: &'module_environment Module, - types: &'module_environment ModuleTypesBuilder, + pub(crate) isa: &'module_environment (dyn TargetIsa + 'module_environment), + pub(crate) module: &'module_environment Module, + pub(crate) types: &'module_environment ModuleTypesBuilder, wasm_func_ty: &'module_environment WasmFuncType, sig_ref_to_ty: SecondaryMap>, needs_gc_heap: bool, @@ -138,7 +138,7 @@ pub struct FuncEnvironment<'module_environment> { pcc_vmctx_memtype: Option, /// Caches of signatures for builtin functions. - builtin_functions: BuiltinFunctions, + pub(crate) builtin_functions: BuiltinFunctions, /// Offsets to struct fields accessed by JIT code. pub(crate) offsets: VMOffsets, @@ -174,6 +174,16 @@ pub struct FuncEnvironment<'module_environment> { /// always present even if this is a "leaf" function, as we have to call /// into the host to trap when signal handlers are disabled. pub(crate) stack_limit_at_function_entry: Option, + + /// Used by the stack switching feature. If set, we have a allocated a + /// slot on this function's stack to be used for the + /// current stack's `handler_list` field. + pub(crate) stack_switching_handler_list_buffer: Option, + + /// Used by the stack switching feature. If set, we have a allocated a + /// slot on this function's stack to be used for the + /// current continuation's `values` field. + pub(crate) stack_switching_values_buffer: Option, } impl<'module_environment> FuncEnvironment<'module_environment> { @@ -228,6 +238,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> { translation, stack_limit_at_function_entry: None, + + stack_switching_handler_list_buffer: None, + stack_switching_values_buffer: None, } } @@ -1509,8 +1522,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { return CheckIndirectCallTypeSignature::StaticTrap; } - WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support. - // Engine-indexed types don't show up until runtime and it's a Wasm // validation error to perform a call through a non-function table, // so these cases are dynamically not reachable. @@ -1525,6 +1536,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> { | WasmHeapType::ConcreteArray(_) | WasmHeapType::Struct | WasmHeapType::ConcreteStruct(_) + | WasmHeapType::Cont + | WasmHeapType::ConcreteCont(_) + | WasmHeapType::NoCont | WasmHeapType::None => { unreachable!() } @@ -1754,7 +1768,9 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm let needs_stack_map = match wasm_ty.top() { WasmHeapTopType::Extern | WasmHeapTopType::Any => true, WasmHeapTopType::Func => false, - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + // TODO(#10248) Once continuations can be stored on the GC heap, we + // will need stack maps for continuation objects. + WasmHeapTopType::Cont => false, }; (ty, needs_stack_map) } @@ -1818,22 +1834,42 @@ impl FuncEnvironment<'_> { let mut pos = builder.cursor(); let table = self.table(table_index); let ty = table.ref_type.heap_type; + let vmctx = self.vmctx_val(&mut pos); + let index_type = table.idx_type; + let delta = self.cast_index_to_i64(&mut builder.cursor(), delta, index_type); + let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); + let mut args = vec![vmctx, table_index_arg, delta]; let grow = if ty.is_vmgcref_type() { - gc::builtins::table_grow_gc_ref(self, &mut pos.func)? + args.push(init_value); + gc::builtins::table_grow_gc_ref(self, &mut builder.cursor().func)? } else { - debug_assert_eq!(ty.top(), WasmHeapTopType::Func); - self.builtin_functions.table_grow_func_ref(&mut pos.func) - }; + debug_assert!(matches!( + ty.top(), + WasmHeapTopType::Func | WasmHeapTopType::Cont + )); + match ty.top() { + WasmHeapTopType::Func => { + args.push(init_value); + self.builtin_functions + .table_grow_func_ref(&mut builder.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = stack_switching::fatpointer::deconstruct( + self, + &mut builder.cursor(), + init_value, + ); + args.extend_from_slice(&[contref, revision]); + self.builtin_functions + .table_grow_cont_obj(&mut builder.func) + } - let vmctx = self.vmctx_val(&mut pos); + _ => panic!("unsupported table type."), + } + }; - let index_type = table.idx_type; - let delta = self.cast_index_to_i64(&mut pos, delta, index_type); - let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64); - let call_inst = pos - .ins() - .call(grow, &[vmctx, table_index_arg, delta, init_value]); - let result = pos.func.dfg.first_result(call_inst); + let call_inst = builder.ins().call(grow, &args); + let result = builder.func.dfg.first_result(call_inst); Ok(self.convert_pointer_to_index_type(builder.cursor(), result, index_type, false)) } @@ -1866,7 +1902,16 @@ impl FuncEnvironment<'_> { } // Continuation types. - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + self.ensure_table_exists(builder.func, table_index); + let (table_entry_addr, flags) = table_data.prepare_table_addr(self, builder, index); + Ok(builder.ins().load( + stack_switching::fatpointer::POINTER_TYPE, + flags, + table_entry_addr, + 0, + )) + } } } @@ -1915,7 +1960,11 @@ impl FuncEnvironment<'_> { } // Continuation types. - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); + builder.ins().store(flags, value, elem_addr, 0); + Ok(()) + } } } @@ -1929,22 +1978,36 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let mut pos = builder.cursor(); let table = self.table(table_index); - let index_type = table.idx_type; - let dst = self.cast_index_to_i64(&mut pos, dst, index_type); - let len = self.cast_index_to_i64(&mut pos, len, index_type); let ty = table.ref_type.heap_type; + let vmctx = self.vmctx_val(&mut pos); + let index_type = table.idx_type; + let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); + let dst = self.cast_index_to_i64(&mut builder.cursor(), dst, index_type); + let len = self.cast_index_to_i64(&mut builder.cursor(), len, index_type); + let mut args = vec![vmctx, table_index_arg, dst]; let libcall = if ty.is_vmgcref_type() { - gc::builtins::table_fill_gc_ref(self, &mut pos.func)? + args.push(val); + gc::builtins::table_fill_gc_ref(self, &mut builder.cursor().func)? } else { - debug_assert_eq!(ty.top(), WasmHeapTopType::Func); - self.builtin_functions.table_fill_func_ref(&mut pos.func) + match ty.top() { + WasmHeapTopType::Func => { + args.push(val); + self.builtin_functions + .table_fill_func_ref(&mut builder.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut builder.cursor(), val); + args.extend_from_slice(&[contref, revision]); + self.builtin_functions + .table_fill_cont_obj(&mut builder.func) + } + _ => panic!("unsupported table type"), + } }; + args.push(len); - let vmctx = self.vmctx_val(&mut pos); - - let table_index_arg = pos.ins().iconst(I32, table_index.as_u32() as i64); - pos.ins() - .call(libcall, &[vmctx, table_index_arg, dst, val, len]); + builder.ins().call(libcall, &args); Ok(()) } @@ -2265,7 +2328,10 @@ impl FuncEnvironment<'_> { WasmHeapTopType::Func => pos.ins().iconst(self.pointer_type(), 0), // NB: null GC references don't need to be in stack maps. WasmHeapTopType::Any | WasmHeapTopType::Extern => pos.ins().iconst(types::I32, 0), - WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support. + WasmHeapTopType::Cont => { + let zero = pos.ins().iconst(self.pointer_type(), 0); + stack_switching::fatpointer::construct(self, &mut pos, zero, zero) + } }) } @@ -2274,9 +2340,19 @@ impl FuncEnvironment<'_> { mut pos: cranelift_codegen::cursor::FuncCursor, value: ir::Value, ) -> WasmResult { - let byte_is_null = - pos.ins() - .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0); + let byte_is_null = match pos.func.dfg.value_type(value) { + // continuation + ty if ty == stack_switching::fatpointer::POINTER_TYPE => { + let (_revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut pos, value); + pos.ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, contref, 0) + } + _ => pos + .ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0), + }; + Ok(pos.ins().uextend(ir::types::I32, byte_is_null)) } @@ -3209,6 +3285,133 @@ impl FuncEnvironment<'_> { self.isa.triple().architecture == target_lexicon::Architecture::X86_64 } + pub fn translate_cont_bind( + &mut self, + builder: &mut FunctionBuilder<'_>, + contobj: ir::Value, + args: &[ir::Value], + ) -> ir::Value { + stack_switching::instructions::translate_cont_bind(self, builder, contobj, args) + } + + pub fn translate_cont_new( + &mut self, + builder: &mut FunctionBuilder<'_>, + _state: &FuncTranslationState, + func: ir::Value, + arg_types: &[WasmValType], + return_types: &[WasmValType], + ) -> WasmResult { + stack_switching::instructions::translate_cont_new( + self, + builder, + func, + arg_types, + return_types, + ) + } + + pub fn translate_resume( + &mut self, + builder: &mut FunctionBuilder<'_>, + type_index: u32, + contobj: ir::Value, + resume_args: &[ir::Value], + resumetable: &[(u32, Option)], + ) -> WasmResult> { + stack_switching::instructions::translate_resume( + self, + builder, + type_index, + contobj, + resume_args, + resumetable, + ) + } + + #[allow(dead_code, reason = "TODO")] + pub fn translate_resume_throw( + &mut self, + _pos: FuncCursor, + _state: &FuncTranslationState, + _tag_index: u32, + _cont: ir::Value, + ) -> WasmResult { + // TODO(#10248) + Err(wasmtime_environ::WasmError::Unsupported( + "resume.throw instruction not implemented, yet".to_string(), + )) + } + + pub fn translate_suspend( + &mut self, + builder: &mut FunctionBuilder<'_>, + tag_index: u32, + suspend_args: &[ir::Value], + tag_return_types: &[ir::Type], + ) -> Vec { + stack_switching::instructions::translate_suspend( + self, + builder, + tag_index, + suspend_args, + tag_return_types, + ) + } + + /// Translates switch instructions. + pub fn translate_switch( + &mut self, + builder: &mut FunctionBuilder, + tag_index: u32, + contobj: ir::Value, + switch_args: &[ir::Value], + return_types: &[ir::Type], + ) -> WasmResult> { + stack_switching::instructions::translate_switch( + self, + builder, + tag_index, + contobj, + switch_args, + return_types, + ) + } + + pub fn continuation_arguments(&self, index: TypeIndex) -> &[WasmValType] { + let idx = self.module.types[index].unwrap_module_type_index(); + self.types[self.types[idx] + .unwrap_cont() + .clone() + .unwrap_interned_type_index()] + .unwrap_func() + .params() + } + + pub fn continuation_returns(&self, index: TypeIndex) -> &[WasmValType] { + let idx = self.module.types[index].unwrap_module_type_index(); + self.types[self.types[idx] + .unwrap_cont() + .clone() + .unwrap_interned_type_index()] + .unwrap_func() + .returns() + } + + pub fn tag_params(&self, tag_index: TagIndex) -> &[WasmValType] { + let idx = self.module.tags[tag_index].signature; + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .params() + } + + pub fn tag_returns(&self, tag_index: TagIndex) -> &[WasmValType] { + let idx = self.module.tags[tag_index].signature; + self.types[idx.unwrap_module_type_index()] + .unwrap_func() + .returns() + } + pub fn use_x86_blendv_for_relaxed_laneselect(&self, ty: Type) -> bool { self.isa.has_x86_blendv_lowering(ty) } @@ -3791,17 +3994,3 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type { IndexType::I64 => I64, } } - -/// TODO(10248) This is removed in the next stack switching PR. It stops the -/// compiler from complaining about the stack switching libcalls being dead -/// code. -#[cfg(feature = "stack-switching")] -#[allow( - dead_code, - reason = "Dummy function to supress more dead code warnings" -)] -pub fn use_stack_switching_libcalls() { - let _ = BuiltinFunctions::cont_new; - let _ = BuiltinFunctions::table_grow_cont_obj; - let _ = BuiltinFunctions::table_fill_cont_obj; -} diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 871c20bff916..af2ae8cfc759 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -42,6 +42,7 @@ mod builder; mod compiler; mod debug; mod func_environ; +mod stack_switching; mod translate; use self::compiler::Compiler; @@ -213,11 +214,7 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32, - WasmHeapTopType::Cont => - // TODO(10248) This is added in a follow-up PR - { - unimplemented!("codegen for stack switching types not implemented, yet") - } + WasmHeapTopType::Cont => stack_switching::fatpointer::POINTER_TYPE, } } diff --git a/crates/cranelift/src/stack_switching/control_effect.rs b/crates/cranelift/src/stack_switching/control_effect.rs new file mode 100644 index 000000000000..e2f77d021848 --- /dev/null +++ b/crates/cranelift/src/stack_switching/control_effect.rs @@ -0,0 +1,84 @@ +use cranelift_codegen::ir; +use cranelift_codegen::ir::InstBuilder; +use cranelift_codegen::ir::types::{I32, I64}; +use cranelift_frontend::FunctionBuilder; +use wasmtime_environ::stack_switching as stack_switching_environ; + +/// Universal control effect. This structure encodes return signal, +/// resume signal, suspension signal, and handler index into a +/// u64 value. This instance is used at compile time. There is a runtime +/// counterpart in `continuations/src/lib.rs`. +/// We convert to and from u64 as follows: The low 32 bits of the u64 are the +/// discriminant, the high 32 bits are the handler_index (if `Suspend`) +#[derive(Clone, Copy)] +pub struct ControlEffect(ir::Value); + +impl ControlEffect { + // Returns the discriminant + pub fn signal<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().ushr_imm(self.0, 32) + } + + pub fn from_u64(val: ir::Value) -> Self { + Self(val) + } + + pub fn to_u64(&self) -> ir::Value { + self.0 + } + + pub fn encode_resume<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(stack_switching_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + + Self(val) + } + + pub fn encode_switch<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(stack_switching_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + + Self(val) + } + + pub fn encode_suspend<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + handler_index: ir::Value, + ) -> Self { + let discriminant = builder.ins().iconst( + I64, + i64::from(stack_switching_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT), + ); + let val = builder.ins().ishl_imm(discriminant, 32); + let handler_index = builder.ins().uextend(I64, handler_index); + let val = builder.ins().bor(val, handler_index); + + Self(val) + } + + /// Returns the payload of the `Suspend` variant + pub fn handler_index<'a>( + self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().ireduce(I32, self.0) + } +} diff --git a/crates/cranelift/src/stack_switching/fatpointer.rs b/crates/cranelift/src/stack_switching/fatpointer.rs new file mode 100644 index 000000000000..fc4b4ef3de41 --- /dev/null +++ b/crates/cranelift/src/stack_switching/fatpointer.rs @@ -0,0 +1,71 @@ +use cranelift_codegen::ir; +use cranelift_codegen::ir::InstBuilder; +use cranelift_codegen::ir::types::I64; + +/// The Cranelfift type used to represent all of the following: +/// - wasm values of type `(ref null $ct)` and `(ref $ct)` +/// - equivalently: runtime values of type `Option` and `VMContObj` +/// Note that a `VMContObj` is a fat pointer +/// consisting of a pointer to `VMContRef` and a 64 bit sequence +/// counter. +/// We represent this here as a 128bit value, with the same representation as +/// `core::mem::transmute::`. +pub const POINTER_TYPE: ir::Type = ir::types::I128; + +/// Turns a (possibly null) reference to a continuation object into a tuple +/// (revision, contref_ptr). If `contobj` denotes a wasm null reference, the +/// contref_ptr part will be a null pointer. +pub(crate) fn deconstruct<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + pos: &mut cranelift_codegen::cursor::FuncCursor, + contobj: ir::Value, +) -> (ir::Value, ir::Value) { + debug_assert_eq!(pos.func.dfg.value_type(contobj), POINTER_TYPE); + + let (lsbs, msbs) = pos.ins().isplit(contobj); + + let (revision_counter, contref) = match env.isa.endianness() { + ir::Endianness::Little => (lsbs, msbs), + ir::Endianness::Big => { + let pad_bits = 64 - env.pointer_type().bits(); + let contref = pos.ins().ushr_imm(lsbs, pad_bits as i64); + (msbs, contref) + } + }; + let contref = if env.pointer_type().bits() < I64.bits() { + pos.ins().ireduce(env.pointer_type(), contref) + } else { + contref + }; + (revision_counter, contref) +} + +/// Constructs a continuation object from a given contref and revision pointer. +/// The contref_addr may be 0, to indicate that we want to build a wasm null reference. +pub(crate) fn construct<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + pos: &mut cranelift_codegen::cursor::FuncCursor, + revision_counter: ir::Value, + contref_addr: ir::Value, +) -> ir::Value { + debug_assert_eq!(pos.func.dfg.value_type(contref_addr), env.pointer_type()); + debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ir::types::I64); + let contref_addr = if env.pointer_type().bits() < I64.bits() { + pos.ins().uextend(I64, contref_addr) + } else { + contref_addr + }; + let (msbs, lsbs) = match env.isa.endianness() { + ir::Endianness::Little => (contref_addr, revision_counter), + ir::Endianness::Big => { + let pad_bits = 64 - env.pointer_type().bits(); + let lsbs = pos.ins().ishl_imm(contref_addr, pad_bits as i64); + (revision_counter, lsbs) + } + }; + + let lsbs = pos.ins().uextend(ir::types::I128, lsbs); + let msbs = pos.ins().uextend(ir::types::I128, msbs); + let msbs = pos.ins().ishl_imm(msbs, 64); + pos.ins().bor(lsbs, msbs) +} diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs new file mode 100644 index 000000000000..21c261ae5985 --- /dev/null +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -0,0 +1,2543 @@ +use cranelift_codegen::ir::BlockArg; +use itertools::{Either, Itertools}; + +use cranelift_codegen::ir::condcodes::*; +use cranelift_codegen::ir::types::*; +use cranelift_codegen::ir::{self, MemFlags}; +use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; +use cranelift_frontend::FunctionBuilder; +use wasmtime_environ::stack_switching as stack_switching_environ; +use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType}; + +pub const DEBUG_ASSERT_TRAP_CODE: crate::TrapCode = crate::TRAP_DELETE_ME_DEBUG_ASSERTION; + +// TODO(frank-emrich) This is the size for x64 Linux. Once we support different +// platforms for stack switching, must select appropriate value for target. +pub const CONTROL_CONTEXT_SIZE: usize = 24; + +use super::control_effect::ControlEffect; +use super::fatpointer; + +// FIXME(frank-emrich) The debugging facilities in this module are very unsafe +// (see comment on `emit_debug_print`). They are not supposed to be part of the +// final, upstreamed code, but deleted beforehand. +#[macro_use] +pub(crate) mod delete_me { + use cranelift_codegen::ir; + use cranelift_codegen::ir::InstBuilder; + use cranelift_codegen::ir::condcodes::IntCC; + use cranelift_codegen::ir::types::*; + use cranelift_frontend::FunctionBuilder; + + macro_rules! call_builtin { + ( $builder:ident, $env:ident, $f:ident( $($args:expr),* ) ) => ( + { + let fname = $env.builtin_functions.$f(&mut $builder.func); + let vmctx = $env.vmctx_val(&mut $builder.cursor()); + $builder.ins().call(fname, &[vmctx, $( $args ), * ]); + } + ); + } + + /// FIXME(frank-emrich) This printing functionality is inherently unsafe: It + /// hard-codes the addresses of the string literals it uses, without any + /// relocation information. Therefore, it will immediately crash and burn if + /// the compiled code is ever used in a different execution of wasmtime than + /// the one producing it. + /// As a result it is not supposed to be part of the final, upstreamed code. + /// + /// Low-level implementation of debug printing. Do not use directly; see + /// `emit_debug_println!` macro for doing actual printing. + /// + /// Takes a string literal which may contain placeholders similarly to those + /// supported by `std::fmt`. + /// + /// Currently supported placeholders: + /// {} for unsigned integers + /// {:p} for printing pointers (in hex form) + /// + /// When printing, we replace them with the corresponding values in `vals`. + /// Thus, the number of placeholders in `s` must match the number of entries + /// in `vals`. + pub fn emit_debug_print<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + s: &'static str, + vals: &[ir::Value], + ) { + let print_s_infix = |env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + start: usize, + end: usize| { + if start < end { + let s: &'static str = &s[start..end]; + // This is quite dodgy, which is why we can only do this for + // debugging purposes: + // At jit time, we take a pointer to the slice of the (static) + // string, thus yielding an address within wasmtime's DATA + // section. This pointer is hard-code into generated code. We do + // not emit any kind of relocation information, which means that + // this breaks if we were to store the generated code and use it + // during subsequent executions of wasmtime (e.g., when using + // wasmtime compile). + let ptr = s.as_ptr(); + let ptr = builder.ins().iconst(env.pointer_type(), ptr as i64); + let len = s.len(); + let len = builder.ins().iconst(I64, len as i64); + + call_builtin!(builder, env, delete_me_print_str(ptr, len)); + } + }; + let print_int = |env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + val: ir::Value| { + let ty = builder.func.dfg.value_type(val); + let val = match ty { + I8 | I32 => builder.ins().uextend(I64, val), + I64 => val, + _ => panic!("Cannot print type {ty}"), + }; + call_builtin!(builder, env, delete_me_print_int(val)); + }; + let print_pointer = |env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ptr: ir::Value| { + call_builtin!(builder, env, delete_me_print_pointer(ptr)); + }; + + if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { + let mut prev_end = 0; + let mut i = 0; + + let mut ph_matches: Vec<(usize, &'static str)> = s + .match_indices("{}") + .chain(s.match_indices("{:p}")) + .collect(); + ph_matches.sort_by_key(|(index, _)| *index); + + for (start, matched_ph) in ph_matches { + let end = start + matched_ph.len(); + + assert!( + i < vals.len(), + "Must supply as many entries in vals as there are placeholders in the string" + ); + + print_s_infix(env, builder, prev_end, start); + match matched_ph { + "{}" => print_int(env, builder, vals[i]), + "{:p}" => print_pointer(env, builder, vals[i]), + u => panic!("Unsupported placeholder in debug_print input string: {u}"), + } + prev_end = end; + i += 1; + } + assert_eq!( + i, + vals.len(), + "Must supply as many entries in vals as there are placeholders in the string" + ); + + print_s_infix(env, builder, prev_end, s.len()); + } + } + + /// Emits code to print debug information. Only actually prints in debug + /// builds and if debug printing flag is enabled. The third and all + /// following arguments are like those to println!: A string literal with + /// placeholders followed by the actual values. + /// + /// Summary of arguments: + /// * `env` - Type &mut crate::func_environ::FuncEnvironment<'a> + /// * `builder` - Type &mut FunctionBuilder, + /// * `msg` : String literal, containing placeholders like those supported by println! + /// * remaining arguments: ir::Values filled into the placeholders in `msg` + #[allow(unused_macros, reason = "Only used in certain debug builds")] + macro_rules! emit_debug_println { + ($env : expr, $builder : expr, $msg : literal, $( $arg:expr ),*) => { + let msg_newline : &'static str= std::concat!( + $msg, + "\n" + ); + emit_debug_print($env, $builder, msg_newline, &[$($arg),*]); + } + } + + /// Low-level implementation of assertion mechanism. Use emit_debug_* macros + /// instead. + /// + /// If `ENABLE_DEBUG_PRINTING` is enabled, `error_str` is printed before + /// trapping in case of an assertion violation. + pub fn emit_debug_assert_generic<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + condition: ir::Value, + error_str: &'static str, + ) { + if cfg!(debug_assertions) { + if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { + let failure_block = builder.create_block(); + let continue_block = builder.create_block(); + + builder + .ins() + .brif(condition, continue_block, &[], failure_block, &[]); + + builder.switch_to_block(failure_block); + builder.seal_block(failure_block); + + emit_debug_print(env, builder, error_str, &[]); + builder.ins().debugtrap(); + builder.ins().jump(continue_block, &[]); + + builder.switch_to_block(continue_block); + builder.seal_block(continue_block); + } else { + builder + .ins() + .trapz(condition, super::DEBUG_ASSERT_TRAP_CODE); + } + } + } + + /// Low-level implementation of assertion mechanism. Use emit_debug_* macros + /// instead. + /// + /// If `ENABLE_DEBUG_PRINTING` is enabled, `error_str` is printed before + /// trapping in case of an assertion violation. Here, `error_str` is expected + /// to contain two placeholders, such as {} or {:p}, which are replaced with + /// `v1` and `v2` when printing. + pub fn emit_debug_assert_icmp<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + operator: IntCC, + v1: ir::Value, + v2: ir::Value, + error_str: &'static str, + ) { + if cfg!(debug_assertions) { + let cmp_res = builder.ins().icmp(operator, v1, v2); + + if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { + let failure_block = builder.create_block(); + let continue_block = builder.create_block(); + + builder + .ins() + .brif(cmp_res, continue_block, &[], failure_block, &[]); + + builder.switch_to_block(failure_block); + builder.seal_block(failure_block); + + emit_debug_print(env, builder, error_str, &[v1, v2]); + builder.ins().debugtrap(); + builder.ins().jump(continue_block, &[]); + + builder.switch_to_block(continue_block); + builder.seal_block(continue_block); + } else { + builder.ins().trapz(cmp_res, super::DEBUG_ASSERT_TRAP_CODE); + } + } + } + + /// Used to implement other macros, do not use directly. + macro_rules! emit_debug_assert_icmp { + ( $env : expr, + $builder: expr, + $operator : expr, + $operator_string : expr, + $v1 : expr, + $v2 : expr) => { + let msg: &'static str = std::concat!( + "assertion failure in ", + std::file!(), + ", line ", + std::line!(), + ": {} ", + $operator_string, + " {} does not hold\n" + ); + emit_debug_assert_icmp($env, $builder, $operator, $v1, $v2, msg); + }; + } + + macro_rules! emit_debug_assert { + ($env: expr, $builder: expr, $condition: expr) => { + let msg: &'static str = std::concat!( + "assertion failure in ", + std::file!(), + ", line ", + std::line!(), + "\n" + ); + // This makes the borrow checker happy if $condition uses env or builder. + let c = $condition; + emit_debug_assert_generic($env, $builder, c, msg); + }; + } + + macro_rules! emit_debug_assert_eq { + ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { + emit_debug_assert_icmp!($env, $builder, IntCC::Equal, "==", $v1, $v2); + }; + } + + macro_rules! emit_debug_assert_ne { + ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { + emit_debug_assert_icmp!($env, $builder, IntCC::NotEqual, "!=", $v1, $v2); + }; + } + + macro_rules! emit_debug_assert_ule { + ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { + emit_debug_assert_icmp!( + $env, + $builder, + IntCC::UnsignedLessThanOrEqual, + "<=", + $v1, + $v2 + ); + }; + } +} +use delete_me::*; + +/// This module contains compile-time counterparts to types defined elsewhere. +pub(crate) mod stack_switching_helpers { + use super::delete_me::*; + use core::marker::PhantomData; + use cranelift_codegen::ir; + use cranelift_codegen::ir::InstBuilder; + use cranelift_codegen::ir::condcodes::IntCC; + use cranelift_codegen::ir::types::*; + use cranelift_codegen::ir::{StackSlot, StackSlotKind::*}; + use cranelift_frontend::FunctionBuilder; + use std::mem; + use wasmtime_environ::PtrSize; + + #[derive(Copy, Clone)] + pub struct VMContRef { + pub address: ir::Value, + } + + #[derive(Copy, Clone)] + pub struct VMArray { + /// Base address of this object, which must be shifted by `offset` below. + base: ir::Value, + + /// Adding this (statically) known offset gets us the overall address. + offset: i32, + + /// The type parameter T is never used in the fields above. We still + /// want to have it for consistency with + /// `stack_switching_environ::Vector` and to use it in the associated + /// functions. + phantom: PhantomData, + } + + pub type VMPayloads = VMArray; + + // Actually a vector of *mut VMTagDefinition + pub type VMHandlerList = VMArray<*mut u8>; + + /// Compile-time representation of stack_switching_environ::VMStackChain, + /// consisting of two `ir::Value`s. + pub struct VMStackChain { + discriminant: ir::Value, + payload: ir::Value, + } + + pub struct VMCommonStackInformation { + pub address: ir::Value, + } + + /// Compile-time representation of `crate::runtime::vm::stack::VMContinuationStack`. + pub struct VMContinuationStack { + /// This is NOT the "top of stack" address of the stack itself. In line + /// with how the (runtime) `FiberStack` type works, this is a pointer to + /// the TOS address. + tos_ptr: ir::Value, + } + + impl VMContRef { + pub fn new(address: ir::Value) -> VMContRef { + VMContRef { address } + } + + pub fn args<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMPayloads { + let offset = env.offsets.ptr.vmcontref_args() as i32; + VMPayloads::new(self.address, offset) + } + + pub fn values<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMPayloads { + let offset = env.offsets.ptr.vmcontref_values() as i32; + VMPayloads::new(self.address, offset) + } + + pub fn common_stack_information<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMCommonStackInformation { + let offset = env.offsets.ptr.vmcontref_common_stack_information() as i64; + let address = builder.ins().iadd_imm(self.address, offset); + VMCommonStackInformation { address } + } + + /// Stores the parent of this continuation, which may either be another + /// continuation or the initial stack. It is therefore represented as a + /// `VMStackChain` element. + pub fn set_parent_stack_chain<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + new_stack_chain: &VMStackChain, + ) { + let offset = env.offsets.ptr.vmcontref_parent_chain() as i32; + new_stack_chain.store(env, builder, self.address, offset) + } + + /// Loads the parent of this continuation, which may either be another + /// continuation or the initial stack. It is therefore represented as a + /// `VMStackChain` element. + pub fn get_parent_stack_chain<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMStackChain { + let offset = env.offsets.ptr.vmcontref_parent_chain() as i32; + VMStackChain::load(env, builder, self.address, offset, env.pointer_type()) + } + + pub fn set_last_ancestor<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + last_ancestor: ir::Value, + ) { + let offset = env.offsets.ptr.vmcontref_last_ancestor() as i32; + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .store(mem_flags, last_ancestor, self.address, offset); + } + + pub fn get_last_ancestor<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = env.offsets.ptr.vmcontref_last_ancestor() as i32; + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .load(env.pointer_type(), mem_flags, self.address, offset) + } + + /// Gets the revision counter the a given continuation + /// reference. + pub fn get_revision<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let offset = env.offsets.ptr.vmcontref_revision() as i32; + let revision = builder.ins().load(I64, mem_flags, self.address, offset); + revision + } + + /// Sets the revision counter on the given continuation + /// reference to `revision + 1`. + + pub fn incr_revision<'a>( + &mut self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + revision: ir::Value, + ) -> ir::Value { + if cfg!(debug_assertions) { + let actual_revision = self.get_revision(env, builder); + emit_debug_assert_eq!(env, builder, revision, actual_revision); + } + let mem_flags = ir::MemFlags::trusted(); + let offset = env.offsets.ptr.vmcontref_revision() as i32; + let revision_plus1 = builder.ins().iadd_imm(revision, 1); + builder + .ins() + .store(mem_flags, revision_plus1, self.address, offset); + if cfg!(debug_assertions) { + let new_revision = self.get_revision(env, builder); + emit_debug_assert_eq!(env, builder, revision_plus1, new_revision); + // Check for overflow: + emit_debug_assert_ule!(env, builder, revision, revision_plus1); + } + revision_plus1 + } + + pub fn get_fiber_stack<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMContinuationStack { + // The top of stack field is stored at offset 0 of the `FiberStack`. + let offset = env.offsets.ptr.vmcontref_stack() as i64; + let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm(self.address, offset); + VMContinuationStack::new(fiber_stack_top_of_stack_ptr) + } + } + + impl VMArray { + pub(crate) fn new(base: ir::Value, offset: i32) -> Self { + Self { + base, + offset, + phantom: PhantomData::default(), + } + } + + fn get(&self, builder: &mut FunctionBuilder, ty: ir::Type, offset: i32) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .load(ty, mem_flags, self.base, self.offset + offset) + } + + fn set(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) { + debug_assert_eq!( + builder.func.dfg.value_type(value), + Type::int_with_byte_size(u16::try_from(std::mem::size_of::()).unwrap()).unwrap() + ); + let mem_flags = ir::MemFlags::trusted(); + builder + .ins() + .store(mem_flags, value, self.base, self.offset + offset); + } + + pub fn get_data<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = env.offsets.ptr.vmarray_data() as i32; + self.get(builder, env.pointer_type(), offset) + } + + fn get_capacity<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Array capacity is stored as u32. + let offset = env.offsets.ptr.vmarray_capacity() as i32; + self.get(builder, I32, offset) + } + + pub fn get_length<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Array length is stored as u32. + let offset = env.offsets.ptr.vmarray_length() as i32; + self.get(builder, I32, offset) + } + + fn set_length<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + length: ir::Value, + ) { + // Array length is stored as u32. + let offset = env.offsets.ptr.vmarray_length() as i32; + self.set::(builder, offset, length); + } + + fn set_capacity<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + capacity: ir::Value, + ) { + // Array capacity is stored as u32. + let offset = env.offsets.ptr.vmarray_capacity() as i32; + self.set::(builder, offset, capacity); + } + + fn set_data<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + data: ir::Value, + ) { + let offset = env.offsets.ptr.vmarray_data() as i32; + self.set::<*mut T>(builder, offset, data); + } + + /// Returns pointer to next empty slot in data buffer and marks the + /// subsequent `arg_count` slots as occupied. + pub fn occupy_next_slots<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + arg_count: i32, + ) -> ir::Value { + let data = self.get_data(env, builder); + let original_length = self.get_length(env, builder); + let new_length = builder.ins().iadd_imm(original_length, arg_count as i64); + self.set_length(env, builder, new_length); + + if cfg!(debug_assertions) { + let capacity = self.get_capacity(env, builder); + emit_debug_assert_ule!(env, builder, new_length, capacity); + } + + let value_size = mem::size_of::() as i64; + let original_length = builder.ins().uextend(I64, original_length); + let byte_offset = builder.ins().imul_imm(original_length, value_size); + builder.ins().iadd(data, byte_offset) + } + + pub fn allocate_or_reuse_stack_slot<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + required_capacity: u32, + existing_slot: Option, + ) -> StackSlot { + let zero = builder.ins().iconst(ir::types::I32, 0); + if cfg!(debug_assertions) { + // We must only allocate while there is no data in the buffer. + let length = self.get_length(env, builder); + emit_debug_assert_eq!(env, builder, length, zero); + let capacity = self.get_capacity(env, builder); + emit_debug_assert_eq!(env, builder, capacity, zero); + } + + let align = u8::try_from(std::mem::align_of::()).unwrap(); + let entry_size = u32::try_from(std::mem::size_of::()).unwrap(); + let required_size = required_capacity * entry_size; + + match existing_slot { + Some(slot) if builder.func.get_stack_slot_data(slot).size >= required_size => { + let slot_data = builder.func.get_stack_slot_data(slot).clone(); + let existing_capacity = slot_data.size / entry_size; + + let capacity_value = builder.ins().iconst(I32, existing_capacity as i64); + emit_debug_println!( + env, + builder, + "[Array::allocate_or_reuse_stack_slot] Reusing existing buffer with capacity {}", + capacity_value + ); + debug_assert!(align <= builder.func.get_stack_slot_data(slot).align_shift); + debug_assert_eq!(builder.func.get_stack_slot_data(slot).kind, ExplicitSlot); + + let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, existing_data); + + slot + } + _ => { + let capacity_value = builder.ins().iconst(I32, required_capacity as i64); + emit_debug_assert_ne!(env, builder, capacity_value, zero); + + emit_debug_println!( + env, + builder, + "[Array::allocate_or_reuse_stack_slot] allocating stack slot with capacity {}", + capacity_value + ); + + let slot_size = ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + required_size, + align, + ); + let slot = builder.create_sized_stack_slot(slot_size); + let new_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + self.set_capacity(env, builder, capacity_value); + self.set_data(env, builder, new_data); + + slot + } + } + } + + /// Loads n entries from this Vector object, where n is the length of + /// `load_types`, which also gives the types of the values to load. + /// Loading starts at index 0 of the Vector object. + pub fn load_data_entries<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + load_types: &[ir::Type], + ) -> Vec { + if cfg!(debug_assertions) { + let length = self.get_length(env, builder); + let load_count = builder.ins().iconst(I32, load_types.len() as i64); + emit_debug_assert_ule!(env, builder, load_count, length); + } + + let memflags = ir::MemFlags::trusted(); + + let data_start_pointer = self.get_data(env, builder); + let mut values = vec![]; + let mut offset = 0; + let entry_size = i32::try_from(std::mem::size_of::()).unwrap(); + for valtype in load_types { + let val = builder + .ins() + .load(*valtype, memflags, data_start_pointer, offset); + values.push(val); + offset += entry_size; + } + values + } + + /// Stores the given `values` in this Vector object, beginning at + /// index 0. This expects the Vector object to be empty (i.e., current + /// length is 0), and to be of sufficient capacity to store |`values`| + /// entries. + /// If `allow_smaller` is true, we allow storing values whose type has a + /// smaller size than T's. In that case, such values will be stored at + /// the beginning of a `T`-sized slot. + pub fn store_data_entries<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + values: &[ir::Value], + allow_smaller: bool, + ) { + let store_count = builder.ins().iconst(I32, values.len() as i64); + + if cfg!(debug_assertions) { + for val in values { + let ty = builder.func.dfg.value_type(*val); + if allow_smaller { + debug_assert!(ty.bytes() as usize <= std::mem::size_of::()); + } else { + debug_assert!(ty.bytes() as usize == std::mem::size_of::()); + } + } + + let capacity = self.get_capacity(env, builder); + let length = self.get_length(env, builder); + let zero = builder.ins().iconst(I32, 0); + emit_debug_assert_ule!(env, builder, store_count, capacity); + emit_debug_assert_eq!(env, builder, length, zero); + } + + let memflags = ir::MemFlags::trusted(); + + let data_start_pointer = self.get_data(env, builder); + + let entry_size = i32::try_from(std::mem::size_of::()).unwrap(); + let mut offset = 0; + for value in values { + builder + .ins() + .store(memflags, *value, data_start_pointer, offset); + offset += entry_size; + } + + self.set_length(env, builder, store_count); + } + + pub fn clear<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + discard_buffer: bool, + ) { + let zero32 = builder.ins().iconst(I32, 0); + self.set_length(env, builder, zero32); + + if discard_buffer { + let zero32 = builder.ins().iconst(I32, 0); + self.set_capacity(env, builder, zero32); + + let zero_ptr = builder.ins().iconst(env.pointer_type(), 0); + self.set_data(env, builder, zero_ptr); + } + } + } + + impl VMStackChain { + /// Creates a `Self` corressponding to `VMStackChain::Continuation(contref)`. + pub fn from_continuation<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + contref: ir::Value, + ) -> VMStackChain { + debug_assert_eq!( + env.offsets.ptr.size_of_vmstack_chain(), + 2 * env.offsets.ptr.size() + ); + let discriminant = + super::stack_switching_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), discriminant as i64); + VMStackChain { + discriminant, + payload: contref, + } + } + + /// Creates a `Self` corressponding to `VMStackChain::Absent`. + pub fn absent<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMStackChain { + debug_assert_eq!( + env.offsets.ptr.size_of_vmstack_chain(), + 2 * env.offsets.ptr.size() + ); + let discriminant = super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), discriminant as i64); + let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64); + VMStackChain { + discriminant, + payload: zero_filler, + } + } + + /// For debugging purposes. Emits an assertion that `self` does not correspond to + /// `VMStackChain::Absent`. + pub fn assert_not_absent<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; + let discriminant = builder + .ins() + .iconst(env.pointer_type(), discriminant as i64); + emit_debug_assert_ne!(env, builder, self.discriminant, discriminant); + } + + pub fn is_initial_stack<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + super::stack_switching_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT as i64, + ) + } + + pub fn is_absent<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT as i64, + ) + } + + /// Return the two raw `ir::Value`s that represent this VMStackChain. + pub fn to_raw_parts(&self) -> [ir::Value; 2] { + [self.discriminant, self.payload] + } + + /// Construct a `Self` from two raw `ir::Value`s. + pub fn from_raw_parts(raw_data: [ir::Value; 2]) -> VMStackChain { + VMStackChain { + discriminant: raw_data[0], + payload: raw_data[1], + } + } + + /// Load a `VMStackChain` object from the given address. + pub fn load<'a>( + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + pointer: ir::Value, + initial_offset: i32, + pointer_type: ir::Type, + ) -> VMStackChain { + let memflags = ir::MemFlags::trusted(); + let mut offset = initial_offset; + let mut data = vec![]; + for _ in 0..2 { + data.push(builder.ins().load(pointer_type, memflags, pointer, offset)); + offset += pointer_type.bytes() as i32; + } + let data = <[ir::Value; 2]>::try_from(data).unwrap(); + Self::from_raw_parts(data) + } + + /// Store this `VMStackChain` object at the given address. + pub fn store<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + target_pointer: ir::Value, + initial_offset: i32, + ) { + let memflags = ir::MemFlags::trusted(); + let mut offset = initial_offset; + let data = self.to_raw_parts(); + + for value in data { + debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type()); + builder.ins().store(memflags, value, target_pointer, offset); + offset += env.pointer_type().bytes() as i32; + } + } + + /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`. + pub fn unchecked_get_continuation<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + if cfg!(debug_assertions) { + let continuation_discriminant = + super::stack_switching_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; + let is_continuation = builder.ins().icmp_imm( + IntCC::Equal, + self.discriminant, + continuation_discriminant as i64, + ); + emit_debug_assert!(env, builder, is_continuation); + } + self.payload + } + + /// Must only be called if `self` represents a `InitialStack` or + /// `Continuation` variant. Returns a pointer to the associated + /// `CommonStackInformation` object. + pub fn get_common_stack_information<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> VMCommonStackInformation { + self.assert_not_absent(env, builder); + + // `self` corresponds to a VMStackChain::InitialStack or + // VMStackChain::Continuation. + // In both cases, the payload is a pointer. + let address = self.payload; + + // `obj` is now a pointer to the beginning of either + // 1. A `VMContRef` struct (in the case of a + // VMStackChain::Continuation) + // 2. A CommonStackInformation struct (in the case of + // VMStackChain::InitialStack) + // + // Since a `VMContRef` starts with an (inlined) CommonStackInformation + // object at offset 0, we actually have in both cases that `ptr` is + // now the address of the beginning of a VMStackLimits object. + debug_assert_eq!(env.offsets.ptr.vmcontref_common_stack_information(), 0); + VMCommonStackInformation { address } + } + } + + impl VMCommonStackInformation { + fn get_state_ptr<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = _env.offsets.ptr.vmcommon_stack_information_state() as i64; + + builder.ins().iadd_imm(self.address, offset) + } + + fn get_stack_limits_ptr<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let offset = _env.offsets.ptr.vmcommon_stack_information_limits() as i64; + + builder.ins().iadd_imm(self.address, offset) + } + + fn load_state<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + let state_ptr = self.get_state_ptr(env, builder); + + builder.ins().load(I32, mem_flags, state_ptr, 0) + } + + fn set_state_no_payload<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + discriminant: u32, + ) { + let discriminant = builder.ins().iconst(I32, discriminant as i64); + emit_debug_println!( + env, + builder, + "setting state of CommonStackInformation {:p} to {}", + self.address, + discriminant + ); + + let mem_flags = ir::MemFlags::trusted(); + let state_ptr = self.get_state_ptr(env, builder); + + builder.ins().store(mem_flags, discriminant, state_ptr, 0); + } + + pub fn set_state_running<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_parent<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::stack_switching::STACK_STATE_PARENT_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_returned<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn set_state_suspended<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) { + let discriminant = + wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT; + self.set_state_no_payload(env, builder, discriminant); + } + + pub fn has_state_any_of<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + state_discriminants: &[u32], + ) -> ir::Value { + let actual_state = self.load_state(env, builder); + let zero = builder.ins().iconst(I8, 0); + let mut res = zero; + for state_discriminant in state_discriminants { + let eq = + builder + .ins() + .icmp_imm(IntCC::Equal, actual_state, *state_discriminant as i64); + res = builder.ins().bor(res, eq); + } + res + } + + pub fn has_state_returned<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + self.has_state_any_of( + env, + builder, + &[wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT], + ) + } + + pub fn has_state_running<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + self.has_state_any_of( + env, + builder, + &[wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT], + ) + } + + // pub fn has_state<'a>( + // &self, + // env: &mut crate::func_environ::FuncEnvironment<'a>, + // builder: &mut FunctionBuilder, + // state: super::stack_switching_environ::VMStackState, + // ) -> ir::Value { + // self.has_state_any_of(env, builder, &[state]) + // } + + /// Checks whether the `VMStackState` reflects that the stack has ever been + /// active (instead of just having been allocated, but never resumed). + pub fn was_invoked<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let actual_state = self.load_state(env, builder); + let allocated = wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT; + builder + .ins() + .icmp_imm(IntCC::NotEqual, actual_state, allocated as i64) + } + + pub fn get_handler_list<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, + ) -> VMHandlerList { + let offset = env.offsets.ptr.vmcommon_stack_information_handlers() as i32; + VMHandlerList::new(self.address, offset) + } + + pub fn get_first_switch_handler_index<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + // Field first_switch_handler_index has type u32 + let memflags = ir::MemFlags::trusted(); + let offset = + env.offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() as i32; + builder.ins().load(I32, memflags, self.address, offset) + } + + pub fn set_first_switch_handler_index<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + value: ir::Value, + ) { + // Field first_switch_handler_index has type u32 + let memflags = ir::MemFlags::trusted(); + let offset = + env.offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() as i32; + builder.ins().store(memflags, value, self.address, offset); + } + + /// Sets `last_wasm_entry_sp` and `stack_limit` fields in + /// `VMRuntimelimits` using the values from the `VMStackLimits` of this + /// object. + pub fn write_limits_to_vmcontext<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmruntime_limits_ptr: ir::Value, + ) { + let stack_limits_ptr = self.get_stack_limits_ptr(env, builder); + + let memflags = ir::MemFlags::trusted(); + + let mut copy_to_vm_runtime_limits = |our_offset, their_offset| { + let our_value = builder.ins().load( + env.pointer_type(), + memflags, + stack_limits_ptr, + i32::try_from(our_offset).unwrap(), + ); + builder.ins().store( + memflags, + our_value, + vmruntime_limits_ptr, + their_offset as i32, + ); + }; + + let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit() as i32; + let last_wasm_entry_fp_offset = + env.offsets.ptr.vmstack_limits_last_wasm_entry_fp() as i32; + copy_to_vm_runtime_limits( + stack_limit_offset, + pointer_size.vmstore_context_stack_limit(), + ); + copy_to_vm_runtime_limits( + last_wasm_entry_fp_offset, + pointer_size.vmstore_context_last_wasm_entry_fp(), + ); + } + + /// Overwrites the `last_wasm_entry_fp` field of the `VMStackLimits` + /// object in the `VMStackLimits` of this object by loading the corresponding + /// field from the `VMRuntimeLimits`. + /// If `load_stack_limit` is true, we do the same for the `stack_limit` + /// field. + pub fn load_limits_from_vmcontext<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmruntime_limits_ptr: ir::Value, + load_stack_limit: bool, + ) { + let stack_limits_ptr = self.get_stack_limits_ptr(env, builder); + + let memflags = ir::MemFlags::trusted(); + let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); + + let mut copy = |runtime_limits_offset, stack_limits_offset| { + let from_vm_runtime_limits = builder.ins().load( + env.pointer_type(), + memflags, + vmruntime_limits_ptr, + runtime_limits_offset, + ); + builder.ins().store( + memflags, + from_vm_runtime_limits, + stack_limits_ptr, + i32::try_from(stack_limits_offset).unwrap(), + ); + }; + + let last_wasm_entry_fp_offset = + env.offsets.ptr.vmstack_limits_last_wasm_entry_fp() as i32; + copy( + pointer_size.vmstore_context_last_wasm_entry_fp(), + last_wasm_entry_fp_offset, + ); + + if load_stack_limit { + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit() as i32; + copy( + pointer_size.vmstore_context_stack_limit(), + stack_limit_offset, + ); + } + } + } + + impl VMContinuationStack { + /// The parameter is NOT the "top of stack" address of the stack itself. In line + /// with how the (runtime) `FiberStack` type works, this is a pointer to + /// the TOS address. + pub fn new(tos_ptr: ir::Value) -> Self { + Self { tos_ptr } + } + + fn load_top_of_stack<'a>( + &self, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let mem_flags = ir::MemFlags::trusted(); + builder.ins().load(I64, mem_flags, self.tos_ptr, 0) + } + + /// Returns address of the control context stored in the stack memory, + /// as used by stack_switch instructions. + pub fn load_control_context<'a>( + &self, + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + ) -> ir::Value { + let tos = self.load_top_of_stack(env, builder); + // Control context begins 24 bytes below top of stack (see unix.rs) + builder.ins().iadd_imm(tos, -0x18) + } + } +} + +use helpers::VMStackChain; +use stack_switching_environ::{ + CONTROL_EFFECT_RESUME_DISCRIMINANT, CONTROL_EFFECT_SWITCH_DISCRIMINANT, +}; +use stack_switching_helpers as helpers; + +/// Stores the given arguments in the appropriate `VMPayloads` object in the +/// continuation. If the continuation was never invoked, use the `args` object. +/// Otherwise, use the `values` object. +pub(crate) fn vmcontref_store_payloads<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + values: &[ir::Value], + contref: ir::Value, +) { + let count = + i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32"); + if values.len() > 0 { + let use_args_block = builder.create_block(); + let use_payloads_block = builder.create_block(); + let store_data_block = builder.create_block(); + builder.append_block_param(store_data_block, env.pointer_type()); + + let co = helpers::VMContRef::new(contref); + let csi = co.common_stack_information(env, builder); + let was_invoked = csi.was_invoked(env, builder); + builder + .ins() + .brif(was_invoked, use_payloads_block, &[], use_args_block, &[]); + + { + builder.switch_to_block(use_args_block); + builder.seal_block(use_args_block); + + let args = co.args(env, builder); + let ptr = args.occupy_next_slots(env, builder, count); + + builder + .ins() + .jump(store_data_block, &[BlockArg::Value(ptr)]); + } + + { + builder.switch_to_block(use_payloads_block); + builder.seal_block(use_payloads_block); + + let payloads = co.values(env, builder); + + // This also checks that the buffer is large enough to hold + // `values.len()` more elements. + let ptr = payloads.occupy_next_slots(env, builder, count); + builder + .ins() + .jump(store_data_block, &[BlockArg::Value(ptr)]); + } + + { + builder.switch_to_block(store_data_block); + builder.seal_block(store_data_block); + + let ptr = builder.block_params(store_data_block)[0]; + + // Store the values. + let memflags = ir::MemFlags::trusted(); + let mut offset = 0; + for value in values { + builder.ins().store(memflags, *value, ptr, offset); + offset += env.offsets.ptr.maximum_value_size() as i32; + } + } + } +} + +pub(crate) fn tag_address<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + index: u32, +) -> ir::Value { + let vmctx = env.vmctx_val(&mut builder.cursor()); + let tag_index = wasmtime_environ::TagIndex::from_u32(index); + let pointer_type = env.pointer_type(); + if let Some(def_index) = env.module.defined_tag_index(tag_index) { + let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap(); + builder.ins().iadd_imm(vmctx, offset as i64) + } else { + let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap(); + builder.ins().load( + pointer_type, + ir::MemFlags::trusted().with_readonly(), + vmctx, + ir::immediates::Offset32::new(offset), + ) + } +} + +/// Returns the stack chain saved in the given `VMContext`. Note that the +/// head of the list is the actively running stack (initial stack or +/// continuation). +pub fn vmctx_load_stack_chain<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, +) -> VMStackChain { + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain() as i32; + + // First we need to get the `VMStoreContext`. + let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); + let vm_store_context = builder.ins().load( + env.pointer_type(), + MemFlags::trusted(), + vmctx, + vm_store_context_offset, + ); + + VMStackChain::load( + env, + builder, + vm_store_context, + stack_chain_offset, + env.pointer_type(), + ) +} + +/// Stores the given stack chain saved in the `VMContext`, overwriting the +/// exsiting one. +pub fn vmctx_store_stack_chain<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, + stack_chain: &VMStackChain, +) { + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain() as i32; + + // First we need to get the `VMStoreContext`. + let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); + let vm_store_context = builder.ins().load( + env.pointer_type(), + MemFlags::trusted(), + vmctx, + vm_store_context_offset, + ); + + stack_chain.store(env, builder, vm_store_context, stack_chain_offset) +} + +/// Similar to `vmctx_store_stack_chain`, but instead of storing an arbitrary +/// `VMStackChain`, stores VMStackChain::Continuation(contref)`. +pub fn vmctx_set_active_continuation<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, + contref: ir::Value, +) { + let chain = VMStackChain::from_continuation(env, builder, contref); + vmctx_store_stack_chain(env, builder, vmctx, &chain) +} + +pub fn vmctx_load_vm_runtime_limits_ptr<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + vmctx: ir::Value, +) -> ir::Value { + let pointer_type = env.pointer_type(); + let offset = i32::from(env.offsets.ptr.vmctx_store_context()); + + // The *pointer* to the VMRuntimeLimits does not change within the + // same function, allowing us to set the `read_only` flag. + let flags = ir::MemFlags::trusted().with_readonly(); + + builder.ins().load(pointer_type, flags, vmctx, offset) +} + +/// This function generates code that searches for a handler for `tag_address`, +/// which must be a `*mut VMTagDefinition`. The search walks up the chain of +/// continuations beginning at `start`. +/// +/// The flag `search_suspend_handlers` determines whether we search for a +/// suspend or switch handler. Concretely, this influences which part of each +/// handler list we will search. +/// +/// We trap if no handler was found. +/// +/// The returned values are: +/// 1. The stack (continuation or initial stack, represented as a VMStackChain) in +/// whose handler list we found the tag (i.e., the stack that performed the +/// resume instruction that installed handler for the tag). +/// 2. The continuation whose parent is the stack mentioned in 1. +/// 3. The index of the handler in the handler list. +/// +/// In pseudo-code, the generated code's behavior can be expressed as +/// follows: +/// +/// chain_link = start +/// while !chain_link.is_initial_stack() { +/// contref = chain_link.get_contref() +/// parent_link = contref.parent +/// parent_csi = parent_link.get_common_stack_information(); +/// handlers = parent_csi.handlers; +/// (begin_range, end_range) = if search_suspend_handlers { +/// (0, parent_csi.first_switch_handler_index) +/// } else { +/// (parent_csi.first_switch_handler_index, handlers.length) +/// }; +/// for index in begin_range..end_range { +/// if handlers[index] == tag_address { +/// goto on_match(contref, index) +/// } +/// } +/// chain_link = parent_link +/// } +/// trap(unhandled_tag) +/// +/// on_match(conref : VMContRef, handler_index : u32) +/// ... execution continues here here ... +/// +fn search_handler<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + start: &helpers::VMStackChain, + tag_address: ir::Value, + search_suspend_handlers: bool, +) -> (VMStackChain, ir::Value, ir::Value) { + let handle_link = builder.create_block(); + let begin_search_handler_list = builder.create_block(); + let try_index = builder.create_block(); + let compare_tags = builder.create_block(); + let on_match = builder.create_block(); + let on_no_match = builder.create_block(); + let block_args = start.to_raw_parts().map(|v| BlockArg::Value(v)); + + // Terminate previous block: + builder.ins().jump(handle_link, &block_args); + + // Block handle_link + let chain_link = { + builder.append_block_param(handle_link, env.pointer_type()); + builder.append_block_param(handle_link, env.pointer_type()); + builder.switch_to_block(handle_link); + + let raw_parts = builder.block_params(handle_link); + let chain_link = helpers::VMStackChain::from_raw_parts([raw_parts[0], raw_parts[1]]); + let is_initial_stack = chain_link.is_initial_stack(env, builder); + builder.ins().brif( + is_initial_stack, + on_no_match, + &[], + begin_search_handler_list, + &[], + ); + chain_link + }; + + // Block begin_search_handler_list + let (contref, parent_link, handler_list_data_ptr, end_range) = { + builder.switch_to_block(begin_search_handler_list); + let contref = chain_link.unchecked_get_continuation(env, builder); + let contref = helpers::VMContRef::new(contref); + + let parent_link = contref.get_parent_stack_chain(env, builder); + + emit_debug_println!( + env, + builder, + "[search_handler] beginning search in parent of continuation {:p}", + contref.address + ); + + let parent_csi = parent_link.get_common_stack_information(env, builder); + + let handlers = parent_csi.get_handler_list(env, builder); + let handler_list_data_ptr = handlers.get_data(env, builder); + + let first_switch_handler_index = parent_csi.get_first_switch_handler_index(env, builder); + + // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range). + let (begin_range, end_range) = if search_suspend_handlers { + let zero = builder.ins().iconst(I32, 0); + if cfg!(debug_assertions) { + let length = handlers.get_length(env, builder); + emit_debug_assert_ule!(env, builder, first_switch_handler_index, length); + } + (zero, first_switch_handler_index) + } else { + let length = handlers.get_length(env, builder); + (first_switch_handler_index, length) + }; + + builder + .ins() + .jump(try_index, &[BlockArg::Value(begin_range)]); + + (contref, parent_link, handler_list_data_ptr, end_range) + }; + + // Block try_index + let index = { + builder.append_block_param(try_index, I32); + builder.switch_to_block(try_index); + let index = builder.block_params(try_index)[0]; + + let in_bounds = builder + .ins() + .icmp(IntCC::UnsignedLessThan, index, end_range); + let block_args = parent_link.to_raw_parts().map(|v| BlockArg::Value(v)); + builder + .ins() + .brif(in_bounds, compare_tags, &[], handle_link, &block_args); + index + }; + + // Block compare_tags + { + builder.switch_to_block(compare_tags); + + let base = handler_list_data_ptr; + let entry_size = std::mem::size_of::<*mut u8>(); + let offset = builder.ins().imul_imm(index, entry_size as i64); + let offset = builder.ins().uextend(I64, offset); + let entry_address = builder.ins().iadd(base, offset); + + let memflags = ir::MemFlags::trusted(); + + let handled_tag = builder + .ins() + .load(env.pointer_type(), memflags, entry_address, 0); + + let tags_match = builder.ins().icmp(IntCC::Equal, handled_tag, tag_address); + let incremented_index = builder.ins().iadd_imm(index, 1); + builder.ins().brif( + tags_match, + on_match, + &[], + try_index, + &[BlockArg::Value(incremented_index)], + ); + } + + // Block on_no_match + { + builder.switch_to_block(on_no_match); + builder.set_cold_block(on_no_match); + builder.ins().trap(crate::TRAP_UNHANDLED_TAG); + } + + builder.seal_block(handle_link); + builder.seal_block(begin_search_handler_list); + builder.seal_block(try_index); + builder.seal_block(compare_tags); + builder.seal_block(on_match); + builder.seal_block(on_no_match); + + // final block: on_match + builder.switch_to_block(on_match); + + emit_debug_println!( + env, + builder, + "[search_handler] found handler at stack chain ({}, {:p}), whose child continuation is {:p}, index is {}", + parent_link.to_raw_parts()[0], + parent_link.to_raw_parts()[1], + contref.address, + index + ); + + (parent_link, contref.address, index) +} + +pub(crate) fn translate_cont_bind<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + contobj: ir::Value, + args: &[ir::Value], +) -> ir::Value { + let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj); + + // The typing rules for cont.bind allow a null reference to be passed to it. + builder.ins().trapz(contref, crate::TRAP_NULL_REFERENCE); + + let mut vmcontref = helpers::VMContRef::new(contref); + let revision = vmcontref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, witness, revision); + emit_debug_println!( + env, + builder, + "[cont_bind] witness = {}, revision = {}, evidence = {}", + witness, + revision, + evidence + ); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + + vmcontref_store_payloads(env, builder, args, contref); + + let revision = vmcontref.incr_revision(env, builder, revision); + emit_debug_println!(env, builder, "new revision = {}", revision); + let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref); + emit_debug_println!(env, builder, "[cont_bind] contref = {:p}", contref); + contobj +} + +pub(crate) fn translate_cont_new<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + func: ir::Value, + arg_types: &[WasmValType], + return_types: &[WasmValType], +) -> WasmResult { + // The typing rules for cont.new allow a null reference to be passed to it. + builder.ins().trapz(func, crate::TRAP_NULL_REFERENCE); + + let nargs = builder.ins().iconst(I32, arg_types.len() as i64); + let nreturns = builder.ins().iconst(I32, return_types.len() as i64); + + let cont_new_func = env.builtin_functions.cont_new(&mut builder.func); + let vmctx = env.vmctx_val(&mut builder.cursor()); + let call_inst = builder + .ins() + .call(cont_new_func, &[vmctx, func, nargs, nreturns]); + let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap(); + + let tag = helpers::VMContRef::new(contref).get_revision(env, builder); + let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref); + emit_debug_println!(env, builder, "[cont_new] contref = {:p}", contref); + Ok(contobj) +} + +pub(crate) fn translate_resume<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + type_index: u32, + resume_contobj: ir::Value, + resume_args: &[ir::Value], + resumetable: &[(u32, Option)], +) -> WasmResult> { + // The resume instruction is the most involved instruction to + // compile as it is responsible for both continuation application + // and control tag dispatch. + // + // Here we translate a resume instruction into several basic + // blocks as follows: + // + // previous block + // | + // | + // resume_block + // / \ + // / \ + // | | + // return_block | + // suspend block + // | + // dispatch block + // + // * resume_block handles continuation arguments and performs + // actual stack switch. On ordinary return from resume, it jumps + // to the `return_block`, whereas on suspension it jumps to the + // `suspend_block`. + // * suspend_block is used on suspension, jumps onward to + // `dispatch_block`. + // * dispatch_block uses a jump table to dispatch to actual + // user-defined handler blocks, based on the handler index + // provided on suspension. Note that we do not jump to the + // handler blocks directly. Instead, each handler block has a + // corresponding premable block, which we jump to in order to + // reach a particular handler block. The preamble block prepares + // the arguments and continuation object to be passed to the + // actual handler block. + // + let resume_block = builder.create_block(); + let return_block = builder.create_block(); + let suspend_block = builder.create_block(); + let dispatch_block = builder.create_block(); + + let vmctx = env.vmctx_val(&mut builder.cursor()); + + // Split the resumetable into suspend handlers (each represented by the tag + // index and handler block) and the switch handlers (represented just by the + // tag index). Note that we currently don't remove duplicate tags. + let (suspend_handlers, switch_tags): (Vec<(u32, Block)>, Vec) = resumetable + .iter() + .partition_map(|(tag_index, block_opt)| match block_opt { + Some(block) => Either::Left((*tag_index, *block)), + None => Either::Right(*tag_index), + }); + + // Technically, there is no need to have a dedicated resume block, we could + // just put all of its contents into the current block. + builder.ins().jump(resume_block, &[]); + + // Resume block: actually resume the continuation chain ending at `resume_contref`. + let (resume_result, vm_runtime_limits_ptr, original_stack_chain, new_stack_chain) = { + builder.switch_to_block(resume_block); + builder.seal_block(resume_block); + + let (witness, resume_contref) = + fatpointer::deconstruct(env, &mut builder.cursor(), resume_contobj); + + // The typing rules for resume allow a null reference to be passed to it. + builder + .ins() + .trapz(resume_contref, crate::TRAP_NULL_REFERENCE); + + let mut vmcontref = helpers::VMContRef::new(resume_contref); + + let revision = vmcontref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); + emit_debug_println!( + env, + builder, + "[resume] resume_contref = {:p} witness = {}, revision = {}, evidence = {}", + resume_contref, + witness, + revision, + evidence + ); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + let next_revision = vmcontref.incr_revision(env, builder, revision); + emit_debug_println!(env, builder, "[resume] new revision = {}", next_revision); + + if cfg!(debug_assertions) { + // This should be impossible due to the linearity check. + let zero = builder.ins().iconst(I8, 0); + let csi = vmcontref.common_stack_information(env, builder); + let has_returned = csi.has_state_returned(env, builder); + emit_debug_assert_eq!(env, builder, has_returned, zero); + } + + if resume_args.len() > 0 { + // We store the arguments in the `VMContRef` to be resumed. + vmcontref_store_payloads(env, builder, resume_args, resume_contref); + } + + // Splice together stack chains: + // Connect the end of the chain starting at `resume_contref` to the currently active chain. + let mut last_ancestor = helpers::VMContRef::new(vmcontref.get_last_ancestor(env, builder)); + + // Make the currently running continuation (if any) the parent of the one we are about to resume. + let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + original_stack_chain.assert_not_absent(env, builder); + if cfg!(debug_assertions) { + // The continuation we are about to resume should have its chain broken up at last_ancestor. + let last_ancestor_chain = last_ancestor.get_parent_stack_chain(env, builder); + let is_absent = last_ancestor_chain.is_absent(env, builder); + emit_debug_assert!(env, builder, is_absent); + } + last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain); + + emit_debug_println!( + env, + builder, + "[resume] spliced together stack chains: parent of {:p} (last ancestor of {:p}) is now pointing to ({}, {:p})", + last_ancestor.address, + vmcontref.address, + original_stack_chain.to_raw_parts()[0], + original_stack_chain.to_raw_parts()[1] + ); + + // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field. + let zero = builder.ins().iconst(env.pointer_type(), 0); + vmcontref.set_last_ancestor(env, builder, zero); + + // We mark `resume_contref` as the currently running one + vmctx_set_active_continuation(env, builder, vmctx, resume_contref); + + // Note that the resume_contref libcall a few lines further below + // manipulates the stack limits as follows: + // 1. Copy stack_limit, last_wasm_entry_sp and last_wasm_exit* values from + // VMRuntimeLimits into the currently active continuation (i.e., the + // one that will become the parent of the to-be-resumed one) + // + // 2. Copy `stack_limit` and `last_wasm_entry_sp` in the + // `VMStackLimits` of `resume_contref` into the `VMRuntimeLimits`. + // + // See the comment on `stack_switching_environ::VMStackChain` for a + // description of the invariants that we maintain for the various stack + // limits. + + // `resume_contref` is now active, and its parent is suspended. + let resume_contref = helpers::VMContRef::new(resume_contref); + let resume_csi = resume_contref.common_stack_information(env, builder); + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + resume_csi.set_state_running(env, builder); + parent_csi.set_state_parent(env, builder); + + // We update the `VMStackLimits` of the parent of the continuation to be resumed + // as well as the `VMRuntimeLimits`. + // See the comment on `stack_switching_environ::VMStackChain` for a description + // of the invariants that we maintain for the various stack limits. + let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); + parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true); + resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + // Install handlers in (soon to be) parent's VMHandlerList: + // Let the i-th handler clause be (on $tag $block). + // Then the i-th entry of the VMHandlerList will be the address of $tag. + let handler_list = parent_csi.get_handler_list(env, builder); + + if resumetable.len() > 0 { + // Total number of handlers (suspend and switch). + let handler_count = u32::try_from(resumetable.len()).unwrap(); + // Populate the Array's data ptr with a pointer to a sufficiently + // large area on this stack. + env.stack_switching_handler_list_buffer = + Some(handler_list.allocate_or_reuse_stack_slot( + env, + builder, + handler_count, + env.stack_switching_handler_list_buffer, + )); + + let suspend_handler_count = suspend_handlers.len(); + + // All handlers, represented by the indices of the tags they handle. + // All the suspend handlers come first, followed by all the switch handlers. + let all_handlers = suspend_handlers + .iter() + .map(|(tag_index, _block)| *tag_index) + .chain(switch_tags); + + // Translate all tag indices to tag addresses (i.e., the corresponding *mut VMTagDefinition). + let all_tag_addresses: Vec = all_handlers + .map(|tag_index| tag_address(env, builder, tag_index)) + .collect(); + + // Store all tag addresess in the handler list. + handler_list.store_data_entries(env, builder, &all_tag_addresses, false); + + // To enable distinguishing switch and suspend handlers when searching the handler list: + // Store at which index the switch handlers start. + let first_switch_handler_index = + builder.ins().iconst(I32, suspend_handler_count as i64); + parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index); + } + + let resume_payload = ControlEffect::encode_resume(env, builder).to_u64(); + + // Note that the control context we use for switching is not the one in + // (the stack of) resume_contref, but in (the stack of) last_ancestor! + let fiber_stack = last_ancestor.get_fiber_stack(env, builder); + let control_context_ptr = fiber_stack.load_control_context(env, builder); + + emit_debug_println!( + env, + builder, + "[resume] about to execute stack_switch, control_context_ptr is {:p}", + control_context_ptr + ); + + let result = + builder + .ins() + .stack_switch(control_context_ptr, control_context_ptr, resume_payload); + + emit_debug_println!( + env, + builder, + "[resume] continuing after stack_switch in frame with parent_stack_chain ({}, {:p}), result is {:p}", + original_stack_chain.to_raw_parts()[0], + original_stack_chain.to_raw_parts()[1], + result + ); + + // At this point we know nothing about the continuation that just + // suspended or returned. In particular, it does not have to be what we + // called `resume_contref` earlier on. We must reload the information + // about the now active continuation from the VMContext. + let new_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + + // Now the parent contref (or initial stack) is active again + vmctx_store_stack_chain(env, builder, vmctx, &original_stack_chain); + parent_csi.set_state_running(env, builder); + + // Just for consistency: Clear the handler list. + handler_list.clear(env, builder, true); + parent_csi.set_first_switch_handler_index(env, builder, zero); + + // Extract the result and signal bit. + let result = ControlEffect::from_u64(result); + let signal = result.signal(env, builder); + + emit_debug_println!( + env, + builder, + "[resume] in resume block, signal is {}", + signal + ); + + // Jump to the return block if the result signal is 0, otherwise jump to + // the suspend block. + builder + .ins() + .brif(signal, suspend_block, &[], return_block, &[]); + + ( + result, + vm_runtime_limits_ptr, + original_stack_chain, + new_stack_chain, + ) + }; + + // The suspend block: Only used when we suspended, not for returns. + // Here we extract the index of the handler to use. + let (handler_index, suspended_contref, suspended_contobj) = { + builder.switch_to_block(suspend_block); + builder.seal_block(suspend_block); + + let suspended_continuation = new_stack_chain.unchecked_get_continuation(env, builder); + let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation); + let suspended_csi = suspended_continuation.common_stack_information(env, builder); + + // Note that at the suspend site, we already + // 1. Set the state of suspended_continuation to Suspended + // 2. Set suspended_continuation.last_ancestor + // 3. Broke the continuation chain at suspended_continuation.last_ancestor + + // We store parts of the VMRuntimeLimits into the continuation that just suspended. + suspended_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); + + // Afterwards (!), restore parts of the VMRuntimeLimits from the + // parent of the suspended continuation (which is now active). + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + // Extract the handler index + let handler_index = ControlEffect::handler_index(resume_result, env, builder); + + let revision = suspended_continuation.get_revision(env, builder); + let suspended_contobj = fatpointer::construct( + env, + &mut builder.cursor(), + revision, + suspended_continuation.address, + ); + + emit_debug_println!( + env, + builder, + "[resume] in suspend block, handler index is {}, new continuation is {:p}, with existing revision {}", + handler_index, + suspended_continuation.address, + revision + ); + + // We need to terminate this block before being allowed to switch to + // another one. + builder.ins().jump(dispatch_block, &[]); + + (handler_index, suspended_continuation, suspended_contobj) + }; + + // For technical reasons, the jump table needs to have a default + // block. In our case, it should be unreachable, since the handler + // index we dispatch on should correspond to a an actual handler + // block in the jump table. + let jt_default_block = builder.create_block(); + { + builder.switch_to_block(jt_default_block); + builder.set_cold_block(jt_default_block); + + builder.ins().trap(crate::TRAP_UNREACHABLE); + } + + // We create a preamble block for each of the actual handler blocks: It + // reads the necessary arguments and passes them to the actual handler + // block, together with the continuation object. + let target_preamble_blocks = { + let mut preamble_blocks = vec![]; + + for &(handle_tag, target_block) in &suspend_handlers { + let preamble_block = builder.create_block(); + preamble_blocks.push(preamble_block); + builder.switch_to_block(preamble_block); + + let param_types = env.tag_params(TagIndex::from_u32(handle_tag)); + let param_types: Vec = param_types + .iter() + .map(|wty| crate::value_type(env.isa, *wty)) + .collect(); + + let values = suspended_contref.values(env, builder); + let mut suspend_args: Vec = values + .load_data_entries(env, builder, ¶m_types) + .into_iter() + .map(|v| BlockArg::Value(v)) + .collect(); + + // At the suspend site, we store the suspend args in the the + // `values` buffer of the VMContRef that was active at the time that + // the suspend instruction was performed. + suspend_args.push(BlockArg::Value(suspended_contobj)); + + // We clear the suspend args. This is mostly for consistency. Note + // that we don't zero out the data buffer, we still need it for the + + values.clear(env, builder, false); + + builder.ins().jump(target_block, &suspend_args); + } + + preamble_blocks + }; + + // Dispatch block. All it does is jump to the right premable block based on + // the handler index. + { + builder.switch_to_block(dispatch_block); + builder.seal_block(dispatch_block); + + let default_bc = builder.func.dfg.block_call(jt_default_block, &[]); + + let adapter_bcs: Vec = target_preamble_blocks + .iter() + .map(|b| builder.func.dfg.block_call(*b, &[])) + .collect(); + + let jt_data = JumpTableData::new(default_bc, &adapter_bcs); + let jt = builder.create_jump_table(jt_data); + + builder.ins().br_table(handler_index, jt); + + for preamble_block in target_preamble_blocks { + builder.seal_block(preamble_block); + } + builder.seal_block(jt_default_block); + } + + // Return block: Jumped to by resume block if continuation + // returned normally. + { + builder.switch_to_block(return_block); + builder.seal_block(return_block); + + // If we got a return signal, a continuation must have been running. + let returned_contref = new_stack_chain.unchecked_get_continuation(env, builder); + let returned_contref = helpers::VMContRef::new(returned_contref); + + // Restore parts of the VMRuntimeLimits from the parent of the + // returned continuation (which is now active). + let parent_csi = original_stack_chain.get_common_stack_information(env, builder); + parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + + let returned_csi = returned_contref.common_stack_information(env, builder); + returned_csi.set_state_returned(env, builder); + + // Load the values returned by the continuation. + let return_types: Vec<_> = env + .continuation_returns(TypeIndex::from_u32(type_index)) + .iter() + .map(|ty| crate::value_type(env.isa, *ty)) + .collect(); + let payloads = returned_contref.args(env, builder); + let return_values = payloads.load_data_entries(env, builder, &return_types); + payloads.clear(env, builder, true); + + Ok(return_values) + } +} + +pub(crate) fn translate_suspend<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + tag_index: u32, + suspend_args: &[ir::Value], + tag_return_types: &[ir::Type], +) -> Vec { + let tag_addr = tag_address(env, builder, tag_index); + emit_debug_println!(env, builder, "[suspend] suspending with tag {:p}", tag_addr); + + let vmctx = env.vmctx_val(&mut builder.cursor()); + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + + let (_, end_of_chain_contref, handler_index) = + search_handler(env, builder, &active_stack_chain, tag_addr, true); + + emit_debug_println!( + env, + builder, + "[suspend] found handler: end of chain contref is {:p}, handler index is {}", + end_of_chain_contref, + handler_index + ); + + // If we get here, the search_handler logic succeeded (i.e., did not trap). + // Thus, there is at least one parent, so we are not on the initial stack. + // Can therefore extract continuation directly. + let active_contref = active_stack_chain.unchecked_get_continuation(env, builder); + let active_contref = helpers::VMContRef::new(active_contref); + let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref); + + active_contref.set_last_ancestor(env, builder, end_of_chain_contref.address); + + // In the active_contref's `values` buffer, stack-allocate enough room so that we can + // later store the following: + // 1. The suspend arguments + // 2. Afterwards, the tag return values + let values = active_contref.values(env, builder); + let required_capacity = + u32::try_from(std::cmp::max(suspend_args.len(), tag_return_types.len())) + .expect("Number of stack switching payloads should fit in u32"); + + if required_capacity > 0 { + env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + env, + builder, + required_capacity, + env.stack_switching_values_buffer, + )); + } + + if suspend_args.len() > 0 { + values.store_data_entries(env, builder, suspend_args, true) + } + + // Set current continuation to suspended and break up handler chain. + let active_contref_csi = active_contref.common_stack_information(env, builder); + if cfg!(debug_assertions) { + let is_running = active_contref_csi.has_state_running(env, builder); + emit_debug_assert!(env, builder, is_running); + } + + active_contref_csi.set_state_suspended(env, builder); + let absent_chain_link = VMStackChain::absent(env, builder); + end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link); + + let suspend_payload = ControlEffect::encode_suspend(env, builder, handler_index).to_u64(); + + // Note that the control context we use for switching is the one + // at the end of the chain, not the one in active_contref! + // This also means that stack_switch saves the information about + // the current stack in the control context located in the stack + // of end_of_chain_contref. + let fiber_stack = end_of_chain_contref.get_fiber_stack(env, builder); + let control_context_ptr = fiber_stack.load_control_context(env, builder); + + builder + .ins() + .stack_switch(control_context_ptr, control_context_ptr, suspend_payload); + + // The return values of the suspend instruction are the tag return values, saved in the `args` buffer. + let values = active_contref.values(env, builder); + let return_values = values.load_data_entries(env, builder, tag_return_types); + // We effectively consume the values and discard the stack allocated buffer. + values.clear(env, builder, true); + + return_values +} + +pub(crate) fn translate_switch<'a>( + env: &mut crate::func_environ::FuncEnvironment<'a>, + builder: &mut FunctionBuilder, + tag_index: u32, + switchee_contobj: ir::Value, + switch_args: &[ir::Value], + return_types: &[ir::Type], +) -> WasmResult> { + let vmctx = env.vmctx_val(&mut builder.cursor()); + + // Check and increment revision on switchee continuation object (i.e., the + // one being switched to). Logically, the switchee continuation extends from + // `switchee_contref` to `switchee_contref.last_ancestor` (i.e., the end of + // the parent chain starting at `switchee_contref`). + let switchee_contref = { + let (witness, target_contref) = + fatpointer::deconstruct(env, &mut builder.cursor(), switchee_contobj); + + // The typing rules for switch allow a null reference to be passed to it. + builder + .ins() + .trapz(target_contref, crate::TRAP_NULL_REFERENCE); + + let mut target_contref = helpers::VMContRef::new(target_contref); + + let revision = target_contref.get_revision(env, builder); + let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); + emit_debug_println!( + env, + builder, + "[switch] target_contref = {:p} witness = {}, revision = {}, evidence = {}", + target_contref.address, + witness, + revision, + evidence + ); + builder + .ins() + .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); + let _next_revision = target_contref.incr_revision(env, builder, revision); + target_contref + }; + + // We create the "switcher continuation" (i.e., the one executing switch) + // from the current execution context: Logically, it extends from the + // continuation reference executing `switch` (subsequently called + // `switcher_contref`) to the immediate child (called + // `switcher_contref_last_ancestor`) of the stack with the corresponding + // handler (saved in `handler_stack_chain`). + let ( + switcher_contref, + switcher_contobj, + switcher_contref_last_ancestor, + handler_stack_chain, + vm_runtime_limits_ptr, + ) = { + let tag_addr = tag_address(env, builder, tag_index); + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + let (handler_stack_chain, last_ancestor, _handler_index) = + search_handler(env, builder, &active_stack_chain, tag_addr, false); + let mut last_ancestor = helpers::VMContRef::new(last_ancestor); + + // If we get here, the search_handler logic succeeded (i.e., did not trap). + // Thus, there is at least one parent, so we are not on the initial stack. + // Can therefore extract continuation directly. + let switcher_contref = active_stack_chain.unchecked_get_continuation(env, builder); + let mut switcher_contref = helpers::VMContRef::new(switcher_contref); + + switcher_contref.set_last_ancestor(env, builder, last_ancestor.address); + + // In the switcher_contref's `values` buffer, stack-allocate enough room so that we can + // later store `tag_return_types.len()` when resuming the continuation. + let values = switcher_contref.values(env, builder); + let required_capacity = u32::try_from(return_types.len()).unwrap(); + if required_capacity > 0 { + env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot( + env, + builder, + required_capacity, + env.stack_switching_values_buffer, + )); + } + + let switcher_contref_csi = switcher_contref.common_stack_information(env, builder); + emit_debug_assert!( + env, + builder, + switcher_contref_csi.has_state_running(env, builder) + ); + switcher_contref_csi.set_state_suspended(env, builder); + // We break off `switcher_contref` from the chain of active + // continuations, by separating the link between `last_ancestor` and its + // parent stack. + let absent = VMStackChain::absent(env, builder); + last_ancestor.set_parent_stack_chain(env, builder, &absent); + + // Load current runtime limits from `VMContext` and store in the + // switcher continuation. + let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); + switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false); + + let revision = switcher_contref.get_revision(env, builder); + let new_contobj = fatpointer::construct( + env, + &mut builder.cursor(), + revision, + switcher_contref.address, + ); + + emit_debug_println!( + env, + builder, + "[switch] created new contref = {:p}, revision = {}", + switcher_contref.address, + revision + ); + + ( + switcher_contref, + new_contobj, + last_ancestor, + handler_stack_chain, + vm_runtime_limits_ptr, + ) + }; + + // Prepare switchee continuation: + // - Store "ordinary" switch arguments as well as the contobj just + // synthesized from the current context (i.e., `switcher_contobj`) in the + // switchee continuation's payload buffer. + // - Splice switchee's continuation chain with handler stack to form new + // overall chain of active continuations. + let (switchee_contref_csi, switchee_contref_last_ancestor) = { + let mut combined_payloads = switch_args.to_vec(); + combined_payloads.push(switcher_contobj); + vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address); + + let switchee_contref_csi = switchee_contref.common_stack_information(env, builder); + + emit_debug_assert!( + env, + builder, + switchee_contref_csi.has_state_any_of( + env, + builder, + &[ + wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT, + wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT + ] + ) + ); + switchee_contref_csi.set_state_running(env, builder); + + let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder); + let mut switchee_contref_last_ancestor = + helpers::VMContRef::new(switchee_contref_last_ancestor); + + switchee_contref_last_ancestor.set_parent_stack_chain(env, builder, &handler_stack_chain); + + (switchee_contref_csi, switchee_contref_last_ancestor) + }; + + // Update VMContext/Store: Update active continuation and `VMRuntimeLimits`. + { + vmctx_set_active_continuation(env, builder, vmctx, switchee_contref.address); + + switchee_contref_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); + } + + // Perform actual stack switch + { + let switcher_last_ancestor_fs = + switcher_contref_last_ancestor.get_fiber_stack(env, builder); + let switcher_last_ancestor_cc = + switcher_last_ancestor_fs.load_control_context(env, builder); + + let switchee_last_ancestor_fs = + switchee_contref_last_ancestor.get_fiber_stack(env, builder); + let switchee_last_ancestor_cc = + switchee_last_ancestor_fs.load_control_context(env, builder); + + // The stack switch involves the following control contexts (e.g., IP, + // SP, FP, ...): + // - `switchee_last_ancestor_cc` contains the information to continue + // execution in the switchee/target continuation. + // - `switcher_last_ancestor_cc` contains the information about how to + // continue execution once we suspend/return to the stack with the + // switch handler. + // + // In total, the following needs to happen: + // 1. Load control context at `switchee_last_ancestor_cc` to perform + // stack switch. + // 2. Move control context at `switcher_last_ancestor_cc` over to + // `switchee_last_ancestor_cc`. + // 3. Upon actual switch, save current control context at + // `switcher_last_ancestor_cc`. + // + // We implement this as follows: + // 1. We copy `switchee_last_ancestor_cc` to a temporary area on the + // stack (`tmp_control_context`). + // 2. We copy `switcher_last_ancestor_cc` over to + // `switchee_last_ancestor_cc`. + // 3. We invoke the stack switch instruction such that it reads from the + // temporary area, and writes to `switcher_last_ancestor_cc`. + // + // Note that the temporary area is only accessed once by the + // `stack_switch` instruction emitted later in this block, meaning that we + // don't have to worry about its lifetime. + // + // NOTE(frank-emrich) The implementation below results in one stack slot + // being created per switch instruction, even though multiple switch + // instructions in the same function could safely re-use the same stack + // slot. Thus, we could implement logic for sharing the stack slot by + // adding an appropriate field to `FuncEnvironment`. + // + // NOTE(frank-emrich) We could avoid the copying to a temporary area by + // making `stack_switch` do all of the necessary moving itself. However, + // that would be a rather ad-hoc change to how the instruction uses the + // two pointers given to it. + + let slot_size = ir::StackSlotData::new( + ir::StackSlotKind::ExplicitSlot, + u32::try_from(CONTROL_CONTEXT_SIZE).unwrap(), + u8::try_from(env.pointer_type().bytes()).unwrap(), + ); + let slot = builder.create_sized_stack_slot(slot_size); + let tmp_control_context = builder.ins().stack_addr(env.pointer_type(), slot, 0); + + let flags = MemFlags::trusted(); + let mut offset: i32 = 0; + while offset < i32::try_from(CONTROL_CONTEXT_SIZE).unwrap() { + // switchee_last_ancestor_cc -> tmp control context + let tmp1 = + builder + .ins() + .load(env.pointer_type(), flags, switchee_last_ancestor_cc, offset); + builder + .ins() + .store(flags, tmp1, tmp_control_context, offset); + + // switcher_last_ancestor_cc -> switchee_last_ancestor_cc + let tmp2 = + builder + .ins() + .load(env.pointer_type(), flags, switcher_last_ancestor_cc, offset); + builder + .ins() + .store(flags, tmp2, switchee_last_ancestor_cc, offset); + + offset += env.pointer_type().bytes() as i32; + } + + let switch_payload = ControlEffect::encode_switch(env, builder).to_u64(); + + emit_debug_println!( + env, + builder, + "[switch] about to execute stack_switch, store_control_context_ptr is {:p}, load_control_context_ptr {:p}, tmp_control_context is {:p}", + switcher_last_ancestor_cc, + switchee_last_ancestor_cc, + tmp_control_context + ); + + let result = builder.ins().stack_switch( + switcher_last_ancestor_cc, + tmp_control_context, + switch_payload, + ); + + emit_debug_println!( + env, + builder, + "[switch] continuing after stack_switch in frame with stack chain ({}, {:p}), result is {:p}", + handler_stack_chain.to_raw_parts()[0], + handler_stack_chain.to_raw_parts()[1], + result + ); + + if cfg!(debug_assertions) { + // The only way to switch back to this point is by using resume or switch instructions. + let result_control_effect = ControlEffect::from_u64(result); + let result_discriminant = result_control_effect.signal(env, builder); + let is_resume = builder.ins().icmp_imm( + IntCC::Equal, + result_discriminant, + CONTROL_EFFECT_RESUME_DISCRIMINANT as i64, + ); + let is_switch = builder.ins().icmp_imm( + IntCC::Equal, + result_discriminant, + CONTROL_EFFECT_SWITCH_DISCRIMINANT as i64, + ); + let is_switch_or_resume = builder.ins().bor(is_switch, is_resume); + emit_debug_assert!(env, builder, is_switch_or_resume); + } + } + + // After switching back to the original stack: Load return values, they are + // stored on the switcher continuation. + let return_values = { + if cfg!(debug_assertions) { + // The originally active continuation (before the switch) should be active again. + let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); + // This has a debug assertion that also checks that the `active_stack_chain` is indeed a continuation. + let active_contref = active_stack_chain.unchecked_get_continuation(env, builder); + emit_debug_assert_eq!(env, builder, switcher_contref.address, active_contref); + } + + let payloads = switcher_contref.values(env, builder); + let return_values = payloads.load_data_entries(env, builder, return_types); + // We consume the values and discard the buffer (allocated on this stack) + payloads.clear(env, builder, true); + return_values + }; + + Ok(return_values) +} diff --git a/crates/cranelift/src/stack_switching/mod.rs b/crates/cranelift/src/stack_switching/mod.rs new file mode 100644 index 000000000000..653191a769fa --- /dev/null +++ b/crates/cranelift/src/stack_switching/mod.rs @@ -0,0 +1,3 @@ +mod control_effect; +pub(crate) mod fatpointer; +pub(crate) mod instructions; diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 01bdd307b053..5972e57d854b 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -93,8 +93,8 @@ use std::collections::{HashMap, hash_map}; use std::vec::Vec; use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources}; use wasmtime_environ::{ - DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, Signed, TableIndex, TypeConvert, - TypeIndex, Unsigned, WasmRefType, WasmResult, wasm_unsupported, + DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, Signed, TableIndex, TagIndex, + TypeConvert, TypeIndex, Unsigned, WasmHeapType, WasmRefType, WasmResult, wasm_unsupported, }; /// Given a `Reachability`, unwrap the inner `T` or, when unreachable, set @@ -2898,54 +2898,150 @@ pub fn translate_operator( // representation, so we don't actually need to do anything. } - Operator::ContNew { cont_type_index: _ } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + Operator::ContNew { cont_type_index } => { + let cont_type_index = TypeIndex::from_u32(*cont_type_index); + let arg_types = environ.continuation_arguments(cont_type_index).to_vec(); + let result_types = environ.continuation_returns(cont_type_index).to_vec(); + let r = state.pop1(); + let contobj = + environ.translate_cont_new(builder, state, r, &arg_types, &result_types)?; + state.push1(contobj); } Operator::ContBind { - argument_index: _, - result_index: _, + argument_index, + result_index, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + let src_types = environ.continuation_arguments(TypeIndex::from_u32(*argument_index)); + let dst_arity = environ + .continuation_arguments(TypeIndex::from_u32(*result_index)) + .len(); + let arg_count = src_types.len() - dst_arity; + + let arg_types = &src_types[0..arg_count]; + for arg_type in arg_types { + // We can't bind GC objects using cont.bind at the moment: We + // don't have the necessary infrastructure to traverse the + // buffers used by cont.bind when looking for GC roots. Thus, + // this crude check ensures that these buffers can never contain + // GC roots to begin with. + if arg_type.is_vmgcref_type_and_not_i31() { + return Err(wasmtime_environ::WasmError::Unsupported( + "cont.bind does not support GC types at the moment".into(), + )); + } + } + + let (original_contobj, args) = state.peekn(arg_count + 1).split_last().unwrap(); + + let new_contobj = environ.translate_cont_bind(builder, *original_contobj, args); + + state.popn(arg_count + 1); + state.push1(new_contobj); } - Operator::Suspend { tag_index: _ } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + Operator::Suspend { tag_index } => { + let tag_index = TagIndex::from_u32(*tag_index); + let param_types = environ.tag_params(tag_index).to_vec(); + let return_types: Vec<_> = environ + .tag_returns(tag_index) + .iter() + .map(|ty| crate::value_type(environ.isa, *ty)) + .collect(); + + let params = state.peekn(param_types.len()); + let param_count = params.len(); + + let return_values = + environ.translate_suspend(builder, tag_index.as_u32(), params, &return_types); + + state.popn(param_count); + state.pushn(&return_values); } Operator::Resume { - cont_type_index: _, - resume_table: _, + cont_type_index, + resume_table, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + // We translate the block indices in the resumetable to actual Blocks. + let mut resumetable = vec![]; + for handle in &resume_table.handlers { + match handle { + wasmparser::Handle::OnLabel { tag, label } => { + let i = state.control_stack.len() - 1 - (*label as usize); + let frame = &mut state.control_stack[i]; + // This is side-effecting! + frame.set_branched_to_exit(); + resumetable.push((*tag, Some(frame.br_destination()))); + } + wasmparser::Handle::OnSwitch { tag } => { + resumetable.push((*tag, None)); + } + } + } + + let cont_type_index = TypeIndex::from_u32(*cont_type_index); + let arity = environ.continuation_arguments(cont_type_index).len(); + let (contobj, call_args) = state.peekn(arity + 1).split_last().unwrap(); + + let cont_return_vals = environ.translate_resume( + builder, + cont_type_index.as_u32(), + *contobj, + call_args, + resumetable.as_slice(), + )?; + + state.popn(arity + 1); // arguments + continuation + state.pushn(&cont_return_vals); } Operator::ResumeThrow { cont_type_index: _, tag_index: _, resume_table: _, - } => { - // TODO(10248) This depends on exception handling - return Err(wasmtime_environ::WasmError::Unsupported( - "resume.throw instructions not supported, yet".to_string(), - )); - } + } => todo!("unimplemented stack switching instruction"), Operator::Switch { - cont_type_index: _, - tag_index: _, + cont_type_index, + tag_index, } => { - // TODO(10248) This is added in a follow-up PR - return Err(wasmtime_environ::WasmError::Unsupported( - "codegen for stack switching instructions not implemented, yet".to_string(), - )); + // Arguments of the continuation we are going to switch to + let continuation_argument_types = environ + .continuation_arguments(TypeIndex::from_u32(*cont_type_index)) + .to_vec(); + // Arity includes the continuation argument + let arity = continuation_argument_types.len(); + let (contobj, switch_args) = state.peekn(arity).split_last().unwrap(); + + // Type of the continuation we are going to create by suspending the + // currently running stack + let current_continuation_type = continuation_argument_types.last().unwrap(); + let current_continuation_type = current_continuation_type.unwrap_ref_type(); + + // Argument types of current_continuation_type. These will in turn + // be the types of the arguments we receive when someone switches + // back to this switch instruction + let current_continuation_arg_types: Vec<_> = match current_continuation_type.heap_type { + WasmHeapType::ConcreteCont(index) => { + let mti = index + .as_module_type_index() + .expect("Only supporting module type indices on switch for now"); + + environ + .continuation_arguments(TypeIndex::from_u32(mti.as_u32())) + .iter() + .map(|ty| crate::value_type(environ.isa, *ty)) + .collect() + } + _ => panic!("Invalid type on switch"), + }; + + let switch_return_values = environ.translate_switch( + builder, + *tag_index, + *contobj, + switch_args, + ¤t_continuation_arg_types, + )?; + + state.popn(arity); + state.pushn(&switch_return_values) } Operator::GlobalAtomicGet { .. } From 0853a160e601ba9b2570bc46131a14219630b1a5 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 5 Jun 2025 20:58:26 +0000 Subject: [PATCH 02/34] cranelift: stack-switching updates pass 1 This first set of changes updates the base pr in order to compiled and pass basic checks (compile, clippy, fmt) with the biggest part of the change being to eliminate injection of tracing/assertions in JIT'ed code. --- crates/cranelift/src/func_environ.rs | 4 +- .../src/stack_switching/control_effect.rs | 7 +- .../src/stack_switching/instructions.rs | 699 +----------------- crates/environ/src/vmoffsets.rs | 6 + 4 files changed, 37 insertions(+), 679 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 6b67fe3da2fe..dad67b88368d 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3383,7 +3383,7 @@ impl FuncEnvironment<'_> { self.types[self.types[idx] .unwrap_cont() .clone() - .unwrap_interned_type_index()] + .unwrap_module_type_index()] .unwrap_func() .params() } @@ -3393,7 +3393,7 @@ impl FuncEnvironment<'_> { self.types[self.types[idx] .unwrap_cont() .clone() - .unwrap_interned_type_index()] + .unwrap_module_type_index()] .unwrap_func() .returns() } diff --git a/crates/cranelift/src/stack_switching/control_effect.rs b/crates/cranelift/src/stack_switching/control_effect.rs index e2f77d021848..bd93b6feebc0 100644 --- a/crates/cranelift/src/stack_switching/control_effect.rs +++ b/crates/cranelift/src/stack_switching/control_effect.rs @@ -2,7 +2,6 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::InstBuilder; use cranelift_codegen::ir::types::{I32, I64}; use cranelift_frontend::FunctionBuilder; -use wasmtime_environ::stack_switching as stack_switching_environ; /// Universal control effect. This structure encodes return signal, /// resume signal, suspension signal, and handler index into a @@ -37,7 +36,7 @@ impl ControlEffect { ) -> Self { let discriminant = builder.ins().iconst( I64, - i64::from(stack_switching_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT), + i64::from(wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT), ); let val = builder.ins().ishl_imm(discriminant, 32); @@ -50,7 +49,7 @@ impl ControlEffect { ) -> Self { let discriminant = builder.ins().iconst( I64, - i64::from(stack_switching_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT), + i64::from(wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT), ); let val = builder.ins().ishl_imm(discriminant, 32); @@ -64,7 +63,7 @@ impl ControlEffect { ) -> Self { let discriminant = builder.ins().iconst( I64, - i64::from(stack_switching_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT), + i64::from(wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT), ); let val = builder.ins().ishl_imm(discriminant, 32); let handler_index = builder.ins().uextend(I64, handler_index); diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 21c261ae5985..6a3eb587d878 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -6,11 +6,8 @@ use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, MemFlags}; use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; use cranelift_frontend::FunctionBuilder; -use wasmtime_environ::stack_switching as stack_switching_environ; use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType}; -pub const DEBUG_ASSERT_TRAP_CODE: crate::TrapCode = crate::TRAP_DELETE_ME_DEBUG_ASSERTION; - // TODO(frank-emrich) This is the size for x64 Linux. Once we support different // platforms for stack switching, must select appropriate value for target. pub const CONTROL_CONTEXT_SIZE: usize = 24; @@ -18,295 +15,8 @@ pub const CONTROL_CONTEXT_SIZE: usize = 24; use super::control_effect::ControlEffect; use super::fatpointer; -// FIXME(frank-emrich) The debugging facilities in this module are very unsafe -// (see comment on `emit_debug_print`). They are not supposed to be part of the -// final, upstreamed code, but deleted beforehand. -#[macro_use] -pub(crate) mod delete_me { - use cranelift_codegen::ir; - use cranelift_codegen::ir::InstBuilder; - use cranelift_codegen::ir::condcodes::IntCC; - use cranelift_codegen::ir::types::*; - use cranelift_frontend::FunctionBuilder; - - macro_rules! call_builtin { - ( $builder:ident, $env:ident, $f:ident( $($args:expr),* ) ) => ( - { - let fname = $env.builtin_functions.$f(&mut $builder.func); - let vmctx = $env.vmctx_val(&mut $builder.cursor()); - $builder.ins().call(fname, &[vmctx, $( $args ), * ]); - } - ); - } - - /// FIXME(frank-emrich) This printing functionality is inherently unsafe: It - /// hard-codes the addresses of the string literals it uses, without any - /// relocation information. Therefore, it will immediately crash and burn if - /// the compiled code is ever used in a different execution of wasmtime than - /// the one producing it. - /// As a result it is not supposed to be part of the final, upstreamed code. - /// - /// Low-level implementation of debug printing. Do not use directly; see - /// `emit_debug_println!` macro for doing actual printing. - /// - /// Takes a string literal which may contain placeholders similarly to those - /// supported by `std::fmt`. - /// - /// Currently supported placeholders: - /// {} for unsigned integers - /// {:p} for printing pointers (in hex form) - /// - /// When printing, we replace them with the corresponding values in `vals`. - /// Thus, the number of placeholders in `s` must match the number of entries - /// in `vals`. - pub fn emit_debug_print<'a>( - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - s: &'static str, - vals: &[ir::Value], - ) { - let print_s_infix = |env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - start: usize, - end: usize| { - if start < end { - let s: &'static str = &s[start..end]; - // This is quite dodgy, which is why we can only do this for - // debugging purposes: - // At jit time, we take a pointer to the slice of the (static) - // string, thus yielding an address within wasmtime's DATA - // section. This pointer is hard-code into generated code. We do - // not emit any kind of relocation information, which means that - // this breaks if we were to store the generated code and use it - // during subsequent executions of wasmtime (e.g., when using - // wasmtime compile). - let ptr = s.as_ptr(); - let ptr = builder.ins().iconst(env.pointer_type(), ptr as i64); - let len = s.len(); - let len = builder.ins().iconst(I64, len as i64); - - call_builtin!(builder, env, delete_me_print_str(ptr, len)); - } - }; - let print_int = |env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - val: ir::Value| { - let ty = builder.func.dfg.value_type(val); - let val = match ty { - I8 | I32 => builder.ins().uextend(I64, val), - I64 => val, - _ => panic!("Cannot print type {ty}"), - }; - call_builtin!(builder, env, delete_me_print_int(val)); - }; - let print_pointer = |env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ptr: ir::Value| { - call_builtin!(builder, env, delete_me_print_pointer(ptr)); - }; - - if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { - let mut prev_end = 0; - let mut i = 0; - - let mut ph_matches: Vec<(usize, &'static str)> = s - .match_indices("{}") - .chain(s.match_indices("{:p}")) - .collect(); - ph_matches.sort_by_key(|(index, _)| *index); - - for (start, matched_ph) in ph_matches { - let end = start + matched_ph.len(); - - assert!( - i < vals.len(), - "Must supply as many entries in vals as there are placeholders in the string" - ); - - print_s_infix(env, builder, prev_end, start); - match matched_ph { - "{}" => print_int(env, builder, vals[i]), - "{:p}" => print_pointer(env, builder, vals[i]), - u => panic!("Unsupported placeholder in debug_print input string: {u}"), - } - prev_end = end; - i += 1; - } - assert_eq!( - i, - vals.len(), - "Must supply as many entries in vals as there are placeholders in the string" - ); - - print_s_infix(env, builder, prev_end, s.len()); - } - } - - /// Emits code to print debug information. Only actually prints in debug - /// builds and if debug printing flag is enabled. The third and all - /// following arguments are like those to println!: A string literal with - /// placeholders followed by the actual values. - /// - /// Summary of arguments: - /// * `env` - Type &mut crate::func_environ::FuncEnvironment<'a> - /// * `builder` - Type &mut FunctionBuilder, - /// * `msg` : String literal, containing placeholders like those supported by println! - /// * remaining arguments: ir::Values filled into the placeholders in `msg` - #[allow(unused_macros, reason = "Only used in certain debug builds")] - macro_rules! emit_debug_println { - ($env : expr, $builder : expr, $msg : literal, $( $arg:expr ),*) => { - let msg_newline : &'static str= std::concat!( - $msg, - "\n" - ); - emit_debug_print($env, $builder, msg_newline, &[$($arg),*]); - } - } - - /// Low-level implementation of assertion mechanism. Use emit_debug_* macros - /// instead. - /// - /// If `ENABLE_DEBUG_PRINTING` is enabled, `error_str` is printed before - /// trapping in case of an assertion violation. - pub fn emit_debug_assert_generic<'a>( - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - condition: ir::Value, - error_str: &'static str, - ) { - if cfg!(debug_assertions) { - if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { - let failure_block = builder.create_block(); - let continue_block = builder.create_block(); - - builder - .ins() - .brif(condition, continue_block, &[], failure_block, &[]); - - builder.switch_to_block(failure_block); - builder.seal_block(failure_block); - - emit_debug_print(env, builder, error_str, &[]); - builder.ins().debugtrap(); - builder.ins().jump(continue_block, &[]); - - builder.switch_to_block(continue_block); - builder.seal_block(continue_block); - } else { - builder - .ins() - .trapz(condition, super::DEBUG_ASSERT_TRAP_CODE); - } - } - } - - /// Low-level implementation of assertion mechanism. Use emit_debug_* macros - /// instead. - /// - /// If `ENABLE_DEBUG_PRINTING` is enabled, `error_str` is printed before - /// trapping in case of an assertion violation. Here, `error_str` is expected - /// to contain two placeholders, such as {} or {:p}, which are replaced with - /// `v1` and `v2` when printing. - pub fn emit_debug_assert_icmp<'a>( - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - operator: IntCC, - v1: ir::Value, - v2: ir::Value, - error_str: &'static str, - ) { - if cfg!(debug_assertions) { - let cmp_res = builder.ins().icmp(operator, v1, v2); - - if super::stack_switching_environ::ENABLE_DEBUG_PRINTING { - let failure_block = builder.create_block(); - let continue_block = builder.create_block(); - - builder - .ins() - .brif(cmp_res, continue_block, &[], failure_block, &[]); - - builder.switch_to_block(failure_block); - builder.seal_block(failure_block); - - emit_debug_print(env, builder, error_str, &[v1, v2]); - builder.ins().debugtrap(); - builder.ins().jump(continue_block, &[]); - - builder.switch_to_block(continue_block); - builder.seal_block(continue_block); - } else { - builder.ins().trapz(cmp_res, super::DEBUG_ASSERT_TRAP_CODE); - } - } - } - - /// Used to implement other macros, do not use directly. - macro_rules! emit_debug_assert_icmp { - ( $env : expr, - $builder: expr, - $operator : expr, - $operator_string : expr, - $v1 : expr, - $v2 : expr) => { - let msg: &'static str = std::concat!( - "assertion failure in ", - std::file!(), - ", line ", - std::line!(), - ": {} ", - $operator_string, - " {} does not hold\n" - ); - emit_debug_assert_icmp($env, $builder, $operator, $v1, $v2, msg); - }; - } - - macro_rules! emit_debug_assert { - ($env: expr, $builder: expr, $condition: expr) => { - let msg: &'static str = std::concat!( - "assertion failure in ", - std::file!(), - ", line ", - std::line!(), - "\n" - ); - // This makes the borrow checker happy if $condition uses env or builder. - let c = $condition; - emit_debug_assert_generic($env, $builder, c, msg); - }; - } - - macro_rules! emit_debug_assert_eq { - ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { - emit_debug_assert_icmp!($env, $builder, IntCC::Equal, "==", $v1, $v2); - }; - } - - macro_rules! emit_debug_assert_ne { - ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { - emit_debug_assert_icmp!($env, $builder, IntCC::NotEqual, "!=", $v1, $v2); - }; - } - - macro_rules! emit_debug_assert_ule { - ($env: expr, $builder: expr, $v1 : expr, $v2: expr) => { - emit_debug_assert_icmp!( - $env, - $builder, - IntCC::UnsignedLessThanOrEqual, - "<=", - $v1, - $v2 - ); - }; - } -} -use delete_me::*; - /// This module contains compile-time counterparts to types defined elsewhere. pub(crate) mod stack_switching_helpers { - use super::delete_me::*; use core::marker::PhantomData; use cranelift_codegen::ir; use cranelift_codegen::ir::InstBuilder; @@ -332,7 +42,7 @@ pub(crate) mod stack_switching_helpers { /// The type parameter T is never used in the fields above. We still /// want to have it for consistency with - /// `stack_switching_environ::Vector` and to use it in the associated + /// `wasmtime_environ::Vector` and to use it in the associated /// functions. phantom: PhantomData, } @@ -342,7 +52,7 @@ pub(crate) mod stack_switching_helpers { // Actually a vector of *mut VMTagDefinition pub type VMHandlerList = VMArray<*mut u8>; - /// Compile-time representation of stack_switching_environ::VMStackChain, + /// Compile-time representation of wasmtime_environ::VMStackChain, /// consisting of two `ir::Value`s. pub struct VMStackChain { discriminant: ir::Value, @@ -466,22 +176,12 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, revision: ir::Value, ) -> ir::Value { - if cfg!(debug_assertions) { - let actual_revision = self.get_revision(env, builder); - emit_debug_assert_eq!(env, builder, revision, actual_revision); - } let mem_flags = ir::MemFlags::trusted(); let offset = env.offsets.ptr.vmcontref_revision() as i32; let revision_plus1 = builder.ins().iadd_imm(revision, 1); builder .ins() .store(mem_flags, revision_plus1, self.address, offset); - if cfg!(debug_assertions) { - let new_revision = self.get_revision(env, builder); - emit_debug_assert_eq!(env, builder, revision_plus1, new_revision); - // Check for overflow: - emit_debug_assert_ule!(env, builder, revision, revision_plus1); - } revision_plus1 } @@ -533,16 +233,6 @@ pub(crate) mod stack_switching_helpers { self.get(builder, env.pointer_type(), offset) } - fn get_capacity<'a>( - &self, - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> ir::Value { - // Array capacity is stored as u32. - let offset = env.offsets.ptr.vmarray_capacity() as i32; - self.get(builder, I32, offset) - } - pub fn get_length<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, @@ -598,11 +288,6 @@ pub(crate) mod stack_switching_helpers { let new_length = builder.ins().iadd_imm(original_length, arg_count as i64); self.set_length(env, builder, new_length); - if cfg!(debug_assertions) { - let capacity = self.get_capacity(env, builder); - emit_debug_assert_ule!(env, builder, new_length, capacity); - } - let value_size = mem::size_of::() as i64; let original_length = builder.ins().uextend(I64, original_length); let byte_offset = builder.ins().imul_imm(original_length, value_size); @@ -616,15 +301,6 @@ pub(crate) mod stack_switching_helpers { required_capacity: u32, existing_slot: Option, ) -> StackSlot { - let zero = builder.ins().iconst(ir::types::I32, 0); - if cfg!(debug_assertions) { - // We must only allocate while there is no data in the buffer. - let length = self.get_length(env, builder); - emit_debug_assert_eq!(env, builder, length, zero); - let capacity = self.get_capacity(env, builder); - emit_debug_assert_eq!(env, builder, capacity, zero); - } - let align = u8::try_from(std::mem::align_of::()).unwrap(); let entry_size = u32::try_from(std::mem::size_of::()).unwrap(); let required_size = required_capacity * entry_size; @@ -635,12 +311,6 @@ pub(crate) mod stack_switching_helpers { let existing_capacity = slot_data.size / entry_size; let capacity_value = builder.ins().iconst(I32, existing_capacity as i64); - emit_debug_println!( - env, - builder, - "[Array::allocate_or_reuse_stack_slot] Reusing existing buffer with capacity {}", - capacity_value - ); debug_assert!(align <= builder.func.get_stack_slot_data(slot).align_shift); debug_assert_eq!(builder.func.get_stack_slot_data(slot).kind, ExplicitSlot); @@ -653,15 +323,6 @@ pub(crate) mod stack_switching_helpers { } _ => { let capacity_value = builder.ins().iconst(I32, required_capacity as i64); - emit_debug_assert_ne!(env, builder, capacity_value, zero); - - emit_debug_println!( - env, - builder, - "[Array::allocate_or_reuse_stack_slot] allocating stack slot with capacity {}", - capacity_value - ); - let slot_size = ir::StackSlotData::new( ir::StackSlotKind::ExplicitSlot, required_size, @@ -687,12 +348,6 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, load_types: &[ir::Type], ) -> Vec { - if cfg!(debug_assertions) { - let length = self.get_length(env, builder); - let load_count = builder.ins().iconst(I32, load_types.len() as i64); - emit_debug_assert_ule!(env, builder, load_count, length); - } - let memflags = ir::MemFlags::trusted(); let data_start_pointer = self.get_data(env, builder); @@ -725,6 +380,7 @@ pub(crate) mod stack_switching_helpers { ) { let store_count = builder.ins().iconst(I32, values.len() as i64); + // TODO(posborne): allow_smaller only used in debug_assert! if cfg!(debug_assertions) { for val in values { let ty = builder.func.dfg.value_type(*val); @@ -734,12 +390,6 @@ pub(crate) mod stack_switching_helpers { debug_assert!(ty.bytes() as usize == std::mem::size_of::()); } } - - let capacity = self.get_capacity(env, builder); - let length = self.get_length(env, builder); - let zero = builder.ins().iconst(I32, 0); - emit_debug_assert_ule!(env, builder, store_count, capacity); - emit_debug_assert_eq!(env, builder, length, zero); } let memflags = ir::MemFlags::trusted(); @@ -788,8 +438,7 @@ pub(crate) mod stack_switching_helpers { env.offsets.ptr.size_of_vmstack_chain(), 2 * env.offsets.ptr.size() ); - let discriminant = - super::stack_switching_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; + let discriminant = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; let discriminant = builder .ins() .iconst(env.pointer_type(), discriminant as i64); @@ -808,7 +457,7 @@ pub(crate) mod stack_switching_helpers { env.offsets.ptr.size_of_vmstack_chain(), 2 * env.offsets.ptr.size() ); - let discriminant = super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; + let discriminant = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; let discriminant = builder .ins() .iconst(env.pointer_type(), discriminant as i64); @@ -819,20 +468,6 @@ pub(crate) mod stack_switching_helpers { } } - /// For debugging purposes. Emits an assertion that `self` does not correspond to - /// `VMStackChain::Absent`. - pub fn assert_not_absent<'a>( - &self, - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) { - let discriminant = super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; - let discriminant = builder - .ins() - .iconst(env.pointer_type(), discriminant as i64); - emit_debug_assert_ne!(env, builder, self.discriminant, discriminant); - } - pub fn is_initial_stack<'a>( &self, _env: &mut crate::func_environ::FuncEnvironment<'a>, @@ -841,19 +476,7 @@ pub(crate) mod stack_switching_helpers { builder.ins().icmp_imm( IntCC::Equal, self.discriminant, - super::stack_switching_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT as i64, - ) - } - - pub fn is_absent<'a>( - &self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> ir::Value { - builder.ins().icmp_imm( - IntCC::Equal, - self.discriminant, - super::stack_switching_environ::STACK_CHAIN_ABSENT_DISCRIMINANT as i64, + wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT as i64, ) } @@ -911,19 +534,10 @@ pub(crate) mod stack_switching_helpers { /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`. pub fn unchecked_get_continuation<'a>( &self, - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, + _env: &mut crate::func_environ::FuncEnvironment<'a>, + _builder: &mut FunctionBuilder, ) -> ir::Value { - if cfg!(debug_assertions) { - let continuation_discriminant = - super::stack_switching_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; - let is_continuation = builder.ins().icmp_imm( - IntCC::Equal, - self.discriminant, - continuation_discriminant as i64, - ); - emit_debug_assert!(env, builder, is_continuation); - } + // TODO(posborne): this used to have emitted assertions but now does not do much. self.payload } @@ -933,10 +547,8 @@ pub(crate) mod stack_switching_helpers { pub fn get_common_stack_information<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, + _builder: &mut FunctionBuilder, ) -> VMCommonStackInformation { - self.assert_not_absent(env, builder); - // `self` corresponds to a VMStackChain::InitialStack or // VMStackChain::Continuation. // In both cases, the payload is a pointer. @@ -995,14 +607,6 @@ pub(crate) mod stack_switching_helpers { discriminant: u32, ) { let discriminant = builder.ins().iconst(I32, discriminant as i64); - emit_debug_println!( - env, - builder, - "setting state of CommonStackInformation {:p} to {}", - self.address, - discriminant - ); - let mem_flags = ir::MemFlags::trusted(); let state_ptr = self.get_state_ptr(env, builder); @@ -1014,7 +618,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) { - let discriminant = wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT; + let discriminant = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT; self.set_state_no_payload(env, builder, discriminant); } @@ -1023,7 +627,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) { - let discriminant = wasmtime_environ::stack_switching::STACK_STATE_PARENT_DISCRIMINANT; + let discriminant = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT; self.set_state_no_payload(env, builder, discriminant); } @@ -1032,7 +636,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) { - let discriminant = wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT; + let discriminant = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT; self.set_state_no_payload(env, builder, discriminant); } @@ -1041,63 +645,10 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) { - let discriminant = - wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT; + let discriminant = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT; self.set_state_no_payload(env, builder, discriminant); } - pub fn has_state_any_of<'a>( - &self, - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - state_discriminants: &[u32], - ) -> ir::Value { - let actual_state = self.load_state(env, builder); - let zero = builder.ins().iconst(I8, 0); - let mut res = zero; - for state_discriminant in state_discriminants { - let eq = - builder - .ins() - .icmp_imm(IntCC::Equal, actual_state, *state_discriminant as i64); - res = builder.ins().bor(res, eq); - } - res - } - - pub fn has_state_returned<'a>( - &self, - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> ir::Value { - self.has_state_any_of( - env, - builder, - &[wasmtime_environ::stack_switching::STACK_STATE_RETURNED_DISCRIMINANT], - ) - } - - pub fn has_state_running<'a>( - &self, - env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> ir::Value { - self.has_state_any_of( - env, - builder, - &[wasmtime_environ::stack_switching::STACK_STATE_RUNNING_DISCRIMINANT], - ) - } - - // pub fn has_state<'a>( - // &self, - // env: &mut crate::func_environ::FuncEnvironment<'a>, - // builder: &mut FunctionBuilder, - // state: super::stack_switching_environ::VMStackState, - // ) -> ir::Value { - // self.has_state_any_of(env, builder, &[state]) - // } - /// Checks whether the `VMStackState` reflects that the stack has ever been /// active (instead of just having been allocated, but never resumed). pub fn was_invoked<'a>( @@ -1106,7 +657,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, ) -> ir::Value { let actual_state = self.load_state(env, builder); - let allocated = wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT; + let allocated = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT; builder .ins() .icmp_imm(IntCC::NotEqual, actual_state, allocated as i64) @@ -1164,12 +715,10 @@ pub(crate) mod stack_switching_helpers { let memflags = ir::MemFlags::trusted(); let mut copy_to_vm_runtime_limits = |our_offset, their_offset| { - let our_value = builder.ins().load( - env.pointer_type(), - memflags, - stack_limits_ptr, - i32::try_from(our_offset).unwrap(), - ); + let our_value = + builder + .ins() + .load(env.pointer_type(), memflags, stack_limits_ptr, our_offset); builder.ins().store( memflags, our_value, @@ -1220,7 +769,7 @@ pub(crate) mod stack_switching_helpers { memflags, from_vm_runtime_limits, stack_limits_ptr, - i32::try_from(stack_limits_offset).unwrap(), + stack_limits_offset, ); }; @@ -1273,9 +822,6 @@ pub(crate) mod stack_switching_helpers { } use helpers::VMStackChain; -use stack_switching_environ::{ - CONTROL_EFFECT_RESUME_DISCRIMINANT, CONTROL_EFFECT_SWITCH_DISCRIMINANT, -}; use stack_switching_helpers as helpers; /// Stores the given arguments in the appropriate `VMPayloads` object in the @@ -1339,7 +885,7 @@ pub(crate) fn vmcontref_store_payloads<'a>( let mut offset = 0; for value in values { builder.ins().store(memflags, *value, ptr, offset); - offset += env.offsets.ptr.maximum_value_size() as i32; + offset += i32::from(env.offsets.ptr.maximum_value_size()); } } } @@ -1355,7 +901,7 @@ pub(crate) fn tag_address<'a>( let pointer_type = env.pointer_type(); if let Some(def_index) = env.module.defined_tag_index(tag_index) { let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap(); - builder.ins().iadd_imm(vmctx, offset as i64) + builder.ins().iadd_imm(vmctx, i64::from(offset)) } else { let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap(); builder.ins().load( @@ -1531,14 +1077,6 @@ fn search_handler<'a>( let contref = helpers::VMContRef::new(contref); let parent_link = contref.get_parent_stack_chain(env, builder); - - emit_debug_println!( - env, - builder, - "[search_handler] beginning search in parent of continuation {:p}", - contref.address - ); - let parent_csi = parent_link.get_common_stack_information(env, builder); let handlers = parent_csi.get_handler_list(env, builder); @@ -1549,10 +1087,6 @@ fn search_handler<'a>( // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range). let (begin_range, end_range) = if search_suspend_handlers { let zero = builder.ins().iconst(I32, 0); - if cfg!(debug_assertions) { - let length = handlers.get_length(env, builder); - emit_debug_assert_ule!(env, builder, first_switch_handler_index, length); - } (zero, first_switch_handler_index) } else { let length = handlers.get_length(env, builder); @@ -1626,16 +1160,6 @@ fn search_handler<'a>( // final block: on_match builder.switch_to_block(on_match); - emit_debug_println!( - env, - builder, - "[search_handler] found handler at stack chain ({}, {:p}), whose child continuation is {:p}, index is {}", - parent_link.to_raw_parts()[0], - parent_link.to_raw_parts()[1], - contref.address, - index - ); - (parent_link, contref.address, index) } @@ -1653,14 +1177,6 @@ pub(crate) fn translate_cont_bind<'a>( let mut vmcontref = helpers::VMContRef::new(contref); let revision = vmcontref.get_revision(env, builder); let evidence = builder.ins().icmp(IntCC::Equal, witness, revision); - emit_debug_println!( - env, - builder, - "[cont_bind] witness = {}, revision = {}, evidence = {}", - witness, - revision, - evidence - ); builder .ins() .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); @@ -1668,9 +1184,7 @@ pub(crate) fn translate_cont_bind<'a>( vmcontref_store_payloads(env, builder, args, contref); let revision = vmcontref.incr_revision(env, builder, revision); - emit_debug_println!(env, builder, "new revision = {}", revision); let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref); - emit_debug_println!(env, builder, "[cont_bind] contref = {:p}", contref); contobj } @@ -1696,7 +1210,6 @@ pub(crate) fn translate_cont_new<'a>( let tag = helpers::VMContRef::new(contref).get_revision(env, builder); let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref); - emit_debug_println!(env, builder, "[cont_new] contref = {:p}", contref); Ok(contobj) } @@ -1780,28 +1293,10 @@ pub(crate) fn translate_resume<'a>( let revision = vmcontref.get_revision(env, builder); let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); - emit_debug_println!( - env, - builder, - "[resume] resume_contref = {:p} witness = {}, revision = {}, evidence = {}", - resume_contref, - witness, - revision, - evidence - ); builder .ins() .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); - let next_revision = vmcontref.incr_revision(env, builder, revision); - emit_debug_println!(env, builder, "[resume] new revision = {}", next_revision); - - if cfg!(debug_assertions) { - // This should be impossible due to the linearity check. - let zero = builder.ins().iconst(I8, 0); - let csi = vmcontref.common_stack_information(env, builder); - let has_returned = csi.has_state_returned(env, builder); - emit_debug_assert_eq!(env, builder, has_returned, zero); - } + let _next_revision = vmcontref.incr_revision(env, builder, revision); if resume_args.len() > 0 { // We store the arguments in the `VMContRef` to be resumed. @@ -1814,25 +1309,8 @@ pub(crate) fn translate_resume<'a>( // Make the currently running continuation (if any) the parent of the one we are about to resume. let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); - original_stack_chain.assert_not_absent(env, builder); - if cfg!(debug_assertions) { - // The continuation we are about to resume should have its chain broken up at last_ancestor. - let last_ancestor_chain = last_ancestor.get_parent_stack_chain(env, builder); - let is_absent = last_ancestor_chain.is_absent(env, builder); - emit_debug_assert!(env, builder, is_absent); - } last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain); - emit_debug_println!( - env, - builder, - "[resume] spliced together stack chains: parent of {:p} (last ancestor of {:p}) is now pointing to ({}, {:p})", - last_ancestor.address, - vmcontref.address, - original_stack_chain.to_raw_parts()[0], - original_stack_chain.to_raw_parts()[1] - ); - // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field. let zero = builder.ins().iconst(env.pointer_type(), 0); vmcontref.set_last_ancestor(env, builder, zero); @@ -1849,7 +1327,7 @@ pub(crate) fn translate_resume<'a>( // 2. Copy `stack_limit` and `last_wasm_entry_sp` in the // `VMStackLimits` of `resume_contref` into the `VMRuntimeLimits`. // - // See the comment on `stack_switching_environ::VMStackChain` for a + // See the comment on `wasmtime_environ::VMStackChain` for a // description of the invariants that we maintain for the various stack // limits. @@ -1862,7 +1340,7 @@ pub(crate) fn translate_resume<'a>( // We update the `VMStackLimits` of the parent of the continuation to be resumed // as well as the `VMRuntimeLimits`. - // See the comment on `stack_switching_environ::VMStackChain` for a description + // See the comment on `wasmtime_environ::VMStackChain` for a description // of the invariants that we maintain for the various stack limits. let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx); parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true); @@ -1917,27 +1395,11 @@ pub(crate) fn translate_resume<'a>( let fiber_stack = last_ancestor.get_fiber_stack(env, builder); let control_context_ptr = fiber_stack.load_control_context(env, builder); - emit_debug_println!( - env, - builder, - "[resume] about to execute stack_switch, control_context_ptr is {:p}", - control_context_ptr - ); - let result = builder .ins() .stack_switch(control_context_ptr, control_context_ptr, resume_payload); - emit_debug_println!( - env, - builder, - "[resume] continuing after stack_switch in frame with parent_stack_chain ({}, {:p}), result is {:p}", - original_stack_chain.to_raw_parts()[0], - original_stack_chain.to_raw_parts()[1], - result - ); - // At this point we know nothing about the continuation that just // suspended or returned. In particular, it does not have to be what we // called `resume_contref` earlier on. We must reload the information @@ -1956,13 +1418,6 @@ pub(crate) fn translate_resume<'a>( let result = ControlEffect::from_u64(result); let signal = result.signal(env, builder); - emit_debug_println!( - env, - builder, - "[resume] in resume block, signal is {}", - signal - ); - // Jump to the return block if the result signal is 0, otherwise jump to // the suspend block. builder @@ -2011,15 +1466,6 @@ pub(crate) fn translate_resume<'a>( suspended_continuation.address, ); - emit_debug_println!( - env, - builder, - "[resume] in suspend block, handler index is {}, new continuation is {:p}, with existing revision {}", - handler_index, - suspended_continuation.address, - revision - ); - // We need to terminate this block before being allowed to switch to // another one. builder.ins().jump(dispatch_block, &[]); @@ -2143,7 +1589,6 @@ pub(crate) fn translate_suspend<'a>( tag_return_types: &[ir::Type], ) -> Vec { let tag_addr = tag_address(env, builder, tag_index); - emit_debug_println!(env, builder, "[suspend] suspending with tag {:p}", tag_addr); let vmctx = env.vmctx_val(&mut builder.cursor()); let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); @@ -2151,14 +1596,6 @@ pub(crate) fn translate_suspend<'a>( let (_, end_of_chain_contref, handler_index) = search_handler(env, builder, &active_stack_chain, tag_addr, true); - emit_debug_println!( - env, - builder, - "[suspend] found handler: end of chain contref is {:p}, handler index is {}", - end_of_chain_contref, - handler_index - ); - // If we get here, the search_handler logic succeeded (i.e., did not trap). // Thus, there is at least one parent, so we are not on the initial stack. // Can therefore extract continuation directly. @@ -2192,11 +1629,6 @@ pub(crate) fn translate_suspend<'a>( // Set current continuation to suspended and break up handler chain. let active_contref_csi = active_contref.common_stack_information(env, builder); - if cfg!(debug_assertions) { - let is_running = active_contref_csi.has_state_running(env, builder); - emit_debug_assert!(env, builder, is_running); - } - active_contref_csi.set_state_suspended(env, builder); let absent_chain_link = VMStackChain::absent(env, builder); end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link); @@ -2251,15 +1683,6 @@ pub(crate) fn translate_switch<'a>( let revision = target_contref.get_revision(env, builder); let evidence = builder.ins().icmp(IntCC::Equal, revision, witness); - emit_debug_println!( - env, - builder, - "[switch] target_contref = {:p} witness = {}, revision = {}, evidence = {}", - target_contref.address, - witness, - revision, - evidence - ); builder .ins() .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED); @@ -2308,11 +1731,6 @@ pub(crate) fn translate_switch<'a>( } let switcher_contref_csi = switcher_contref.common_stack_information(env, builder); - emit_debug_assert!( - env, - builder, - switcher_contref_csi.has_state_running(env, builder) - ); switcher_contref_csi.set_state_suspended(env, builder); // We break off `switcher_contref` from the chain of active // continuations, by separating the link between `last_ancestor` and its @@ -2333,14 +1751,6 @@ pub(crate) fn translate_switch<'a>( switcher_contref.address, ); - emit_debug_println!( - env, - builder, - "[switch] created new contref = {:p}, revision = {}", - switcher_contref.address, - revision - ); - ( switcher_contref, new_contobj, @@ -2362,19 +1772,6 @@ pub(crate) fn translate_switch<'a>( vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address); let switchee_contref_csi = switchee_contref.common_stack_information(env, builder); - - emit_debug_assert!( - env, - builder, - switchee_contref_csi.has_state_any_of( - env, - builder, - &[ - wasmtime_environ::stack_switching::STACK_STATE_FRESH_DISCRIMINANT, - wasmtime_environ::stack_switching::STACK_STATE_SUSPENDED_DISCRIMINANT - ] - ) - ); switchee_contref_csi.set_state_running(env, builder); let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder); @@ -2478,60 +1875,16 @@ pub(crate) fn translate_switch<'a>( let switch_payload = ControlEffect::encode_switch(env, builder).to_u64(); - emit_debug_println!( - env, - builder, - "[switch] about to execute stack_switch, store_control_context_ptr is {:p}, load_control_context_ptr {:p}, tmp_control_context is {:p}", - switcher_last_ancestor_cc, - switchee_last_ancestor_cc, - tmp_control_context - ); - - let result = builder.ins().stack_switch( + let _result = builder.ins().stack_switch( switcher_last_ancestor_cc, tmp_control_context, switch_payload, ); - - emit_debug_println!( - env, - builder, - "[switch] continuing after stack_switch in frame with stack chain ({}, {:p}), result is {:p}", - handler_stack_chain.to_raw_parts()[0], - handler_stack_chain.to_raw_parts()[1], - result - ); - - if cfg!(debug_assertions) { - // The only way to switch back to this point is by using resume or switch instructions. - let result_control_effect = ControlEffect::from_u64(result); - let result_discriminant = result_control_effect.signal(env, builder); - let is_resume = builder.ins().icmp_imm( - IntCC::Equal, - result_discriminant, - CONTROL_EFFECT_RESUME_DISCRIMINANT as i64, - ); - let is_switch = builder.ins().icmp_imm( - IntCC::Equal, - result_discriminant, - CONTROL_EFFECT_SWITCH_DISCRIMINANT as i64, - ); - let is_switch_or_resume = builder.ins().bor(is_switch, is_resume); - emit_debug_assert!(env, builder, is_switch_or_resume); - } } // After switching back to the original stack: Load return values, they are // stored on the switcher continuation. let return_values = { - if cfg!(debug_assertions) { - // The originally active continuation (before the switch) should be active again. - let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx); - // This has a debug assertion that also checks that the `active_stack_chain` is indeed a continuation. - let active_contref = active_stack_chain.unchecked_get_continuation(env, builder); - emit_debug_assert_eq!(env, builder, switcher_contref.address, active_contref); - } - let payloads = switcher_contref.values(env, builder); let return_values = payloads.load_data_entries(env, builder, return_types); // We consume the values and discard the buffer (allocated on this stack) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 517450c7cbb2..a7f4271600be 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -162,6 +162,12 @@ pub trait PtrSize { 4 } + /// This is the size of the largest value type (i.e. a V128). + #[inline] + fn maximum_value_size(&self) -> u8 { + self.size_of_vmglobal_definition() + } + // Offsets within `VMStoreContext` /// Return the offset of the `fuel_consumed` field of `VMStoreContext` From 858b22ab6404f34ab8c70b55deb570bbc8f5ba4f Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 6 Jun 2025 20:20:24 +0000 Subject: [PATCH 03/34] cranelift: stack-switching: restore original visibility for a few func_environ members --- crates/cranelift/src/func_environ.rs | 4 ++-- crates/cranelift/src/stack_switching/fatpointer.rs | 4 ++-- crates/cranelift/src/stack_switching/instructions.rs | 4 ++-- crates/cranelift/src/translate/code_translator.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index dad67b88368d..037070503f53 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -94,9 +94,9 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`. pub struct FuncEnvironment<'module_environment> { compiler: &'module_environment Compiler, - pub(crate) isa: &'module_environment (dyn TargetIsa + 'module_environment), + isa: &'module_environment (dyn TargetIsa + 'module_environment), pub(crate) module: &'module_environment Module, - pub(crate) types: &'module_environment ModuleTypesBuilder, + types: &'module_environment ModuleTypesBuilder, wasm_func_ty: &'module_environment WasmFuncType, sig_ref_to_ty: SecondaryMap>, needs_gc_heap: bool, diff --git a/crates/cranelift/src/stack_switching/fatpointer.rs b/crates/cranelift/src/stack_switching/fatpointer.rs index fc4b4ef3de41..028c79bbc3ce 100644 --- a/crates/cranelift/src/stack_switching/fatpointer.rs +++ b/crates/cranelift/src/stack_switching/fatpointer.rs @@ -24,7 +24,7 @@ pub(crate) fn deconstruct<'a>( let (lsbs, msbs) = pos.ins().isplit(contobj); - let (revision_counter, contref) = match env.isa.endianness() { + let (revision_counter, contref) = match env.isa().endianness() { ir::Endianness::Little => (lsbs, msbs), ir::Endianness::Big => { let pad_bits = 64 - env.pointer_type().bits(); @@ -55,7 +55,7 @@ pub(crate) fn construct<'a>( } else { contref_addr }; - let (msbs, lsbs) = match env.isa.endianness() { + let (msbs, lsbs) = match env.isa().endianness() { ir::Endianness::Little => (contref_addr, revision_counter), ir::Endianness::Big => { let pad_bits = 64 - env.pointer_type().bits(); diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 6a3eb587d878..d06fe5a240e5 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -1499,7 +1499,7 @@ pub(crate) fn translate_resume<'a>( let param_types = env.tag_params(TagIndex::from_u32(handle_tag)); let param_types: Vec = param_types .iter() - .map(|wty| crate::value_type(env.isa, *wty)) + .map(|wty| crate::value_type(env.isa(), *wty)) .collect(); let values = suspended_contref.values(env, builder); @@ -1571,7 +1571,7 @@ pub(crate) fn translate_resume<'a>( let return_types: Vec<_> = env .continuation_returns(TypeIndex::from_u32(type_index)) .iter() - .map(|ty| crate::value_type(env.isa, *ty)) + .map(|ty| crate::value_type(env.isa(), *ty)) .collect(); let payloads = returned_contref.args(env, builder); let return_values = payloads.load_data_entries(env, builder, &return_types); diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 5972e57d854b..ad6c02c41b23 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -2944,7 +2944,7 @@ pub fn translate_operator( let return_types: Vec<_> = environ .tag_returns(tag_index) .iter() - .map(|ty| crate::value_type(environ.isa, *ty)) + .map(|ty| crate::value_type(environ.isa(), *ty)) .collect(); let params = state.peekn(param_types.len()); @@ -3026,7 +3026,7 @@ pub fn translate_operator( environ .continuation_arguments(TypeIndex::from_u32(mti.as_u32())) .iter() - .map(|ty| crate::value_type(environ.isa, *ty)) + .map(|ty| crate::value_type(environ.isa(), *ty)) .collect() } _ => panic!("Invalid type on switch"), From 43a39723d6c719e514f4cb75a2be87db29d6ee5d Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 6 Jun 2025 16:44:48 +0000 Subject: [PATCH 04/34] cranelift: stack-switching conditional compilation At this point, the only bit we really branch on is what we do in order to avoid problems tying into wasmtime_environ. This is basd on the approach and macro used by the gc code for converting presence/absence of the cranelift feature flag to cranelift compile time. This is a bit of a half-measure for now as we still compile most stack-switching code in cranelift, but this does enough to avoid causing problems with missing definitions in wasmtime_environ. --- crates/cranelift/src/func_environ.rs | 6 ++-- .../src/stack_switching/instructions.rs | 2 +- crates/cranelift/src/stack_switching/mod.rs | 34 +++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 037070503f53..dc74a87e8444 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1860,8 +1860,7 @@ impl FuncEnvironment<'_> { init_value, ); args.extend_from_slice(&[contref, revision]); - self.builtin_functions - .table_grow_cont_obj(&mut builder.func) + stack_switching::builtins::table_grow_cont_obj(self, &mut builder.func)? } _ => panic!("unsupported table type."), @@ -1999,8 +1998,7 @@ impl FuncEnvironment<'_> { let (revision, contref) = stack_switching::fatpointer::deconstruct(self, &mut builder.cursor(), val); args.extend_from_slice(&[contref, revision]); - self.builtin_functions - .table_fill_cont_obj(&mut builder.func) + stack_switching::builtins::table_fill_cont_obj(self, &mut builder.func)? } _ => panic!("unsupported table type"), } diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index d06fe5a240e5..7a1ce46e1b0c 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -1201,7 +1201,7 @@ pub(crate) fn translate_cont_new<'a>( let nargs = builder.ins().iconst(I32, arg_types.len() as i64); let nreturns = builder.ins().iconst(I32, return_types.len() as i64); - let cont_new_func = env.builtin_functions.cont_new(&mut builder.func); + let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?; let vmctx = env.vmctx_val(&mut builder.cursor()); let call_inst = builder .ins() diff --git a/crates/cranelift/src/stack_switching/mod.rs b/crates/cranelift/src/stack_switching/mod.rs index 653191a769fa..25381f727942 100644 --- a/crates/cranelift/src/stack_switching/mod.rs +++ b/crates/cranelift/src/stack_switching/mod.rs @@ -1,3 +1,37 @@ mod control_effect; pub(crate) mod fatpointer; pub(crate) mod instructions; + +pub(crate) mod builtins { + macro_rules! define_builtin_accessors { + ( $( $name:ident , )* ) => { + $( + #[inline] + pub fn $name( + func_env: &mut crate::func_environ::FuncEnvironment<'_>, + func: &mut crate::ir::Function, + ) -> wasmtime_environ::WasmResult { + #[cfg(feature = "stack-switching")] + { + return Ok(func_env.builtin_functions.$name(func)); + } + + #[cfg(not(feature = "stack-switching"))] + { + let _ = (func, func_env); + return Err(wasmtime_environ::wasm_unsupported!( + "support for Wasm Stack Switching disabled at compile time because the `stack-switching` cargo \ + feature was not enabled" + )); + } + } + )* + }; + } + + define_builtin_accessors! { + cont_new, + table_grow_cont_obj, + table_fill_cont_obj, + } +} From 100d621fd2103b94b78fd73c7472b0f016aad2b0 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 6 Jun 2025 17:53:13 +0000 Subject: [PATCH 05/34] cranelift: avoid "as" casts in stack-switching Replace either with infallible From or fallible, panicing TryFrom alternatives where required. --- .../src/stack_switching/instructions.rs | 136 ++++++++++-------- 1 file changed, 75 insertions(+), 61 deletions(-) diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 7a1ce46e1b0c..d3a59ea68d8b 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -81,7 +81,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, _builder: &mut FunctionBuilder, ) -> VMPayloads { - let offset = env.offsets.ptr.vmcontref_args() as i32; + let offset = env.offsets.ptr.vmcontref_args().into(); VMPayloads::new(self.address, offset) } @@ -90,7 +90,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, _builder: &mut FunctionBuilder, ) -> VMPayloads { - let offset = env.offsets.ptr.vmcontref_values() as i32; + let offset = env.offsets.ptr.vmcontref_values().into(); VMPayloads::new(self.address, offset) } @@ -99,7 +99,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> VMCommonStackInformation { - let offset = env.offsets.ptr.vmcontref_common_stack_information() as i64; + let offset: i64 = env.offsets.ptr.vmcontref_common_stack_information().into(); let address = builder.ins().iadd_imm(self.address, offset); VMCommonStackInformation { address } } @@ -113,7 +113,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, new_stack_chain: &VMStackChain, ) { - let offset = env.offsets.ptr.vmcontref_parent_chain() as i32; + let offset = env.offsets.ptr.vmcontref_parent_chain().into(); new_stack_chain.store(env, builder, self.address, offset) } @@ -125,7 +125,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> VMStackChain { - let offset = env.offsets.ptr.vmcontref_parent_chain() as i32; + let offset = env.offsets.ptr.vmcontref_parent_chain().into(); VMStackChain::load(env, builder, self.address, offset, env.pointer_type()) } @@ -135,7 +135,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, last_ancestor: ir::Value, ) { - let offset = env.offsets.ptr.vmcontref_last_ancestor() as i32; + let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into(); let mem_flags = ir::MemFlags::trusted(); builder .ins() @@ -147,7 +147,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> ir::Value { - let offset = env.offsets.ptr.vmcontref_last_ancestor() as i32; + let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into(); let mem_flags = ir::MemFlags::trusted(); builder .ins() @@ -162,7 +162,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, ) -> ir::Value { let mem_flags = ir::MemFlags::trusted(); - let offset = env.offsets.ptr.vmcontref_revision() as i32; + let offset: i32 = env.offsets.ptr.vmcontref_revision().into(); let revision = builder.ins().load(I64, mem_flags, self.address, offset); revision } @@ -177,7 +177,7 @@ pub(crate) mod stack_switching_helpers { revision: ir::Value, ) -> ir::Value { let mem_flags = ir::MemFlags::trusted(); - let offset = env.offsets.ptr.vmcontref_revision() as i32; + let offset: i32 = env.offsets.ptr.vmcontref_revision().into(); let revision_plus1 = builder.ins().iadd_imm(revision, 1); builder .ins() @@ -191,7 +191,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, ) -> VMContinuationStack { // The top of stack field is stored at offset 0 of the `FiberStack`. - let offset = env.offsets.ptr.vmcontref_stack() as i64; + let offset: i64 = env.offsets.ptr.vmcontref_stack().into(); let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm(self.address, offset); VMContinuationStack::new(fiber_stack_top_of_stack_ptr) } @@ -229,7 +229,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> ir::Value { - let offset = env.offsets.ptr.vmarray_data() as i32; + let offset = env.offsets.ptr.vmarray_data().into(); self.get(builder, env.pointer_type(), offset) } @@ -239,7 +239,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, ) -> ir::Value { // Array length is stored as u32. - let offset = env.offsets.ptr.vmarray_length() as i32; + let offset = env.offsets.ptr.vmarray_length().into(); self.get(builder, I32, offset) } @@ -250,7 +250,7 @@ pub(crate) mod stack_switching_helpers { length: ir::Value, ) { // Array length is stored as u32. - let offset = env.offsets.ptr.vmarray_length() as i32; + let offset = env.offsets.ptr.vmarray_length().into(); self.set::(builder, offset, length); } @@ -261,7 +261,7 @@ pub(crate) mod stack_switching_helpers { capacity: ir::Value, ) { // Array capacity is stored as u32. - let offset = env.offsets.ptr.vmarray_capacity() as i32; + let offset = env.offsets.ptr.vmarray_capacity().into(); self.set::(builder, offset, capacity); } @@ -271,7 +271,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, data: ir::Value, ) { - let offset = env.offsets.ptr.vmarray_data() as i32; + let offset = env.offsets.ptr.vmarray_data().into(); self.set::<*mut T>(builder, offset, data); } @@ -285,10 +285,12 @@ pub(crate) mod stack_switching_helpers { ) -> ir::Value { let data = self.get_data(env, builder); let original_length = self.get_length(env, builder); - let new_length = builder.ins().iadd_imm(original_length, arg_count as i64); + let new_length = builder + .ins() + .iadd_imm(original_length, i64::from(arg_count)); self.set_length(env, builder, new_length); - let value_size = mem::size_of::() as i64; + let value_size: i64 = mem::size_of::().try_into().unwrap(); let original_length = builder.ins().uextend(I64, original_length); let byte_offset = builder.ins().imul_imm(original_length, value_size); builder.ins().iadd(data, byte_offset) @@ -310,7 +312,7 @@ pub(crate) mod stack_switching_helpers { let slot_data = builder.func.get_stack_slot_data(slot).clone(); let existing_capacity = slot_data.size / entry_size; - let capacity_value = builder.ins().iconst(I32, existing_capacity as i64); + let capacity_value = builder.ins().iconst(I32, i64::from(existing_capacity)); debug_assert!(align <= builder.func.get_stack_slot_data(slot).align_shift); debug_assert_eq!(builder.func.get_stack_slot_data(slot).kind, ExplicitSlot); @@ -322,7 +324,7 @@ pub(crate) mod stack_switching_helpers { slot } _ => { - let capacity_value = builder.ins().iconst(I32, required_capacity as i64); + let capacity_value = builder.ins().iconst(I32, i64::from(required_capacity)); let slot_size = ir::StackSlotData::new( ir::StackSlotKind::ExplicitSlot, required_size, @@ -378,16 +380,19 @@ pub(crate) mod stack_switching_helpers { values: &[ir::Value], allow_smaller: bool, ) { - let store_count = builder.ins().iconst(I32, values.len() as i64); + let store_count = builder + .ins() + .iconst(I32, i64::try_from(values.len()).unwrap()); // TODO(posborne): allow_smaller only used in debug_assert! if cfg!(debug_assertions) { for val in values { let ty = builder.func.dfg.value_type(*val); + let size = usize::try_from(ty.bytes()).unwrap(); if allow_smaller { - debug_assert!(ty.bytes() as usize <= std::mem::size_of::()); + debug_assert!(size <= std::mem::size_of::()); } else { - debug_assert!(ty.bytes() as usize == std::mem::size_of::()); + debug_assert!(size == std::mem::size_of::()); } } } @@ -441,7 +446,7 @@ pub(crate) mod stack_switching_helpers { let discriminant = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT; let discriminant = builder .ins() - .iconst(env.pointer_type(), discriminant as i64); + .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap()); VMStackChain { discriminant, payload: contref, @@ -460,7 +465,7 @@ pub(crate) mod stack_switching_helpers { let discriminant = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT; let discriminant = builder .ins() - .iconst(env.pointer_type(), discriminant as i64); + .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap()); let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64); VMStackChain { discriminant, @@ -476,7 +481,7 @@ pub(crate) mod stack_switching_helpers { builder.ins().icmp_imm( IntCC::Equal, self.discriminant, - wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT as i64, + i64::try_from(wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT).unwrap(), ) } @@ -506,7 +511,7 @@ pub(crate) mod stack_switching_helpers { let mut data = vec![]; for _ in 0..2 { data.push(builder.ins().load(pointer_type, memflags, pointer, offset)); - offset += pointer_type.bytes() as i32; + offset += i32::try_from(pointer_type.bytes()).unwrap(); } let data = <[ir::Value; 2]>::try_from(data).unwrap(); Self::from_raw_parts(data) @@ -527,7 +532,7 @@ pub(crate) mod stack_switching_helpers { for value in data { debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type()); builder.ins().store(memflags, value, target_pointer, offset); - offset += env.pointer_type().bytes() as i32; + offset += i32::try_from(env.pointer_type().bytes()).unwrap(); } } @@ -571,20 +576,20 @@ pub(crate) mod stack_switching_helpers { impl VMCommonStackInformation { fn get_state_ptr<'a>( &self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, + env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> ir::Value { - let offset = _env.offsets.ptr.vmcommon_stack_information_state() as i64; + let offset: i64 = env.offsets.ptr.vmcommon_stack_information_state().into(); builder.ins().iadd_imm(self.address, offset) } fn get_stack_limits_ptr<'a>( &self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, + env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> ir::Value { - let offset = _env.offsets.ptr.vmcommon_stack_information_limits() as i64; + let offset: i64 = env.offsets.ptr.vmcommon_stack_information_limits().into(); builder.ins().iadd_imm(self.address, offset) } @@ -606,7 +611,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, discriminant: u32, ) { - let discriminant = builder.ins().iconst(I32, discriminant as i64); + let discriminant = builder.ins().iconst(I32, i64::from(discriminant)); let mem_flags = ir::MemFlags::trusted(); let state_ptr = self.get_state_ptr(env, builder); @@ -660,7 +665,7 @@ pub(crate) mod stack_switching_helpers { let allocated = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT; builder .ins() - .icmp_imm(IntCC::NotEqual, actual_state, allocated as i64) + .icmp_imm(IntCC::NotEqual, actual_state, i64::from(allocated)) } pub fn get_handler_list<'a>( @@ -668,7 +673,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, _builder: &mut FunctionBuilder, ) -> VMHandlerList { - let offset = env.offsets.ptr.vmcommon_stack_information_handlers() as i32; + let offset = env.offsets.ptr.vmcommon_stack_information_handlers().into(); VMHandlerList::new(self.address, offset) } @@ -679,10 +684,11 @@ pub(crate) mod stack_switching_helpers { ) -> ir::Value { // Field first_switch_handler_index has type u32 let memflags = ir::MemFlags::trusted(); - let offset = - env.offsets - .ptr - .vmcommon_stack_information_first_switch_handler_index() as i32; + let offset: i32 = env + .offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + .into(); builder.ins().load(I32, memflags, self.address, offset) } @@ -694,10 +700,11 @@ pub(crate) mod stack_switching_helpers { ) { // Field first_switch_handler_index has type u32 let memflags = ir::MemFlags::trusted(); - let offset = - env.offsets - .ptr - .vmcommon_stack_information_first_switch_handler_index() as i32; + let offset: i32 = env + .offsets + .ptr + .vmcommon_stack_information_first_switch_handler_index() + .into(); builder.ins().store(memflags, value, self.address, offset); } @@ -715,22 +722,23 @@ pub(crate) mod stack_switching_helpers { let memflags = ir::MemFlags::trusted(); let mut copy_to_vm_runtime_limits = |our_offset, their_offset| { - let our_value = - builder - .ins() - .load(env.pointer_type(), memflags, stack_limits_ptr, our_offset); + let our_value = builder.ins().load( + env.pointer_type(), + memflags, + stack_limits_ptr, + i32::from(our_offset), + ); builder.ins().store( memflags, our_value, vmruntime_limits_ptr, - their_offset as i32, + i32::from(their_offset), ); }; let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap(); - let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit() as i32; - let last_wasm_entry_fp_offset = - env.offsets.ptr.vmstack_limits_last_wasm_entry_fp() as i32; + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit(); + let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp(); copy_to_vm_runtime_limits( stack_limit_offset, pointer_size.vmstore_context_stack_limit(), @@ -773,15 +781,14 @@ pub(crate) mod stack_switching_helpers { ); }; - let last_wasm_entry_fp_offset = - env.offsets.ptr.vmstack_limits_last_wasm_entry_fp() as i32; + let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp(); copy( pointer_size.vmstore_context_last_wasm_entry_fp(), last_wasm_entry_fp_offset, ); if load_stack_limit { - let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit() as i32; + let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit(); copy( pointer_size.vmstore_context_stack_limit(), stack_limit_offset, @@ -921,7 +928,7 @@ pub fn vmctx_load_stack_chain<'a>( builder: &mut FunctionBuilder, vmctx: ir::Value, ) -> VMStackChain { - let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain() as i32; + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into(); // First we need to get the `VMStoreContext`. let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); @@ -949,7 +956,7 @@ pub fn vmctx_store_stack_chain<'a>( vmctx: ir::Value, stack_chain: &VMStackChain, ) { - let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain() as i32; + let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into(); // First we need to get the `VMStoreContext`. let vm_store_context_offset = env.offsets.ptr.vmctx_store_context(); @@ -1122,7 +1129,9 @@ fn search_handler<'a>( let base = handler_list_data_ptr; let entry_size = std::mem::size_of::<*mut u8>(); - let offset = builder.ins().imul_imm(index, entry_size as i64); + let offset = builder + .ins() + .imul_imm(index, i64::try_from(entry_size).unwrap()); let offset = builder.ins().uextend(I64, offset); let entry_address = builder.ins().iadd(base, offset); @@ -1198,8 +1207,12 @@ pub(crate) fn translate_cont_new<'a>( // The typing rules for cont.new allow a null reference to be passed to it. builder.ins().trapz(func, crate::TRAP_NULL_REFERENCE); - let nargs = builder.ins().iconst(I32, arg_types.len() as i64); - let nreturns = builder.ins().iconst(I32, return_types.len() as i64); + let nargs = builder + .ins() + .iconst(I32, i64::try_from(arg_types.len()).unwrap()); + let nreturns = builder + .ins() + .iconst(I32, i64::try_from(return_types.len()).unwrap()); let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?; let vmctx = env.vmctx_val(&mut builder.cursor()); @@ -1383,8 +1396,9 @@ pub(crate) fn translate_resume<'a>( // To enable distinguishing switch and suspend handlers when searching the handler list: // Store at which index the switch handlers start. - let first_switch_handler_index = - builder.ins().iconst(I32, suspend_handler_count as i64); + let first_switch_handler_index = builder + .ins() + .iconst(I32, i64::try_from(suspend_handler_count).unwrap()); parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index); } @@ -1870,7 +1884,7 @@ pub(crate) fn translate_switch<'a>( .ins() .store(flags, tmp2, switchee_last_ancestor_cc, offset); - offset += env.pointer_type().bytes() as i32; + offset += i32::try_from(env.pointer_type().bytes()).unwrap(); } let switch_payload = ControlEffect::encode_switch(env, builder).to_u64(); From 00c2b56e841f3f8d9a653dfed70c319bf4c44445 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 6 Jun 2025 19:41:00 +0000 Subject: [PATCH 06/34] cranelift: cleanup stack-switching control_effect signatures After removing emission of runtime trace logging and assertions, there were several unused parameters. Remove those from the ControlEffect signatures completely. --- .../src/stack_switching/control_effect.rs | 28 ++++--------------- .../src/stack_switching/instructions.rs | 10 +++---- 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/crates/cranelift/src/stack_switching/control_effect.rs b/crates/cranelift/src/stack_switching/control_effect.rs index bd93b6feebc0..558fdfb87043 100644 --- a/crates/cranelift/src/stack_switching/control_effect.rs +++ b/crates/cranelift/src/stack_switching/control_effect.rs @@ -14,11 +14,7 @@ pub struct ControlEffect(ir::Value); impl ControlEffect { // Returns the discriminant - pub fn signal<'a>( - &self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> ir::Value { + pub fn signal(&self, builder: &mut FunctionBuilder) -> ir::Value { builder.ins().ushr_imm(self.0, 32) } @@ -30,10 +26,7 @@ impl ControlEffect { self.0 } - pub fn encode_resume<'a>( - _env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> Self { + pub fn encode_resume(builder: &mut FunctionBuilder) -> Self { let discriminant = builder.ins().iconst( I64, i64::from(wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT), @@ -43,10 +36,7 @@ impl ControlEffect { Self(val) } - pub fn encode_switch<'a>( - _env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> Self { + pub fn encode_switch(builder: &mut FunctionBuilder) -> Self { let discriminant = builder.ins().iconst( I64, i64::from(wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT), @@ -56,11 +46,7 @@ impl ControlEffect { Self(val) } - pub fn encode_suspend<'a>( - _env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - handler_index: ir::Value, - ) -> Self { + pub fn encode_suspend(builder: &mut FunctionBuilder, handler_index: ir::Value) -> Self { let discriminant = builder.ins().iconst( I64, i64::from(wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT), @@ -73,11 +59,7 @@ impl ControlEffect { } /// Returns the payload of the `Suspend` variant - pub fn handler_index<'a>( - self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, - builder: &mut FunctionBuilder, - ) -> ir::Value { + pub fn handler_index(self, builder: &mut FunctionBuilder) -> ir::Value { builder.ins().ireduce(I32, self.0) } } diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index d3a59ea68d8b..8a8be0748228 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -1402,7 +1402,7 @@ pub(crate) fn translate_resume<'a>( parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index); } - let resume_payload = ControlEffect::encode_resume(env, builder).to_u64(); + let resume_payload = ControlEffect::encode_resume(builder).to_u64(); // Note that the control context we use for switching is not the one in // (the stack of) resume_contref, but in (the stack of) last_ancestor! @@ -1430,7 +1430,7 @@ pub(crate) fn translate_resume<'a>( // Extract the result and signal bit. let result = ControlEffect::from_u64(result); - let signal = result.signal(env, builder); + let signal = result.signal(builder); // Jump to the return block if the result signal is 0, otherwise jump to // the suspend block. @@ -1470,7 +1470,7 @@ pub(crate) fn translate_resume<'a>( parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr); // Extract the handler index - let handler_index = ControlEffect::handler_index(resume_result, env, builder); + let handler_index = resume_result.handler_index(builder); let revision = suspended_continuation.get_revision(env, builder); let suspended_contobj = fatpointer::construct( @@ -1647,7 +1647,7 @@ pub(crate) fn translate_suspend<'a>( let absent_chain_link = VMStackChain::absent(env, builder); end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link); - let suspend_payload = ControlEffect::encode_suspend(env, builder, handler_index).to_u64(); + let suspend_payload = ControlEffect::encode_suspend(builder, handler_index).to_u64(); // Note that the control context we use for switching is the one // at the end of the chain, not the one in active_contref! @@ -1887,7 +1887,7 @@ pub(crate) fn translate_switch<'a>( offset += i32::try_from(env.pointer_type().bytes()).unwrap(); } - let switch_payload = ControlEffect::encode_switch(env, builder).to_u64(); + let switch_payload = ControlEffect::encode_switch(builder).to_u64(); let _result = builder.ins().stack_switch( switcher_last_ancestor_cc, From e9fa92d1fd989a10b584124ca07acee836123b4d Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 6 Jun 2025 21:45:13 +0000 Subject: [PATCH 07/34] cranelift: rename stack-switching VMArray to VMHostArray This matches a change to the mirrored runtime type in the upstream changes. --- .../src/stack_switching/instructions.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 8a8be0748228..9d2462ab703a 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -33,7 +33,7 @@ pub(crate) mod stack_switching_helpers { } #[derive(Copy, Clone)] - pub struct VMArray { + pub struct VMHostArray { /// Base address of this object, which must be shifted by `offset` below. base: ir::Value, @@ -47,10 +47,10 @@ pub(crate) mod stack_switching_helpers { phantom: PhantomData, } - pub type VMPayloads = VMArray; + pub type VMPayloads = VMHostArray; // Actually a vector of *mut VMTagDefinition - pub type VMHandlerList = VMArray<*mut u8>; + pub type VMHandlerList = VMHostArray<*mut u8>; /// Compile-time representation of wasmtime_environ::VMStackChain, /// consisting of two `ir::Value`s. @@ -197,7 +197,7 @@ pub(crate) mod stack_switching_helpers { } } - impl VMArray { + impl VMHostArray { pub(crate) fn new(base: ir::Value, offset: i32) -> Self { Self { base, @@ -229,7 +229,7 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> ir::Value { - let offset = env.offsets.ptr.vmarray_data().into(); + let offset = env.offsets.ptr.vmhostarray_data().into(); self.get(builder, env.pointer_type(), offset) } @@ -239,7 +239,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, ) -> ir::Value { // Array length is stored as u32. - let offset = env.offsets.ptr.vmarray_length().into(); + let offset = env.offsets.ptr.vmhostarray_length().into(); self.get(builder, I32, offset) } @@ -250,7 +250,7 @@ pub(crate) mod stack_switching_helpers { length: ir::Value, ) { // Array length is stored as u32. - let offset = env.offsets.ptr.vmarray_length().into(); + let offset = env.offsets.ptr.vmhostarray_length().into(); self.set::(builder, offset, length); } @@ -261,7 +261,7 @@ pub(crate) mod stack_switching_helpers { capacity: ir::Value, ) { // Array capacity is stored as u32. - let offset = env.offsets.ptr.vmarray_capacity().into(); + let offset = env.offsets.ptr.vmhostarray_capacity().into(); self.set::(builder, offset, capacity); } @@ -271,7 +271,7 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, data: ir::Value, ) { - let offset = env.offsets.ptr.vmarray_data().into(); + let offset = env.offsets.ptr.vmhostarray_data().into(); self.set::<*mut T>(builder, offset, data); } From 866fbffbb03ccfa436aca5032abee306d0be10da Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 13:27:05 -0500 Subject: [PATCH 08/34] stack-switching: fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Daniel Hillerström --- crates/cranelift/src/stack_switching/fatpointer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cranelift/src/stack_switching/fatpointer.rs b/crates/cranelift/src/stack_switching/fatpointer.rs index 028c79bbc3ce..8c8f9e7cd553 100644 --- a/crates/cranelift/src/stack_switching/fatpointer.rs +++ b/crates/cranelift/src/stack_switching/fatpointer.rs @@ -2,7 +2,7 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::InstBuilder; use cranelift_codegen::ir::types::I64; -/// The Cranelfift type used to represent all of the following: +/// The Cranelift type used to represent all of the following: /// - wasm values of type `(ref null $ct)` and `(ref $ct)` /// - equivalently: runtime values of type `Option` and `VMContObj` /// Note that a `VMContObj` is a fat pointer From 115e503b5c47056633a1342205baa2bfaa330252 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 18:32:37 +0000 Subject: [PATCH 09/34] stack-switching: used Index impl for get_stack_slot_data --- cranelift/codegen/src/ir/function.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index b078b8935303..02c615400183 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -257,9 +257,7 @@ impl FunctionStencil { /// Find the data for the given stack slot pub fn get_stack_slot_data(&self, stack_slot: StackSlot) -> &StackSlotData { - self.sized_stack_slots - .get(stack_slot) - .expect("undeclared stack slot: {stack_slot}") + &self.sized_stack_slots[stack_slot] } /// Get a concrete `Type` from a user defined `DynamicType`. From 11c6b4a2eae3b226390ea2043d29ea08ef739f37 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 19:18:19 +0000 Subject: [PATCH 10/34] stack-switching: use smallvec over vec in several cases --- crates/cranelift/src/func_environ.rs | 6 +++--- crates/cranelift/src/translate/code_translator.rs | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index dc74a87e8444..9e83746ac8c9 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -18,7 +18,7 @@ use cranelift_entity::packed_option::ReservedValue; use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap}; use cranelift_frontend::Variable; use cranelift_frontend::{FuncInstBuilder, FunctionBuilder}; -use smallvec::SmallVec; +use smallvec::{SmallVec, smallvec}; use std::mem; use wasmparser::{Operator, WasmFeatures}; use wasmtime_environ::{ @@ -1838,7 +1838,7 @@ impl FuncEnvironment<'_> { let index_type = table.idx_type; let delta = self.cast_index_to_i64(&mut builder.cursor(), delta, index_type); let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); - let mut args = vec![vmctx, table_index_arg, delta]; + let mut args: SmallVec<[_; 6]> = smallvec![vmctx, table_index_arg, delta]; let grow = if ty.is_vmgcref_type() { args.push(init_value); gc::builtins::table_grow_gc_ref(self, &mut builder.cursor().func)? @@ -1983,7 +1983,7 @@ impl FuncEnvironment<'_> { let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); let dst = self.cast_index_to_i64(&mut builder.cursor(), dst, index_type); let len = self.cast_index_to_i64(&mut builder.cursor(), len, index_type); - let mut args = vec![vmctx, table_index_arg, dst]; + let mut args: SmallVec<[_; 6]> = smallvec![vmctx, table_index_arg, dst]; let libcall = if ty.is_vmgcref_type() { args.push(val); gc::builtins::table_fill_gc_ref(self, &mut builder.cursor().func)? diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index ad6c02c41b23..70a1ff58390b 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -88,7 +88,7 @@ use cranelift_codegen::ir::{BlockArg, types::*}; use cranelift_codegen::packed_option::ReservedValue; use cranelift_frontend::{FunctionBuilder, Variable}; use itertools::Itertools; -use smallvec::SmallVec; +use smallvec::{SmallVec, ToSmallVec}; use std::collections::{HashMap, hash_map}; use std::vec::Vec; use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources}; @@ -2900,8 +2900,8 @@ pub fn translate_operator( Operator::ContNew { cont_type_index } => { let cont_type_index = TypeIndex::from_u32(*cont_type_index); - let arg_types = environ.continuation_arguments(cont_type_index).to_vec(); - let result_types = environ.continuation_returns(cont_type_index).to_vec(); + let arg_types: SmallVec<[_; 8]> = environ.continuation_arguments(cont_type_index).to_smallvec(); + let result_types: SmallVec<[_; 8]> = environ.continuation_returns(cont_type_index).to_smallvec(); let r = state.pop1(); let contobj = environ.translate_cont_new(builder, state, r, &arg_types, &result_types)?; @@ -2941,7 +2941,7 @@ pub fn translate_operator( Operator::Suspend { tag_index } => { let tag_index = TagIndex::from_u32(*tag_index); let param_types = environ.tag_params(tag_index).to_vec(); - let return_types: Vec<_> = environ + let return_types: SmallVec<[_; 8]> = environ .tag_returns(tag_index) .iter() .map(|ty| crate::value_type(environ.isa(), *ty)) @@ -3002,9 +3002,9 @@ pub fn translate_operator( tag_index, } => { // Arguments of the continuation we are going to switch to - let continuation_argument_types = environ + let continuation_argument_types: SmallVec<[_; 8]> = environ .continuation_arguments(TypeIndex::from_u32(*cont_type_index)) - .to_vec(); + .to_smallvec(); // Arity includes the continuation argument let arity = continuation_argument_types.len(); let (contobj, switch_args) = state.peekn(arity).split_last().unwrap(); @@ -3017,7 +3017,7 @@ pub fn translate_operator( // Argument types of current_continuation_type. These will in turn // be the types of the arguments we receive when someone switches // back to this switch instruction - let current_continuation_arg_types: Vec<_> = match current_continuation_type.heap_type { + let current_continuation_arg_types: SmallVec<[_; 8]> = match current_continuation_type.heap_type { WasmHeapType::ConcreteCont(index) => { let mti = index .as_module_type_index() From fa8001e05a924172f0195ed4d5ed778e9e06d29b Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 19:29:33 +0000 Subject: [PATCH 11/34] stack-switching: avoid resumetable naming confusion --- .../src/translate/code_translator.rs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs index 70a1ff58390b..94dad35f0037 100644 --- a/crates/cranelift/src/translate/code_translator.rs +++ b/crates/cranelift/src/translate/code_translator.rs @@ -2958,21 +2958,21 @@ pub fn translate_operator( } Operator::Resume { cont_type_index, - resume_table, + resume_table: wasm_resume_table, } => { - // We translate the block indices in the resumetable to actual Blocks. - let mut resumetable = vec![]; - for handle in &resume_table.handlers { + // We translate the block indices in the wasm resume_table to actual Blocks. + let mut clif_resume_table = vec![]; + for handle in &wasm_resume_table.handlers { match handle { wasmparser::Handle::OnLabel { tag, label } => { let i = state.control_stack.len() - 1 - (*label as usize); let frame = &mut state.control_stack[i]; // This is side-effecting! frame.set_branched_to_exit(); - resumetable.push((*tag, Some(frame.br_destination()))); + clif_resume_table.push((*tag, Some(frame.br_destination()))); } wasmparser::Handle::OnSwitch { tag } => { - resumetable.push((*tag, None)); + clif_resume_table.push((*tag, None)); } } } @@ -2986,7 +2986,7 @@ pub fn translate_operator( cont_type_index.as_u32(), *contobj, call_args, - resumetable.as_slice(), + &clif_resume_table )?; state.popn(arity + 1); // arguments + continuation @@ -2996,7 +2996,12 @@ pub fn translate_operator( cont_type_index: _, tag_index: _, resume_table: _, - } => todo!("unimplemented stack switching instruction"), + } => { + // TODO(10248) This depends on exception handling + return Err(wasmtime_environ::WasmError::Unsupported( + "resume.throw instructions not supported, yet".to_string(), + )); + } Operator::Switch { cont_type_index, tag_index, From 51017f26f8220e4b7fb2dae6d4a22246833f0128 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 19:33:15 +0000 Subject: [PATCH 12/34] stack-switching: cleanup unused params from unchecked_get_continuation The extra parameters here used to be used for emitting runtime assertions, but with those gone we just had unused params and lifetimes, clean those out. --- .../src/stack_switching/instructions.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 9d2462ab703a..105a531863d2 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -537,12 +537,7 @@ pub(crate) mod stack_switching_helpers { } /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`. - pub fn unchecked_get_continuation<'a>( - &self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, - _builder: &mut FunctionBuilder, - ) -> ir::Value { - // TODO(posborne): this used to have emitted assertions but now does not do much. + pub fn unchecked_get_continuation(&self) -> ir::Value { self.payload } @@ -1080,7 +1075,7 @@ fn search_handler<'a>( // Block begin_search_handler_list let (contref, parent_link, handler_list_data_ptr, end_range) = { builder.switch_to_block(begin_search_handler_list); - let contref = chain_link.unchecked_get_continuation(env, builder); + let contref = chain_link.unchecked_get_continuation(); let contref = helpers::VMContRef::new(contref); let parent_link = contref.get_parent_stack_chain(env, builder); @@ -1452,7 +1447,7 @@ pub(crate) fn translate_resume<'a>( builder.switch_to_block(suspend_block); builder.seal_block(suspend_block); - let suspended_continuation = new_stack_chain.unchecked_get_continuation(env, builder); + let suspended_continuation = new_stack_chain.unchecked_get_continuation(); let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation); let suspended_csi = suspended_continuation.common_stack_information(env, builder); @@ -1570,7 +1565,7 @@ pub(crate) fn translate_resume<'a>( builder.seal_block(return_block); // If we got a return signal, a continuation must have been running. - let returned_contref = new_stack_chain.unchecked_get_continuation(env, builder); + let returned_contref = new_stack_chain.unchecked_get_continuation(); let returned_contref = helpers::VMContRef::new(returned_contref); // Restore parts of the VMRuntimeLimits from the parent of the @@ -1613,7 +1608,7 @@ pub(crate) fn translate_suspend<'a>( // If we get here, the search_handler logic succeeded (i.e., did not trap). // Thus, there is at least one parent, so we are not on the initial stack. // Can therefore extract continuation directly. - let active_contref = active_stack_chain.unchecked_get_continuation(env, builder); + let active_contref = active_stack_chain.unchecked_get_continuation(); let active_contref = helpers::VMContRef::new(active_contref); let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref); @@ -1726,7 +1721,7 @@ pub(crate) fn translate_switch<'a>( // If we get here, the search_handler logic succeeded (i.e., did not trap). // Thus, there is at least one parent, so we are not on the initial stack. // Can therefore extract continuation directly. - let switcher_contref = active_stack_chain.unchecked_get_continuation(env, builder); + let switcher_contref = active_stack_chain.unchecked_get_continuation(); let mut switcher_contref = helpers::VMContRef::new(switcher_contref); switcher_contref.set_last_ancestor(env, builder, last_ancestor.address); From 44c2b347e0246ae13b2f3dcde9d26949e0e2af38 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 19:40:58 +0000 Subject: [PATCH 13/34] stack_switching: simplify store_data_entries assertion --- .../src/stack_switching/instructions.rs | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 105a531863d2..5ebebd94852b 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -378,24 +378,16 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, values: &[ir::Value], - allow_smaller: bool, ) { let store_count = builder .ins() .iconst(I32, i64::try_from(values.len()).unwrap()); - // TODO(posborne): allow_smaller only used in debug_assert! - if cfg!(debug_assertions) { - for val in values { - let ty = builder.func.dfg.value_type(*val); - let size = usize::try_from(ty.bytes()).unwrap(); - if allow_smaller { - debug_assert!(size <= std::mem::size_of::()); - } else { - debug_assert!(size == std::mem::size_of::()); - } - } - } + debug_assert!(values.iter().all(|val| { + let ty = builder.func.dfg.value_type(*val); + let size = usize::try_from(ty.bytes()).unwrap(); + size <= std::mem::size_of::() + })); let memflags = ir::MemFlags::trusted(); @@ -1387,7 +1379,7 @@ pub(crate) fn translate_resume<'a>( .collect(); // Store all tag addresess in the handler list. - handler_list.store_data_entries(env, builder, &all_tag_addresses, false); + handler_list.store_data_entries(env, builder, &all_tag_addresses); // To enable distinguishing switch and suspend handlers when searching the handler list: // Store at which index the switch handlers start. @@ -1633,7 +1625,7 @@ pub(crate) fn translate_suspend<'a>( } if suspend_args.len() > 0 { - values.store_data_entries(env, builder, suspend_args, true) + values.store_data_entries(env, builder, suspend_args) } // Set current continuation to suspended and break up handler chain. From b8fb8b3e79552a5f388fd9e1abe04aa0d6a39a36 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 11 Jun 2025 19:50:38 +0000 Subject: [PATCH 14/34] stack-switching: simplify translate_table_{grow,fill} control flow --- crates/cranelift/src/func_environ.rs | 76 +++++++++++++--------------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9e83746ac8c9..559e306a18ac 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1839,31 +1839,25 @@ impl FuncEnvironment<'_> { let delta = self.cast_index_to_i64(&mut builder.cursor(), delta, index_type); let table_index_arg = builder.ins().iconst(I32, table_index.as_u32() as i64); let mut args: SmallVec<[_; 6]> = smallvec![vmctx, table_index_arg, delta]; - let grow = if ty.is_vmgcref_type() { - args.push(init_value); - gc::builtins::table_grow_gc_ref(self, &mut builder.cursor().func)? - } else { - debug_assert!(matches!( - ty.top(), - WasmHeapTopType::Func | WasmHeapTopType::Cont - )); - match ty.top() { - WasmHeapTopType::Func => { - args.push(init_value); - self.builtin_functions - .table_grow_func_ref(&mut builder.func) - } - WasmHeapTopType::Cont => { - let (revision, contref) = stack_switching::fatpointer::deconstruct( - self, - &mut builder.cursor(), - init_value, - ); - args.extend_from_slice(&[contref, revision]); - stack_switching::builtins::table_grow_cont_obj(self, &mut builder.func)? - } - _ => panic!("unsupported table type."), + let grow = match ty.top() { + WasmHeapTopType::Extern | WasmHeapTopType::Any => { + args.push(init_value); + gc::builtins::table_grow_gc_ref(self, &mut builder.cursor().func)? + } + WasmHeapTopType::Func => { + args.push(init_value); + self.builtin_functions + .table_grow_func_ref(&mut builder.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = stack_switching::fatpointer::deconstruct( + self, + &mut builder.cursor(), + init_value, + ); + args.extend_from_slice(&[contref, revision]); + stack_switching::builtins::table_grow_cont_obj(self, &mut builder.func)? } }; @@ -1984,27 +1978,25 @@ impl FuncEnvironment<'_> { let dst = self.cast_index_to_i64(&mut builder.cursor(), dst, index_type); let len = self.cast_index_to_i64(&mut builder.cursor(), len, index_type); let mut args: SmallVec<[_; 6]> = smallvec![vmctx, table_index_arg, dst]; - let libcall = if ty.is_vmgcref_type() { - args.push(val); - gc::builtins::table_fill_gc_ref(self, &mut builder.cursor().func)? - } else { - match ty.top() { - WasmHeapTopType::Func => { - args.push(val); - self.builtin_functions - .table_fill_func_ref(&mut builder.func) - } - WasmHeapTopType::Cont => { - let (revision, contref) = - stack_switching::fatpointer::deconstruct(self, &mut builder.cursor(), val); - args.extend_from_slice(&[contref, revision]); - stack_switching::builtins::table_fill_cont_obj(self, &mut builder.func)? - } - _ => panic!("unsupported table type"), + let libcall = match ty.top() { + WasmHeapTopType::Any | WasmHeapTopType::Extern => { + args.push(val); + gc::builtins::table_fill_gc_ref(self, &mut builder.cursor().func)? + } + WasmHeapTopType::Func => { + args.push(val); + self.builtin_functions + .table_fill_func_ref(&mut builder.func) + } + WasmHeapTopType::Cont => { + let (revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut builder.cursor(), val); + args.extend_from_slice(&[contref, revision]); + stack_switching::builtins::table_fill_cont_obj(self, &mut builder.func)? } }; - args.push(len); + args.push(len); builder.ins().call(libcall, &args); Ok(()) From a693350f22ed9711de466bafd22865f4908537a0 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 12 Jun 2025 16:45:01 +0000 Subject: [PATCH 15/34] stack-switching: remove translate_resume_throw stub There's already a stub elsewhere and this is not called, when exceptions are added and it is time to revisit, this method can be restored. --- crates/cranelift/src/func_environ.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 559e306a18ac..9c32455078ce 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -3319,20 +3319,6 @@ impl FuncEnvironment<'_> { ) } - #[allow(dead_code, reason = "TODO")] - pub fn translate_resume_throw( - &mut self, - _pos: FuncCursor, - _state: &FuncTranslationState, - _tag_index: u32, - _cont: ir::Value, - ) -> WasmResult { - // TODO(#10248) - Err(wasmtime_environ::WasmError::Unsupported( - "resume.throw instruction not implemented, yet".to_string(), - )) - } - pub fn translate_suspend( &mut self, builder: &mut FunctionBuilder<'_>, From eeab1b1d8268e5eb12924602c9043367a927b053 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 12 Jun 2025 17:09:00 +0000 Subject: [PATCH 16/34] stack-switching: compute control_context_size based on target triple --- .../src/stack_switching/instructions.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 5ebebd94852b..7f684be07c35 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -6,11 +6,14 @@ use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, MemFlags}; use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData}; use cranelift_frontend::FunctionBuilder; -use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType}; +use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported}; -// TODO(frank-emrich) This is the size for x64 Linux. Once we support different -// platforms for stack switching, must select appropriate value for target. -pub const CONTROL_CONTEXT_SIZE: usize = 24; +fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult { + match (triple.architecture, triple.operating_system) { + (target_lexicon::Architecture::X86_64, target_lexicon::OperatingSystem::Linux) => Ok(24), + _ => Err(wasm_unsupported!("stack switching not support on {triple}")), + } +} use super::control_effect::ControlEffect; use super::fatpointer; @@ -1842,9 +1845,10 @@ pub(crate) fn translate_switch<'a>( // that would be a rather ad-hoc change to how the instruction uses the // two pointers given to it. + let cctx_size = control_context_size(env.isa().triple())?; let slot_size = ir::StackSlotData::new( ir::StackSlotKind::ExplicitSlot, - u32::try_from(CONTROL_CONTEXT_SIZE).unwrap(), + u32::from(cctx_size), u8::try_from(env.pointer_type().bytes()).unwrap(), ); let slot = builder.create_sized_stack_slot(slot_size); @@ -1852,7 +1856,7 @@ pub(crate) fn translate_switch<'a>( let flags = MemFlags::trusted(); let mut offset: i32 = 0; - while offset < i32::try_from(CONTROL_CONTEXT_SIZE).unwrap() { + while offset < i32::from(cctx_size) { // switchee_last_ancestor_cc -> tmp control context let tmp1 = builder From 3bd138fe65e1987c799115cf702aa28f2a177d13 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 16 Jun 2025 16:17:11 +0000 Subject: [PATCH 17/34] stack-switching: VMHostArrayRef updates Rename VMHostArray -> VMHostArrayRef Change impl to compute address with offset upfront rather than on each load. --- .../src/stack_switching/instructions.rs | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/stack_switching/instructions.rs index 7f684be07c35..b32bae51da74 100644 --- a/crates/cranelift/src/stack_switching/instructions.rs +++ b/crates/cranelift/src/stack_switching/instructions.rs @@ -36,12 +36,9 @@ pub(crate) mod stack_switching_helpers { } #[derive(Copy, Clone)] - pub struct VMHostArray { - /// Base address of this object, which must be shifted by `offset` below. - base: ir::Value, - - /// Adding this (statically) known offset gets us the overall address. - offset: i32, + pub struct VMHostArrayRef { + /// Address of the VMHostArray we are referencing + address: ir::Value, /// The type parameter T is never used in the fields above. We still /// want to have it for consistency with @@ -50,10 +47,10 @@ pub(crate) mod stack_switching_helpers { phantom: PhantomData, } - pub type VMPayloads = VMHostArray; + pub type VMPayloads = VMHostArrayRef; // Actually a vector of *mut VMTagDefinition - pub type VMHandlerList = VMHostArray<*mut u8>; + pub type VMHandlerList = VMHostArrayRef<*mut u8>; /// Compile-time representation of wasmtime_environ::VMStackChain, /// consisting of two `ir::Value`s. @@ -82,19 +79,21 @@ pub(crate) mod stack_switching_helpers { pub fn args<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, - _builder: &mut FunctionBuilder, + builder: &mut FunctionBuilder, ) -> VMPayloads { - let offset = env.offsets.ptr.vmcontref_args().into(); - VMPayloads::new(self.address, offset) + let offset: i64 = env.offsets.ptr.vmcontref_args().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMPayloads::new(address) } pub fn values<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, - _builder: &mut FunctionBuilder, + builder: &mut FunctionBuilder, ) -> VMPayloads { - let offset = env.offsets.ptr.vmcontref_values().into(); - VMPayloads::new(self.address, offset) + let offset: i64 = env.offsets.ptr.vmcontref_values().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMPayloads::new(address) } pub fn common_stack_information<'a>( @@ -200,11 +199,10 @@ pub(crate) mod stack_switching_helpers { } } - impl VMHostArray { - pub(crate) fn new(base: ir::Value, offset: i32) -> Self { + impl VMHostArrayRef { + pub(crate) fn new(address: ir::Value) -> Self { Self { - base, - offset, + address, phantom: PhantomData::default(), } } @@ -213,7 +211,7 @@ pub(crate) mod stack_switching_helpers { let mem_flags = ir::MemFlags::trusted(); builder .ins() - .load(ty, mem_flags, self.base, self.offset + offset) + .load(ty, mem_flags, self.address, offset) } fn set(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) { @@ -222,9 +220,7 @@ pub(crate) mod stack_switching_helpers { Type::int_with_byte_size(u16::try_from(std::mem::size_of::()).unwrap()).unwrap() ); let mem_flags = ir::MemFlags::trusted(); - builder - .ins() - .store(mem_flags, value, self.base, self.offset + offset); + builder.ins().store(mem_flags, value, self.address, offset); } pub fn get_data<'a>( @@ -661,10 +657,11 @@ pub(crate) mod stack_switching_helpers { pub fn get_handler_list<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, - _builder: &mut FunctionBuilder, + builder: &mut FunctionBuilder, ) -> VMHandlerList { - let offset = env.offsets.ptr.vmcommon_stack_information_handlers().into(); - VMHandlerList::new(self.address, offset) + let offset: i64 = env.offsets.ptr.vmcommon_stack_information_handlers().into(); + let address = builder.ins().iadd_imm(self.address, offset); + VMHandlerList::new(address) } pub fn get_first_switch_handler_index<'a>( From 49313bccc344e8475c739d31cccf3f673aa573d1 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 18 Jun 2025 19:21:06 +0000 Subject: [PATCH 18/34] stack-switching: move cranelift code to live under func_environ This matches the directory structure for gc and aids in visibility for a few members required by stack-switching code in cranelift. --- crates/cranelift/src/func_environ.rs | 5 +++-- .../src/{ => func_environ}/stack_switching/control_effect.rs | 0 .../src/{ => func_environ}/stack_switching/fatpointer.rs | 0 .../src/{ => func_environ}/stack_switching/instructions.rs | 0 .../cranelift/src/{ => func_environ}/stack_switching/mod.rs | 0 crates/cranelift/src/lib.rs | 3 +-- 6 files changed, 4 insertions(+), 4 deletions(-) rename crates/cranelift/src/{ => func_environ}/stack_switching/control_effect.rs (100%) rename crates/cranelift/src/{ => func_environ}/stack_switching/fatpointer.rs (100%) rename crates/cranelift/src/{ => func_environ}/stack_switching/instructions.rs (100%) rename crates/cranelift/src/{ => func_environ}/stack_switching/mod.rs (100%) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9c32455078ce..ea74e345e12b 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1,11 +1,12 @@ mod gc; +pub(crate) mod stack_switching; use crate::compiler::Compiler; use crate::translate::{ FuncTranslationState, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize, TargetEnvironment, }; -use crate::{BuiltinFunctionSignatures, TRAP_INTERNAL_ASSERT, stack_switching}; +use crate::{BuiltinFunctionSignatures, TRAP_INTERNAL_ASSERT}; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::{Imm64, Offset32, V128Imm}; @@ -95,7 +96,7 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); pub struct FuncEnvironment<'module_environment> { compiler: &'module_environment Compiler, isa: &'module_environment (dyn TargetIsa + 'module_environment), - pub(crate) module: &'module_environment Module, + module: &'module_environment Module, types: &'module_environment ModuleTypesBuilder, wasm_func_ty: &'module_environment WasmFuncType, sig_ref_to_ty: SecondaryMap>, diff --git a/crates/cranelift/src/stack_switching/control_effect.rs b/crates/cranelift/src/func_environ/stack_switching/control_effect.rs similarity index 100% rename from crates/cranelift/src/stack_switching/control_effect.rs rename to crates/cranelift/src/func_environ/stack_switching/control_effect.rs diff --git a/crates/cranelift/src/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs similarity index 100% rename from crates/cranelift/src/stack_switching/fatpointer.rs rename to crates/cranelift/src/func_environ/stack_switching/fatpointer.rs diff --git a/crates/cranelift/src/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs similarity index 100% rename from crates/cranelift/src/stack_switching/instructions.rs rename to crates/cranelift/src/func_environ/stack_switching/instructions.rs diff --git a/crates/cranelift/src/stack_switching/mod.rs b/crates/cranelift/src/func_environ/stack_switching/mod.rs similarity index 100% rename from crates/cranelift/src/stack_switching/mod.rs rename to crates/cranelift/src/func_environ/stack_switching/mod.rs diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index af2ae8cfc759..720e33f9d54e 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -42,7 +42,6 @@ mod builder; mod compiler; mod debug; mod func_environ; -mod stack_switching; mod translate; use self::compiler::Compiler; @@ -214,7 +213,7 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32, - WasmHeapTopType::Cont => stack_switching::fatpointer::POINTER_TYPE, + WasmHeapTopType::Cont => func_environ::stack_switching::fatpointer::POINTER_TYPE, } } From c86b06a8536b01dc2ac48de8356ed60444fed2e4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 18 Jun 2025 20:17:54 +0000 Subject: [PATCH 19/34] stack-switching: formatting fix --- .../src/func_environ/stack_switching/instructions.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index b32bae51da74..c43e9e8f9a60 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -209,9 +209,7 @@ pub(crate) mod stack_switching_helpers { fn get(&self, builder: &mut FunctionBuilder, ty: ir::Type, offset: i32) -> ir::Value { let mem_flags = ir::MemFlags::trusted(); - builder - .ins() - .load(ty, mem_flags, self.address, offset) + builder.ins().load(ty, mem_flags, self.address, offset) } fn set(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) { From ab50f6c26968beb03513af3975e9a55736435999 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 25 Jun 2025 23:45:48 +0000 Subject: [PATCH 20/34] stack-switching: reduce visibility on a few additional items --- crates/cranelift/src/func_environ.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 5cf313b288dd..f1633a2a66df 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -139,7 +139,7 @@ pub struct FuncEnvironment<'module_environment> { pcc_vmctx_memtype: Option, /// Caches of signatures for builtin functions. - pub(crate) builtin_functions: BuiltinFunctions, + builtin_functions: BuiltinFunctions, /// Offsets to struct fields accessed by JIT code. pub(crate) offsets: VMOffsets, @@ -184,7 +184,7 @@ pub struct FuncEnvironment<'module_environment> { /// Used by the stack switching feature. If set, we have a allocated a /// slot on this function's stack to be used for the /// current continuation's `values` field. - pub(crate) stack_switching_values_buffer: Option, + pub stack_switching_values_buffer: Option, } impl<'module_environment> FuncEnvironment<'module_environment> { From 6797ea2ae082214c7dcd71ebc21f1a9b46e8e305 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 26 Jun 2025 00:04:26 +0000 Subject: [PATCH 21/34] stack-switching: simplify contobj fatptr con/de-struction --- .../stack_switching/fatpointer.rs | 44 +++++-------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs index 8c8f9e7cd553..6dd394318746 100644 --- a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs +++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs @@ -1,6 +1,5 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::InstBuilder; -use cranelift_codegen::ir::types::I64; /// The Cranelift type used to represent all of the following: /// - wasm values of type `(ref null $ct)` and `(ref $ct)` @@ -21,22 +20,13 @@ pub(crate) fn deconstruct<'a>( contobj: ir::Value, ) -> (ir::Value, ir::Value) { debug_assert_eq!(pos.func.dfg.value_type(contobj), POINTER_TYPE); + let ptr_ty = env.pointer_type(); + assert!(ptr_ty.bits() <= 64); - let (lsbs, msbs) = pos.ins().isplit(contobj); + let contref = pos.ins().ireduce(ptr_ty, contobj); + let shifted = pos.ins().ushr_imm(contobj, 64); + let revision_counter = pos.ins().ireduce(ir::types::I64, shifted); - let (revision_counter, contref) = match env.isa().endianness() { - ir::Endianness::Little => (lsbs, msbs), - ir::Endianness::Big => { - let pad_bits = 64 - env.pointer_type().bits(); - let contref = pos.ins().ushr_imm(lsbs, pad_bits as i64); - (msbs, contref) - } - }; - let contref = if env.pointer_type().bits() < I64.bits() { - pos.ins().ireduce(env.pointer_type(), contref) - } else { - contref - }; (revision_counter, contref) } @@ -50,22 +40,12 @@ pub(crate) fn construct<'a>( ) -> ir::Value { debug_assert_eq!(pos.func.dfg.value_type(contref_addr), env.pointer_type()); debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ir::types::I64); - let contref_addr = if env.pointer_type().bits() < I64.bits() { - pos.ins().uextend(I64, contref_addr) - } else { - contref_addr - }; - let (msbs, lsbs) = match env.isa().endianness() { - ir::Endianness::Little => (contref_addr, revision_counter), - ir::Endianness::Big => { - let pad_bits = 64 - env.pointer_type().bits(); - let lsbs = pos.ins().ishl_imm(contref_addr, pad_bits as i64); - (revision_counter, lsbs) - } - }; + assert!(env.pointer_type().bits() <= 64); - let lsbs = pos.ins().uextend(ir::types::I128, lsbs); - let msbs = pos.ins().uextend(ir::types::I128, msbs); - let msbs = pos.ins().ishl_imm(msbs, 64); - pos.ins().bor(lsbs, msbs) + let contref_addr = pos.ins().uextend(ir::types::I128, contref_addr); + let revision_counter = pos.ins().uextend(ir::types::I128, revision_counter); + let shifted_counter = pos.ins().ishl_imm(revision_counter, 64); + let contobj = pos.ins().bor(shifted_counter, contref_addr); + + contobj } From a6a3ff8bf8894746e596f859b45d96b2428c18dc Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 30 Jun 2025 15:50:16 +0000 Subject: [PATCH 22/34] stack-switching: add disas tests to cover new instructions --- .../stack_switching/instructions.rs | 4 +- .../resume-suspend-data-passing.wat | 311 +++++++++++++ .../disas/stack-switching/resume-suspend.wat | 263 +++++++++++ .../stack-switching/symmetric-switch.wat | 418 ++++++++++++++++++ 4 files changed, 995 insertions(+), 1 deletion(-) create mode 100644 tests/disas/stack-switching/resume-suspend-data-passing.wat create mode 100644 tests/disas/stack-switching/resume-suspend.wat create mode 100644 tests/disas/stack-switching/symmetric-switch.wat diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index c43e9e8f9a60..b2529ed7331e 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -11,7 +11,9 @@ use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wa fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult { match (triple.architecture, triple.operating_system) { (target_lexicon::Architecture::X86_64, target_lexicon::OperatingSystem::Linux) => Ok(24), - _ => Err(wasm_unsupported!("stack switching not support on {triple}")), + _ => Err(wasm_unsupported!( + "stack switching not supported on {triple}" + )), } } diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat new file mode 100644 index 000000000000..821520abedc0 --- /dev/null +++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat @@ -0,0 +1,311 @@ +;;! target = "x86_64-unknown-linux-gnu" +;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y" +;;! test = "optimize" + +(module + (type $ft (func)) + (tag $t (param i32)) + (type $ct (cont $ft)) + + (func $countdown + (local $i i32) + (local.set $i (i32.const 10)) + (loop $loop + ;; suspend and pass countdown to our cosnumer + (suspend $t (local.get $i)) + ;; decrement i; break if we're at 0 + (local.tee $i (i32.sub (local.get $i) (i32.const 1))) + (br_if $loop) + ) + ) + (elem declare func $countdown) + + (func (export "main") + (local $c (ref $ct)) + (local.set $c (cont.new $ct (ref.func $countdown))) + (loop $loop + (block $on_gen (result i32 (ref $ct)) + (resume $ct (on $t $on_gen) (local.get $c)) + ;; no more data, return + (return) + ) + ;; stack contains [i32 (ref $ct)] + (local.set $c) + (drop) ;; could print here + (br $loop) + ) + ) +) + +;; function u0:0(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 16, align = 65536 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003c v3 = iconst.i32 10 +;; v61 = iconst.i64 120 +;; @0044 v30 = stack_addr.i64 ss0 +;; v59 = iconst.i64 16 +;; v60 = iconst.i64 0 +;; v57 = iconst.i64 80 +;; v56 = iconst.i64 -24 +;; v70 = iconst.i64 0x0002_0000_0000 +;; @0040 jump block2(v3) ; v3 = 10 +;; +;; block2(v4: i32): +;; @0044 v8 = load.i64 notrap aligned v0+8 +;; @0044 v9 = load.i64 notrap aligned v8+64 +;; @0044 v10 = load.i64 notrap aligned v8+72 +;; v65 = iconst.i64 1 +;; v64 = iconst.i64 24 +;; @003a v2 = iconst.i32 0 +;; @0044 jump block4(v9, v10, v4) +;; +;; block4(v11: i64, v12: i64, v52: i32): +;; v73 = iconst.i64 1 +;; v74 = icmp eq v11, v73 ; v73 = 1 +;; @0044 trapnz v74, user22 +;; @0044 jump block5 +;; +;; block5: +;; @0044 v14 = load.i64 notrap aligned v12+48 +;; @0044 v15 = load.i64 notrap aligned v12+56 +;; v75 = iconst.i64 24 +;; v76 = iadd v15, v75 ; v75 = 24 +;; @0044 v17 = load.i64 notrap aligned v76+8 +;; @0044 v18 = load.i32 notrap aligned v15+40 +;; v77 = iconst.i32 0 +;; v67 = iconst.i32 3 +;; v66 = iconst.i64 48 +;; @0044 v6 = iadd.i64 v0, v66 ; v66 = 48 +;; v62 = iconst.i32 1 +;; @0044 jump block6(v77) ; v77 = 0 +;; +;; block6(v20: i32): +;; @0044 v21 = icmp ult v20, v18 +;; @0044 brif v21, block7, block4(v14, v15, v52) +;; +;; block7: +;; v78 = iconst.i32 3 +;; v79 = ishl.i32 v20, v78 ; v78 = 3 +;; @0044 v23 = uextend.i64 v79 +;; @0044 v24 = iadd.i64 v17, v23 +;; @0044 v25 = load.i64 notrap aligned v24 +;; v80 = iadd.i64 v0, v66 ; v66 = 48 +;; v81 = icmp eq v25, v80 +;; v82 = iconst.i32 1 +;; v83 = iadd.i32 v20, v82 ; v82 = 1 +;; @0044 brif v81, block8, block6(v83) +;; +;; block8: +;; @0044 store.i64 notrap aligned v12, v10+64 +;; v84 = iconst.i32 1 +;; v85 = iconst.i64 120 +;; v86 = iadd.i64 v10, v85 ; v85 = 120 +;; @0044 store notrap aligned v84, v86+4 ; v84 = 1 +;; @0044 store.i64 notrap aligned v30, v86+8 +;; @0044 store.i32 notrap aligned v4, v30 +;; @0044 store notrap aligned v84, v86 ; v84 = 1 +;; v87 = iconst.i32 3 +;; v88 = iconst.i64 16 +;; v89 = iadd.i64 v10, v88 ; v88 = 16 +;; @0044 store notrap aligned v87, v89 ; v87 = 3 +;; v90 = iconst.i64 0 +;; @0044 store notrap aligned v90, v12+48 ; v90 = 0 +;; @0044 store notrap aligned v90, v12+56 ; v90 = 0 +;; v91 = iconst.i64 80 +;; v92 = iadd.i64 v12, v91 ; v91 = 80 +;; @0044 v43 = load.i64 notrap aligned v92 +;; v93 = iconst.i64 -24 +;; v94 = iadd v43, v93 ; v93 = -24 +;; @0044 v40 = uextend.i64 v20 +;; v95 = iconst.i64 0x0002_0000_0000 +;; v96 = bor v40, v95 ; v95 = 0x0002_0000_0000 +;; @0044 v45 = stack_switch v94, v94, v96 +;; @0044 v47 = load.i64 notrap aligned v86+8 +;; v97 = iconst.i32 0 +;; @0044 store notrap aligned v97, v86 ; v97 = 0 +;; @0044 store notrap aligned v97, v86+4 ; v97 = 0 +;; @0044 store notrap aligned v90, v86+8 ; v90 = 0 +;; v98 = isub.i32 v52, v84 ; v84 = 1 +;; @004d brif v98, block2(v98), block10 +;; +;; block10: +;; @004f jump block3 +;; +;; block3: +;; @0050 jump block1 +;; +;; block1: +;; @0050 return +;; } +;; +;; function u0:1(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 8, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1:7 sig0 +;; fn1 = colocated u1:53 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0056 v2 = iconst.i32 0 +;; @0056 v4 = call fn0(v0, v2) ; v2 = 0 +;; @0058 trapz v4, user16 +;; @0058 v8 = call fn1(v0, v4, v2, v2) ; v2 = 0, v2 = 0 +;; @0058 v9 = load.i64 notrap aligned v8+72 +;; @0058 v11 = uextend.i128 v9 +;; v119 = iconst.i64 64 +;; v121 = ishl v11, v119 ; v119 = 64 +;; @0058 v10 = uextend.i128 v8 +;; @0058 v13 = bor v121, v10 +;; v116 = iconst.i64 1 +;; @0062 v28 = iconst.i64 0 +;; @0062 v29 = iconst.i64 2 +;; @0062 v32 = iconst.i32 1 +;; v114 = iconst.i64 16 +;; @0062 v34 = iconst.i32 2 +;; v110 = iconst.i64 24 +;; @0062 v45 = stack_addr.i64 ss0 +;; v109 = iconst.i64 48 +;; @0062 v47 = iadd v0, v109 ; v109 = 48 +;; v107 = iconst.i64 80 +;; v106 = iconst.i64 -24 +;; v125 = iconst.i64 0x0001_0000_0000 +;; v108 = iconst.i64 32 +;; @005c jump block2(v13) +;; +;; block2(v16: i128): +;; @0062 jump block5 +;; +;; block5: +;; @0062 v18 = ireduce.i64 v16 +;; @0062 trapz v18, user16 +;; @0062 v21 = load.i64 notrap aligned v18+72 +;; v128 = iconst.i64 64 +;; v129 = ushr.i128 v16, v128 ; v128 = 64 +;; @0062 v20 = ireduce.i64 v129 +;; @0062 v22 = icmp eq v21, v20 +;; @0062 trapz v22, user23 +;; v130 = iconst.i64 1 +;; v131 = iadd v21, v130 ; v130 = 1 +;; @0062 store notrap aligned v131, v18+72 +;; @0062 v24 = load.i64 notrap aligned v18+64 +;; @0062 v25 = load.i64 notrap aligned v0+8 +;; @0062 v26 = load.i64 notrap aligned v25+64 +;; @0062 v27 = load.i64 notrap aligned v25+72 +;; @0062 store notrap aligned v26, v24+48 +;; @0062 store notrap aligned v27, v24+56 +;; v132 = iconst.i64 0 +;; @0062 store notrap aligned v132, v18+64 ; v132 = 0 +;; @0062 v30 = load.i64 notrap aligned v0+8 +;; v133 = iconst.i64 2 +;; @0062 store notrap aligned v133, v30+64 ; v133 = 2 +;; @0062 store notrap aligned v18, v30+72 +;; v134 = iconst.i32 1 +;; v135 = iconst.i64 16 +;; v136 = iadd v18, v135 ; v135 = 16 +;; @0062 store notrap aligned v134, v136 ; v134 = 1 +;; v137 = iconst.i32 2 +;; v138 = iadd v27, v135 ; v135 = 16 +;; @0062 store notrap aligned v137, v138 ; v137 = 2 +;; @0062 v36 = load.i64 notrap aligned readonly v0+8 +;; @0062 v38 = load.i64 notrap aligned v36+56 +;; @0062 store notrap aligned v38, v27+8 +;; @0062 v39 = load.i64 notrap aligned v36+16 +;; @0062 store notrap aligned v39, v27 +;; @0062 v41 = load.i64 notrap aligned v18 +;; @0062 store notrap aligned v41, v36+16 +;; @0062 v42 = load.i64 notrap aligned v18+8 +;; @0062 store notrap aligned v42, v36+56 +;; v139 = iconst.i64 24 +;; v140 = iadd v27, v139 ; v139 = 24 +;; @0062 store notrap aligned v134, v140+4 ; v134 = 1 +;; @0062 store.i64 notrap aligned v45, v140+8 +;; v141 = iadd.i64 v0, v109 ; v109 = 48 +;; @0062 store notrap aligned v141, v45 +;; @0062 store notrap aligned v134, v140 ; v134 = 1 +;; @0062 store notrap aligned v134, v27+40 ; v134 = 1 +;; v142 = iconst.i64 80 +;; v143 = iadd v24, v142 ; v142 = 80 +;; @0062 v54 = load.i64 notrap aligned v143 +;; v144 = iconst.i64 -24 +;; v145 = iadd v54, v144 ; v144 = -24 +;; v146 = iconst.i64 0x0001_0000_0000 +;; @0062 v56 = stack_switch v145, v145, v146 ; v146 = 0x0001_0000_0000 +;; @0062 v57 = load.i64 notrap aligned v0+8 +;; @0062 v58 = load.i64 notrap aligned v57+64 +;; @0062 v59 = load.i64 notrap aligned v57+72 +;; @0062 store notrap aligned v26, v57+64 +;; @0062 store notrap aligned v27, v57+72 +;; @0062 store notrap aligned v134, v138 ; v134 = 1 +;; v147 = iconst.i32 0 +;; @0062 store notrap aligned v147, v140 ; v147 = 0 +;; @0062 store notrap aligned v147, v140+4 ; v147 = 0 +;; @0062 store notrap aligned v132, v140+8 ; v132 = 0 +;; @0062 store notrap aligned v132, v27+40 ; v132 = 0 +;; v148 = iconst.i64 32 +;; v149 = ushr v56, v148 ; v148 = 32 +;; @0062 brif v149, block7, block6 +;; +;; block7: +;; @0062 v69 = load.i64 notrap aligned v36+56 +;; @0062 store notrap aligned v69, v59+8 +;; @0062 v71 = load.i64 notrap aligned v27 +;; @0062 store notrap aligned v71, v36+16 +;; @0062 v72 = load.i64 notrap aligned v27+8 +;; @0062 store notrap aligned v72, v36+56 +;; @0062 v74 = load.i64 notrap aligned v59+72 +;; @0062 jump block8 +;; +;; block9 cold: +;; @0062 trap user11 +;; +;; block10: +;; v98 = iconst.i64 120 +;; @0062 v79 = iadd.i64 v59, v98 ; v98 = 120 +;; @0062 v80 = load.i64 notrap aligned v79+8 +;; @0062 v81 = load.i32 notrap aligned v80 +;; v154 = iconst.i32 0 +;; @0062 store notrap aligned v154, v79 ; v154 = 0 +;; @0062 jump block4 +;; +;; block8: +;; @0062 v73 = ireduce.i32 v56 +;; @0062 br_table v73, block9, [block10] +;; +;; block6: +;; @0062 v84 = load.i64 notrap aligned v27 +;; @0062 store notrap aligned v84, v36+16 +;; @0062 v85 = load.i64 notrap aligned v27+8 +;; @0062 store notrap aligned v85, v36+56 +;; @0062 v87 = iconst.i32 4 +;; v150 = iconst.i64 16 +;; v151 = iadd.i64 v59, v150 ; v150 = 16 +;; @0062 store notrap aligned v87, v151 ; v87 = 4 +;; v94 = iconst.i64 104 +;; @0062 v89 = iadd.i64 v59, v94 ; v94 = 104 +;; @0062 v90 = load.i64 notrap aligned v89+8 +;; v152 = iconst.i32 0 +;; @0062 store notrap aligned v152, v89 ; v152 = 0 +;; @0062 store notrap aligned v152, v89+4 ; v152 = 0 +;; v153 = iconst.i64 0 +;; @0062 store notrap aligned v153, v89+8 ; v153 = 0 +;; @0068 return +;; +;; block4: +;; @0062 v76 = uextend.i128 v74 +;; v155 = iconst.i64 64 +;; v156 = ishl v76, v155 ; v155 = 64 +;; @0062 v75 = uextend.i128 v59 +;; @0062 v78 = bor v156, v75 +;; @006d jump block2(v78) +;; } diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat new file mode 100644 index 000000000000..09e95f55fc22 --- /dev/null +++ b/tests/disas/stack-switching/resume-suspend.wat @@ -0,0 +1,263 @@ +;;! target = "x86_64-unknown-linux-gnu" +;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y" +;;! test = "optimize" + +(module + (type $ft (func)) + (tag $t (type $ft)) + (type $ct (cont $ft)) + + (func $target (suspend $t)) + (elem declare func $target) + + (func (export "minimal_suspend") + (local $k (ref null $ct)) + (local.set $k (cont.new $ct (ref.func $target))) + (block $h (result (ref null $ct)) + (resume $ct (on $t $h) (local.get $k)) + ;; continuation suspended back... + (ref.null $ct) + ) + (drop) + ) +) + +;; function u0:0(i64 vmctx, i64) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003b v5 = load.i64 notrap aligned v0+8 +;; @003b v6 = load.i64 notrap aligned v5+64 +;; @003b v7 = load.i64 notrap aligned v5+72 +;; v54 = iconst.i64 1 +;; v53 = iconst.i64 24 +;; @003b v16 = iconst.i32 0 +;; @003b jump block2(v6, v7) +;; +;; block2(v8: i64, v9: i64): +;; v62 = iconst.i64 1 +;; v63 = icmp eq v8, v62 ; v62 = 1 +;; @003b trapnz v63, user22 +;; @003b jump block3 +;; +;; block3: +;; @003b v11 = load.i64 notrap aligned v9+48 +;; @003b v12 = load.i64 notrap aligned v9+56 +;; v64 = iconst.i64 24 +;; v65 = iadd v12, v64 ; v64 = 24 +;; @003b v14 = load.i64 notrap aligned v65+8 +;; @003b v15 = load.i32 notrap aligned v12+40 +;; v66 = iconst.i32 0 +;; v56 = iconst.i32 3 +;; v55 = iconst.i64 48 +;; @003b v3 = iadd.i64 v0, v55 ; v55 = 48 +;; v51 = iconst.i32 1 +;; @003b jump block4(v66) ; v66 = 0 +;; +;; block4(v17: i32): +;; @003b v18 = icmp ult v17, v15 +;; @003b brif v18, block5, block2(v11, v12) +;; +;; block5: +;; v67 = iconst.i32 3 +;; v68 = ishl.i32 v17, v67 ; v67 = 3 +;; @003b v20 = uextend.i64 v68 +;; @003b v21 = iadd.i64 v14, v20 +;; @003b v22 = load.i64 notrap aligned v21 +;; v69 = iadd.i64 v0, v55 ; v55 = 48 +;; v70 = icmp eq v22, v69 +;; v71 = iconst.i32 1 +;; v72 = iadd.i32 v17, v71 ; v71 = 1 +;; @003b brif v70, block6, block4(v72) +;; +;; block6: +;; @003b store.i64 notrap aligned v9, v7+64 +;; v73 = iconst.i32 3 +;; v48 = iconst.i64 16 +;; @003b v28 = iadd.i64 v7, v48 ; v48 = 16 +;; @003b store notrap aligned v73, v28 ; v73 = 3 +;; v49 = iconst.i64 0 +;; @003b store notrap aligned v49, v9+48 ; v49 = 0 +;; @003b store notrap aligned v49, v9+56 ; v49 = 0 +;; v46 = iconst.i64 80 +;; @003b v35 = iadd.i64 v9, v46 ; v46 = 80 +;; @003b v36 = load.i64 notrap aligned v35 +;; v45 = iconst.i64 -24 +;; @003b v37 = iadd v36, v45 ; v45 = -24 +;; @003b v33 = uextend.i64 v17 +;; v59 = iconst.i64 0x0002_0000_0000 +;; v60 = bor v33, v59 ; v59 = 0x0002_0000_0000 +;; @003b v38 = stack_switch v37, v37, v60 +;; v50 = iconst.i64 120 +;; @003b v25 = iadd.i64 v7, v50 ; v50 = 120 +;; @003b v40 = load.i64 notrap aligned v25+8 +;; v74 = iconst.i32 0 +;; @003b store notrap aligned v74, v25 ; v74 = 0 +;; @003b store notrap aligned v74, v25+4 ; v74 = 0 +;; @003b store notrap aligned v49, v25+8 ; v49 = 0 +;; @003d jump block1 +;; +;; block1: +;; @003d return +;; } +;; +;; function u0:1(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 8, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1:7 sig0 +;; fn1 = colocated u1:53 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0043 v7 = iconst.i32 0 +;; @0043 v9 = call fn0(v0, v7) ; v7 = 0 +;; @0045 trapz v9, user16 +;; @0045 v13 = call fn1(v0, v9, v7, v7) ; v7 = 0, v7 = 0 +;; @0045 v14 = load.i64 notrap aligned v13+72 +;; @004e jump block3 +;; +;; block3: +;; @0045 v16 = uextend.i128 v14 +;; v130 = iconst.i64 64 +;; v134 = ishl v16, v130 ; v130 = 64 +;; v136 = ireduce.i64 v134 +;; v138 = bor v136, v13 +;; @004e trapz v138, user16 +;; @004e v24 = load.i64 notrap aligned v138+72 +;; @0045 v15 = uextend.i128 v13 +;; @0045 v18 = bor v134, v15 +;; v140 = ushr v18, v130 ; v130 = 64 +;; @004e v23 = ireduce.i64 v140 +;; @004e v25 = icmp eq v24, v23 +;; @004e trapz v25, user23 +;; v125 = iconst.i64 1 +;; @004e v26 = iadd v24, v125 ; v125 = 1 +;; @004e store notrap aligned v26, v138+72 +;; @004e v27 = load.i64 notrap aligned v138+64 +;; @004e v28 = load.i64 notrap aligned v0+8 +;; @004e v29 = load.i64 notrap aligned v28+64 +;; @004e v30 = load.i64 notrap aligned v28+72 +;; @004e store notrap aligned v29, v27+48 +;; @004e store notrap aligned v30, v27+56 +;; @0040 v2 = iconst.i64 0 +;; @004e store notrap aligned v2, v138+64 ; v2 = 0 +;; @004e v33 = load.i64 notrap aligned v0+8 +;; @004e v32 = iconst.i64 2 +;; @004e store notrap aligned v32, v33+64 ; v32 = 2 +;; @004e store notrap aligned v138, v33+72 +;; @004e v35 = iconst.i32 1 +;; v123 = iconst.i64 16 +;; @004e v36 = iadd v138, v123 ; v123 = 16 +;; @004e store notrap aligned v35, v36 ; v35 = 1 +;; @004e v37 = iconst.i32 2 +;; @004e v38 = iadd v30, v123 ; v123 = 16 +;; @004e store notrap aligned v37, v38 ; v37 = 2 +;; @004e v39 = load.i64 notrap aligned readonly v0+8 +;; @004e v41 = load.i64 notrap aligned v39+56 +;; @004e store notrap aligned v41, v30+8 +;; @004e v42 = load.i64 notrap aligned v39+16 +;; @004e store notrap aligned v42, v30 +;; @004e v44 = load.i64 notrap aligned v138 +;; @004e store notrap aligned v44, v39+16 +;; @004e v45 = load.i64 notrap aligned v138+8 +;; @004e store notrap aligned v45, v39+56 +;; v119 = iconst.i64 24 +;; @004e v46 = iadd v30, v119 ; v119 = 24 +;; @004e store notrap aligned v35, v46+4 ; v35 = 1 +;; @004e v48 = stack_addr.i64 ss0 +;; @004e store notrap aligned v48, v46+8 +;; v118 = iconst.i64 48 +;; @004e v50 = iadd.i64 v0, v118 ; v118 = 48 +;; @004e store notrap aligned v50, v48 +;; @004e store notrap aligned v35, v46 ; v35 = 1 +;; @004e store notrap aligned v35, v30+40 ; v35 = 1 +;; v116 = iconst.i64 80 +;; @004e v56 = iadd v27, v116 ; v116 = 80 +;; @004e v57 = load.i64 notrap aligned v56 +;; v115 = iconst.i64 -24 +;; @004e v58 = iadd v57, v115 ; v115 = -24 +;; v142 = iconst.i64 0x0001_0000_0000 +;; @004e v59 = stack_switch v58, v58, v142 ; v142 = 0x0001_0000_0000 +;; @004e v60 = load.i64 notrap aligned v0+8 +;; @004e v61 = load.i64 notrap aligned v60+64 +;; @004e v62 = load.i64 notrap aligned v60+72 +;; @004e store notrap aligned v29, v60+64 +;; @004e store notrap aligned v30, v60+72 +;; @004e store notrap aligned v35, v38 ; v35 = 1 +;; v145 = iconst.i32 0 +;; @004e store notrap aligned v145, v46 ; v145 = 0 +;; @004e store notrap aligned v145, v46+4 ; v145 = 0 +;; @004e store notrap aligned v2, v46+8 ; v2 = 0 +;; @004e store notrap aligned v2, v30+40 ; v2 = 0 +;; v117 = iconst.i64 32 +;; @004e v69 = ushr v59, v117 ; v117 = 32 +;; @004e brif v69, block5, block4 +;; +;; block5: +;; @004e v72 = load.i64 notrap aligned v39+56 +;; @004e store notrap aligned v72, v62+8 +;; @004e v74 = load.i64 notrap aligned v30 +;; @004e store notrap aligned v74, v39+16 +;; @004e v75 = load.i64 notrap aligned v30+8 +;; @004e store notrap aligned v75, v39+56 +;; @004e v77 = load.i64 notrap aligned v62+72 +;; @004e jump block6 +;; +;; block7 cold: +;; @004e trap user11 +;; +;; block8: +;; v107 = iconst.i64 120 +;; @004e v82 = iadd.i64 v62, v107 ; v107 = 120 +;; @004e v83 = load.i64 notrap aligned v82+8 +;; v153 = iconst.i32 0 +;; @004e store notrap aligned v153, v82 ; v153 = 0 +;; @004e v79 = uextend.i128 v77 +;; v154 = iconst.i64 64 +;; v155 = ishl v79, v154 ; v154 = 64 +;; @004e v78 = uextend.i128 v62 +;; @004e v81 = bor v155, v78 +;; @004e jump block2(v81) +;; +;; block6: +;; @004e v76 = ireduce.i32 v59 +;; @004e br_table v76, block7, [block8] +;; +;; block4: +;; @004e v86 = load.i64 notrap aligned v30 +;; @004e store notrap aligned v86, v39+16 +;; @004e v87 = load.i64 notrap aligned v30+8 +;; @004e store notrap aligned v87, v39+56 +;; @004e v89 = iconst.i32 4 +;; v146 = iconst.i64 16 +;; v147 = iadd.i64 v62, v146 ; v146 = 16 +;; @004e store notrap aligned v89, v147 ; v89 = 4 +;; v103 = iconst.i64 104 +;; @004e v91 = iadd.i64 v62, v103 ; v103 = 104 +;; @004e v92 = load.i64 notrap aligned v91+8 +;; v148 = iconst.i32 0 +;; @004e store notrap aligned v148, v91 ; v148 = 0 +;; @004e store notrap aligned v148, v91+4 ; v148 = 0 +;; v149 = iconst.i64 0 +;; @004e store notrap aligned v149, v91+8 ; v149 = 0 +;; v150 = uextend.i128 v149 ; v149 = 0 +;; v151 = iconst.i64 64 +;; v152 = ishl v150, v151 ; v151 = 64 +;; @0040 v6 = bor v152, v150 +;; @0056 jump block2(v6) +;; +;; block2(v19: i128): +;; @0058 jump block1 +;; +;; block1: +;; @0058 return +;; } diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat new file mode 100644 index 000000000000..d852d36feb3f --- /dev/null +++ b/tests/disas/stack-switching/symmetric-switch.wat @@ -0,0 +1,418 @@ +;;! target = "x86_64-unknown-linux-gnu" +;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y" + +(module + (type $fta (func)) + (type $cta (cont $fta)) + + (type $ftb (func (param (ref $cta)))) + (type $ctb (cont $ftb)) + + (tag $yield) + + (func $task_a (type $fta) + (cont.new $ctb (ref.func $task_b)) + (switch $ctb $yield) + ) + + (func $task_b (type $ftb)) + + (elem declare func $task_a $task_b) + + (func (export "entry") + (cont.new $cta (ref.func $task_a)) + (resume $cta (on $yield switch)) + ) +) + +;; function u0:0(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 24, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1:7 sig0 +;; fn1 = colocated u1:53 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @003a v2 = iconst.i32 1 +;; @003a v4 = call fn0(v0, v2) ; v2 = 1 +;; @003c trapz v4, user16 +;; @003c v5 = iconst.i32 1 +;; @003c v6 = iconst.i32 0 +;; @003c v8 = call fn1(v0, v4, v5, v6) ; v5 = 1, v6 = 0 +;; @003c v9 = load.i64 notrap aligned v8+72 +;; @003c v10 = uextend.i128 v8 +;; @003c v11 = uextend.i128 v9 +;; v138 = iconst.i64 64 +;; v139 = uextend.i128 v138 ; v138 = 64 +;; @003c v12 = ishl v11, v139 +;; @003c v13 = bor v12, v10 +;; @003e v15 = ireduce.i64 v13 +;; v136 = iconst.i64 64 +;; v137 = uextend.i128 v136 ; v136 = 64 +;; @003e v16 = ushr v13, v137 +;; @003e v17 = ireduce.i64 v16 +;; @003e trapz v15, user16 +;; @003e v18 = load.i64 notrap aligned v15+72 +;; @003e v19 = icmp eq v18, v17 +;; @003e trapz v19, user23 +;; v135 = iconst.i64 1 +;; @003e v20 = iadd v18, v135 ; v135 = 1 +;; @003e store notrap aligned v20, v15+72 +;; v134 = iconst.i64 48 +;; @003e v22 = iadd v0, v134 ; v134 = 48 +;; @003e v23 = load.i64 notrap aligned v0+8 +;; @003e v24 = load.i64 notrap aligned v23+64 +;; @003e v25 = load.i64 notrap aligned v23+72 +;; @003e jump block2(v24, v25) +;; +;; block2(v26: i64, v27: i64): +;; v133 = iconst.i64 1 +;; @003e v28 = icmp eq v26, v133 ; v133 = 1 +;; @003e trapnz v28, user22 +;; @003e jump block3 +;; +;; block3: +;; @003e v29 = load.i64 notrap aligned v27+48 +;; @003e v30 = load.i64 notrap aligned v27+56 +;; v132 = iconst.i64 24 +;; @003e v31 = iadd v30, v132 ; v132 = 24 +;; @003e v32 = load.i64 notrap aligned v31+8 +;; @003e v33 = load.i32 notrap aligned v30+40 +;; @003e v34 = load.i32 notrap aligned v31 +;; @003e jump block4(v33) +;; +;; block4(v35: i32): +;; @003e v36 = icmp ult v35, v34 +;; @003e brif v36, block5, block2(v29, v30) +;; +;; block5: +;; v131 = iconst.i32 8 +;; @003e v37 = imul.i32 v35, v131 ; v131 = 8 +;; @003e v38 = uextend.i64 v37 +;; @003e v39 = iadd.i64 v32, v38 +;; @003e v40 = load.i64 notrap aligned v39 +;; @003e v41 = icmp eq v40, v22 +;; v130 = iconst.i32 1 +;; @003e v42 = iadd.i32 v35, v130 ; v130 = 1 +;; @003e brif v41, block6, block4(v42) +;; +;; block6: +;; @003e store.i64 notrap aligned v27, v25+64 +;; v129 = iconst.i64 120 +;; @003e v43 = iadd.i64 v25, v129 ; v129 = 120 +;; v128 = iconst.i64 0 +;; @003e v44 = iadd.i64 v25, v128 ; v128 = 0 +;; @003e v45 = iconst.i32 3 +;; v127 = iconst.i64 16 +;; @003e v46 = iadd v44, v127 ; v127 = 16 +;; @003e store notrap aligned v45, v46 ; v45 = 3 +;; @003e v47 = iconst.i64 0 +;; @003e v48 = iconst.i64 0 +;; @003e store notrap aligned v47, v27+48 ; v47 = 0 +;; @003e store notrap aligned v48, v27+56 ; v48 = 0 +;; @003e v49 = load.i64 notrap aligned readonly v0+8 +;; v126 = iconst.i64 0 +;; @003e v50 = iadd v44, v126 ; v126 = 0 +;; @003e v51 = load.i64 notrap aligned v49+56 +;; @003e store notrap aligned v51, v50+8 +;; @003e v52 = load.i64 notrap aligned v25+72 +;; @003e v53 = uextend.i128 v25 +;; @003e v54 = uextend.i128 v52 +;; v124 = iconst.i64 64 +;; v125 = uextend.i128 v124 ; v124 = 64 +;; @003e v55 = ishl v54, v125 +;; @003e v56 = bor v55, v53 +;; v123 = iconst.i64 0 +;; @003e v58 = iadd.i64 v15, v123 ; v123 = 0 +;; v122 = iconst.i64 16 +;; @003e v59 = iadd v58, v122 ; v122 = 16 +;; @003e v60 = load.i32 notrap aligned v59 +;; v121 = iconst.i32 0 +;; @003e v61 = icmp ne v60, v121 ; v121 = 0 +;; @003e brif v61, block9, block8 +;; +;; block8: +;; v120 = iconst.i64 104 +;; @003e v62 = iadd.i64 v15, v120 ; v120 = 104 +;; @003e v63 = load.i64 notrap aligned v62+8 +;; @003e v64 = load.i32 notrap aligned v62 +;; v119 = iconst.i32 1 +;; @003e v65 = iadd v64, v119 ; v119 = 1 +;; @003e store notrap aligned v65, v62 +;; @003e v66 = uextend.i64 v64 +;; v118 = iconst.i64 16 +;; @003e v67 = imul v66, v118 ; v118 = 16 +;; @003e v68 = iadd v63, v67 +;; @003e jump block10(v68) +;; +;; block9: +;; v117 = iconst.i64 120 +;; @003e v69 = iadd.i64 v15, v117 ; v117 = 120 +;; @003e v70 = load.i64 notrap aligned v69+8 +;; @003e v71 = load.i32 notrap aligned v69 +;; v116 = iconst.i32 1 +;; @003e v72 = iadd v71, v116 ; v116 = 1 +;; @003e store notrap aligned v72, v69 +;; @003e v73 = uextend.i64 v71 +;; v115 = iconst.i64 16 +;; @003e v74 = imul v73, v115 ; v115 = 16 +;; @003e v75 = iadd v70, v74 +;; @003e jump block10(v75) +;; +;; block10(v57: i64): +;; @003e store.i128 notrap aligned v56, v57 +;; v114 = iconst.i64 0 +;; @003e v76 = iadd.i64 v15, v114 ; v114 = 0 +;; @003e v77 = iconst.i32 1 +;; v113 = iconst.i64 16 +;; @003e v78 = iadd v76, v113 ; v113 = 16 +;; @003e store notrap aligned v77, v78 ; v77 = 1 +;; @003e v79 = load.i64 notrap aligned v15+64 +;; @003e store.i64 notrap aligned v29, v79+48 +;; @003e store.i64 notrap aligned v30, v79+56 +;; @003e v80 = iconst.i64 2 +;; @003e v81 = load.i64 notrap aligned v0+8 +;; @003e store notrap aligned v80, v81+64 ; v80 = 2 +;; @003e store.i64 notrap aligned v15, v81+72 +;; v112 = iconst.i64 0 +;; @003e v82 = iadd v76, v112 ; v112 = 0 +;; @003e v83 = load.i64 notrap aligned v82 +;; @003e store notrap aligned v83, v49+16 +;; @003e v84 = load.i64 notrap aligned v82+8 +;; @003e store notrap aligned v84, v49+56 +;; v111 = iconst.i64 80 +;; @003e v85 = iadd.i64 v27, v111 ; v111 = 80 +;; @003e v86 = load.i64 notrap aligned v85 +;; v110 = iconst.i64 -24 +;; @003e v87 = iadd v86, v110 ; v110 = -24 +;; v109 = iconst.i64 80 +;; @003e v88 = iadd v79, v109 ; v109 = 80 +;; @003e v89 = load.i64 notrap aligned v88 +;; v108 = iconst.i64 -24 +;; @003e v90 = iadd v89, v108 ; v108 = -24 +;; @003e v91 = stack_addr.i64 ss0 +;; @003e v92 = load.i64 notrap aligned v90 +;; @003e store notrap aligned v92, v91 +;; @003e v93 = load.i64 notrap aligned v87 +;; @003e store notrap aligned v93, v90 +;; @003e v94 = load.i64 notrap aligned v90+8 +;; @003e store notrap aligned v94, v91+8 +;; @003e v95 = load.i64 notrap aligned v87+8 +;; @003e store notrap aligned v95, v90+8 +;; @003e v96 = load.i64 notrap aligned v90+16 +;; @003e store notrap aligned v96, v91+16 +;; @003e v97 = load.i64 notrap aligned v87+16 +;; @003e store notrap aligned v97, v90+16 +;; @003e v98 = iconst.i64 3 +;; v107 = iconst.i64 32 +;; @003e v99 = ishl v98, v107 ; v98 = 3, v107 = 32 +;; @003e v100 = stack_switch v87, v91, v99 +;; v106 = iconst.i64 120 +;; @003e v101 = iadd.i64 v25, v106 ; v106 = 120 +;; @003e v102 = load.i64 notrap aligned v101+8 +;; @003e v103 = iconst.i32 0 +;; @003e store notrap aligned v103, v101 ; v103 = 0 +;; @003e v104 = iconst.i32 0 +;; @003e store notrap aligned v104, v101+4 ; v104 = 0 +;; @003e v105 = iconst.i64 0 +;; @003e store notrap aligned v105, v101+8 ; v105 = 0 +;; @0041 jump block1 +;; +;; block1: +;; @0041 return +;; } +;; +;; function u0:1(i64 vmctx, i64, i128) tail { +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i128): +;; @0044 jump block1 +;; +;; block1: +;; @0044 return +;; } +;; +;; function u0:2(i64 vmctx, i64) tail { +;; ss0 = explicit_slot 8, align = 256 +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly gv0+8 +;; gv2 = load.i64 notrap aligned gv1+16 +;; gv3 = vmctx +;; sig0 = (i64 vmctx, i32) -> i64 tail +;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail +;; fn0 = colocated u1:7 sig0 +;; fn1 = colocated u1:53 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64): +;; @0047 v2 = iconst.i32 0 +;; @0047 v4 = call fn0(v0, v2) ; v2 = 0 +;; @0049 trapz v4, user16 +;; @0049 v5 = iconst.i32 0 +;; @0049 v6 = iconst.i32 0 +;; @0049 v8 = call fn1(v0, v4, v5, v6) ; v5 = 0, v6 = 0 +;; @0049 v9 = load.i64 notrap aligned v8+72 +;; @0049 v10 = uextend.i128 v8 +;; @0049 v11 = uextend.i128 v9 +;; v111 = iconst.i64 64 +;; v112 = uextend.i128 v111 ; v111 = 64 +;; @0049 v12 = ishl v11, v112 +;; @0049 v13 = bor v12, v10 +;; @004b jump block2 +;; +;; block2: +;; @004b v15 = ireduce.i64 v13 +;; v109 = iconst.i64 64 +;; v110 = uextend.i128 v109 ; v109 = 64 +;; @004b v16 = ushr.i128 v13, v110 +;; @004b v17 = ireduce.i64 v16 +;; @004b trapz v15, user16 +;; @004b v18 = load.i64 notrap aligned v15+72 +;; @004b v19 = icmp eq v18, v17 +;; @004b trapz v19, user23 +;; v108 = iconst.i64 1 +;; @004b v20 = iadd v18, v108 ; v108 = 1 +;; @004b store notrap aligned v20, v15+72 +;; @004b v21 = load.i64 notrap aligned v15+64 +;; @004b v22 = load.i64 notrap aligned v0+8 +;; @004b v23 = load.i64 notrap aligned v22+64 +;; @004b v24 = load.i64 notrap aligned v22+72 +;; @004b store notrap aligned v23, v21+48 +;; @004b store notrap aligned v24, v21+56 +;; @004b v25 = iconst.i64 0 +;; @004b store notrap aligned v25, v15+64 ; v25 = 0 +;; @004b v26 = iconst.i64 2 +;; @004b v27 = load.i64 notrap aligned v0+8 +;; @004b store notrap aligned v26, v27+64 ; v26 = 2 +;; @004b store notrap aligned v15, v27+72 +;; v107 = iconst.i64 0 +;; @004b v28 = iadd v15, v107 ; v107 = 0 +;; @004b v29 = iconst.i32 1 +;; v106 = iconst.i64 16 +;; @004b v30 = iadd v28, v106 ; v106 = 16 +;; @004b store notrap aligned v29, v30 ; v29 = 1 +;; @004b v31 = iconst.i32 2 +;; v105 = iconst.i64 16 +;; @004b v32 = iadd v24, v105 ; v105 = 16 +;; @004b store notrap aligned v31, v32 ; v31 = 2 +;; @004b v33 = load.i64 notrap aligned readonly v0+8 +;; v104 = iconst.i64 0 +;; @004b v34 = iadd v24, v104 ; v104 = 0 +;; @004b v35 = load.i64 notrap aligned v33+56 +;; @004b store notrap aligned v35, v34+8 +;; @004b v36 = load.i64 notrap aligned v33+16 +;; @004b store notrap aligned v36, v34 +;; v103 = iconst.i64 0 +;; @004b v37 = iadd v28, v103 ; v103 = 0 +;; @004b v38 = load.i64 notrap aligned v37 +;; @004b store notrap aligned v38, v33+16 +;; @004b v39 = load.i64 notrap aligned v37+8 +;; @004b store notrap aligned v39, v33+56 +;; v102 = iconst.i64 24 +;; @004b v40 = iadd v24, v102 ; v102 = 24 +;; @004b v41 = iconst.i32 1 +;; @004b v42 = stack_addr.i64 ss0 +;; @004b store notrap aligned v41, v40+4 ; v41 = 1 +;; @004b store notrap aligned v42, v40+8 +;; v101 = iconst.i64 48 +;; @004b v44 = iadd.i64 v0, v101 ; v101 = 48 +;; @004b v45 = iconst.i32 1 +;; @004b v46 = load.i64 notrap aligned v40+8 +;; @004b store notrap aligned v44, v46 +;; @004b store notrap aligned v45, v40 ; v45 = 1 +;; @004b v47 = iconst.i32 0 +;; @004b store notrap aligned v47, v24+40 ; v47 = 0 +;; @004b v48 = iconst.i64 1 +;; v100 = iconst.i64 32 +;; @004b v49 = ishl v48, v100 ; v48 = 1, v100 = 32 +;; v99 = iconst.i64 80 +;; @004b v50 = iadd v21, v99 ; v99 = 80 +;; @004b v51 = load.i64 notrap aligned v50 +;; v98 = iconst.i64 -24 +;; @004b v52 = iadd v51, v98 ; v98 = -24 +;; @004b v53 = stack_switch v52, v52, v49 +;; @004b v54 = load.i64 notrap aligned v0+8 +;; @004b v55 = load.i64 notrap aligned v54+64 +;; @004b v56 = load.i64 notrap aligned v54+72 +;; @004b v57 = load.i64 notrap aligned v0+8 +;; @004b store notrap aligned v23, v57+64 +;; @004b store notrap aligned v24, v57+72 +;; @004b v58 = iconst.i32 1 +;; v97 = iconst.i64 16 +;; @004b v59 = iadd v24, v97 ; v97 = 16 +;; @004b store notrap aligned v58, v59 ; v58 = 1 +;; @004b v60 = iconst.i32 0 +;; @004b store notrap aligned v60, v40 ; v60 = 0 +;; @004b v61 = iconst.i32 0 +;; @004b store notrap aligned v61, v40+4 ; v61 = 0 +;; @004b v62 = iconst.i64 0 +;; @004b store notrap aligned v62, v40+8 ; v62 = 0 +;; @004b store notrap aligned v25, v24+40 ; v25 = 0 +;; v96 = iconst.i64 32 +;; @004b v63 = ushr v53, v96 ; v96 = 32 +;; @004b brif v63, block4, block3 +;; +;; block4: +;; v95 = iconst.i64 0 +;; @004b v64 = iadd.i64 v56, v95 ; v95 = 0 +;; v94 = iconst.i64 0 +;; @004b v65 = iadd v64, v94 ; v94 = 0 +;; @004b v66 = load.i64 notrap aligned v33+56 +;; @004b store notrap aligned v66, v65+8 +;; v93 = iconst.i64 0 +;; @004b v67 = iadd.i64 v24, v93 ; v93 = 0 +;; @004b v68 = load.i64 notrap aligned v67 +;; @004b store notrap aligned v68, v33+16 +;; @004b v69 = load.i64 notrap aligned v67+8 +;; @004b store notrap aligned v69, v33+56 +;; @004b v70 = ireduce.i32 v53 +;; @004b v71 = load.i64 notrap aligned v56+72 +;; @004b v72 = uextend.i128 v56 +;; @004b v73 = uextend.i128 v71 +;; v91 = iconst.i64 64 +;; v92 = uextend.i128 v91 ; v91 = 64 +;; @004b v74 = ishl v73, v92 +;; @004b v75 = bor v74, v72 +;; @004b jump block5 +;; +;; block6 cold: +;; @004b trap user11 +;; +;; block5: +;; @004b br_table v70, block6, [] +;; +;; block3: +;; v90 = iconst.i64 0 +;; @004b v76 = iadd.i64 v24, v90 ; v90 = 0 +;; @004b v77 = load.i64 notrap aligned v76 +;; @004b store notrap aligned v77, v33+16 +;; @004b v78 = load.i64 notrap aligned v76+8 +;; @004b store notrap aligned v78, v33+56 +;; v89 = iconst.i64 0 +;; @004b v79 = iadd.i64 v56, v89 ; v89 = 0 +;; @004b v80 = iconst.i32 4 +;; v88 = iconst.i64 16 +;; @004b v81 = iadd v79, v88 ; v88 = 16 +;; @004b store notrap aligned v80, v81 ; v80 = 4 +;; v87 = iconst.i64 104 +;; @004b v82 = iadd.i64 v56, v87 ; v87 = 104 +;; @004b v83 = load.i64 notrap aligned v82+8 +;; @004b v84 = iconst.i32 0 +;; @004b store notrap aligned v84, v82 ; v84 = 0 +;; @004b v85 = iconst.i32 0 +;; @004b store notrap aligned v85, v82+4 ; v85 = 0 +;; @004b v86 = iconst.i64 0 +;; @004b store notrap aligned v86, v82+8 ; v86 = 0 +;; @0050 jump block1 +;; +;; block1: +;; @0050 return +;; } From ba013f92742f993627ae32c01661b9f54355dc6d Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 2 Jul 2025 20:03:14 +0000 Subject: [PATCH 23/34] stack-switching: fix layout of VMContObj In the course of the various runtime updates, the layout of the runtime VMContObj got switched around. This resulted in failures when doing certain table operations on continuations. This change fixes that layout problem and adds some tests with offsets to avoid the problem. Due to the way that we interact with the VMContObj in cranelift, we don't use these offsets outside of the tests. --- crates/environ/src/vmoffsets.rs | 17 +++++++++++++ .../src/runtime/vm/stack_switching.rs | 24 +++++++++++++++---- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 41dde2c6a530..c8bd8e1d7b54 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -338,6 +338,23 @@ pub trait PtrSize { .unwrap() } + // Offsets within `VMContObj` + + /// Return the offset of `VMContObj::contref` + fn vmcontobj_contref(&self) -> u8 { + 0 + } + + /// Return the offset of `VMContObj::revision` + fn vmcontobj_revision(&self) -> u8 { + 8 + } + + /// Return the size of `VMHostArray`. + fn size_of_vmcontobj(&self) -> u8 { + 16 + } + // Offsets within `VMContRef` /// Return the offset of `VMContRef::common_stack_information`. diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 25a3ee846a0c..0e1093a4e3f6 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -39,15 +39,11 @@ pub use stack::*; /// For performance reasons, the VMContRef at the bottom of this chain /// (i.e., the one pointed to by the VMContObj) has a pointer to the /// other end of the chain (i.e., its last ancestor). -// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any -// more? Now that we use I128 on the Cranelift side (see -// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it -// should be fine to use the natural alignment of the type. #[repr(C, align(16))] #[derive(Debug, Clone, Copy)] pub struct VMContObj { - pub revision: u64, pub contref: NonNull, + pub revision: u64, } impl VMContObj { @@ -649,6 +645,24 @@ mod tests { ); } + #[test] + fn check_vm_contobj_offsets() { + let module = Module::new(); + let offsets = VMOffsets::new(HostPtr, &module); + assert_eq!( + offset_of!(VMContObj, contref), + usize::from(offsets.ptr.vmcontobj_contref()) + ); + assert_eq!( + offset_of!(VMContObj, revision), + usize::from(offsets.ptr.vmcontobj_revision()) + ); + assert_eq!( + size_of::(), + usize::from(offsets.ptr.size_of_vmcontobj()) + ) + } + #[test] fn check_vm_contref_offsets() { let module = Module::new(); From 8d0629729b4f5c6b1a5de073603c288b273a4cab Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 14 Jul 2025 17:53:55 +0000 Subject: [PATCH 24/34] Fix formatting of merge conflict resolution --- crates/cranelift/src/func_environ.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 2083f578d45b..41c151b3e992 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1828,7 +1828,7 @@ impl FuncEnvironment<'_> { self.table_vmctx_and_defined_index(&mut pos, table_index); let index_type = table.idx_type; let delta = self.cast_index_to_i64(&mut pos, delta, index_type); - + let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, defined_table_index, delta]; let grow = match ty.top() { WasmHeapTopType::Extern | WasmHeapTopType::Any => { @@ -1837,15 +1837,11 @@ impl FuncEnvironment<'_> { } WasmHeapTopType::Func => { args.push(init_value); - self.builtin_functions - .table_grow_func_ref(pos.func) + self.builtin_functions.table_grow_func_ref(pos.func) } WasmHeapTopType::Cont => { - let (revision, contref) = stack_switching::fatpointer::deconstruct( - self, - &mut pos, - init_value, - ); + let (revision, contref) = + stack_switching::fatpointer::deconstruct(self, &mut pos, init_value); args.extend_from_slice(&[contref, revision]); stack_switching::builtins::table_grow_cont_obj(self, pos.func)? } From cae487894995405ef175117066328f18565b9237 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Mon, 14 Jul 2025 19:10:14 +0000 Subject: [PATCH 25/34] cranelift: remove ir::function::get_stack_slot_data This method isn't required as sized_stack_slots is already pub. --- cranelift/codegen/src/ir/function.rs | 5 ----- .../src/func_environ/stack_switching/instructions.rs | 9 ++++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 02c615400183..99ca7d39f70a 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -255,11 +255,6 @@ impl FunctionStencil { self.get_dyn_scale(dyn_ty) } - /// Find the data for the given stack slot - pub fn get_stack_slot_data(&self, stack_slot: StackSlot) -> &StackSlotData { - &self.sized_stack_slots[stack_slot] - } - /// Get a concrete `Type` from a user defined `DynamicType`. pub fn get_concrete_dynamic_ty(&self, ty: DynamicType) -> Option { self.dfg diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index b2529ed7331e..092128e2fa26 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -307,14 +307,13 @@ pub(crate) mod stack_switching_helpers { let required_size = required_capacity * entry_size; match existing_slot { - Some(slot) if builder.func.get_stack_slot_data(slot).size >= required_size => { - let slot_data = builder.func.get_stack_slot_data(slot).clone(); + Some(slot) if builder.func.sized_stack_slots[slot].size >= required_size => { + let slot_data = &builder.func.sized_stack_slots[slot]; + debug_assert!(align <= slot_data.align_shift); + debug_assert_eq!(slot_data.kind, ExplicitSlot); let existing_capacity = slot_data.size / entry_size; let capacity_value = builder.ins().iconst(I32, i64::from(existing_capacity)); - debug_assert!(align <= builder.func.get_stack_slot_data(slot).align_shift); - debug_assert_eq!(builder.func.get_stack_slot_data(slot).kind, ExplicitSlot); - let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0); self.set_capacity(env, builder, capacity_value); From 61d22a61ca54e887bad80768854ca20bfac8b739 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 26 Aug 2025 19:11:04 +0000 Subject: [PATCH 26/34] stack-switching: reduce visibility of a couple func_environ methods --- crates/cranelift/src/func_environ.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 700bbb5c6a1b..24912ce0cf26 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -176,12 +176,12 @@ pub struct FuncEnvironment<'module_environment> { /// Used by the stack switching feature. If set, we have a allocated a /// slot on this function's stack to be used for the /// current stack's `handler_list` field. - pub(crate) stack_switching_handler_list_buffer: Option, + stack_switching_handler_list_buffer: Option, /// Used by the stack switching feature. If set, we have a allocated a /// slot on this function's stack to be used for the /// current continuation's `values` field. - pub stack_switching_values_buffer: Option, + stack_switching_values_buffer: Option, } impl<'module_environment> FuncEnvironment<'module_environment> { From 078002aa024fe825f148f9dbcf864049bcf708ba Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 26 Aug 2025 20:48:15 +0000 Subject: [PATCH 27/34] stack-switching: define VMContObj as two words This change migrates VMContObj and its usages in cranelift and runtime to work with the VMContObj fat pointer as two words in order to better target different architectures (still gated to x86_64 for now). To support this, a size type was plumbed into the builtins function signature types (as is done for component types) that maps to usize. --- crates/cranelift/src/func_environ.rs | 2 +- .../stack_switching/fatpointer.rs | 38 ++++++++++--------- crates/cranelift/src/lib.rs | 9 ++++- crates/environ/src/builtin.rs | 4 +- crates/environ/src/vmoffsets.rs | 11 ++++-- crates/wasmtime/src/runtime/vm/libcalls.rs | 5 ++- .../src/runtime/vm/stack_switching.rs | 10 ++--- crates/wasmtime/src/runtime/vm/vmcontext.rs | 1 + 8 files changed, 49 insertions(+), 31 deletions(-) diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 24912ce0cf26..877ecc14adbe 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -2397,7 +2397,7 @@ impl FuncEnvironment<'_> { WasmHeapTopType::Cont => { let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index); Ok(builder.ins().load( - stack_switching::fatpointer::POINTER_TYPE, + stack_switching::fatpointer::fatpointer_type(self), flags, elem_addr, 0, diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs index 6dd394318746..e2fe365acee5 100644 --- a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs +++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs @@ -1,15 +1,16 @@ use cranelift_codegen::ir; use cranelift_codegen::ir::InstBuilder; -/// The Cranelift type used to represent all of the following: +/// Returns the Cranelift type used to represent all of the following: /// - wasm values of type `(ref null $ct)` and `(ref $ct)` /// - equivalently: runtime values of type `Option` and `VMContObj` -/// Note that a `VMContObj` is a fat pointer -/// consisting of a pointer to `VMContRef` and a 64 bit sequence -/// counter. -/// We represent this here as a 128bit value, with the same representation as -/// `core::mem::transmute::`. -pub const POINTER_TYPE: ir::Type = ir::types::I128; +/// Note that a `VMContObj` is a fat pointer consisting of a pointer to +/// `VMContRef` and a pointer-sized revision counter. We represent this as 2 words +/// (pointer and usize). +pub fn fatpointer_type(env: &crate::func_environ::FuncEnvironment) -> ir::Type { + let ptr_bits = env.pointer_type().bits(); + ir::Type::int((2 * ptr_bits).try_into().unwrap()).unwrap() +} /// Turns a (possibly null) reference to a continuation object into a tuple /// (revision, contref_ptr). If `contobj` denotes a wasm null reference, the @@ -19,13 +20,13 @@ pub(crate) fn deconstruct<'a>( pos: &mut cranelift_codegen::cursor::FuncCursor, contobj: ir::Value, ) -> (ir::Value, ir::Value) { - debug_assert_eq!(pos.func.dfg.value_type(contobj), POINTER_TYPE); + debug_assert_eq!(pos.func.dfg.value_type(contobj), fatpointer_type(env)); let ptr_ty = env.pointer_type(); - assert!(ptr_ty.bits() <= 64); + let ptr_bits = ptr_ty.bits(); let contref = pos.ins().ireduce(ptr_ty, contobj); - let shifted = pos.ins().ushr_imm(contobj, 64); - let revision_counter = pos.ins().ireduce(ir::types::I64, shifted); + let shifted = pos.ins().ushr_imm(contobj, i64::from(ptr_bits)); + let revision_counter = pos.ins().ireduce(ptr_ty, shifted); (revision_counter, contref) } @@ -38,13 +39,16 @@ pub(crate) fn construct<'a>( revision_counter: ir::Value, contref_addr: ir::Value, ) -> ir::Value { - debug_assert_eq!(pos.func.dfg.value_type(contref_addr), env.pointer_type()); - debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ir::types::I64); - assert!(env.pointer_type().bits() <= 64); + let ptr_ty = env.pointer_type(); + let ptr_bits = ptr_ty.bits(); + let fat_ptr_ty = fatpointer_type(env); + + debug_assert_eq!(pos.func.dfg.value_type(contref_addr), ptr_ty); + debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ptr_ty); - let contref_addr = pos.ins().uextend(ir::types::I128, contref_addr); - let revision_counter = pos.ins().uextend(ir::types::I128, revision_counter); - let shifted_counter = pos.ins().ishl_imm(revision_counter, 64); + let contref_addr = pos.ins().uextend(fat_ptr_ty, contref_addr); + let revision_counter = pos.ins().uextend(fat_ptr_ty, revision_counter); + let shifted_counter = pos.ins().ishl_imm(revision_counter, i64::from(ptr_bits)); let contobj = pos.ins().bor(shifted_counter, contref_addr); contobj diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 2977a9c7a2c1..eba725ebfc32 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -213,7 +213,10 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type { match wasm_ht.top() { WasmHeapTopType::Func => pointer_type, WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => ir::types::I32, - WasmHeapTopType::Cont => func_environ::stack_switching::fatpointer::POINTER_TYPE, + WasmHeapTopType::Cont => { + // VMContObj is 2 * pointer_size (pointer + usize revision) + ir::Type::int((2 * pointer_type.bits()).try_into().unwrap()).unwrap() + } } } @@ -372,6 +375,10 @@ impl BuiltinFunctionSignatures { AbiParam::new(ir::types::I8) } + fn size(&self) -> AbiParam { + AbiParam::new(self.pointer_type) + } + fn wasm_signature(&self, builtin: BuiltinFunctionIndex) -> Signature { let mut _cur = 0; macro_rules! iter { diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index 299ed239e2c3..65f0225db05a 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -233,12 +233,12 @@ macro_rules! foreach_builtin_function { // denote the continuation being `None`, `init_contref` // may be 0. #[cfg(feature = "stack-switching")] - table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer; + table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: size) -> pointer; // `value_contref` and `value_revision` together encode // the Option, as in previous libcall. #[cfg(feature = "stack-switching")] - table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool; + table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: size, len: u64) -> bool; // Return the instance ID for a given vmctx. #[cfg(feature = "gc")] diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 8e67688ebef0..7a25a3c808b3 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -348,12 +348,17 @@ pub trait PtrSize { /// Return the offset of `VMContObj::revision` fn vmcontobj_revision(&self) -> u8 { - 8 + self.size() } - /// Return the size of `VMHostArray`. + /// Return the size of `VMContObj`. fn size_of_vmcontobj(&self) -> u8 { - 16 + u8::try_from(align( + u32::from(self.vmcontobj_revision()) + + u32::try_from(core::mem::size_of::()).unwrap(), + u32::from(self.size()), + )) + .unwrap() } // Offsets within `VMContRef` diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs index 395c22b52a78..8180ccf89ff8 100644 --- a/crates/wasmtime/src/runtime/vm/libcalls.rs +++ b/crates/wasmtime/src/runtime/vm/libcalls.rs @@ -157,6 +157,7 @@ pub mod raw { (@ty f64x2) => (f64x2); (@ty bool) => (bool); (@ty pointer) => (*mut u8); + (@ty size) => (usize); } wasmtime_environ::foreach_builtin_function!(libcall); @@ -339,7 +340,7 @@ unsafe fn table_grow_cont_obj( // The following two values together form the initial Option. // A None value is indicated by the pointer being null. init_value_contref: *mut u8, - init_value_revision: u64, + init_value_revision: usize, ) -> Result> { let defined_table_index = DefinedTableIndex::from_u32(defined_table_index); let element = unsafe { VMContObj::from_raw_parts(init_value_contref, init_value_revision) }; @@ -416,7 +417,7 @@ unsafe fn table_fill_cont_obj( table_index: u32, dst: u64, value_contref: *mut u8, - value_revision: u64, + value_revision: usize, len: u64, ) -> Result<()> { let instance = store.instance_mut(instance); diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs index 4796e1a7fb80..32c2c275339a 100644 --- a/crates/wasmtime/src/runtime/vm/stack_switching.rs +++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs @@ -39,15 +39,15 @@ pub use stack::*; /// For performance reasons, the VMContRef at the bottom of this chain /// (i.e., the one pointed to by the VMContObj) has a pointer to the /// other end of the chain (i.e., its last ancestor). -#[repr(C, align(16))] +#[repr(C)] #[derive(Debug, Clone, Copy)] pub struct VMContObj { pub contref: NonNull, - pub revision: u64, + pub revision: usize, } impl VMContObj { - pub fn new(contref: NonNull, revision: u64) -> Self { + pub fn new(contref: NonNull, revision: usize) -> Self { Self { contref, revision } } @@ -59,7 +59,7 @@ impl VMContObj { /// /// Behavior will be undefined if a pointer to data that is not a /// VMContRef is provided. - pub unsafe fn from_raw_parts(contref: *mut u8, revision: u64) -> Option { + pub unsafe fn from_raw_parts(contref: *mut u8, revision: usize) -> Option { NonNull::new(contref.cast::()).map(|contref| Self::new(contref, revision)) } } @@ -204,7 +204,7 @@ pub struct VMContRef { pub last_ancestor: *mut VMContRef, /// Revision counter. - pub revision: u64, + pub revision: usize, /// The underlying stack. pub stack: VMContinuationStack, diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 818b08940873..220e973082bc 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -1056,6 +1056,7 @@ macro_rules! define_builtin_array { (@ty f64x2) => (f64x2); (@ty bool) => (bool); (@ty pointer) => (*mut u8); + (@ty size) => (usize); (@ty vmctx) => (NonNull); } From e1c704c884bffc491aa731f1e5077299809c3bc6 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 27 Aug 2025 16:20:38 +0000 Subject: [PATCH 28/34] fixup! stack-switching: define VMContObj as two words --- crates/cranelift/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index eba725ebfc32..021c042ebd47 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -375,6 +375,7 @@ impl BuiltinFunctionSignatures { AbiParam::new(ir::types::I8) } + #[cfg(feature = "stack-switching")] fn size(&self) -> AbiParam { AbiParam::new(self.pointer_type) } From 7d4e6783d7820155efa1cf7dee2d80cc99507813 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Wed, 27 Aug 2025 22:43:34 +0000 Subject: [PATCH 29/34] stack-switching: add stub Val::ContRef This type is not fully complete until continuation/gc integration is revisited (#10248) but without these changes, test cases are now failing on panics as we need some representation of continuation references in the runtime Val enumeration. Runtime errors with TODO notes are added for the stubbed code paths to revisit later. --- crates/environ/src/gc.rs | 5 --- crates/wasmtime/src/runtime/coredump.rs | 6 ++++ .../wasmtime/src/runtime/externals/global.rs | 10 ++++++ .../wasmtime/src/runtime/trampoline/global.rs | 10 ++++++ crates/wasmtime/src/runtime/values.rs | 34 +++++++++++++++++-- .../src/runtime/vm/gc/enabled/arrayref.rs | 12 +++++++ .../src/runtime/vm/gc/enabled/structref.rs | 12 +++++++ crates/wasmtime/src/runtime/wave/core.rs | 1 + src/commands/run.rs | 2 ++ 9 files changed, 85 insertions(+), 7 deletions(-) diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 92104ad181d1..d9e771b02fd6 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -40,15 +40,10 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4; /// Get the byte size of the given Wasm type when it is stored inside the GC /// heap. pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 { - use crate::{WasmHeapType::*, WasmRefType}; match ty { WasmStorageType::I8 => 1, WasmStorageType::I16 => 2, WasmStorageType::Val(ty) => match ty { - WasmValType::Ref(WasmRefType { - nullable: _, - heap_type: ConcreteCont(_) | Cont, - }) => unimplemented!("Stack switching feature not compatbile with GC, yet"), WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4, WasmValType::I64 | WasmValType::F64 => 8, WasmValType::V128 => 16, diff --git a/crates/wasmtime/src/runtime/coredump.rs b/crates/wasmtime/src/runtime/coredump.rs index 517c2fbb241a..ac557b7f602d 100644 --- a/crates/wasmtime/src/runtime/coredump.rs +++ b/crates/wasmtime/src/runtime/coredump.rs @@ -210,6 +210,12 @@ impl WasmCoreDump { ty: wasm_encoder::AbstractHeapType::Exn, }) } + Val::ContRef(_) => { + wasm_encoder::ConstExpr::ref_null(wasm_encoder::HeapType::Abstract { + shared: false, + ty: wasm_encoder::AbstractHeapType::Cont, + }) + } }; globals.global( wasm_encoder::GlobalType { diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs index ce87e2616592..708ded32927d 100644 --- a/crates/wasmtime/src/runtime/externals/global.rs +++ b/crates/wasmtime/src/runtime/externals/global.rs @@ -274,6 +274,16 @@ impl Global { let new = new.as_ref(); definition.write_gc_ref(&mut store, new); } + Val::ContRef(None) => { + // Allow null continuation references for globals - these are just placeholders + // Non-null references are not supported yet + } + Val::ContRef(Some(_)) => { + // TODO(#10248): Implement non-null global continuation reference handling + return Err(anyhow::anyhow!( + "setting non-null continuation references in globals not yet supported" + )); + } } } Ok(()) diff --git a/crates/wasmtime/src/runtime/trampoline/global.rs b/crates/wasmtime/src/runtime/trampoline/global.rs index 2fad1d81aa80..b6fbca85626f 100644 --- a/crates/wasmtime/src/runtime/trampoline/global.rs +++ b/crates/wasmtime/src/runtime/trampoline/global.rs @@ -70,6 +70,16 @@ pub fn generate_global_export( let new = new.as_ref(); global.write_gc_ref(&mut store, new); } + Val::ContRef(None) => { + // Allow null continuation references for trampoline globals - these are just placeholders + // Non-null references are not supported yet + } + Val::ContRef(Some(_)) => { + // TODO(#10248): Implement non-null trampoline continuation reference handling + return Err(anyhow::anyhow!( + "non-null continuation references in trampoline globals not yet supported" + )); + } } } diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs index e572b2dfb157..15aa627b8bd4 100644 --- a/crates/wasmtime/src/runtime/values.rs +++ b/crates/wasmtime/src/runtime/values.rs @@ -8,6 +8,13 @@ use wasmtime_environ::WasmHeapTopType; pub use crate::runtime::vm::ValRaw; +/// A stub implementation for continuation references. +/// +/// This is a placeholder until continuation objects are fully integrated +/// with the GC system (see #10248). +#[derive(Debug, Clone, Copy)] +pub struct ContRef; + /// Possible runtime values that a WebAssembly module can either consume or /// produce. /// @@ -50,6 +57,12 @@ pub enum Val { /// An exception reference. ExnRef(Option>), + + /// A continuation reference. + /// + /// Note: This is currently a stub implementation as continuation objects + /// are not yet fully integrated with the GC system. See #10248. + ContRef(Option), } macro_rules! accessors { @@ -118,7 +131,7 @@ impl Val { WasmHeapTopType::Extern => Val::ExternRef(None), WasmHeapTopType::Any => Val::AnyRef(None), WasmHeapTopType::Exn => Val::ExnRef(None), - WasmHeapTopType::Cont => todo!(), // FIXME(#10248) + WasmHeapTopType::Cont => Val::ContRef(None), } } @@ -177,6 +190,12 @@ impl Val { Val::AnyRef(Some(a)) => ValType::Ref(RefType::new(false, a._ty(store)?)), Val::ExnRef(None) => ValType::NULLEXNREF, Val::ExnRef(Some(e)) => ValType::Ref(RefType::new(false, e._ty(store)?.into())), + Val::ContRef(_) => { + // TODO(#10248): Return proper continuation reference type when available + return Err(anyhow::anyhow!( + "continuation references not yet supported in embedder API" + )); + } }) } @@ -216,7 +235,8 @@ impl Val { | (Val::FuncRef(_), _) | (Val::ExternRef(_), _) | (Val::AnyRef(_), _) - | (Val::ExnRef(_), _) => false, + | (Val::ExnRef(_), _) + | (Val::ContRef(_), _) => false, }) } @@ -268,6 +288,12 @@ impl Val { Some(f) => f.to_raw(store), None => ptr::null_mut(), })), + Val::ContRef(_) => { + // TODO(#10248): Implement proper continuation reference to_raw conversion + Err(anyhow::anyhow!( + "continuation references not yet supported in to_raw conversion" + )) + } } } @@ -363,6 +389,7 @@ impl Val { Val::AnyRef(a) => Some(Ref::Any(a)), Val::ExnRef(e) => Some(Ref::Exn(e)), Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => None, + Val::ContRef(_) => None, // TODO(#10248): Return proper Ref::Cont when available } } @@ -509,6 +536,9 @@ impl Val { // particular store, so they're always considered as "yes I came // from that store", Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => true, + + // Continuation references are not yet associated with stores + Val::ContRef(_) => true, // TODO(#10248): Proper store association when implemented } } } diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs index 04982d3c49b7..5d6f525f8a5f 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs @@ -276,6 +276,12 @@ impl VMArrayRef { .gc_object_data(self.as_gc_ref()) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement array continuation reference element handling + return Err(anyhow::anyhow!( + "setting continuation references in array elements not yet supported" + )); + } } Ok(()) } @@ -383,6 +389,12 @@ impl VMArrayRef { .gc_object_data(self.as_gc_ref()) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement array continuation reference init handling + return Err(anyhow::anyhow!( + "initializing continuation references in array elements not yet supported" + )); + } } Ok(()) } diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs index 364faf1c0194..a8ecfea074e4 100644 --- a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs +++ b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs @@ -233,6 +233,12 @@ impl VMStructRef { .gc_object_data(self.as_gc_ref()) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement struct continuation reference field handling + return Err(anyhow::anyhow!( + "setting continuation references in struct fields not yet supported" + )); + } } Ok(()) } @@ -386,6 +392,12 @@ pub(crate) fn initialize_field_impl( .gc_object_data(gc_ref) .write_u32(offset, id.into_raw()); } + Val::ContRef(_) => { + // TODO(#10248): Implement struct continuation reference field init handling + return Err(anyhow::anyhow!( + "initializing continuation references in struct fields not yet supported" + )); + } } Ok(()) } diff --git a/crates/wasmtime/src/runtime/wave/core.rs b/crates/wasmtime/src/runtime/wave/core.rs index 338ce4b2d888..fd89da83d7b9 100644 --- a/crates/wasmtime/src/runtime/wave/core.rs +++ b/crates/wasmtime/src/runtime/wave/core.rs @@ -40,6 +40,7 @@ impl WasmValue for crate::Val { Self::ExternRef(_) => WasmTypeKind::Unsupported, Self::AnyRef(_) => WasmTypeKind::Unsupported, Self::ExnRef(_) => WasmTypeKind::Unsupported, + Self::ContRef(_) => WasmTypeKind::Unsupported, } } diff --git a/src/commands/run.rs b/src/commands/run.rs index d42e573b4c05..e3fd86d27b82 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -746,6 +746,8 @@ impl RunCommand { Val::AnyRef(Some(_)) => println!(""), Val::ExnRef(None) => println!(""), Val::ExnRef(Some(_)) => println!(""), + Val::ContRef(None) => println!(""), + Val::ContRef(Some(_)) => println!(""), } } From e929cb7b64f651637db8187b349ecff707dd10bd Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 28 Aug 2025 16:21:19 +0000 Subject: [PATCH 30/34] fixup! stack-switching: add stub Val::ContRef --- crates/c-api/src/val.rs | 2 ++ crates/fuzzing/src/generators/value.rs | 7 +++++++ crates/fuzzing/src/oracles/diff_spec.rs | 3 ++- crates/fuzzing/src/oracles/diff_v8.rs | 2 ++ crates/fuzzing/src/oracles/diff_wasmi.rs | 1 + crates/fuzzing/src/oracles/diff_wasmtime.rs | 5 +++++ 6 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/c-api/src/val.rs b/crates/c-api/src/val.rs index efc53cf52968..c99b7cd41f66 100644 --- a/crates/c-api/src/val.rs +++ b/crates/c-api/src/val.rs @@ -98,6 +98,7 @@ impl wasm_val_t { Val::ExternRef(_) => crate::abort("creating a wasm_val_t from an externref"), Val::ExnRef(_) => crate::abort("creating a wasm_val_t from an exnref"), Val::V128(_) => crate::abort("creating a wasm_val_t from a v128"), + Val::ContRef(_) => crate::abort("creating a wasm_val_t from a contref"), } } @@ -259,6 +260,7 @@ impl wasmtime_val_t { v128: val.as_u128().to_le_bytes(), }, }, + Val::ContRef(_) => crate::abort("contrefs not yet supported in C API (#10248)"), } } diff --git a/crates/fuzzing/src/generators/value.rs b/crates/fuzzing/src/generators/value.rs index 697982a12013..685c655400f1 100644 --- a/crates/fuzzing/src/generators/value.rs +++ b/crates/fuzzing/src/generators/value.rs @@ -18,6 +18,7 @@ pub enum DiffValue { ExternRef { null: bool }, AnyRef { null: bool }, ExnRef { null: bool }, + ContRef { null: bool }, } impl DiffValue { @@ -32,6 +33,7 @@ impl DiffValue { DiffValue::ExternRef { .. } => DiffValueType::ExternRef, DiffValue::AnyRef { .. } => DiffValueType::AnyRef, DiffValue::ExnRef { .. } => DiffValueType::ExnRef, + DiffValue::ContRef { .. } => DiffValueType::ContRef, } } @@ -189,6 +191,7 @@ impl DiffValue { ExternRef => DiffValue::ExternRef { null: true }, AnyRef => DiffValue::AnyRef { null: true }, ExnRef => DiffValue::ExnRef { null: true }, + ContRef => DiffValue::ContRef { null: true }, }; arbitrary::Result::Ok(val) } @@ -236,6 +239,7 @@ impl Hash for DiffValue { DiffValue::FuncRef { null } => null.hash(state), DiffValue::AnyRef { null } => null.hash(state), DiffValue::ExnRef { null } => null.hash(state), + DiffValue::ContRef { null } => null.hash(state), } } } @@ -273,6 +277,7 @@ impl PartialEq for DiffValue { (Self::ExternRef { null: a }, Self::ExternRef { null: b }) => a == b, (Self::AnyRef { null: a }, Self::AnyRef { null: b }) => a == b, (Self::ExnRef { null: a }, Self::ExnRef { null: b }) => a == b, + (Self::ContRef { null: a }, Self::ContRef { null: b }) => a == b, _ => false, } } @@ -291,6 +296,7 @@ pub enum DiffValueType { ExternRef, AnyRef, ExnRef, + ContRef, } impl TryFrom for DiffValueType { @@ -310,6 +316,7 @@ impl TryFrom for DiffValueType { (true, HeapType::I31) => Ok(Self::AnyRef), (true, HeapType::None) => Ok(Self::AnyRef), (true, HeapType::Exn) => Ok(Self::ExnRef), + (true, HeapType::Cont) => Ok(Self::ContRef), _ => Err("non-null reference types are not supported yet"), }, } diff --git a/crates/fuzzing/src/oracles/diff_spec.rs b/crates/fuzzing/src/oracles/diff_spec.rs index c5b9edbafc38..85fab4b5d7a7 100644 --- a/crates/fuzzing/src/oracles/diff_spec.rs +++ b/crates/fuzzing/src/oracles/diff_spec.rs @@ -107,7 +107,8 @@ impl From<&DiffValue> for SpecValue { DiffValue::FuncRef { .. } | DiffValue::ExternRef { .. } | DiffValue::AnyRef { .. } - | DiffValue::ExnRef { .. } => { + | DiffValue::ExnRef { .. } + | DiffValue::ContRef { .. } => { unimplemented!() } } diff --git a/crates/fuzzing/src/oracles/diff_v8.rs b/crates/fuzzing/src/oracles/diff_v8.rs index 7e0ed12e21ec..eaf612a4be2c 100644 --- a/crates/fuzzing/src/oracles/diff_v8.rs +++ b/crates/fuzzing/src/oracles/diff_v8.rs @@ -191,6 +191,7 @@ impl DiffInstance for V8Instance { DiffValue::V128(_) => return Ok(None), DiffValue::AnyRef { .. } => unimplemented!(), DiffValue::ExnRef { .. } => unimplemented!(), + DiffValue::ContRef { .. } => unimplemented!(), }); } // JS doesn't support v128 return values @@ -316,6 +317,7 @@ fn get_diff_value( DiffValueType::AnyRef => unimplemented!(), DiffValueType::ExnRef => unimplemented!(), DiffValueType::V128 => unreachable!(), + DiffValueType::ContRef => unimplemented!(), } } diff --git a/crates/fuzzing/src/oracles/diff_wasmi.rs b/crates/fuzzing/src/oracles/diff_wasmi.rs index c591c218cdf8..dbed7f2f2700 100644 --- a/crates/fuzzing/src/oracles/diff_wasmi.rs +++ b/crates/fuzzing/src/oracles/diff_wasmi.rs @@ -195,6 +195,7 @@ impl From<&DiffValue> for wasmi::Val { } DiffValue::AnyRef { .. } => unimplemented!(), DiffValue::ExnRef { .. } => unimplemented!(), + DiffValue::ContRef { .. } => unimplemented!(), } } } diff --git a/crates/fuzzing/src/oracles/diff_wasmtime.rs b/crates/fuzzing/src/oracles/diff_wasmtime.rs index 53c54b73a239..f600005164b4 100644 --- a/crates/fuzzing/src/oracles/diff_wasmtime.rs +++ b/crates/fuzzing/src/oracles/diff_wasmtime.rs @@ -227,6 +227,10 @@ impl From<&DiffValue> for Val { assert!(null); Val::ExnRef(None) } + DiffValue::ContRef { null } => { + assert!(null); + Val::ExnRef(None) + } } } } @@ -243,6 +247,7 @@ impl From for DiffValue { Val::FuncRef(r) => DiffValue::FuncRef { null: r.is_none() }, Val::AnyRef(r) => DiffValue::AnyRef { null: r.is_none() }, Val::ExnRef(e) => DiffValue::ExnRef { null: e.is_none() }, + Val::ContRef(c) => DiffValue::ContRef { null: c.is_none() }, } } } From e258e35a4c4c7bd890660967163eb44cba9c11df Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 28 Aug 2025 20:07:58 +0000 Subject: [PATCH 31/34] fixup! stack-switching: add stub Val::ContRef --- crates/wasmtime/src/runtime/externals/global.rs | 2 +- crates/wasmtime/src/runtime/trampoline/global.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs index 708ded32927d..2223c114334c 100644 --- a/crates/wasmtime/src/runtime/externals/global.rs +++ b/crates/wasmtime/src/runtime/externals/global.rs @@ -276,7 +276,7 @@ impl Global { } Val::ContRef(None) => { // Allow null continuation references for globals - these are just placeholders - // Non-null references are not supported yet + definition.write_gc_ref(&mut store, None); } Val::ContRef(Some(_)) => { // TODO(#10248): Implement non-null global continuation reference handling diff --git a/crates/wasmtime/src/runtime/trampoline/global.rs b/crates/wasmtime/src/runtime/trampoline/global.rs index b6fbca85626f..5ef0a6ddf2cc 100644 --- a/crates/wasmtime/src/runtime/trampoline/global.rs +++ b/crates/wasmtime/src/runtime/trampoline/global.rs @@ -72,7 +72,7 @@ pub fn generate_global_export( } Val::ContRef(None) => { // Allow null continuation references for trampoline globals - these are just placeholders - // Non-null references are not supported yet + global.write_gc_ref(&mut store, None); } Val::ContRef(Some(_)) => { // TODO(#10248): Implement non-null trampoline continuation reference handling From 5e2433771d575d5489e7def8b63cda9be2920d07 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Thu, 28 Aug 2025 23:39:19 +0000 Subject: [PATCH 32/34] fixup! stack-switching: define VMContObj as two words prtest:full --- crates/environ/src/vmoffsets.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index 7a25a3c808b3..71fb7b045da1 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -390,7 +390,7 @@ pub trait PtrSize { /// Return the offset of `VMContRef::stack`. fn vmcontref_stack(&self) -> u8 { - self.vmcontref_revision() + 8 + self.vmcontref_revision() + self.size() } /// Return the offset of `VMContRef::args`. From ed4b01000e685828ea1fbe7ead806e6226f0eea4 Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Fri, 29 Aug 2025 17:47:07 +0000 Subject: [PATCH 33/34] stack-switching: don't conflate host and target pointer sizes Disas tests were failing on i686 targeting x86_64 as the size of the host pointer was leaking into what we were using to do codegen in a few paths. This patch is a bit of a hack as it seems like using a generic for T: *mut u8 (as an example) is a bit questionable. To keep things small, I do a hacky typecheck to map pointers to the target pointer size here. --- .../stack_switching/instructions.rs | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index 092128e2fa26..e78c2066de7b 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -201,7 +201,7 @@ pub(crate) mod stack_switching_helpers { } } - impl VMHostArrayRef { + impl VMHostArrayRef { pub(crate) fn new(address: ir::Value) -> Self { Self { address, @@ -270,8 +270,10 @@ pub(crate) mod stack_switching_helpers { builder: &mut FunctionBuilder, data: ir::Value, ) { - let offset = env.offsets.ptr.vmhostarray_data().into(); - self.set::<*mut T>(builder, offset, data); + debug_assert_eq!(builder.func.dfg.value_type(data), env.pointer_type()); + let offset: i32 = env.offsets.ptr.vmhostarray_data().into(); + let mem_flags = ir::MemFlags::trusted(); + builder.ins().store(mem_flags, data, self.address, offset); } /// Returns pointer to next empty slot in data buffer and marks the @@ -302,8 +304,19 @@ pub(crate) mod stack_switching_helpers { required_capacity: u32, existing_slot: Option, ) -> StackSlot { - let align = u8::try_from(std::mem::align_of::()).unwrap(); - let entry_size = u32::try_from(std::mem::size_of::()).unwrap(); + // TODO: hack around host pointer size being mixed up with target + let (align, entry_size) = + if core::any::TypeId::of::() == core::any::TypeId::of::<*mut u8>() { + ( + u8::try_from(env.pointer_type().bytes()).unwrap(), + env.pointer_type().bytes(), + ) + } else { + ( + u8::try_from(std::mem::align_of::()).unwrap(), + u32::try_from(std::mem::size_of::()).unwrap(), + ) + }; let required_size = required_capacity * entry_size; match existing_slot { @@ -368,14 +381,12 @@ pub(crate) mod stack_switching_helpers { /// index 0. This expects the Vector object to be empty (i.e., current /// length is 0), and to be of sufficient capacity to store |`values`| /// entries. - /// If `allow_smaller` is true, we allow storing values whose type has a - /// smaller size than T's. In that case, such values will be stored at - /// the beginning of a `T`-sized slot. pub fn store_data_entries<'a>( &self, env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, values: &[ir::Value], + entry_size: u32, ) { let store_count = builder .ins() @@ -383,21 +394,20 @@ pub(crate) mod stack_switching_helpers { debug_assert!(values.iter().all(|val| { let ty = builder.func.dfg.value_type(*val); - let size = usize::try_from(ty.bytes()).unwrap(); - size <= std::mem::size_of::() + let size = ty.bytes(); + size <= entry_size })); let memflags = ir::MemFlags::trusted(); let data_start_pointer = self.get_data(env, builder); - let entry_size = i32::try_from(std::mem::size_of::()).unwrap(); let mut offset = 0; for value in values { builder .ins() .store(memflags, *value, data_start_pointer, offset); - offset += entry_size; + offset += i32::try_from(entry_size).unwrap(); } self.set_length(env, builder, store_count); @@ -793,11 +803,13 @@ pub(crate) mod stack_switching_helpers { fn load_top_of_stack<'a>( &self, - _env: &mut crate::func_environ::FuncEnvironment<'a>, + env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, ) -> ir::Value { let mem_flags = ir::MemFlags::trusted(); - builder.ins().load(I64, mem_flags, self.tos_ptr, 0) + builder + .ins() + .load(env.pointer_type(), mem_flags, self.tos_ptr, 0) } /// Returns address of the control context stored in the stack memory, @@ -1114,10 +1126,8 @@ fn search_handler<'a>( builder.switch_to_block(compare_tags); let base = handler_list_data_ptr; - let entry_size = std::mem::size_of::<*mut u8>(); - let offset = builder - .ins() - .imul_imm(index, i64::try_from(entry_size).unwrap()); + let entry_size = env.pointer_type().bytes(); + let offset = builder.ins().imul_imm(index, i64::from(entry_size)); let offset = builder.ins().uextend(I64, offset); let entry_address = builder.ins().iadd(base, offset); @@ -1378,7 +1388,8 @@ pub(crate) fn translate_resume<'a>( .collect(); // Store all tag addresess in the handler list. - handler_list.store_data_entries(env, builder, &all_tag_addresses); + let entry_size = env.pointer_type().bytes(); + handler_list.store_data_entries(env, builder, &all_tag_addresses, entry_size); // To enable distinguishing switch and suspend handlers when searching the handler list: // Store at which index the switch handlers start. @@ -1624,7 +1635,8 @@ pub(crate) fn translate_suspend<'a>( } if suspend_args.len() > 0 { - values.store_data_entries(env, builder, suspend_args) + let entry_size: u32 = std::mem::size_of::().try_into().unwrap(); + values.store_data_entries(env, builder, suspend_args, entry_size) } // Set current continuation to suspended and break up handler chain. From 9343ab99f2c07fb3f2a4dd778142422da9202d3f Mon Sep 17 00:00:00 2001 From: Paul Osborne Date: Tue, 2 Sep 2025 18:50:03 +0000 Subject: [PATCH 34/34] stack-switching: VMHostArray entry sizes based off env PtrSize Revisiting the previous commit with an approach that should be less brittle. --- .../stack_switching/instructions.rs | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs index e78c2066de7b..383da04f93e7 100644 --- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs +++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs @@ -29,9 +29,27 @@ pub(crate) mod stack_switching_helpers { use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{StackSlot, StackSlotKind::*}; use cranelift_frontend::FunctionBuilder; - use std::mem; use wasmtime_environ::PtrSize; + /// Provides information about the layout of a type when it is used as an + /// element in a host array. This is used for `VMHostArrayRef`. + pub(crate) trait VMHostArrayEntry { + /// Returns `(align, size)` in bytes. + fn vmhostarray_entry_layout(p: &P) -> (u8, u32); + } + + impl VMHostArrayEntry for u128 { + fn vmhostarray_entry_layout(_p: &P) -> (u8, u32) { + (16, 16) + } + } + + impl VMHostArrayEntry for *mut T { + fn vmhostarray_entry_layout(p: &P) -> (u8, u32) { + (p.size(), p.size().into()) + } + } + #[derive(Copy, Clone)] pub struct VMContRef { pub address: ir::Value, @@ -201,7 +219,7 @@ pub(crate) mod stack_switching_helpers { } } - impl VMHostArrayRef { + impl VMHostArrayRef { pub(crate) fn new(address: ir::Value) -> Self { Self { address, @@ -217,7 +235,8 @@ pub(crate) mod stack_switching_helpers { fn set(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) { debug_assert_eq!( builder.func.dfg.value_type(value), - Type::int_with_byte_size(u16::try_from(std::mem::size_of::()).unwrap()).unwrap() + Type::int_with_byte_size(u16::try_from(core::mem::size_of::()).unwrap()) + .unwrap() ); let mem_flags = ir::MemFlags::trusted(); builder.ins().store(mem_flags, value, self.address, offset); @@ -291,9 +310,11 @@ pub(crate) mod stack_switching_helpers { .iadd_imm(original_length, i64::from(arg_count)); self.set_length(env, builder, new_length); - let value_size: i64 = mem::size_of::().try_into().unwrap(); + let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); let original_length = builder.ins().uextend(I64, original_length); - let byte_offset = builder.ins().imul_imm(original_length, value_size); + let byte_offset = builder + .ins() + .imul_imm(original_length, i64::from(entry_size)); builder.ins().iadd(data, byte_offset) } @@ -304,19 +325,7 @@ pub(crate) mod stack_switching_helpers { required_capacity: u32, existing_slot: Option, ) -> StackSlot { - // TODO: hack around host pointer size being mixed up with target - let (align, entry_size) = - if core::any::TypeId::of::() == core::any::TypeId::of::<*mut u8>() { - ( - u8::try_from(env.pointer_type().bytes()).unwrap(), - env.pointer_type().bytes(), - ) - } else { - ( - u8::try_from(std::mem::align_of::()).unwrap(), - u32::try_from(std::mem::size_of::()).unwrap(), - ) - }; + let (align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); let required_size = required_capacity * entry_size; match existing_slot { @@ -366,13 +375,13 @@ pub(crate) mod stack_switching_helpers { let data_start_pointer = self.get_data(env, builder); let mut values = vec![]; let mut offset = 0; - let entry_size = i32::try_from(std::mem::size_of::()).unwrap(); + let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); for valtype in load_types { let val = builder .ins() .load(*valtype, memflags, data_start_pointer, offset); values.push(val); - offset += entry_size; + offset += i32::try_from(entry_size).unwrap(); } values } @@ -386,12 +395,13 @@ pub(crate) mod stack_switching_helpers { env: &mut crate::func_environ::FuncEnvironment<'a>, builder: &mut FunctionBuilder, values: &[ir::Value], - entry_size: u32, ) { let store_count = builder .ins() .iconst(I32, i64::try_from(values.len()).unwrap()); + let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr); + debug_assert!(values.iter().all(|val| { let ty = builder.func.dfg.value_type(*val); let size = ty.bytes(); @@ -1388,8 +1398,7 @@ pub(crate) fn translate_resume<'a>( .collect(); // Store all tag addresess in the handler list. - let entry_size = env.pointer_type().bytes(); - handler_list.store_data_entries(env, builder, &all_tag_addresses, entry_size); + handler_list.store_data_entries(env, builder, &all_tag_addresses); // To enable distinguishing switch and suspend handlers when searching the handler list: // Store at which index the switch handlers start. @@ -1635,8 +1644,7 @@ pub(crate) fn translate_suspend<'a>( } if suspend_args.len() > 0 { - let entry_size: u32 = std::mem::size_of::().try_into().unwrap(); - values.store_data_entries(env, builder, suspend_args, entry_size) + values.store_data_entries(env, builder, suspend_args); } // Set current continuation to suspended and break up handler chain.