From f29c5168b205bfa51316ab79d04c143f1937edef Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Sat, 27 Dec 2025 22:35:40 +0800 Subject: [PATCH 01/11] Add test for never type fallback in try blocks with Into Closes #125364 This test verifies that the never type fallback behavior in try blocks with `Into` bounds works correctly across different editions: - Edition 2021: Fallback to `()` causes type error - Edition 2024: Fallback to `!` compiles successfully --- ...try-block-never-type-fallback.e2021.stderr | 20 +++++++++++++++ .../try-block-never-type-fallback.rs | 25 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/ui/never_type/try-block-never-type-fallback.e2021.stderr create mode 100644 tests/ui/never_type/try-block-never-type-fallback.rs diff --git a/tests/ui/never_type/try-block-never-type-fallback.e2021.stderr b/tests/ui/never_type/try-block-never-type-fallback.e2021.stderr new file mode 100644 index 0000000000000..7e23c86bcbccc --- /dev/null +++ b/tests/ui/never_type/try-block-never-type-fallback.e2021.stderr @@ -0,0 +1,20 @@ +error[E0277]: the trait bound `!: From<()>` is not satisfied + --> $DIR/try-block-never-type-fallback.rs:20:9 + | +LL | bar(try { x? }); + | --- ^^^^^^^^^^ the trait `From<()>` is not implemented for `!` + | | + | required by a bound introduced by this call + | + = note: this error might have been caused by changes to Rust's type-inference algorithm (see issue #148922 for more information) + = help: you might have intended to use the type `()` here instead + = note: required for `()` to implement `Into` +note: required by a bound in `bar` + --> $DIR/try-block-never-type-fallback.rs:15:23 + | +LL | fn bar(_: Result, u32>) { + | ^^^^^^^ required by this bound in `bar` + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/never_type/try-block-never-type-fallback.rs b/tests/ui/never_type/try-block-never-type-fallback.rs new file mode 100644 index 0000000000000..94107a0ff103c --- /dev/null +++ b/tests/ui/never_type/try-block-never-type-fallback.rs @@ -0,0 +1,25 @@ +//@ revisions: e2021 e2024 +//@[e2021] edition: 2021 +//@[e2024] edition: 2024 +//@[e2024] check-pass + +// Issue #125364: Bad interaction between never_type, try_blocks, and From/Into +// +// In edition 2021, the never type in try blocks falls back to (), +// causing a type error (since (): Into does not hold). +// In edition 2024, it falls back to !, allowing the code to compile correctly. + +#![feature(never_type)] +#![feature(try_blocks)] + +fn bar(_: Result, u32>) { + unimplemented!() +} + +fn foo(x: Result) { + bar(try { x? }); + //[e2021]~^ ERROR the trait bound `!: From<()>` is not satisfied +} + +fn main() { +} From cf8d0c82dc2e3de67d29a09389dbf035c5723c57 Mon Sep 17 00:00:00 2001 From: Deadbeef Date: Wed, 24 Dec 2025 14:42:14 -0500 Subject: [PATCH 02/11] fix rustfmt on `const impl Ty {}` --- src/tools/rustfmt/src/items.rs | 2 +- src/tools/rustfmt/tests/source/const_trait.rs | 14 ++++++++++++++ src/tools/rustfmt/tests/target/const_trait.rs | 9 +++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 src/tools/rustfmt/tests/source/const_trait.rs diff --git a/src/tools/rustfmt/src/items.rs b/src/tools/rustfmt/src/items.rs index d1778f324c6aa..c9e32492c50b0 100644 --- a/src/tools/rustfmt/src/items.rs +++ b/src/tools/rustfmt/src/items.rs @@ -971,7 +971,7 @@ fn format_impl_ref_and_type( result.push_str(format_defaultness(of_trait.defaultness)); result.push_str(format_safety(of_trait.safety)); } else { - result.push_str(format_constness_right(*constness)); + result.push_str(format_constness(*constness)); } let shape = if context.config.style_edition() >= StyleEdition::Edition2024 { diff --git a/src/tools/rustfmt/tests/source/const_trait.rs b/src/tools/rustfmt/tests/source/const_trait.rs new file mode 100644 index 0000000000000..99414a74f250c --- /dev/null +++ b/src/tools/rustfmt/tests/source/const_trait.rs @@ -0,0 +1,14 @@ +#![feature(trait_alias, const_trait_impl)] + +const trait Bar {} + +const trait Foo = Bar; + +impl const Bar for () {} + +// const impl gets reformatted to impl const.. for now +const impl Bar for u8 {} + +struct X; + +const impl X {} diff --git a/src/tools/rustfmt/tests/target/const_trait.rs b/src/tools/rustfmt/tests/target/const_trait.rs index c4c88b17fc500..337a4cf4140e8 100644 --- a/src/tools/rustfmt/tests/target/const_trait.rs +++ b/src/tools/rustfmt/tests/target/const_trait.rs @@ -3,3 +3,12 @@ const trait Bar {} const trait Foo = Bar; + +impl const Bar for () {} + +// const impl gets reformatted to impl const.. for now +impl const Bar for u8 {} + +struct X; + +const impl X {} From 4f7086a629f23865558a075e7562893da60968ba Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 5 Jun 2025 11:33:03 +0000 Subject: [PATCH 03/11] Split out ABI handling for LLVM intrinsics --- compiler/rustc_codegen_ssa/src/mir/block.rs | 86 ++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index cce33107e2c27..2d779c504d8f9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -14,7 +14,7 @@ use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_span::source_map::Spanned; -use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -1036,6 +1036,90 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => bug!("{} is not callable", callee.layout.ty), }; + if let Some(instance) = instance + && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name + && name.as_str().starts_with("llvm.") + { + let mut llargs = Vec::with_capacity(args.len()); + + let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; + let return_dest = if dest_ty.is_unit() { + ReturnDest::Nothing + } else if let Some(index) = destination.as_local() { + match self.locals[index] { + LocalRef::Place(dest) => ReturnDest::Store(dest), + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + LocalRef::PendingOperand => { + // Handle temporary places, specifically `Operand` ones, as + // they don't have `alloca`s. + ReturnDest::DirectOperand(index) + } + LocalRef::Operand(_) => bug!("place local already assigned to"), + } + } else { + ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) + }; + + for arg in args { + let op = self.codegen_operand(bx, &arg.node); + + match op.val { + ZeroSized => {} + Immediate(_) => llargs.push(op.immediate()), + Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = bx.load(bx.backend_type(op.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = op.layout.backend_repr { + if scalar.is_bool() { + bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = bx.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + let fn_ptr = bx.get_fn_addr(instance); + self.set_debug_loc(bx, source_info); + + // FIXME remove usage of fn_abi + let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = bx.fn_decl_backend_type(fn_abi); + + let llret = bx.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if self.mir[helper.bb].is_cleanup { + bx.apply_attrs_to_cleanup_callsite(llret); + } + + if let Some(target) = target { + self.store_return( + bx, + return_dest, + &ArgAbi { + layout: bx.layout_of(dest_ty), + mode: PassMode::Direct(ArgAttributes::new()), + }, + llret, + ); + return helper.funclet_br(self, bx, target, mergeable_succ); + } else { + bx.unreachable(); + return MergingSucc::False; + } + } + // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. From 62e17e920cd577440c1a91ebf665d6b9b5e0a453 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 18 Sep 2025 13:25:01 +0000 Subject: [PATCH 04/11] Move llvm intrinsic call to backend --- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 52 +++++++++++++++++- compiler/rustc_codegen_llvm/src/intrinsic.rs | 54 ++++++++++++++++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 45 ++-------------- .../rustc_codegen_ssa/src/traits/intrinsic.rs | 7 +++ 4 files changed, 114 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index d2714ba7914f3..75ea39c231140 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -9,7 +9,7 @@ use gccjit::Type; use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] use rustc_abi::ExternAbi; -use rustc_abi::{BackendRepr, HasDataLayout}; +use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::common::IntPredicate; @@ -20,7 +20,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; use rustc_middle::bug; #[cfg(feature = "master")] @@ -609,6 +609,54 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let fn_ptr = self.get_fn_addr(instance); + let fn_ty = fn_ptr.get_type(); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + // FIXME directly use the llvm intrinsic adjustment functions here + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 5503f7689304b..f159161deee8e 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,7 +1,9 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; -use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; +use rustc_abi::{ + Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, +}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; @@ -633,6 +635,56 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let fn_ptr = self.get_fn_addr(instance); + // FIXME remove usage of fn_abi + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = self.fn_decl_backend_type(fn_abi); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { self.call_intrinsic("llvm.trap", &[], &[]); } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 2d779c504d8f9..7f1ae81a15ce5 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1040,8 +1040,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { - let mut llargs = Vec::with_capacity(args.len()); - let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; let return_dest = if dest_ty.is_unit() { ReturnDest::Nothing @@ -1060,48 +1058,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) }; - for arg in args { - let op = self.codegen_operand(bx, &arg.node); - - match op.val { - ZeroSized => {} - Immediate(_) => llargs.push(op.immediate()), - Pair(a, b) => { - llargs.push(a); - llargs.push(b); - } - Ref(op_place_val) => { - let mut llval = op_place_val.llval; - // We can't use `PlaceRef::load` here because the argument - // may have a type we don't treat as immediate, but the ABI - // used for this call is passing it by-value. In that case, - // the load would just produce `OperandValue::Ref` instead - // of the `OperandValue::Immediate` we need for the call. - llval = bx.load(bx.backend_type(op.layout), llval, op_place_val.align); - if let BackendRepr::Scalar(scalar) = op.layout.backend_repr { - if scalar.is_bool() { - bx.range_metadata(llval, WrappingRange { start: 0, end: 1 }); - } - // We store bools as `i8` so we need to truncate to `i1`. - llval = bx.to_immediate_scalar(llval, scalar); - } - llargs.push(llval); - } - } - } + let args = + args.into_iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect::>(); - let fn_ptr = bx.get_fn_addr(instance); self.set_debug_loc(bx, source_info); - // FIXME remove usage of fn_abi - let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty()); - assert!(!fn_abi.ret.is_indirect()); - let fn_ty = bx.fn_decl_backend_type(fn_abi); - - let llret = bx.call(fn_ty, None, None, fn_ptr, &llargs, None, None); - if self.mir[helper.bb].is_cleanup { - bx.apply_attrs_to_cleanup_callsite(llret); - } + let llret = + bx.codegen_llvm_intrinsic_call(instance, &args, self.mir[helper.bb].is_cleanup); if let Some(target) = target { self.store_return( diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index 187e4b90656ab..04183c2801e7a 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -25,6 +25,13 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { span: Span, ) -> Result<(), ty::Instance<'tcx>>; + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value; + fn abort(&mut self); fn assume(&mut self, val: Self::Value); fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; From 17f6652641c86b2e199dc4639ed9585291fabf61 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:29:05 +0000 Subject: [PATCH 05/11] Fix for uninhabited llvm intrinsic return type --- compiler/rustc_codegen_ssa/src/mir/block.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 7f1ae81a15ce5..560880a76e51a 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1040,8 +1040,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { - let dest_ty = destination.ty(&self.mir.local_decls, bx.tcx()).ty; - let return_dest = if dest_ty.is_unit() { + let result_layout = + self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); + + let return_dest = if result_layout.is_zst() { ReturnDest::Nothing } else if let Some(index) = destination.as_local() { match self.locals[index] { @@ -1070,10 +1072,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { self.store_return( bx, return_dest, - &ArgAbi { - layout: bx.layout_of(dest_ty), - mode: PassMode::Direct(ArgAttributes::new()), - }, + &ArgAbi { layout: result_layout, mode: PassMode::Direct(ArgAttributes::new()) }, llret, ); return helper.funclet_br(self, bx, target, mergeable_succ); From b40eae863a54a6f8cae9ba563b9e15f4d8a8496f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:03:25 +0100 Subject: [PATCH 06/11] Don't assume get_fn is only called from codegen_mir in cg_gcc --- compiler/rustc_codegen_gcc/src/builder.rs | 1 + compiler/rustc_codegen_gcc/src/context.rs | 4 +--- compiler/rustc_codegen_gcc/src/declare.rs | 8 ++------ 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 79228c20d2927..f57c50eb64d17 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -514,6 +514,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { type CodegenCx = CodegenCx<'gcc, 'tcx>; fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Builder<'a, 'gcc, 'tcx> { + *cx.current_func.borrow_mut() = Some(block.get_function()); Builder::with_cx(cx, block) } diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index dbb89a4ff7dba..67bec261cc2e9 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -391,9 +391,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { } fn get_fn(&self, instance: Instance<'tcx>) -> Function<'gcc> { - let func = get_fn(self, instance); - *self.current_func.borrow_mut() = Some(func); - func + get_fn(self, instance) } fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index 42d6fb17a88bc..fec7d9d0a4038 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -100,18 +100,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let return_type = self.type_i32(); let variadic = false; self.linkage.set(FunctionType::Exported); - let func = declare_raw_fn( + declare_raw_fn( self, name, callconv, return_type, &[self.type_i32(), const_string], variadic, - ); - // NOTE: it is needed to set the current_func here as well, because get_fn() is not called - // for the main function. - *self.current_func.borrow_mut() = Some(func); - func + ) } pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> { From c506e2342635e6076bd03ec03663cd21018e55b9 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:14:03 +0100 Subject: [PATCH 07/11] Pass Function to Builder::function_call in cg_gcc --- compiler/rustc_codegen_gcc/src/builder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index f57c50eb64d17..8b4a8de94bf3f 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -316,12 +316,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { fn function_call( &mut self, - func: RValue<'gcc>, + func: Function<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>, ) -> RValue<'gcc> { - // TODO(antoyo): remove when the API supports a different type for functions. - let func: Function<'gcc> = self.cx.rvalue_as_function(func); let args = self.check_call("call", func, args); // gccjit requires to use the result of functions, even when it's not used. @@ -1766,6 +1764,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { + // TODO(antoyo): remove when the API supports a different type for functions. + let func: Function<'gcc> = self.cx.rvalue_as_function(func); self.function_call(func, args, funclet) } else { // If it's a not function that was defined, it's a function pointer. From ae8ef1f5eba50429499db9d1f3d42e9d9bd2058e Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 11 Nov 2025 11:52:43 +0100 Subject: [PATCH 08/11] Inline BuilderMethods::call for intrinsics Intrinsics only need a fraction of the functionality offered by BuilderMethods::call and in particular don't need the FnAbi to be computed other than (currently) as step towards computing the function value type. --- compiler/rustc_codegen_llvm/src/builder.rs | 2 +- compiler/rustc_codegen_llvm/src/intrinsic.rs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 9a9e4dcdbb3c4..7a49ba64029e5 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1705,7 +1705,7 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { ret.expect("LLVM does not have support for catchret") } - fn check_call<'b>( + pub(crate) fn check_call<'b>( &mut self, typ: &str, fn_ty: &'ll Type, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index f159161deee8e..78ccab78da6c7 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,5 +1,7 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; +use std::ffi::c_uint; +use std::ptr; use rustc_abi::{ Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, @@ -677,7 +679,20 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } - let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + debug!("call intrinsic {:?} with args ({:?})", instance, llargs); + let args = self.check_call("call", fn_ty, fn_ptr, &llargs); + let llret = unsafe { + llvm::LLVMBuildCallWithOperandBundles( + self.llbuilder, + fn_ty, + fn_ptr, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + ptr::dangling(), + 0, + c"".as_ptr(), + ) + }; if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } From 9bcd6ed4c9b70e6c99cb39e27e7b40ad18456efa Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:42:32 +0000 Subject: [PATCH 09/11] Partially inline get_fn_addr/get_fn in codegen_llvm_intrinsic_call This moves all LLVM intrinsic handling out of the regular call path for cg_gcc and makes it easier to hook into this code for future cg_llvm changes. --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_gcc/src/context.rs | 7 ++-- compiler/rustc_codegen_gcc/src/declare.rs | 14 ------- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 42 ++++++++++++++++++- compiler/rustc_codegen_llvm/src/context.rs | 3 ++ compiler/rustc_codegen_llvm/src/intrinsic.rs | 35 +++++++++++++++- compiler/rustc_codegen_ssa/src/mir/block.rs | 5 ++- 7 files changed, 86 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 8b4a8de94bf3f..3def9a5c015cc 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -314,7 +314,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.block.get_function() } - fn function_call( + pub fn function_call( &mut self, func: Function<'gcc>, args: &[RValue<'gcc>], diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index 67bec261cc2e9..d200d5319a936 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -92,6 +92,8 @@ pub struct CodegenCx<'gcc, 'tcx> { pub instances: RefCell, LValue<'gcc>>>, /// Cache function instances of monomorphic and polymorphic items pub function_instances: RefCell, Function<'gcc>>>, + /// Cache function instances of intrinsics + pub intrinsic_instances: RefCell, Function<'gcc>>>, /// Cache generated vtables pub vtables: RefCell, Option>), RValue<'gcc>>>, @@ -280,6 +282,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { linkage: Cell::new(FunctionType::Internal), instances: Default::default(), function_instances: Default::default(), + intrinsic_instances: Default::default(), on_stack_params: Default::default(), on_stack_function_params: Default::default(), vtables: Default::default(), @@ -397,9 +400,7 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { let func_name = self.tcx.symbol_name(instance).name; - let func = if self.intrinsics.borrow().contains_key(func_name) { - self.intrinsics.borrow()[func_name] - } else if let Some(variable) = self.get_declared_value(func_name) { + let func = if let Some(variable) = self.get_declared_value(func_name) { return variable; } else { get_fn(self, instance) diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index fec7d9d0a4038..e4130b221ee3b 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -8,7 +8,6 @@ use rustc_target::callconv::FnAbi; use crate::abi::{FnAbiGcc, FnAbiGccExt}; use crate::context::CodegenCx; -use crate::intrinsic::llvm; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { pub fn get_or_insert_global( @@ -162,19 +161,6 @@ fn declare_raw_fn<'gcc>( param_types: &[Type<'gcc>], variadic: bool, ) -> Function<'gcc> { - if name.starts_with("llvm.") { - let intrinsic = match name { - "llvm.fma.f16" => { - // fma is not a target builtin, but a normal builtin, so we handle it differently - // here. - cx.context.get_builtin_function("fma") - } - _ => llvm::intrinsic(name, cx), - }; - - cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic); - return intrinsic; - } let func = if cx.functions.borrow().contains_key(name) { cx.functions.borrow()[name] } else { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 75ea39c231140..77b88a9690844 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -615,7 +615,47 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc args: &[OperandRef<'tcx, Self::Value>], is_cleanup: bool, ) -> Self::Value { - let fn_ptr = self.get_fn_addr(instance); + let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) { + func + } else { + let sym = self.tcx.symbol_name(instance).name; + + let func = if let Some(func) = self.intrinsics.borrow().get(sym) { + *func + } else { + self.linkage.set(FunctionType::Extern); + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let fn_ty = fn_abi.gcc_type(self); + + let func = match sym { + "llvm.fma.f16" => { + // fma is not a target builtin, but a normal builtin, so we handle it differently + // here. + self.context.get_builtin_function("fma") + } + _ => llvm::intrinsic(sym, self), + }; + + self.intrinsics.borrow_mut().insert(sym.to_string(), func); + + self.on_stack_function_params + .borrow_mut() + .insert(func, fn_ty.on_stack_param_indices); + #[cfg(feature = "master")] + for fn_attr in fn_ty.fn_attributes { + func.add_attribute(fn_attr); + } + + crate::attributes::from_fn_attrs(self, func, instance); + + func + }; + + self.intrinsic_instances.borrow_mut().insert(instance, func); + + func + }; + let fn_ptr = func.get_address(None); let fn_ty = fn_ptr.get_type(); let mut llargs = vec![]; diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 03da70d48f7d6..07b04863af6b7 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -101,6 +101,8 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, &'ll Value>>, + /// Cache instances of intrinsics + pub intrinsic_instances: RefCell, &'ll Value>>, /// Cache generated vtables pub vtables: RefCell, Option>), &'ll Value>>, /// Cache of constant strings, @@ -627,6 +629,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { tls_model, codegen_unit, instances: Default::default(), + intrinsic_instances: Default::default(), vtables: Default::default(), const_str_cache: Default::default(), const_globals: Default::default(), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 78ccab78da6c7..b4057eea735ea 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -32,6 +32,7 @@ use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; use crate::builder::gpu_offload::{gen_call_handling, gen_define_handling}; use crate::context::CodegenCx; +use crate::declare::declare_raw_fn; use crate::errors::{ AutoDiffWithoutEnable, AutoDiffWithoutLto, OffloadWithoutEnable, OffloadWithoutFatLTO, }; @@ -643,11 +644,41 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { args: &[OperandRef<'tcx, Self::Value>], is_cleanup: bool, ) -> Self::Value { - let fn_ptr = self.get_fn_addr(instance); + let tcx = self.tcx(); + // FIXME remove usage of fn_abi let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); assert!(!fn_abi.ret.is_indirect()); - let fn_ty = self.fn_decl_backend_type(fn_abi); + let fn_ty = fn_abi.llvm_type(self); + + let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) { + llfn + } else { + let sym = tcx.symbol_name(instance).name; + + // FIXME use get_intrinsic + let llfn = if let Some(llfn) = self.get_declared_value(sym) { + llfn + } else { + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + sym, + fn_abi.llvm_cconv(self), + llvm::UnnamedAddr::Global, + llvm::Visibility::Default, + fn_ty, + ); + fn_abi.apply_attrs_llfn(self, llfn, Some(instance)); + + llfn + }; + + self.intrinsic_instances.borrow_mut().insert(instance, llfn); + + llfn + }; let mut llargs = vec![]; diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 560880a76e51a..77d55766d837e 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -9,7 +9,7 @@ use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER; use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; @@ -1040,6 +1040,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") { + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); + let result_layout = self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); From cb23b54eb144c5d6258e437cc690c82e78468bc8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 24 Nov 2025 13:11:12 +0000 Subject: [PATCH 10/11] Add a hack for llvm.wasm.throw As this is the only unwinding intrinsic we use and codegen_llvm_intrinsic_call currently doesn't handle unwinding intrinsics, this uses the conventional call path for llvm.wasm.throw. --- compiler/rustc_codegen_ssa/src/mir/block.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 77d55766d837e..d22546dee5654 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -1039,6 +1039,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if let Some(instance) = instance && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name && name.as_str().starts_with("llvm.") + // This is the only LLVM intrinsic we use that unwinds + // FIXME either add unwind support to codegen_llvm_intrinsic_call or replace usage of + // this intrinsic with something else + && name.as_str() != "llvm.wasm.throw" { assert!(!instance.args.has_infer()); assert!(!instance.args.has_escaping_bound_vars()); From aeac6cb2d65499eb4eb541b66dbf292ffe93d770 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 27 Dec 2025 08:49:59 +0000 Subject: [PATCH 11/11] Fix compilation of cg_gcc with master feature disabled --- compiler/rustc_codegen_gcc/src/intrinsic/mod.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 77b88a9690844..36ea76cbc51a0 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -23,16 +23,12 @@ use rustc_codegen_ssa::traits::{ IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; use rustc_middle::bug; -#[cfg(feature = "master")] -use rustc_middle::ty::layout::FnAbiOf; -use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, Ty}; use rustc_span::{Span, Symbol, sym}; use rustc_target::callconv::{ArgAbi, PassMode}; -#[cfg(feature = "master")] -use crate::abi::FnAbiGccExt; -use crate::abi::GccType; +use crate::abi::{FnAbiGccExt, GccType}; use crate::builder::Builder; use crate::common::{SignType, TypeReflection}; use crate::context::CodegenCx;