support c-variadic functions in rustc_const_eval

compiler/rustc_ast_passes/messages.ftl

@@ -82,10 +82,6 @@ ast_passes_c_variadic_no_extern = `...` is not supported for non-extern function
 
 ast_passes_c_variadic_not_supported = the `{$target}` target does not support c-variadic functions
 
-ast_passes_const_and_c_variadic = functions cannot be both `const` and C-variadic
-    .const = `const` because of this
-    .variadic = C-variadic because of this
-
 ast_passes_const_and_coroutine = functions cannot be both `const` and `{$coroutine_kind}`
     .const = `const` because of this
     .coroutine = `{$coroutine_kind}` because of this

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -697,15 +697,6 @@ impl<'a> AstValidator<'a> {
             unreachable!("C variable argument list cannot be used in closures")
         };
 
-        // C-variadics are not yet implemented in const evaluation.
-        if let Const::Yes(const_span) = sig.header.constness {
-            self.dcx().emit_err(errors::ConstAndCVariadic {
-                spans: vec![const_span, variadic_param.span],
-                const_span,
-                variadic_span: variadic_param.span,
-            });
-        }
-
         if let Some(coroutine_kind) = sig.header.coroutine_kind {
             self.dcx().emit_err(errors::CoroutineAndCVariadic {
                 spans: vec![coroutine_kind.span(), variadic_param.span],

compiler/rustc_ast_passes/src/errors.rs

@@ -710,17 +710,6 @@ pub(crate) struct ConstAndCoroutine {
     pub coroutine_kind: &'static str,
 }
 
-#[derive(Diagnostic)]
-#[diag(ast_passes_const_and_c_variadic)]
-pub(crate) struct ConstAndCVariadic {
-    #[primary_span]
-    pub spans: Vec<Span>,
-    #[label(ast_passes_const)]
-    pub const_span: Span,
-    #[label(ast_passes_variadic)]
-    pub variadic_span: Span,
-}
-
 #[derive(Diagnostic)]
 #[diag(ast_passes_coroutine_and_c_variadic)]
 pub(crate) struct CoroutineAndCVariadic {

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -175,6 +175,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                         let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func);
                         fx.bcx.ins().symbol_value(fx.pointer_type, local_data_id)
                     }
+                    GlobalAlloc::VaList => {
+                        bug!("valist allocation should never make it to codegen")
+                    }
                     GlobalAlloc::TypeId { .. } => {
                         return CValue::const_val(
                             fx,
@@ -381,6 +384,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                     GlobalAlloc::Function { .. }
                     | GlobalAlloc::Static(_)
                     | GlobalAlloc::TypeId { .. }
+                    | GlobalAlloc::VaList
                     | GlobalAlloc::VTable(..) => {
                         unreachable!()
                     }
@@ -494,6 +498,9 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                         .principal()
                         .map(|principal| tcx.instantiate_bound_regions_with_erased(principal)),
                 ),
+                GlobalAlloc::VaList => {
+                    bug!("valist allocation should never make it to codegen")
+                }
                 GlobalAlloc::TypeId { .. } => {
                     // Nothing to do, the bytes/offset of this pointer have already been written together with all other bytes,
                     // so we just need to drop this provenance.

compiler/rustc_codegen_gcc/src/common.rs

@@ -4,6 +4,7 @@ use rustc_abi::{self as abi, HasDataLayout};
 use rustc_codegen_ssa::traits::{
     BaseTypeCodegenMethods, ConstCodegenMethods, MiscCodegenMethods, StaticCodegenMethods,
 };
+use rustc_middle::bug;
 use rustc_middle::mir::Mutability;
 use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, PointerArithmetic, Scalar};
 use rustc_middle::ty::layout::LayoutOf;
@@ -281,6 +282,9 @@ impl<'gcc, 'tcx> ConstCodegenMethods for CodegenCx<'gcc, 'tcx> {
                         let init = self.const_data_from_alloc(alloc);
                         self.static_addr_of(init, alloc.inner().align, None)
                     }
+                    GlobalAlloc::VaList => {
+                        bug!("valist allocation should never make it to codegen")
+                    }
                     GlobalAlloc::TypeId { .. } => {
                         let val = self.const_usize(offset.bytes());
                         // This is still a variable of pointer type, even though we only use the provenance

compiler/rustc_codegen_llvm/src/common.rs

@@ -325,6 +325,9 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                         let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false);
                         self.static_addr_of_impl(init, alloc.inner().align, None)
                     }
+                    GlobalAlloc::VaList => {
+                        bug!("valist allocation should never make it to codegen")
+                    }
                     GlobalAlloc::Static(def_id) => {
                         assert!(self.tcx.is_static(def_id));
                         assert!(!self.tcx.is_thread_local_static(def_id));

compiler/rustc_const_eval/messages.ftl

@@ -91,6 +91,8 @@ const_eval_deref_function_pointer =
     accessing {$allocation} which contains a function
 const_eval_deref_typeid_pointer =
     accessing {$allocation} which contains a `TypeId`
+const_eval_deref_va_list_pointer =
+    accessing {$allocation} which contains a variable argument list
 const_eval_deref_vtable_pointer =
     accessing {$allocation} which contains a vtable
 const_eval_division_by_zero =
@@ -432,6 +434,8 @@ const_eval_unwind_past_top =
 
 ## The `front_matter`s here refer to either `const_eval_front_matter_invalid_value` or `const_eval_front_matter_invalid_value_with_path`.
 ## (We'd love to sort this differently to make that more clear but tidy won't let us...)
+const_eval_va_arg_out_of_bounds = more C-variadic arguments read than were passed
+
 const_eval_validation_box_to_uninhabited = {$front_matter}: encountered a box pointing to uninhabited type {$ty}
 
 const_eval_validation_dangling_box_no_provenance = {$front_matter}: encountered a dangling box ({$pointer} has no provenance)

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -23,8 +23,8 @@ use crate::fluent_generated as fluent;
 use crate::interpret::{
     self, AllocId, AllocInit, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
     GlobalAlloc, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, RangeSet, Scalar,
-    compile_time_machine, err_inval, interp_ok, throw_exhaust, throw_inval, throw_ub,
-    throw_ub_custom, throw_unsup, throw_unsup_format,
+    compile_time_machine, err_inval, err_unsup_format, interp_ok, throw_exhaust, throw_inval,
+    throw_ub, throw_ub_custom, throw_unsup, throw_unsup_format,
 };
 
 /// When hitting this many interpreted terminators we emit a deny by default lint
@@ -586,6 +586,58 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                 }
             }
 
+            sym::va_arg => {
+                let ptr_size = ecx.tcx.data_layout.pointer_size();
+
+                // The only argument is a `&mut VaList`.
+                let ap_ref = ecx.read_pointer(&args[0])?;
+
+                // The first bytes of the `VaList` value store a pointer. The `AllocId` of this
+                // pointer is a key into a global map of variable argument lists. The offset is
+                // used as the index of the argument to read.
+                let va_list_ptr = {
+                    let alloc = ecx
+                        .get_ptr_alloc(ap_ref, ptr_size)?
+                        .expect("va_list storage should not be a ZST");
+                    let scalar = alloc.read_pointer(Size::ZERO)?;
+                    scalar.to_pointer(ecx)?
+                };
+
+                let (prov, offset) = va_list_ptr.into_raw_parts();
+                let alloc_id = prov.unwrap().alloc_id();
+                let index = offset.bytes_usize();
+
+                // Update the offset in this `VaList` value so that a subsequent call to `va_arg`
+                // reads the next argument.
+                let new_va_list_ptr = va_list_ptr.wrapping_offset(Size::from_bytes(1), ecx);
+                let addr = Scalar::from_maybe_pointer(new_va_list_ptr, ecx);
+                let mut alloc = ecx
+                    .get_ptr_alloc_mut(ap_ref, ptr_size)?
+                    .expect("va_list storage should not be a ZST");
+                alloc.write_ptr_sized(Size::ZERO, addr)?;
+
+                let arguments = ecx.memory.va_list_map.get(&alloc_id).ok_or_else(|| {
+                    err_unsup_format!("va_arg on unknown va_list allocation {:?}", alloc_id)
+                })?;
+
+                let Some(src_mplace) = arguments.get(index).cloned() else {
+                    throw_ub!(VaArgOutOfBounds)
+                };
+
+                // NOTE: In C some type conversions are allowed (e.g. casting between signed and
+                // unsigned integers). For now we require c-variadic arguments to be read with the
+                // exact type they were passed as.
+                if src_mplace.layout.ty != dest.layout.ty {
+                    throw_unsup_format!(
+                        "va_arg type mismatch: requested `{}`, but next argument is `{}`",
+                        dest.layout.ty,
+                        src_mplace.layout.ty
+                    );
+                }
+
+                ecx.copy_op(&src_mplace, dest)?;
+            }
+
             _ => {
                 // We haven't handled the intrinsic, let's see if we can use a fallback body.
                 if ecx.tcx.intrinsic(instance.def_id()).unwrap().must_be_overridden {

compiler/rustc_const_eval/src/errors.rs

@@ -492,6 +492,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             WriteToReadOnly(_) => const_eval_write_to_read_only,
             DerefFunctionPointer(_) => const_eval_deref_function_pointer,
             DerefVTablePointer(_) => const_eval_deref_vtable_pointer,
+            DerefVaListPointer(_) => const_eval_deref_va_list_pointer,
             DerefTypeIdPointer(_) => const_eval_deref_typeid_pointer,
             InvalidBool(_) => const_eval_invalid_bool,
             InvalidChar(_) => const_eval_invalid_char,
@@ -509,6 +510,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             InvalidNichedEnumVariantWritten { .. } => {
                 const_eval_invalid_niched_enum_variant_written
             }
+            VaArgOutOfBounds => const_eval_va_arg_out_of_bounds,
             AbiMismatchArgument { .. } => const_eval_incompatible_arg_types,
             AbiMismatchReturn { .. } => const_eval_incompatible_return_types,
         }
@@ -535,6 +537,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             | InvalidMeta(InvalidMetaKind::TooBig)
             | InvalidUninitBytes(None)
             | DeadLocal
+            | VaArgOutOfBounds
             | UninhabitedEnumVariantWritten(_)
             | UninhabitedEnumVariantRead(_) => {}
 
@@ -609,6 +612,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             WriteToReadOnly(alloc)
             | DerefFunctionPointer(alloc)
             | DerefVTablePointer(alloc)
+            | DerefVaListPointer(alloc)
             | DerefTypeIdPointer(alloc) => {
                 diag.arg("allocation", alloc);
             }

compiler/rustc_const_eval/src/interpret/call.rs

@@ -4,9 +4,9 @@ use std::assert_matches::assert_matches;
 use std::borrow::Cow;
 
 use either::{Left, Right};
-use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx};
+use rustc_abi::{self as abi, ExternAbi, FieldIdx, HasDataLayout, Integer, Size, VariantIdx};
 use rustc_hir::def_id::DefId;
-use rustc_middle::ty::layout::{IntegerExt, TyAndLayout};
+use rustc_middle::ty::layout::{HasTyCtxt, IntegerExt, TyAndLayout};
 use rustc_middle::ty::{self, AdtDef, Instance, Ty, VariantDef};
 use rustc_middle::{bug, mir, span_bug};
 use rustc_span::sym;
@@ -15,9 +15,9 @@ use tracing::field::Empty;
 use tracing::{info, instrument, trace};
 
 use super::{
-    CtfeProvenance, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy,
-    Projectable, Provenance, ReturnAction, ReturnContinuation, Scalar, StackPopInfo, interp_ok,
-    throw_ub, throw_ub_custom, throw_unsup_format,
+    CtfeProvenance, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, MemoryKind, OpTy,
+    PlaceTy, Pointer, Projectable, Provenance, ReturnAction, ReturnContinuation, Scalar,
+    StackPopInfo, interp_ok, throw_ub, throw_ub_custom,
 };
 use crate::interpret::EnteredTraceSpan;
 use crate::{enter_trace_span, fluent_generated as fluent};
@@ -349,12 +349,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     ) -> InterpResult<'tcx> {
         let _trace = enter_trace_span!(M, step::init_stack_frame, %instance, tracing_separate_thread = Empty);
 
-        // Compute callee information.
-        // FIXME: for variadic support, do we have to somehow determine callee's extra_args?
-        let callee_fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
+        let (fixed_count, c_variadic_args) = if caller_fn_abi.c_variadic {
+            let sig = self.tcx.fn_sig(instance.def_id()).skip_binder();
+            let fixed_count = sig.inputs().skip_binder().len();
+            assert!(caller_fn_abi.args.len() >= fixed_count);
+            let extra_tys: Vec<Ty<'tcx>> =
+                caller_fn_abi.args[fixed_count..].iter().map(|arg_abi| arg_abi.layout.ty).collect();
 
-        if callee_fn_abi.c_variadic || caller_fn_abi.c_variadic {
-            throw_unsup_format!("calling a c-variadic function is not supported");
+            (fixed_count, self.tcx.mk_type_list(&extra_tys))
+        } else {
+            (caller_fn_abi.args.len(), ty::List::empty())
+        };
+
+        let callee_fn_abi = self.fn_abi_of_instance(instance, c_variadic_args)?;
+
+        if callee_fn_abi.c_variadic ^ caller_fn_abi.c_variadic {
+            unreachable!("caller and callee disagree on being c-variadic");
         }
 
         if caller_fn_abi.conv != callee_fn_abi.conv {
@@ -436,16 +446,63 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             // this is a single iterator (that handles `spread_arg`), then
             // `pass_argument` would be the loop body. It takes care to
             // not advance `caller_iter` for ignored arguments.
-            let mut callee_args_abis = callee_fn_abi.args.iter().enumerate();
-            for local in body.args_iter() {
+            let mut callee_args_abis = if caller_fn_abi.c_variadic {
+                callee_fn_abi.args[..fixed_count].iter().enumerate()
+            } else {
+                callee_fn_abi.args.iter().enumerate()
+            };
+
+            let mut it = body.args_iter().peekable();
+            while let Some(local) = it.next() {
                 // Construct the destination place for this argument. At this point all
                 // locals are still dead, so we cannot construct a `PlaceTy`.
                 let dest = mir::Place::from(local);
                 // `layout_of_local` does more than just the instantiation we need to get the
                 // type, but the result gets cached so this avoids calling the instantiation
                 // query *again* the next time this local is accessed.
                 let ty = self.layout_of_local(self.frame(), local, None)?.ty;
-                if Some(local) == body.spread_arg {
+                if caller_fn_abi.c_variadic && it.peek().is_none() {
+                    // The callee's signature has an additional VaList argument, that the caller
+                    // won't actually pass. Here we synthesize a `VaList` value, whose leading bytes
+                    // are a pointer that can be mapped to the corresponding variable argument list.
+                    self.storage_live(local)?;
+
+                    let place = self.eval_place(dest)?;
+                    let mplace = self.force_allocation(&place)?;
+
+                    // This global allocation is used as a key so `va_arg` can look up the variable
+                    // argument list corresponding to a `VaList` value.
+                    let alloc_id = self.tcx().reserve_and_set_va_list_alloc();
+
+                    // Consume the remaining arguments and store them in a global allocation.
+                    let mut varargs = Vec::new();
+                    for (fn_arg, abi) in &mut caller_args {
+                        let op = self.copy_fn_arg(fn_arg);
+                        let mplace = self.allocate(abi.layout, MemoryKind::Stack)?;
+                        self.copy_op(&op, &mplace)?;
+
+                        varargs.push(mplace);
+                    }
+
+                    // When the frame is dropped, this ID is used to deallocate the variable arguments list.
+                    self.frame_mut().va_list = Some(alloc_id);
+
+                    // A global map that is used to implement `va_arg`.
+                    self.memory.va_list_map.insert(alloc_id, varargs);
+
+                    // A VaList is a global allocation, so make sure we get the right root pointer.
+                    // We know this is not an `extern static` so this cannot fail.
+                    let ptr = self.global_root_pointer(Pointer::from(alloc_id)).unwrap();
+                    let addr = Scalar::from_pointer(ptr, self);
+
+                    // Store the pointer to the global variable arguments list allocation in the
+                    // first bytes of the `VaList` value.
+                    let mut alloc = self
+                        .get_ptr_alloc_mut(mplace.ptr(), self.data_layout().pointer_size())?
+                        .expect("not a ZST");
+
+                    alloc.write_ptr_sized(Size::ZERO, addr)?;
+                } else if Some(local) == body.spread_arg {
                     // Make the local live once, then fill in the value field by field.
                     self.storage_live(local)?;
                     // Must be a tuple