Add support for on_stack parameters

src/abi.rs

@@ -1,5 +1,6 @@
-use gccjit::{ToRValue, Type};
+use gccjit::{ToLValue, ToRValue, Type};
 use rustc_codegen_ssa::traits::{AbiBuilderMethods, BaseTypeMethods};
+use rustc_data_structures::stable_set::FxHashSet;
 use rustc_middle::bug;
 use rustc_middle::ty::Ty;
 use rustc_target::abi::call::{CastTarget, FnAbi, PassMode, Reg, RegKind};
@@ -15,9 +16,21 @@ impl<'a, 'gcc, 'tcx> AbiBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn get_param(&mut self, index: usize) -> Self::Value {
-        self.cx.current_func.borrow().expect("current func")
-            .get_param(index as i32)
-            .to_rvalue()
+        let func = self.current_func();
+        let param = func.get_param(index as i32);
+        let on_stack =
+            if let Some(on_stack_param_indices) = self.on_stack_function_params.borrow().get(&func) {
+                on_stack_param_indices.contains(&index)
+            }
+            else {
+                false
+            };
+        if on_stack {
+            param.to_lvalue().get_address(None)
+        }
+        else {
+            param.to_rvalue()
+        }
     }
 }
 
@@ -87,12 +100,13 @@ impl GccType for Reg {
 
 pub trait FnAbiGccExt<'gcc, 'tcx> {
     // TODO(antoyo): return a function pointer type instead?
-    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool);
+    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>);
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
 }
 
 impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
-    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool) {
+    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>) {
+        let mut on_stack_param_indices = FxHashSet::default();
         let args_capacity: usize = self.args.iter().map(|arg|
             if arg.pad.is_some() {
                 1
@@ -144,17 +158,22 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     unimplemented!();
                 }
                 PassMode::Cast(cast) => cast.gcc_type(cx),
-                PassMode::Indirect { extra_attrs: None, .. } => cx.type_ptr_to(arg.memory_ty(cx)),
+                PassMode::Indirect { extra_attrs: None, on_stack: true, .. } => {
+                    on_stack_param_indices.insert(argument_tys.len());
+                    arg.memory_ty(cx)
+                },
+                PassMode::Indirect { extra_attrs: None, on_stack: false, .. } => cx.type_ptr_to(arg.memory_ty(cx)),
             };
             argument_tys.push(arg_ty);
         }
 
-        (return_ty, argument_tys, self.c_variadic)
+        (return_ty, argument_tys, self.c_variadic, on_stack_param_indices)
     }
 
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
-        let (return_type, params, variadic) = self.gcc_type(cx);
+        let (return_type, params, variadic, on_stack_param_indices) = self.gcc_type(cx);
         let pointer_type = cx.context.new_function_pointer_type(None, return_type, &params, variadic);
+        cx.on_stack_params.borrow_mut().insert(pointer_type.dyncast_function_ptr_type().expect("function ptr type"), on_stack_param_indices);
         pointer_type
     }
 }

src/builder.rs

@@ -30,6 +30,7 @@ use rustc_codegen_ssa::traits::{
     OverflowOp,
     StaticBuilderMethods,
 };
+use rustc_data_structures::stable_set::FxHashSet;
 use rustc_middle::ty::{ParamEnv, Ty, TyCtxt};
 use rustc_middle::ty::layout::{FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout};
 use rustc_span::Span;
@@ -207,6 +208,11 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             param_types.push(param);
         }
 
+        let mut on_stack_param_indices = FxHashSet::default();
+        if let Some(indices) = self.on_stack_params.borrow().get(&gcc_func) {
+            on_stack_param_indices = indices.clone();
+        }
+
         if all_args_match {
             return Cow::Borrowed(args);
         }
@@ -215,10 +221,15 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             .into_iter()
             .zip(args.iter())
             .enumerate()
-            .map(|(_i, (expected_ty, &actual_val))| {
+            .map(|(index, (expected_ty, &actual_val))| {
                 let actual_ty = actual_val.get_type();
                 if expected_ty != actual_ty {
-                    self.bitcast(actual_val, expected_ty)
+                    if on_stack_param_indices.contains(&index) {
+                        actual_val.dereference(None).to_rvalue()
+                    }
+                    else {
+                        self.bitcast(actual_val, expected_ty)
+                    }
                 }
                 else {
                     actual_val
@@ -946,14 +957,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     /* Miscellaneous instructions */
-    fn memcpy(&mut self, dst: RValue<'gcc>, dst_align: Align, src: RValue<'gcc>, src_align: Align, size: RValue<'gcc>, flags: MemFlags) {
-        if flags.contains(MemFlags::NONTEMPORAL) {
-            // HACK(nox): This is inefficient but there is no nontemporal memcpy.
-            let val = self.load(src.get_type(), src, src_align);
-            let ptr = self.pointercast(dst, self.type_ptr_to(self.val_ty(val)));
-            self.store_with_flags(val, ptr, dst_align, flags);
-            return;
-        }
+    fn memcpy(&mut self, dst: RValue<'gcc>, _dst_align: Align, src: RValue<'gcc>, _src_align: Align, size: RValue<'gcc>, flags: MemFlags) {
+        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
         let size = self.intcast(size, self.type_size_t(), false);
         let _is_volatile = flags.contains(MemFlags::VOLATILE);
         let dst = self.pointercast(dst, self.type_i8p());

src/context.rs

@@ -1,6 +1,6 @@
 use std::cell::{Cell, RefCell};
 
-use gccjit::{Block, CType, Context, Function, FunctionType, LValue, RValue, Struct, Type};
+use gccjit::{Block, CType, Context, Function, FunctionPtrType, FunctionType, LValue, RValue, Struct, Type};
 use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::traits::{
     BackendTypes,
@@ -81,6 +81,12 @@ pub struct CodegenCx<'gcc, 'tcx> {
     /// Cache generated vtables
     pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), RValue<'gcc>>>,
 
+    // TODO(antoyo): improve the SSA API to not require those.
+    // Mapping from function pointer type to indexes of on stack parameters.
+    pub on_stack_params: RefCell<FxHashMap<FunctionPtrType<'gcc>, FxHashSet<usize>>>,
+    // Mapping from function to indexes of on stack parameters.
+    pub on_stack_function_params: RefCell<FxHashMap<Function<'gcc>, FxHashSet<usize>>>,
+
     /// Cache of emitted const globals (value -> global)
     pub const_globals: RefCell<FxHashMap<RValue<'gcc>, RValue<'gcc>>>,
     /// Map from the address of a global variable (rvalue) to the global variable itself (lvalue).
@@ -208,6 +214,8 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             linkage: Cell::new(FunctionType::Internal),
             instances: Default::default(),
             function_instances: Default::default(),
+            on_stack_params: Default::default(),
+            on_stack_function_params: Default::default(),
             vtables: Default::default(),
             const_globals: Default::default(),
             global_lvalues: Default::default(),

src/declare.rs

@@ -80,8 +80,9 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> RValue<'gcc> {
-        let (return_type, params, variadic) = fn_abi.gcc_type(self);
+        let (return_type, params, variadic, on_stack_param_indices) = fn_abi.gcc_type(self);
         let func = declare_raw_fn(self, name, () /*fn_abi.llvm_cconv()*/, return_type, &params, variadic);
+        self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices);
         // FIXME(antoyo): this is a wrong cast. That requires changing the compiler API.
         unsafe { std::mem::transmute(func) }
     }

src/intrinsic/mod.rs

@@ -464,17 +464,17 @@ impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             val.to_rvalue()
         };
         match self.mode {
-            PassMode::Ignore => {}
+            PassMode::Ignore => {},
             PassMode::Pair(..) => {
                 OperandValue::Pair(next(), next()).store(bx, dst);
-            }
+            },
             PassMode::Indirect { extra_attrs: Some(_), .. } => {
                 OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst);
-            }
+            },
             PassMode::Direct(_) | PassMode::Indirect { extra_attrs: None, .. } | PassMode::Cast(_) => {
                 let next_arg = next();
-                self.store(bx, next_arg.to_rvalue(), dst);
-            }
+                self.store(bx, next_arg, dst);
+            },
         }
     }
 }