Codegen **non-overloaded** LLVM intrinsics using their name

sayantn · sayantn · commit 3e4faa5399b8 · 2025-06-15T01:57:09.000+05:30
diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs
@@ -1,6 +1,6 @@
 use std::fmt::Write;
 
-use gccjit::{Struct, Type};
+use gccjit::{RValue, Struct, Type};
 use rustc_abi as abi;
 use rustc_abi::Primitive::*;
 use rustc_abi::{
@@ -373,7 +373,11 @@ impl<'gcc, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         unimplemented!();
     }
 
-    fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
+    fn fn_decl_backend_type(
+        &self,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+        _fn_ptr: RValue<'gcc>,
+    ) -> Type<'gcc> {
         // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
         let FnAbiGcc { return_type, arguments_type, is_c_variadic, .. } = fn_abi.gcc_type(self);
         self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -1,5 +1,5 @@
 use std::borrow::Borrow;
-use std::cmp;
+use std::{cmp, iter};
 
 use libc::c_uint;
 use rustc_abi::{
@@ -300,8 +300,39 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     }
 }
 
+pub(crate) enum FunctionSignature<'ll> {
+    /// The signature is obtained directly from LLVM, and **may not match the Rust signature**
+    Intrinsic(&'ll Type),
+    /// The name starts with `llvm.`, but can't obtain the intrinsic ID. May be invalid or upgradable
+    MaybeInvalidIntrinsic(&'ll Type),
+    /// Just the Rust signature
+    Rust(&'ll Type),
+}
+
+impl<'ll> FunctionSignature<'ll> {
+    pub(crate) fn fn_ty(&self) -> &'ll Type {
+        match self {
+            FunctionSignature::Intrinsic(fn_ty)
+            | FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
+            | FunctionSignature::Rust(fn_ty) => fn_ty,
+        }
+    }
+}
+
 pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
+    fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
+    fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type>;
+    /// When `do_verify` is set, this function performs checks for the signature of LLVM intrinsics
+    /// and emits a fatal error if it doesn't match. These checks are important,but somewhat expensive
+    /// So they are only used at function definitions, not at callsites
+    fn llvm_type(
+        &self,
+        cx: &CodegenCx<'ll, 'tcx>,
+        name: &[u8],
+        do_verify: bool,
+    ) -> FunctionSignature<'ll>;
+    /// **If this function is an LLVM intrinsic** checks if the LLVM signature provided matches with this
+    fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_ty: &'ll Type) -> bool;
     fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
 
@@ -314,30 +345,38 @@ pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
     );
 
     /// Apply attributes to a function call.
-    fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
+    fn apply_attrs_callsite(
+        &self,
+        bx: &mut Builder<'_, 'll, 'tcx>,
+        callsite: &'ll Value,
+        llfn: &'ll Value,
+    );
 }
 
 impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
-    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
+    fn llvm_return_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
+        match &self.ret.mode {
+            PassMode::Ignore => cx.type_void(),
+            PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
+            PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
+            PassMode::Indirect { .. } => cx.type_void(),
+        }
+    }
+
+    fn llvm_argument_types(&self, cx: &CodegenCx<'ll, 'tcx>) -> Vec<&'ll Type> {
+        let indirect_return = matches!(self.ret.mode, PassMode::Indirect { .. });
+
         // Ignore "extra" args from the call site for C variadic functions.
         // Only the "fixed" args are part of the LLVM function signature.
         let args =
             if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args };
 
-        // This capacity calculation is approximate.
-        let mut llargument_tys = Vec::with_capacity(
-            self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 },
-        );
+        let mut llargument_tys =
+            Vec::with_capacity(args.len() + if indirect_return { 1 } else { 0 });
 
-        let llreturn_ty = match &self.ret.mode {
-            PassMode::Ignore => cx.type_void(),
-            PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
-            PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
-            PassMode::Indirect { .. } => {
-                llargument_tys.push(cx.type_ptr());
-                cx.type_void()
-            }
-        };
+        if indirect_return {
+            llargument_tys.push(cx.type_ptr());
+        }
 
         for arg in args {
             // Note that the exact number of arguments pushed here is carefully synchronized with
@@ -384,10 +423,74 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             llargument_tys.push(llarg_ty);
         }
 
-        if self.c_variadic {
-            cx.type_variadic_func(&llargument_tys, llreturn_ty)
+        llargument_tys
+    }
+
+    fn verify_intrinsic_signature(&self, cx: &CodegenCx<'ll, 'tcx>, llvm_fn_ty: &'ll Type) -> bool {
+        let rust_return_ty = self.llvm_return_type(cx);
+        let rust_argument_tys = self.llvm_argument_types(cx);
+
+        let llvm_return_ty = cx.get_return_type(llvm_fn_ty);
+        let llvm_argument_tys = cx.func_params_types(llvm_fn_ty);
+        let llvm_is_variadic = cx.func_is_variadic(llvm_fn_ty);
+
+        if self.c_variadic != llvm_is_variadic || rust_argument_tys.len() != llvm_argument_tys.len()
+        {
+            return false;
+        }
+
+        // todo: add bypasses for types not accessible from Rust here
+        iter::once((rust_return_ty, llvm_return_ty))
+            .chain(iter::zip(rust_argument_tys, llvm_argument_tys))
+            .all(|(rust_ty, llvm_ty)| rust_ty == llvm_ty)
+    }
+
+    fn llvm_type(
+        &self,
+        cx: &CodegenCx<'ll, 'tcx>,
+        name: &[u8],
+        do_verify: bool,
+    ) -> FunctionSignature<'ll> {
+        let mut maybe_invalid = false;
+
+        if name.starts_with(b"llvm.") {
+            if let Some(intrinsic) = llvm::Intrinsic::lookup(name) {
+                if !intrinsic.is_overloaded() {
+                    // FIXME: also do this for overloaded intrinsics
+                    let llvm_fn_ty = cx.intrinsic_type(intrinsic, &[]);
+                    if do_verify {
+                        if !self.verify_intrinsic_signature(cx, llvm_fn_ty) {
+                            cx.tcx.dcx().fatal(format!(
+                                "Intrinsic signature mismatch for `{}`: expected signature `{llvm_fn_ty:?}`",
+                                str::from_utf8(name).unwrap()
+                            ));
+                        }
+                    }
+                    return FunctionSignature::Intrinsic(llvm_fn_ty);
+                }
+            } else {
+                // it's one of 2 cases,
+                // - either the base name is invalid
+                // - it has been superceded by something else, so the intrinsic was removed entirely
+                // to check for upgrades, we need the `llfn`, so we defer it for now
+
+                maybe_invalid = true;
+            }
+        }
+
+        let return_ty = self.llvm_return_type(cx);
+        let argument_tys = self.llvm_argument_types(cx);
+
+        let fn_ty = if self.c_variadic {
+            cx.type_variadic_func(&argument_tys, return_ty)
         } else {
-            cx.type_func(&llargument_tys, llreturn_ty)
+            cx.type_func(&argument_tys, return_ty)
+        };
+
+        if maybe_invalid {
+            FunctionSignature::MaybeInvalidIntrinsic(fn_ty)
+        } else {
+            FunctionSignature::Rust(fn_ty)
         }
     }
 
@@ -530,7 +633,23 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         }
     }
 
-    fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
+    fn apply_attrs_callsite(
+        &self,
+        bx: &mut Builder<'_, 'll, 'tcx>,
+        callsite: &'ll Value,
+        llfn: &'ll Value,
+    ) {
+        // if we are using the LLVM signature, use the LLVM attributes otherwise it might be problematic
+        let name = llvm::get_value_name(llfn);
+        if name.starts_with(b"llvm.")
+            && let Some(intrinsic) = llvm::Intrinsic::lookup(name)
+        {
+            // FIXME: also do this for overloaded intrinsics
+            if !intrinsic.is_overloaded() {
+                return;
+            }
+        }
+
         let mut func_attrs = SmallVec::<[_; 2]>::new();
         if self.ret.layout.is_uninhabited() {
             func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -379,7 +379,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             )
         };
         if let Some(fn_abi) = fn_abi {
-            fn_abi.apply_attrs_callsite(self, invoke);
+            fn_abi.apply_attrs_callsite(self, invoke, llfn);
         }
         invoke
     }
@@ -1372,7 +1372,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             )
         };
         if let Some(fn_abi) = fn_abi {
-            fn_abi.apply_attrs_callsite(self, call);
+            fn_abi.apply_attrs_callsite(self, call, llfn);
         }
         call
     }
@@ -1694,7 +1694,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             )
         };
         if let Some(fn_abi) = fn_abi {
-            fn_abi.apply_attrs_callsite(self, callbr);
+            fn_abi.apply_attrs_callsite(self, callbr, llfn);
         }
         callbr
     }
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
@@ -1058,6 +1058,21 @@ impl<'ll> CodegenCx<'ll, '_> {
         self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo));
         eh_catch_typeinfo
     }
+
+    pub(crate) fn intrinsic_type(
+        &self,
+        intrinsic: llvm::Intrinsic,
+        type_params: &[&'ll Type],
+    ) -> &'ll Type {
+        unsafe {
+            llvm::LLVMIntrinsicGetType(
+                self.llcx(),
+                intrinsic.id(),
+                type_params.as_ptr(),
+                type_params.len(),
+            )
+        }
+    }
 }
 
 impl CodegenCx<'_, '_> {
diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs
@@ -22,7 +22,7 @@ use rustc_target::callconv::FnAbi;
 use smallvec::SmallVec;
 use tracing::debug;
 
-use crate::abi::FnAbiLlvmExt;
+use crate::abi::{FnAbiLlvmExt, FunctionSignature};
 use crate::common::AsCCharPtr;
 use crate::context::{CodegenCx, GenericCx, SCx, SimpleCx};
 use crate::llvm::AttributePlace::Function;
@@ -150,17 +150,34 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     ) -> &'ll Value {
         debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi);
 
-        // Function addresses in Rust are never significant, allowing functions to
-        // be merged.
-        let llfn = declare_raw_fn(
-            self,
-            name,
-            fn_abi.llvm_cconv(self),
-            llvm::UnnamedAddr::Global,
-            llvm::Visibility::Default,
-            fn_abi.llvm_type(self),
-        );
-        fn_abi.apply_attrs_llfn(self, llfn, instance);
+        let signature = fn_abi.llvm_type(self, name.as_bytes(), true);
+        let llfn;
+
+        if let FunctionSignature::Intrinsic(fn_ty) = signature {
+            // intrinsics have a specified set of attributes, so we don't use the `FnAbi` set for them
+            llfn = declare_simple_fn(
+                self,
+                name,
+                fn_abi.llvm_cconv(self),
+                llvm::UnnamedAddr::Global,
+                llvm::Visibility::Default,
+                fn_ty,
+            );
+        } else {
+            // Function addresses in Rust are never significant, allowing functions to
+            // be merged.
+            llfn = declare_raw_fn(
+                self,
+                name,
+                fn_abi.llvm_cconv(self),
+                llvm::UnnamedAddr::Global,
+                llvm::Visibility::Default,
+                signature.fn_ty(),
+            );
+            fn_abi.apply_attrs_llfn(self, llfn, instance);
+        }
+
+        // todo: check for upgrades, and emit error if not upgradable
 
         if self.tcx.sess.is_sanitizer_cfi_enabled() {
             if let Some(instance) = instance {
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -1054,7 +1054,7 @@ fn gen_fn<'a, 'll, 'tcx>(
     codegen: &mut dyn FnMut(Builder<'a, 'll, 'tcx>),
 ) -> (&'ll Type, &'ll Value) {
     let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
-    let llty = fn_abi.llvm_type(cx);
+    let llty = fn_abi.llvm_type(cx, name.as_bytes(), true).fn_ty();
     let llfn = cx.declare_fn(name, fn_abi, None);
     cx.set_frame_pointer_type(llfn);
     cx.apply_target_cpu_attr(llfn);
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -1060,6 +1060,7 @@ unsafe extern "C" {
     ) -> &'a Type;
     pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint;
     pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type);
+    pub(crate) fn LLVMIsFunctionVarArg(FunctionTy: &Type) -> Bool;
 
     // Operations on struct types
     pub(crate) fn LLVMStructTypeInContext<'a>(
@@ -1206,6 +1207,12 @@ unsafe extern "C" {
         ParamCount: size_t,
         NameLength: *mut size_t,
     ) -> *mut c_char;
+    pub(crate) fn LLVMIntrinsicGetType<'a>(
+        C: &'a Context,
+        ID: NonZero<c_uint>,
+        ParamTypes: *const &'a Type,
+        ParamCount: size_t,
+    ) -> &'a Type;
 
     // Operations on parameters
     pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
@@ -339,6 +339,10 @@ impl Intrinsic {
         NonZero::new(id).map(|id| Self { id })
     }
 
+    pub(crate) fn id(self) -> NonZero<c_uint> {
+        self.id
+    }
+
     pub(crate) fn is_overloaded(self) -> bool {
         unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
     }
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -84,6 +84,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             args
         }
     }
+
+    pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool {
+        unsafe { llvm::LLVMIsFunctionVarArg(ty) == True }
+    }
 }
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub(crate) fn type_bool(&self) -> &'ll Type {
@@ -293,8 +297,12 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type {
         ty.llvm_type(self)
     }
-    fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
-        fn_abi.llvm_type(self)
+    fn fn_decl_backend_type(
+        &self,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+        fn_ptr: &'ll Value,
+    ) -> &'ll Type {
+        fn_abi.llvm_type(self, llvm::get_value_name(fn_ptr), false).fn_ty()
     }
     fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
         fn_abi.ptr_to_llvm_type(self)
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs
diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs

Original file line number	Diff line number	Diff line change
`@@ -379,7 +379,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {`
`379`	`379`	`)`
`380`	`380`	`};`
`381`	`381`	`if let Some(fn_abi) = fn_abi {`
`382`		`- fn_abi.apply_attrs_callsite(self, invoke);`
	`382`	`+ fn_abi.apply_attrs_callsite(self, invoke, llfn);`
`383`	`383`	`}`
`384`	`384`	`invoke`
`385`	`385`	`}`
`@@ -1372,7 +1372,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {`
`1372`	`1372`	`)`
`1373`	`1373`	`};`
`1374`	`1374`	`if let Some(fn_abi) = fn_abi {`
`1375`		`- fn_abi.apply_attrs_callsite(self, call);`
	`1375`	`+ fn_abi.apply_attrs_callsite(self, call, llfn);`
`1376`	`1376`	`}`
`1377`	`1377`	`call`
`1378`	`1378`	`}`
`@@ -1694,7 +1694,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {`
`1694`	`1694`	`)`
`1695`	`1695`	`};`
`1696`	`1696`	`if let Some(fn_abi) = fn_abi {`
`1697`		`- fn_abi.apply_attrs_callsite(self, callbr);`
	`1697`	`+ fn_abi.apply_attrs_callsite(self, callbr, llfn);`
`1698`	`1698`	`}`
`1699`	`1699`	`callbr`
`1700`	`1700`	`}`
Original file line number	Diff line number	Diff line change
`@@ -339,6 +339,10 @@ impl Intrinsic {`
`339`	`339`	`NonZero::new(id).map(\|id\| Self { id })`
`340`	`340`	`}`
`341`	`341`
	`342`	`+ pub(crate) fn id(self) -> NonZero<c_uint> {`
	`343`	`+ self.id`
	`344`	`+ }`
	`345`	`+`
`342`	`346`	`pub(crate) fn is_overloaded(self) -> bool {`
`343`	`347`	`unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }`
`344`	`348`	`}`
Original file line number	Diff line number	Diff line change
`@@ -84,6 +84,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {`
`84`	`84`	`args`
`85`	`85`	`}`
`86`	`86`	`}`
	`87`	`+`
	`88`	`+ pub(crate) fn func_is_variadic(&self, ty: &'ll Type) -> bool {`
	`89`	`+ unsafe { llvm::LLVMIsFunctionVarArg(ty) == True }`
	`90`	`+ }`
`87`	`91`	`}`
`88`	`92`	`impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {`
`89`	`93`	`pub(crate) fn type_bool(&self) -> &'ll Type {`
`@@ -293,8 +297,12 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {`
`293`	`297`	`fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type {`
`294`	`298`	`ty.llvm_type(self)`
`295`	`299`	`}`
`296`		`- fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {`
`297`		`- fn_abi.llvm_type(self)`
	`300`	`+ fn fn_decl_backend_type(`
	`301`	`+ &self,`
	`302`	`+ fn_abi: &FnAbi<'tcx, Ty<'tcx>>,`
	`303`	`+ fn_ptr: &'ll Value,`
	`304`	`+ ) -> &'ll Type {`
	`305`	`+ fn_abi.llvm_type(self, llvm::get_value_name(fn_ptr), false).fn_ty()`
`298`	`306`	`}`
`299`	`307`	`fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {`
`300`	`308`	`fn_abi.ptr_to_llvm_type(self)`