Auto merge of #142597 - folkertdev:abi-cannot-be-called, r=<try>

error on calls to ABIs that cannot be called

We recently added `extern "custom"`, which cannot be called using a rust call expression. But there are more ABIs that can't be called in that way, because the call does not semantically make sense.

More details are in #140566 (comment)

r? `@workingjubilee`

try-job: x86_64-gnu-llvm-19-3

Author: rust-bors[bot]

compiler/rustc_hir_typeck/messages.ftl

@@ -1,6 +1,6 @@
-hir_typeck_abi_custom_call =
-    functions with the `"custom"` ABI cannot be called
-    .note = an `extern "custom"` function can only be called from within inline assembly
+hir_typeck_abi_cannot_be_called =
+    functions with the {$abi} ABI cannot be called
+    .note = an `extern {$abi}` function can only be called using inline assembly
 
 hir_typeck_add_missing_parentheses_in_range = you must surround the range in parentheses to call its `{$func_name}` function
 

compiler/rustc_hir_typeck/src/callee.rs

@@ -1,6 +1,6 @@
 use std::iter;
 
-use rustc_abi::ExternAbi;
+use rustc_abi::{CanonAbi, ExternAbi};
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{self, CtorKind, Namespace, Res};
@@ -16,6 +16,7 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::{bug, span_bug};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, sym};
+use rustc_target::spec::{AbiMap, AbiMapping};
 use rustc_trait_selection::error_reporting::traits::DefIdOrName;
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
@@ -84,7 +85,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         while result.is_none() && autoderef.next().is_some() {
             result = self.try_overloaded_call_step(call_expr, callee_expr, arg_exprs, &autoderef);
         }
-        self.check_call_custom_abi(autoderef.final_ty(false), call_expr.span);
+
+        match autoderef.final_ty(false).kind() {
+            ty::FnDef(def_id, _) => {
+                let abi = self.tcx.fn_sig(def_id).skip_binder().skip_binder().abi;
+                self.check_call_abi(abi, call_expr.span);
+            }
+            ty::FnPtr(_, header) => {
+                self.check_call_abi(header.abi, call_expr.span);
+            }
+            _ => { /* cannot have a non-rust abi */ }
+        }
+
         self.register_predicates(autoderef.into_obligations());
 
         let output = match result {
@@ -137,19 +149,37 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         output
     }
 
-    /// Functions of type `extern "custom" fn(/* ... */)` cannot be called using `ExprKind::Call`.
+    /// Can a function with this ABI be called with a rust call expression?
     ///
-    /// These functions have a calling convention that is unknown to rust, hence it cannot generate
-    /// code for the call. The only way to execute such a function is via inline assembly.
-    fn check_call_custom_abi(&self, callee_ty: Ty<'tcx>, span: Span) {
-        let abi = match callee_ty.kind() {
-            ty::FnDef(def_id, _) => self.tcx.fn_sig(def_id).skip_binder().skip_binder().abi,
-            ty::FnPtr(_, header) => header.abi,
-            _ => return,
+    /// Some ABIs cannot be called from rust, either because rust does not know how to generate
+    /// code for the call, or because a call does not semantically make sense.
+    pub(crate) fn check_call_abi(&self, abi: ExternAbi, span: Span) {
+        let canon_abi = match AbiMap::from_target(&self.sess().target).canonize_abi(abi, false) {
+            AbiMapping::Direct(canon_abi) | AbiMapping::Deprecated(canon_abi) => canon_abi,
+            AbiMapping::Invalid => return,
+        };
+
+        let valid = match canon_abi {
+            // Rust doesn't know how to call functions with this ABI.
+            CanonAbi::Custom => false,
+
+            // These is an entry point for the host, and cannot be called on the GPU.
+            CanonAbi::GpuKernel => false,
+
+            // The interrupt ABIs should only be called by the CPU. They have complex
+            // pre- and postconditions, and can use non-standard instructions like `iret` on x86.
+            CanonAbi::Interrupt(_) => false,
+
+            CanonAbi::C
+            | CanonAbi::Rust
+            | CanonAbi::RustCold
+            | CanonAbi::Arm(_)
+            | CanonAbi::X86(_) => true,
         };
 
-        if let ExternAbi::Custom = abi {
-            self.tcx.dcx().emit_err(errors::AbiCustomCall { span });
+        if !valid {
+            let err = crate::errors::AbiCannotBeCalled { span, abi };
+            self.tcx.dcx().emit_err(err);
         }
     }
 

compiler/rustc_hir_typeck/src/errors.rs

@@ -2,6 +2,7 @@
 
 use std::borrow::Cow;
 
+use rustc_abi::ExternAbi;
 use rustc_ast::Label;
 use rustc_errors::codes::*;
 use rustc_errors::{
@@ -1159,8 +1160,10 @@ pub(crate) struct NakedFunctionsMustNakedAsm {
 }
 
 #[derive(Diagnostic)]
-#[diag(hir_typeck_abi_custom_call)]
-pub(crate) struct AbiCustomCall {
+#[diag(hir_typeck_abi_cannot_be_called)]
+pub(crate) struct AbiCannotBeCalled {
     #[primary_span]
+    #[note]
     pub span: Span,
+    pub abi: ExternAbi,
 }

compiler/rustc_hir_typeck/src/expr.rs

@@ -5,7 +5,7 @@
 //!
 //! See [`rustc_hir_analysis::check`] for more context on type checking in general.
 
-use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx};
+use rustc_abi::{FIRST_VARIANT, FieldIdx};
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -1650,13 +1650,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Some(method.def_id),
                 );
 
-                // Functions of type `extern "custom" fn(/* ... */)` cannot be called using
-                // `ExprKind::MethodCall`. These functions have a calling convention that is
-                // unknown to rust, hence it cannot generate code for the call. The only way
-                // to execute such a function is via inline assembly.
-                if let ExternAbi::Custom = method.sig.abi {
-                    self.tcx.dcx().emit_err(crate::errors::AbiCustomCall { span: expr.span });
-                }
+                self.check_call_abi(method.sig.abi, expr.span);
 
                 method.sig.output()
             }

tests/codegen/naked-asan.rs

@@ -1,22 +1,28 @@
-// Make sure we do not request sanitizers for naked functions.
+//@ add-core-stubs
+//@ needs-llvm-components: x86
+//@ compile-flags: --target x86_64-unknown-linux-gnu -Zsanitizer=address -Ctarget-feature=-crt-static
 
-//@ only-x86_64
-//@ needs-sanitizer-address
-//@ compile-flags: -Zsanitizer=address -Ctarget-feature=-crt-static
+// Make sure we do not request sanitizers for naked functions.
 
 #![crate_type = "lib"]
+#![feature(no_core)]
 #![no_std]
+#![no_core]
 #![feature(abi_x86_interrupt)]
 
+extern crate minicore;
+use minicore::*;
+
+#[no_mangle]
 pub fn caller() {
-    page_fault_handler(1, 2);
+    unsafe { asm!("call {}", sym page_fault_handler) }
 }
 
-// CHECK: declare x86_intrcc void @page_fault_handler(ptr {{.*}}, i64{{.*}}){{.*}}#[[ATTRS:[0-9]+]]
+// CHECK: declare x86_intrcc void @page_fault_handler(){{.*}}#[[ATTRS:[0-9]+]]
 #[unsafe(naked)]
 #[no_mangle]
-pub extern "x86-interrupt" fn page_fault_handler(_: u64, _: u64) {
-    core::arch::naked_asm!("ud2")
+pub extern "x86-interrupt" fn page_fault_handler() {
+    naked_asm!("ud2")
 }
 
 // CHECK: #[[ATTRS]] =

tests/ui/abi/bad-custom.rs

@@ -73,19 +73,19 @@ unsafe extern "custom" {
 
 fn caller(f: unsafe extern "custom" fn(i64) -> i64, mut x: i64) -> i64 {
     unsafe { f(x) }
-    //~^ ERROR functions with the `"custom"` ABI cannot be called
+    //~^ ERROR functions with the "custom" ABI cannot be called
 }
 
 fn caller_by_ref(f: &unsafe extern "custom" fn(i64) -> i64, mut x: i64) -> i64 {
     unsafe { f(x) }
-    //~^ ERROR functions with the `"custom"` ABI cannot be called
+    //~^ ERROR functions with the "custom" ABI cannot be called
 }
 
 type Custom = unsafe extern "custom" fn(i64) -> i64;
 
 fn caller_alias(f: Custom, mut x: i64) -> i64 {
     unsafe { f(x) }
-    //~^ ERROR functions with the `"custom"` ABI cannot be called
+    //~^ ERROR functions with the "custom" ABI cannot be called
 }
 
 #[unsafe(naked)]
@@ -107,15 +107,15 @@ fn no_promotion_to_fn_trait(f: unsafe extern "custom" fn()) -> impl Fn()  {
 pub fn main() {
     unsafe {
         assert_eq!(double(21), 42);
-        //~^ ERROR functions with the `"custom"` ABI cannot be called
+        //~^ ERROR functions with the "custom" ABI cannot be called
 
         assert_eq!(unsafe { increment(41) }, 42);
-        //~^ ERROR functions with the `"custom"` ABI cannot be called
+        //~^ ERROR functions with the "custom" ABI cannot be called
 
         assert!(Thing(41).is_even());
-        //~^ ERROR functions with the `"custom"` ABI cannot be called
+        //~^ ERROR functions with the "custom" ABI cannot be called
 
         assert_eq!(Thing::bitwise_not(42), !42);
-        //~^ ERROR functions with the `"custom"` ABI cannot be called
+        //~^ ERROR functions with the "custom" ABI cannot be called
     }
 }

tests/ui/abi/bad-custom.stderr

@@ -245,43 +245,85 @@ LL +     #[unsafe(naked)]
 LL |     extern "custom" fn negate(a: i64) -> i64 {
    |
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:75:14
+   |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:75:14
    |
 LL |     unsafe { f(x) }
    |              ^^^^
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:80:14
+   |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:80:14
    |
 LL |     unsafe { f(x) }
    |              ^^^^
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:87:14
+   |
+LL |     unsafe { f(x) }
+   |              ^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:87:14
    |
 LL |     unsafe { f(x) }
    |              ^^^^
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:109:20
+   |
+LL |         assert_eq!(double(21), 42);
+   |                    ^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:109:20
    |
 LL |         assert_eq!(double(21), 42);
    |                    ^^^^^^^^^^
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:112:29
+   |
+LL |         assert_eq!(unsafe { increment(41) }, 42);
+   |                             ^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:112:29
    |
 LL |         assert_eq!(unsafe { increment(41) }, 42);
    |                             ^^^^^^^^^^^^^
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:115:17
+   |
+LL |         assert!(Thing(41).is_even());
+   |                 ^^^^^^^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:115:17
    |
 LL |         assert!(Thing(41).is_even());
    |                 ^^^^^^^^^^^^^^^^^^^
 
-error: functions with the `"custom"` ABI cannot be called
+error: functions with the "custom" ABI cannot be called
+  --> $DIR/bad-custom.rs:118:20
+   |
+LL |         assert_eq!(Thing::bitwise_not(42), !42);
+   |                    ^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: an `extern "custom"` function can only be called using inline assembly
   --> $DIR/bad-custom.rs:118:20
    |
 LL |         assert_eq!(Thing::bitwise_not(42), !42);