Auto merge of #141435 - RalfJung:unsupported_calling_conventions, r=<try>

Add (back) `unsupported_calling_conventions` lint to reject more invalid calling conventions

This adds back the `unsupported_calling_conventions` lint that was removed in #129935, in order to start the process of dealing with #137018. Specifically, we are going for the plan laid out [here](#137018 (comment)):
- thiscall, stdcall, fastcall, cdecl should only be accepted on x86-32
- vectorcall should only be accepted on x86-32 and x86-64

The difference to the status quo is that:
- We stop accepting stdcall, fastcall on targets that are windows && non-x86-32 (we already don't accept these on targets that are non-windows && non-x86-32)
- We stop accepting cdecl on targets that are non-x86-32
- (There is no difference for thiscall, this was already a hard error on non-x86-32)
- We stop accepting vectorcall on targets that are windows && non-x86-*

Vectorcall is an unstable ABI so we can just make this a hard error immediately. The others are stable, so we emit the `unsupported_calling_conventions` forward-compat lint. I set up the lint to show up in dependencies via cargo's future-compat report immediately, but we could also make it show up just for the local crate first if that is preferred.

try-job: i686-msvc-1
try-job: x86_64-msvc-1
try-job: test-various

Author: rust-bors[bot]

Cargo.lock

@@ -3786,6 +3786,7 @@ dependencies = [
  "rustc_middle",
  "rustc_session",
  "rustc_span",
+ "rustc_target",
  "rustc_trait_selection",
  "smallvec",
  "tracing",

compiler/rustc_hir_analysis/Cargo.toml

@@ -26,6 +26,7 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"

compiler/rustc_hir_analysis/src/check/check.rs

@@ -4,14 +4,15 @@ use std::ops::ControlFlow;
 use rustc_abi::FieldIdx;
 use rustc_attr_data_structures::ReprAttr::ReprPacked;
 use rustc_data_structures::unord::{UnordMap, UnordSet};
-use rustc_errors::MultiSpan;
 use rustc_errors::codes::*;
+use rustc_errors::{EmissionGuarantee, MultiSpan};
 use rustc_hir::def::{CtorKind, DefKind};
 use rustc_hir::{LangItem, Node, intravisit};
 use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt};
 use rustc_infer::traits::{Obligation, ObligationCauseCode};
 use rustc_lint_defs::builtin::{
-    REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
+    REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, UNSUPPORTED_CALLING_CONVENTIONS,
+    UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
 };
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
@@ -24,6 +25,7 @@ use rustc_middle::ty::{
     TypeVisitable, TypeVisitableExt, fold_regions,
 };
 use rustc_session::lint::builtin::UNINHABITED_STATIC;
+use rustc_target::spec::{AbiMap, AbiMapping};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::error_reporting::traits::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits;
@@ -35,25 +37,56 @@ use {rustc_attr_data_structures as attrs, rustc_hir as hir};
 use super::compare_impl_item::check_type_bounds;
 use super::*;
 
-pub fn check_abi(tcx: TyCtxt<'_>, span: Span, abi: ExternAbi) {
-    if !tcx.sess.target.is_abi_supported(abi) {
-        struct_span_code_err!(
-            tcx.dcx(),
-            span,
-            E0570,
-            "`{abi}` is not a supported ABI for the current target",
-        )
-        .emit();
+pub fn check_abi(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
+    // FIXME: this should be checked earlier, e.g. in `rustc_ast_lowering`, to fix
+    // things like #86232.
+    fn add_help<T: EmissionGuarantee>(abi: ExternAbi, diag: &mut Diag<'_, T>) {
+        if let ExternAbi::Cdecl { unwind } = abi {
+            let c_abi = ExternAbi::C { unwind };
+            diag.help(format!("use `extern {c_abi}` instead",));
+        } else if let ExternAbi::Stdcall { unwind } = abi {
+            let c_abi = ExternAbi::C { unwind };
+            let system_abi = ExternAbi::System { unwind };
+            diag.help(format!(
+                "if you need `extern {abi}` on win32 and `extern {c_abi}` everywhere else, \
+                use `extern {system_abi}`"
+            ));
+        }
+    }
+
+    match AbiMap::from_target(&tcx.sess.target).canonize_abi(abi, false) {
+        AbiMapping::Direct(..) => (),
+        AbiMapping::Invalid => {
+            let mut err = struct_span_code_err!(
+                tcx.dcx(),
+                span,
+                E0570,
+                "`{abi}` is not a supported ABI for the current target",
+            );
+            add_help(abi, &mut err);
+            err.emit();
+        }
+        AbiMapping::Deprecated(..) => {
+            tcx.node_span_lint(UNSUPPORTED_CALLING_CONVENTIONS, hir_id, span, |lint| {
+                lint.primary_message("use of calling convention not supported on this target");
+                add_help(abi, lint);
+            });
+        }
     }
 }
 
 pub fn check_abi_fn_ptr(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
-    if !tcx.sess.target.is_abi_supported(abi) {
-        tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
-            lint.primary_message(format!(
-                "the calling convention {abi} is not supported on this target"
-            ));
-        });
+    // This is always an FCW, even for `AbiMapping::Invalid`, since we started linting later than
+    // in `check_abi` above.
+    match AbiMap::from_target(&tcx.sess.target).canonize_abi(abi, false) {
+        AbiMapping::Direct(..) => (),
+        AbiMapping::Deprecated(..) | AbiMapping::Invalid => {
+            tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
+                lint.primary_message(format!(
+                    "the calling convention {abi} is not supported on this target"
+                ));
+            });
+        }
     }
 }
 
@@ -779,7 +812,7 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
             let hir::ItemKind::ForeignMod { abi, items } = it.kind else {
                 return;
             };
-            check_abi(tcx, it.span, abi);
+            check_abi(tcx, it.hir_id(), it.span, abi);
 
             for item in items {
                 let def_id = item.id.owner_id.def_id;

compiler/rustc_hir_typeck/src/lib.rs

@@ -150,7 +150,7 @@ fn typeck_with_inspect<'tcx>(
             tcx.fn_sig(def_id).instantiate_identity()
         };
 
-        check_abi(tcx, span, fn_sig.abi());
+        check_abi(tcx, id, span, fn_sig.abi());
 
         // Compute the function signature from point of view of inside the fn.
         let mut fn_sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), fn_sig);

compiler/rustc_lint/src/lib.rs

@@ -599,7 +599,6 @@ fn register_builtins(store: &mut LintStore) {
         "converted into hard error, see PR #125380 \
          <https://github.com/rust-lang/rust/pull/125380> for more information",
     );
-    store.register_removed("unsupported_calling_conventions", "converted into hard error");
     store.register_removed(
         "cenum_impl_drop_cast",
         "converted into hard error, \

compiler/rustc_lint_defs/src/builtin.rs

@@ -3621,6 +3621,54 @@ declare_lint! {
     crate_level_only
 }
 
+declare_lint! {
+    /// The `unsupported_calling_conventions` lint is output whenever there is a use of the
+    /// `stdcall`, `fastcall`, and `cdecl` calling conventions (or their unwind
+    /// variants) on targets that cannot meaningfully be supported for the requested target.
+    ///
+    /// For example `stdcall` does not make much sense for a x86_64 or, more apparently, powerpc
+    /// code, because this calling convention was never specified for those targets.
+    ///
+    /// Historically MSVC toolchains have fallen back to the regular C calling convention for
+    /// targets other than x86, but Rust doesn't really see a similar need to introduce a similar
+    /// hack across many more targets.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (needs specific targets)
+    /// extern "stdcall" fn stdcall() {}
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    /// warning: use of calling convention not supported on this target
+    ///   --> $DIR/unsupported.rs:39:1
+    ///    |
+    /// LL | extern "stdcall" fn stdcall() {}
+    ///    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    ///    |
+    ///    = note: `#[warn(unsupported_calling_conventions)]` on by default
+    ///    = warning: this was previously accepted by the compiler but is being phased out;
+    ///               it will become a hard error in a future release!
+    ///    = note: for more information, see issue ...
+    /// ```
+    ///
+    /// ### Explanation
+    ///
+    /// On most of the targets the behaviour of `stdcall` and similar calling conventions is not
+    /// defined at all, but was previously accepted due to a bug in the implementation of the
+    /// compiler.
+    pub UNSUPPORTED_CALLING_CONVENTIONS,
+    Warn,
+    "use of unsupported calling convention",
+    @future_incompatible = FutureIncompatibleInfo {
+        reason: FutureIncompatibilityReason::FutureReleaseError,
+        report_in_deps: true,
+        reference: "issue #137018 <https://github.com/rust-lang/rust/issues/137018>",
+    };
+}
+
 declare_lint! {
     /// The `unsupported_fn_ptr_calling_conventions` lint is output whenever there is a use of
     /// a target dependent calling convention on a target that does not support this calling

compiler/rustc_metadata/messages.ftl

@@ -272,6 +272,9 @@ metadata_raw_dylib_no_nul =
 metadata_raw_dylib_only_windows =
     link kind `raw-dylib` is only supported on Windows targets
 
+metadata_raw_dylib_unsupported_abi =
+    ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture
+
 metadata_renaming_no_link =
     renaming of the library `{$lib_name}` was specified, however this crate contains no `#[link(...)]` attributes referencing this library
 
@@ -319,12 +322,6 @@ metadata_unknown_link_modifier =
 
 metadata_unknown_target_modifier_unsafe_allowed = unknown target modifier `{$flag_name}`, requested by `-Cunsafe-allow-abi-mismatch={$flag_name}`
 
-metadata_unsupported_abi =
-    ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture
-
-metadata_unsupported_abi_i686 =
-    ABI not supported by `#[link(kind = "raw-dylib")]` on i686
-
 metadata_wasm_c_abi =
     older versions of the `wasm-bindgen` crate are incompatible with current versions of Rust; please update to `wasm-bindgen` v0.2.88
 

compiler/rustc_metadata/src/errors.rs

@@ -300,15 +300,8 @@ pub struct NoLinkModOverride {
 }
 
 #[derive(Diagnostic)]
-#[diag(metadata_unsupported_abi_i686)]
-pub struct UnsupportedAbiI686 {
-    #[primary_span]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(metadata_unsupported_abi)]
-pub struct UnsupportedAbi {
+#[diag(metadata_raw_dylib_unsupported_abi)]
+pub struct RawDylibUnsupportedAbi {
     #[primary_span]
     pub span: Span,
 }

compiler/rustc_metadata/src/native_libs.rs

@@ -652,7 +652,13 @@ impl<'tcx> Collector<'tcx> {
     ) -> DllImport {
         let span = self.tcx.def_span(item);
 
-        // this logic is similar to `Target::adjust_abi` (in rustc_target/src/spec/mod.rs) but errors on unsupported inputs
+        // This `extern` block should have been checked for general ABI support before, but let's
+        // double-check that.
+        assert!(self.tcx.sess.target.is_abi_supported(abi));
+
+        // This logic is similar to `AbiMap::canonize_abi` (in rustc_target/src/spec/abi_map.rs) but
+        // we need more detail than those adjustments, and we can't support all ABIs that are
+        // generally supported.
         let calling_convention = if self.tcx.sess.target.arch == "x86" {
             match abi {
                 ExternAbi::C { .. } | ExternAbi::Cdecl { .. } => DllCallingConvention::C,
@@ -679,7 +685,7 @@ impl<'tcx> Collector<'tcx> {
                     DllCallingConvention::Vectorcall(self.i686_arg_list_size(item))
                 }
                 _ => {
-                    self.tcx.dcx().emit_fatal(errors::UnsupportedAbiI686 { span });
+                    self.tcx.dcx().emit_fatal(errors::RawDylibUnsupportedAbi { span });
                 }
             }
         } else {
@@ -688,7 +694,7 @@ impl<'tcx> Collector<'tcx> {
                     DllCallingConvention::C
                 }
                 _ => {
-                    self.tcx.dcx().emit_fatal(errors::UnsupportedAbi { span });
+                    self.tcx.dcx().emit_fatal(errors::RawDylibUnsupportedAbi { span });
                 }
             }
         };

compiler/rustc_target/src/spec/abi_map.rs

@@ -115,9 +115,6 @@ impl AbiMap {
             (ExternAbi::Vectorcall { .. }, Arch::X86 | Arch::X86_64) => {
                 CanonAbi::X86(X86Call::Vectorcall)
             }
-            (ExternAbi::Vectorcall { .. }, _) if os == OsKind::Windows => {
-                return AbiMapping::Deprecated(CanonAbi::C);
-            }
             (ExternAbi::Vectorcall { .. }, _) => return AbiMapping::Invalid,
 
             (ExternAbi::SysV64 { .. }, Arch::X86_64) => CanonAbi::X86(X86Call::SysV64),

compiler/rustc_target/src/spec/mod.rs

@@ -62,7 +62,7 @@ mod abi_map;
 mod base;
 mod json;
 
-pub use abi_map::AbiMap;
+pub use abi_map::{AbiMap, AbiMapping};
 pub use base::apple;
 pub use base::avr::ef_avr_arch;