Auto merge of rust-lang#128068 - goldsteinn:goldsteinn/panic-func-wrappers, r=<try>

panic: Use local functions in `panic!` whenever possible

This is basically extending the idea implemented in `panic_2021!`.

By creating a cold/noinline function that wraps the setup for the call to the internal panic function moves more of the code-size cost of `panic!` to the cold path.

For example: https://godbolt.org/z/T1ndrcq4d

Author: bors

library/core/src/panic.rs

@@ -19,23 +19,50 @@ pub use self::unwind_safe::{AssertUnwindSafe, RefUnwindSafe, UnwindSafe};
 
 #[doc(hidden)]
 #[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")]
-#[allow_internal_unstable(panic_internals, const_format_args)]
+#[allow_internal_unstable(
+    panic_internals,
+    core_intrinsics,
+    const_dispatch,
+    const_eval_select,
+    const_format_args,
+    rustc_attrs
+)]
 #[rustc_diagnostic_item = "core_panic_2015_macro"]
 #[rustc_macro_transparency = "semitransparent"]
 pub macro panic_2015 {
-    () => (
-        $crate::panicking::panic("explicit panic")
-    ),
-    ($msg:literal $(,)?) => (
-        $crate::panicking::panic($msg)
-    ),
+    // For all cases where it is possible, wrap the actual call to the
+    // internal panic implementation function with a local no-inline
+    // cold function. This moves the codegen for setting up the
+    // arguments to the panic implementation function to the
+    // presumably cold panic path.
+    () => ({
+        $crate::panicking::panic_cold_explicit();
+    }),
+    // Special-case for string literal.
+    ($msg:literal $(,)?) => ({
+        #[cold]
+        #[track_caller]
+        #[inline(never)]
+        const fn panic_cold_literal() -> ! {
+            $crate::panicking::panic($msg);
+        }
+        panic_cold_literal();
+    }),
     // Use `panic_str_2015` instead of `panic_display::<&str>` for non_fmt_panic lint.
     ($msg:expr $(,)?) => ({
         $crate::panicking::panic_str_2015($msg);
     }),
     // Special-case the single-argument case for const_panic.
     ("{}", $arg:expr $(,)?) => ({
-        $crate::panicking::panic_display(&$arg);
+        #[cold]
+        #[track_caller]
+        #[inline(never)]
+        #[rustc_const_panic_str] // enforce a &&str argument in const-check and hook this by const-eval
+        #[rustc_do_not_const_check] // hooked by const-eval
+        const fn panic_cold_display<T: $crate::fmt::Display>(arg: &T) -> ! {
+            $crate::panicking::panic_display(arg)
+        }
+        panic_cold_display(&$arg);
     }),
     ($fmt:expr, $($arg:tt)+) => ({
         // Semicolon to prevent temporaries inside the formatting machinery from
@@ -44,27 +71,6 @@ pub macro panic_2015 {
     }),
 }
 
-#[doc(hidden)]
-#[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")]
-#[allow_internal_unstable(panic_internals, const_format_args)]
-#[rustc_diagnostic_item = "core_panic_2021_macro"]
-#[rustc_macro_transparency = "semitransparent"]
-#[cfg(feature = "panic_immediate_abort")]
-pub macro panic_2021 {
-    () => (
-        $crate::panicking::panic("explicit panic")
-    ),
-    // Special-case the single-argument case for const_panic.
-    ("{}", $arg:expr $(,)?) => ({
-        $crate::panicking::panic_display(&$arg);
-    }),
-    ($($t:tt)+) => ({
-        // Semicolon to prevent temporaries inside the formatting machinery from
-        // being considered alive in the caller after the panic_fmt call.
-        $crate::panicking::panic_fmt($crate::const_format_args!($($t)+));
-    }),
-}
-
 #[doc(hidden)]
 #[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")]
 #[allow_internal_unstable(
@@ -77,18 +83,16 @@ pub macro panic_2021 {
 )]
 #[rustc_diagnostic_item = "core_panic_2021_macro"]
 #[rustc_macro_transparency = "semitransparent"]
-#[cfg(not(feature = "panic_immediate_abort"))]
 pub macro panic_2021 {
+    // For all cases where it is possible, wrap the actual call to the
+    // internal panic implementation function with a local no-inline
+    // cold function. This moves the codegen for setting up the
+    // arguments to the panic implementation function to the
+    // presumably cold panic path.
+    // It would be nice to handle literals here, but there are
+    // some issues handling embedded format arguments.
     () => ({
-        // Create a function so that the argument for `track_caller`
-        // can be moved inside if possible.
-        #[cold]
-        #[track_caller]
-        #[inline(never)]
-        const fn panic_cold_explicit() -> ! {
-            $crate::panicking::panic_explicit()
-        }
-        panic_cold_explicit();
+        $crate::panicking::panic_cold_explicit();
     }),
     // Special-case the single-argument case for const_panic.
     ("{}", $arg:expr $(,)?) => ({
@@ -111,13 +115,34 @@ pub macro panic_2021 {
 
 #[doc(hidden)]
 #[unstable(feature = "edition_panic", issue = "none", reason = "use unreachable!() instead")]
-#[allow_internal_unstable(panic_internals)]
+#[allow_internal_unstable(
+    panic_internals,
+    core_intrinsics,
+    const_dispatch,
+    const_eval_select,
+    const_format_args,
+    rustc_attrs
+)]
 #[rustc_diagnostic_item = "unreachable_2015_macro"]
 #[rustc_macro_transparency = "semitransparent"]
 pub macro unreachable_2015 {
+    // For all cases where it is possible, wrap the actual call to the
+    // internal panic implementation function with a local no-inline
+    // cold function. This moves the codegen for setting up the
+    // arguments to the panic implementation function to the
+    // presumably cold panic path.
     () => (
-        $crate::panicking::panic("internal error: entered unreachable code")
+        $crate::panicking::unreachable_cold_explicit()
     ),
+    ($msg:literal $(,)?) => ({
+        #[cold]
+        #[track_caller]
+        #[inline(never)]
+        const fn unreachable_cold_literal() -> ! {
+            $crate::panicking::unreachable_display(&$msg);
+        }
+        unreachable_cold_literal();
+    }),
     // Use of `unreachable_display` for non_fmt_panic lint.
     // NOTE: the message ("internal error ...") is embedded directly in unreachable_display
     ($msg:expr $(,)?) => ({
@@ -130,12 +155,31 @@ pub macro unreachable_2015 {
 
 #[doc(hidden)]
 #[unstable(feature = "edition_panic", issue = "none", reason = "use unreachable!() instead")]
-#[allow_internal_unstable(panic_internals)]
+#[allow_internal_unstable(
+    panic_internals,
+    core_intrinsics,
+    const_dispatch,
+    const_eval_select,
+    const_format_args,
+    rustc_attrs
+)]
 #[rustc_macro_transparency = "semitransparent"]
 pub macro unreachable_2021 {
     () => (
-        $crate::panicking::panic("internal error: entered unreachable code")
+        $crate::panicking::unreachable_cold_explicit()
     ),
+    // Special-case the single-argument case for const_panic.
+    ("{}", $arg:expr $(,)?) => ({
+        #[cold]
+        #[track_caller]
+        #[inline(never)]
+        #[rustc_const_panic_str] // enforce a &&str argument in const-check and hook this by const-eval
+        #[rustc_do_not_const_check] // hooked by const-eval
+        const fn unreachable_cold_display<T: $crate::fmt::Display>(arg: &T) -> ! {
+            $crate::panicking::unreachable_display(arg)
+        }
+        unreachable_cold_display(&$arg);
+    }),
     ($($t:tt)+) => (
         $crate::panic!("internal error: entered unreachable code: {}", $crate::format_args!($($t)+))
     ),

library/core/src/panicking.rs

@@ -239,8 +239,12 @@ pub const fn panic_explicit() -> ! {
 
 #[inline]
 #[track_caller]
+#[rustc_do_not_const_check] // hooked by const-eval
+// enforce a &&str argument in const-check and hook this by const-eval
+#[rustc_const_panic_str]
+#[rustc_const_unstable(feature = "panic_internals", issue = "none")]
 #[rustc_diagnostic_item = "unreachable_display"] // needed for `non-fmt-panics` lint
-pub fn unreachable_display<T: fmt::Display>(x: &T) -> ! {
+pub const fn unreachable_display<T: fmt::Display>(x: &T) -> ! {
     panic_fmt(format_args!("internal error: entered unreachable code: {}", *x));
 }
 
@@ -254,6 +258,22 @@ pub const fn panic_str_2015(expr: &str) -> ! {
     panic_display(&expr);
 }
 
+#[cold]
+#[track_caller]
+#[inline(never)]
+#[rustc_const_unstable(feature = "panic_internals", issue = "none")]
+pub const fn panic_cold_explicit() -> ! {
+    panic("explicit panic");
+}
+
+#[cold]
+#[track_caller]
+#[inline(never)]
+#[rustc_const_unstable(feature = "panic_internals", issue = "none")]
+pub const fn unreachable_cold_explicit() -> ! {
+    panic("internal error: entered unreachable code");
+}
+
 #[inline]
 #[track_caller]
 #[rustc_do_not_const_check] // hooked by const-eval

library/std/src/panic.rs

@@ -212,19 +212,49 @@ impl fmt::Display for PanicHookInfo<'_> {
 
 #[doc(hidden)]
 #[unstable(feature = "edition_panic", issue = "none", reason = "use panic!() instead")]
-#[allow_internal_unstable(libstd_sys_internals, const_format_args, panic_internals, rt)]
+#[allow_internal_unstable(
+    libstd_sys_internals,
+    const_format_args,
+    panic_internals,
+    rt,
+    const_dispatch,
+    const_eval_select,
+    rustc_attrs
+)]
 #[cfg_attr(not(test), rustc_diagnostic_item = "std_panic_2015_macro")]
 #[rustc_macro_transparency = "semitransparent"]
 pub macro panic_2015 {
+    // For all cases where it is possible, wrap the actual call to the
+    // internal panic implementation function with a local no-inline
+    // cold function. This moves the codegen for setting up the
+    // arguments to the panic implementation function to the
+    // presumably cold panic path.
+    // It would be nice to handle literals here specially with a
+    // wrapper function, but unfortunately it results in unclear error
+    // messages when using panic(<non-str>).
     () => ({
-        $crate::rt::begin_panic("explicit panic")
+        #[cold]
+        #[track_caller]
+        #[inline(never)]
+        const fn panic_cold_explicit() -> ! {
+            $crate::rt::begin_panic("explicit panic");
+        }
+        panic_cold_explicit();
     }),
     ($msg:expr $(,)?) => ({
         $crate::rt::begin_panic($msg);
     }),
     // Special-case the single-argument case for const_panic.
     ("{}", $arg:expr $(,)?) => ({
-        $crate::rt::panic_display(&$arg);
+        #[cold]
+        #[track_caller]
+        #[inline(never)]
+        #[rustc_const_panic_str] // enforce a &&str argument in const-check and hook this by const-eval
+        #[rustc_do_not_const_check] // hooked by const-eval
+        const fn panic_cold_display<T: $crate::fmt::Display>(arg: &T) -> ! {
+            $crate::rt::panic_display(arg)
+        }
+        panic_cold_display(&$arg);
     }),
     ($fmt:expr, $($arg:tt)+) => ({
         // Semicolon to prevent temporaries inside the formatting machinery from

tests/mir-opt/building/issue_101867.main.built.after.mir

@@ -8,8 +8,8 @@ fn main() -> () {
     let mut _0: ();
     let _1: std::option::Option<u8> as UserTypeProjection { base: UserType(0), projs: [] };
     let mut _2: !;
-    let _3: ();
-    let mut _4: !;
+    let _3: !;
+    let _4: !;
     let mut _6: isize;
     scope 1 {
         debug x => _1;
@@ -31,14 +31,12 @@ fn main() -> () {
     }
 
     bb1: {
-        StorageLive(_3);
         StorageLive(_4);
-        _4 = begin_panic::<&str>(const "explicit panic") -> bb8;
+        _4 = panic_cold_explicit() -> bb8;
     }
 
     bb2: {
         StorageDead(_4);
-        StorageDead(_3);
         unreachable;
     }
 

tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-abort.diff

@@ -4,7 +4,7 @@
   fn hello() -> () {
       let mut _0: ();
       let mut _1: bool;
-      let mut _2: !;
+      let _2: !;
 
       bb0: {
           StorageLive(_1);
@@ -14,7 +14,7 @@
       }
 
       bb1: {
-          _2 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
+          _2 = panic_cold_explicit() -> unwind unreachable;
       }
 
       bb2: {

tests/mir-opt/const_prop/control_flow_simplification.hello.GVN.panic-unwind.diff

@@ -4,7 +4,7 @@
   fn hello() -> () {
       let mut _0: ();
       let mut _1: bool;
-      let mut _2: !;
+      let _2: !;
 
       bb0: {
           StorageLive(_1);
@@ -14,7 +14,7 @@
       }
 
       bb1: {
-          _2 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
+          _2 = panic_cold_explicit() -> unwind continue;
       }
 
       bb2: {

tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff

@@ -9,6 +9,7 @@
       let mut _4: isize;
       let _5: T;
       let mut _6: !;
+      let _7: !;
       scope 1 {
           debug y => _5;
       }
@@ -31,8 +32,8 @@
       }
 
       bb2: {
-          StorageLive(_6);
-          _6 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
+          StorageLive(_7);
+          _7 = wrap_unwrap::panic_cold_explicit() -> unwind unreachable;
       }
 
       bb3: {

tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff

@@ -9,6 +9,7 @@
       let mut _4: isize;
       let _5: T;
       let mut _6: !;
+      let _7: !;
       scope 1 {
           debug y => _5;
       }
@@ -31,8 +32,8 @@
       }
 
       bb2: {
-          StorageLive(_6);
-          _6 = begin_panic::<&str>(const "explicit panic") -> unwind continue;
+          StorageLive(_7);
+          _7 = wrap_unwrap::panic_cold_explicit() -> unwind continue;
       }
 
       bb3: {

tests/mir-opt/inline/inline_diverging.g.Inline.panic-abort.diff

@@ -10,7 +10,7 @@
       let mut _5: !;
       let _6: !;
 +     scope 1 (inlined panic) {
-+         let mut _7: !;
++         let _7: !;
 +     }
 
       bb0: {
@@ -36,7 +36,7 @@
           StorageLive(_6);
 -         _6 = panic() -> unwind unreachable;
 +         StorageLive(_7);
-+         _7 = begin_panic::<&str>(const "explicit panic") -> unwind unreachable;
++         _7 = panic_cold_explicit() -> unwind unreachable;
       }
   }