Add unstable hotpatch flag.

Author: nebulark

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -373,6 +373,15 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
         to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile"));
     }
 
+    // patchable-function is only implemented on x86 on LLVM
+    if cx.sess().opts.unstable_opts.hotpatch && cx.sess().target.is_x86() {
+        to_add.push(llvm::CreateAttrStringValue(
+            cx.llcx,
+            "patchable-function",
+            "prologue-short-redirect",
+        ));
+    }
+
     // FIXME: none of these functions interact with source level attributes.
     to_add.extend(frame_pointer_type_attr(cx));
     to_add.extend(function_return_attr(cx));

compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs

@@ -35,6 +35,7 @@ impl OwnedTargetMachine {
         emit_stack_size_section: bool,
         relax_elf_relocations: bool,
         use_init_array: bool,
+        is_hotpatchable: bool,
         split_dwarf_file: &CStr,
         output_obj_file: &CStr,
         debug_info_compression: &CStr,
@@ -67,6 +68,7 @@ impl OwnedTargetMachine {
                 emit_stack_size_section,
                 relax_elf_relocations,
                 use_init_array,
+                is_hotpatchable,
                 split_dwarf_file.as_ptr(),
                 output_obj_file.as_ptr(),
                 debug_info_compression.as_ptr(),

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -229,6 +229,11 @@ pub(crate) fn target_machine_factory(
     let use_init_array =
         !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section);
 
+    // this makes LLVM add a hotpatch flag in the codeview S_COMPILE3 record,
+    // which is required by linkers for the functionpadmin option
+    // aarch64 is always hotpatchable
+    let is_hotpatchable = sess.opts.unstable_opts.hotpatch || sess.target.arch.contains("aarch64");
+
     let path_mapping = sess.source_map().path_mapping().clone();
 
     let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
@@ -301,6 +306,7 @@ pub(crate) fn target_machine_factory(
             emit_stack_size_section,
             relax_elf_relocations,
             use_init_array,
+            is_hotpatchable,
             &split_dwarf_file,
             &output_obj_file,
             &debuginfo_compression,

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2219,6 +2219,7 @@ unsafe extern "C" {
         EmitStackSizeSection: bool,
         RelaxELFRelocations: bool,
         UseInitArray: bool,
+        IsHotpatchable: bool,
         SplitDwarfFile: *const c_char,
         OutputObjFile: *const c_char,
         DebugInfoCompression: *const c_char,

compiler/rustc_interface/src/tests.rs

@@ -787,6 +787,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(force_unstable_if_unmarked, true);
     tracked!(function_return, FunctionReturn::ThunkExtern);
     tracked!(function_sections, Some(false));
+    tracked!(hotpatch, true);
     tracked!(human_readable_cgu_names, true);
     tracked!(incremental_ignore_spans, true);
     tracked!(inline_in_all_cgus, Some(true));

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -362,7 +362,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     bool FunctionSections, bool DataSections, bool UniqueSectionNames,
     bool TrapUnreachable, bool Singlethread, bool VerboseAsm,
     bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray,
-    const char *SplitDwarfFile, const char *OutputObjFile,
+    bool IsHotpatchable, const char *SplitDwarfFile, const char *OutputObjFile,
     const char *DebugInfoCompression, bool UseEmulatedTls,
     const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
 
@@ -392,6 +392,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   // Always preserve comments that were written by the user
   Options.MCOptions.PreserveAsmComments = true;
   Options.MCOptions.ABIName = ABIStr;
+  Options.Hotpatch = IsHotpatchable;
   if (SplitDwarfFile) {
     Options.MCOptions.SplitDwarfFile = SplitDwarfFile;
   }

compiler/rustc_session/src/options.rs

@@ -1793,6 +1793,10 @@ options! {
         environment variable `RUSTC_GRAPHVIZ_FONT` (default: `Courier, monospace`)"),
     has_thread_local: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "explicitly enable the `cfg(target_thread_local)` directive"),
+    hotpatch: bool = (false, parse_bool, [TRACKED],
+        "ensures hotpatching is always possible by ensuring that the first instruction of \
+        each function is at least two bytes, and no jump within the function goes to the first instruction. \
+        Should be combined with link-arg passing -functionpadmin to the linker. Currently only supported for x86 (default: false)"),
     human_readable_cgu_names: bool = (false, parse_bool, [TRACKED],
         "generate human-readable, predictable names for codegen units (default: no)"),
     identify_regions: bool = (false, parse_bool, [UNTRACKED],

compiler/rustc_target/src/spec/mod.rs

@@ -2085,6 +2085,9 @@ impl Target {
 
         Ok(dl)
     }
+    pub fn is_x86(&self) -> bool {
+        ["x86", "x86_64"].contains(&&self.arch[..])
+    }
 }
 
 pub trait HasTargetSpec {

src/tools/run-make-support/src/external_deps/llvm.rs

@@ -282,6 +282,13 @@ impl LlvmFilecheck {
         self
     }
 
+    /// Specify the prefix (without :) for patterns to match. By default, these patterns are prefixed with "CHECK:".
+    pub fn check_prefix(&mut self, prefix: &str) -> &mut Self {
+        self.cmd.arg("--check-prefix");
+        self.cmd.arg(prefix);
+        self
+    }
+
     /// `--input-file` option.
     pub fn input_file<P: AsRef<Path>>(&mut self, input_file: P) -> &mut Self {
         self.cmd.arg("--input-file");

tests/codegen/hotpatch.rs

@@ -0,0 +1,15 @@
+// check if functions get the attribute, so that LLVM ensures they are hotpatchable
+// the attribute is only implemented for x86, aarch64 does not require it
+
+//@ revisions: x32 x64
+//@[x32] only-x86
+//@[x64] only-x86_64
+//@ compile-flags: -Z hotpatch
+
+#![crate_type = "lib"]
+
+#[no_mangle]
+pub fn foo() {}
+
+// CHECK-LABEL: @foo() unnamed_addr #0
+// CHECK: attributes #0 = { {{.*}} "patchable-function"="prologue-short-redirect" {{.*}}}

tests/run-make/hotpatch-unaffected/lib.rs

@@ -0,0 +1,25 @@
+// to be able to hotpatch a function there are two requirements:
+// 1. the first instruction of a functin must be at least two bytes long
+// 2. there must not be a jump to the first instruction
+
+// the functions in this file already fulfill the conditions so hotpatch should not affect them
+
+// --------------------------------------------------------------------------------------------
+
+#[no_mangle]
+#[inline(never)]
+pub fn return_42() -> i32 {
+    42
+}
+
+// --------------------------------------------------------------------------------------------
+// This tailcall does not jump to the first instruction so hotpatch should leave it unaffected
+
+#[no_mangle]
+pub fn tailcall(a: i32) -> i32 {
+    if a > 10000 {
+        return a;
+    }
+
+    if a % 2 == 0 { tailcall(a / 2) } else { tailcall(a * 3 + 1) }
+}

tests/run-make/hotpatch-unaffected/rmake.rs

@@ -0,0 +1,48 @@
+// Check if hotpatch leaves the functions that are already hotpatchable untouched
+
+use run_make_support::{assertion_helpers, llvm, rustc};
+
+fn main() {
+    // hotpatch is only implemented for X86 and aarch64
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
+    {
+        fn base_rustc() -> rustc::Rustc {
+            let mut rustc = rustc();
+            rustc.input("lib.rs").crate_type("lib").opt_level("3");
+            rustc
+        }
+
+        fn dump_lib(libname: &str) -> String {
+            llvm::llvm_objdump()
+                .arg("--disassemble-symbols=return_42,tailcall")
+                .input(libname)
+                .run()
+                .stdout_utf8()
+        }
+
+        base_rustc().crate_name("regular").run();
+        let regular_dump = dump_lib("libregular.rlib");
+
+        base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run();
+        let hotpatch_dump = dump_lib("libhotpatch.rlib");
+
+        {
+            let mut lines_regular = regular_dump.lines();
+            let mut lines_hotpatch = hotpatch_dump.lines();
+
+            loop {
+                match (lines_regular.next(), lines_hotpatch.next()) {
+                    (None, None) => break,
+                    (Some(r), Some(h)) => {
+                        if r.contains("libregular.rlib") {
+                            assertion_helpers::assert_contains(h, "libhotpatch.rlib")
+                        } else {
+                            assertion_helpers::assert_equals(&r, &h)
+                        }
+                    }
+                    _ => panic!("expected files to have equal number of lines"),
+                }
+            }
+        }
+    }
+}

tests/run-make/hotpatch/lib.rs

@@ -0,0 +1,27 @@
+// to be able to hotpatch a function there are two requirements:
+// 1) the first instruction of a functin must be at least two bytes long
+// 2) there must not be a jump to the first instruction
+
+// The LLVM attribute we use '"patchable-function", "prologue-short-redirect"' only ensures 1)
+// However in practice 2) rarely matters. Its rare that it occurs and the problems it caused can be
+// avoided by the hotpatch tool.
+// In this test we check if 1) is ensured by inserted nops as needed
+
+// ----------------------------------------------------------------------------------------------
+
+// empty_fn just returns. Note that 'ret' is a single byte instruction, but hotpatch requires
+// a two or more byte instructions to be at the start of the functions.
+// Preferably we would also tests a different single byte instruction,
+// but I was not able to find an example with another one byte intstruction.
+
+// check that if the first instruction is just a single byte, so our test is valid
+// CHECK-LABEL: <empty_fn>:
+// CHECK-NOT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}}
+
+// check that the first instruction is at least 2 bytes long
+// HOTPATCH-LABEL: <empty_fn>:
+// HOTPATCH-NEXT: 0: {{[0-9a-f][0-9a-f]}} {{[0-9a-f][0-9a-f]}} {{.*}}
+
+#[no_mangle]
+#[inline(never)]
+pub fn empty_fn() {}

tests/run-make/hotpatch/rmake.rs

@@ -0,0 +1,42 @@
+// Check if hotpatch makes the functions hotpachable that were not
+// More details in lib.rs
+
+use run_make_support::{llvm, rustc};
+
+fn main() {
+    // hotpatch is only implemented for x86 and aarch64, but for aarch64 functions
+    // are always hotpatchable so we don't need to check it
+    #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
+    {
+        fn base_rustc() -> rustc::Rustc {
+            let mut rustc = rustc();
+            rustc.input("lib.rs").crate_type("lib").opt_level("3");
+            rustc
+        }
+
+        fn dump_lib(libname: &str) -> String {
+            llvm::llvm_objdump()
+                .arg("--disassemble-symbols=empty_fn")
+                .input(libname)
+                .run()
+                .stdout_utf8()
+        }
+
+        {
+            base_rustc().crate_name("regular").run();
+            let regular_dump = dump_lib("libregular.rlib");
+            llvm::llvm_filecheck().patterns("lib.rs").stdin_buf(regular_dump).run();
+        }
+
+        {
+            base_rustc().crate_name("hotpatch").arg("-Zhotpatch").run();
+            let hotpatch_dump = dump_lib("libhotpatch.rlib");
+
+            llvm::llvm_filecheck()
+                .patterns("lib.rs")
+                .check_prefix("HOTPATCH")
+                .stdin_buf(hotpatch_dump)
+                .run();
+        }
+    }
+}

tests/run-make/hotpatch_pdb/main.rs

@@ -0,0 +1,6 @@
+// CHECK: S_OBJNAME{{.*}}hotpatch_pdb{{.*}}.o
+// CHECK: S_COMPILE3
+// CHECK-NOT: S_
+// CHECK: flags = {{.*}}hot patchable
+
+pub fn main() {}

tests/run-make/hotpatch_pdb/rmake.rs

@@ -0,0 +1,31 @@
+// Check if hotpatch flag is present in the Codeview.
+// While this is not strictly neccessary for functionpadmin to work, the currently linker
+// would ignore functionpadmin if this is not pressent
+
+use run_make_support::{llvm, rustc};
+
+fn main() {
+    // PDBs are windows only and hotpatch is only implemented for x86 and aarch64
+    #[cfg(all(
+        target_os = "windows",
+        any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")
+    ))]
+    {
+        let output = rustc()
+            .input("main.rs")
+            .arg("-g")
+            .arg("-Zhotpatch")
+            .crate_name("hotpatch_pdb")
+            .crate_type("bin")
+            .run();
+
+        let pdbutil_output = llvm::llvm_pdbutil()
+            .arg("dump")
+            .arg("-symbols")
+            .input("hotpatch_pdb.pdb")
+            .run()
+            .stdout_utf8();
+
+        llvm::llvm_filecheck().patterns("main.rs").stdin_buf(&pdbutil_output).run();
+    }
+}