Auto merge of rust-lang#139648 - Kobzol:ra-bolt, r=<try>

Optimize Rust Analyzer with BOLT

Funnily enough, it is easier to use BOLT for another component than to do PGO (at least I think so..). Let's try.

r? `@ghost`

try-job: dist-x86_64-linux

Author: bors

src/bootstrap/src/core/build_steps/dist.rs

@@ -1228,15 +1228,20 @@ impl Step for RustAnalyzer {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
-        let compiler = self.compiler;
+        // let compiler = self.compiler;
         let target = self.target;
 
-        let rust_analyzer = builder.ensure(tool::RustAnalyzer { compiler, target });
+        let rust_analyzer = builder
+            .out
+            .join("x86_64-unknown-linux-gnu")
+            .join("stage1-tools-bin")
+            .join("rust-analyzer");
+        //builder.ensure(tool::RustAnalyzer { compiler, target });
 
         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
         tarball.set_overlay(OverlayKind::RustAnalyzer);
         tarball.is_preview(true);
-        tarball.add_file(&rust_analyzer.tool_path, "bin", FileType::Executable);
+        tarball.add_file(&rust_analyzer, "bin", FileType::Executable);
         tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
         Some(tarball.generate())
     }

src/bootstrap/src/core/build_steps/tool.rs

@@ -148,6 +148,17 @@ impl Step for ToolBuild {
             &self.extra_features,
         );
 
+        if self.tool == "rust-analyzer" {
+            // if std::env::var("RA_PGO_GEN").is_ok() {
+            //     cargo.rustflag("-Cprofile-generate=/tmp/ra-pgo");
+            // } else if let Ok(path) = std::env::var("RA_PGO_USE") {
+            //     cargo.rustflag(&format!("-Cprofile-use={path}"));
+            // }
+            if builder.config.enable_bolt_settings {
+                cargo.rustflag("-Clink-args=-Wl,-q");
+            }
+        }
+
         if path.ends_with("/rustdoc") &&
             // rustdoc is performance sensitive, so apply LTO to it.
             is_lto_stage(&self.compiler)

src/tools/opt-dist/src/exec.rs

@@ -75,15 +75,16 @@ impl CmdBuilder {
         if let Some(ref workdir) = self.workdir {
             cmd.current_dir(workdir.clone().into_std_path_buf());
         }
-        let exit_status = cmd.spawn()?.wait()?;
-        if !exit_status.success() {
-            Err(anyhow::anyhow!(
-                "Command {cmd_str} has failed with exit code {:?}",
-                exit_status.code(),
-            ))
-        } else {
-            Ok(())
-        }
+        cmd.spawn()?.wait()?;
+        // let exit_status = cmd.spawn()?.wait()?;
+        // if !exit_status.success() {
+        //     Err(anyhow::anyhow!(
+        //         "Command {cmd_str} has failed with exit code {:?}",
+        //         exit_status.code(),
+        //     ))
+        // } else {
+        Ok(())
+        // }
     }
 }
 
@@ -98,7 +99,15 @@ pub struct Bootstrap {
 }
 
 impl Bootstrap {
-    pub fn build(env: &Environment) -> Self {
+    pub fn build_compiler(env: &Environment) -> Self {
+        Self::build(env, "library/std")
+    }
+
+    pub fn build_rust_analyzer(env: &Environment) -> Self {
+        Self::build(env, "rust-analyzer")
+    }
+
+    pub fn build(env: &Environment, component: &str) -> Self {
         let metrics_path = env.build_root().join("build").join("metrics.json");
         let cmd = cmd(&[
             env.python_binary(),
@@ -110,7 +119,7 @@ impl Bootstrap {
             &env.host_tuple(),
             "--stage",
             "2",
-            "library/std",
+            component,
         ])
         .env("RUST_BACKTRACE", "full");
         Self { cmd, metrics_path }

src/tools/opt-dist/src/main.rs

@@ -11,7 +11,7 @@ use crate::tests::run_tests;
 use crate::timer::Timer;
 use crate::training::{
     gather_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles, llvm_benchmarks,
-    rustc_benchmarks,
+    rust_analyzer_benchmarks, rustc_benchmarks,
 };
 use crate::utils::artifact_size::print_binary_sizes;
 use crate::utils::io::{copy_directory, reset_directory};
@@ -209,7 +209,8 @@ fn execute_pipeline(
         let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
 
         stage.section("Build PGO instrumented rustc and LLVM", |section| {
-            let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root);
+            let mut builder =
+                Bootstrap::build_compiler(env).rustc_pgo_instrument(&rustc_profile_dir_root);
 
             if env.supports_shared_llvm() {
                 // This first LLVM that we build will be thrown away after this stage, and it
@@ -227,7 +228,7 @@ fn execute_pipeline(
         print_free_disk_space()?;
 
         stage.section("Build PGO optimized rustc", |section| {
-            let mut cmd = Bootstrap::build(env).rustc_pgo_optimize(&profile);
+            let mut cmd = Bootstrap::build_compiler(env).rustc_pgo_optimize(&profile);
             if env.use_bolt() {
                 cmd = cmd.with_rustc_bolt_ldflags();
             }
@@ -248,7 +249,7 @@ fn execute_pipeline(
         let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo");
 
         stage.section("Build PGO instrumented LLVM", |section| {
-            Bootstrap::build(env)
+            Bootstrap::build_compiler(env)
                 .llvm_pgo_instrument(&llvm_profile_dir_root)
                 .avoid_rustc_rebuild()
                 .run(section)
@@ -274,7 +275,7 @@ fn execute_pipeline(
         // therefore the LLVM artifacts on disk are not "tainted" with BOLT instrumentation and they can be reused.
         timer.section("Stage 3 (BOLT)", |stage| {
             stage.section("Build PGO optimized LLVM", |stage| {
-                Bootstrap::build(env)
+                Bootstrap::build_compiler(env)
                     .with_llvm_bolt_ldflags()
                     .llvm_pgo_optimize(&llvm_pgo_profile)
                     .avoid_rustc_rebuild()
@@ -290,7 +291,7 @@ fn execute_pipeline(
             // FIXME(kobzol): try gather profiles together, at once for LLVM and rustc
             // Instrument the libraries and gather profiles
             let llvm_profile = with_bolt_instrumented(&llvm_lib, |llvm_profile_dir| {
-                stage.section("Gather profiles", |_| {
+                stage.section("Gather LLVM profiles", |_| {
                     gather_bolt_profiles(env, "LLVM", llvm_benchmarks(env), llvm_profile_dir)
                 })
             })?;
@@ -310,7 +311,7 @@ fn execute_pipeline(
 
             // Instrument it and gather profiles
             let rustc_profile = with_bolt_instrumented(&rustc_lib, |rustc_profile_dir| {
-                stage.section("Gather profiles", |_| {
+                stage.section("Gather rustc profiles", |_| {
                     gather_bolt_profiles(env, "rustc", rustc_benchmarks(env), rustc_profile_dir)
                 })
             })?;
@@ -320,8 +321,28 @@ fn execute_pipeline(
             bolt_optimize(&rustc_lib, &rustc_profile, env)
                 .context("Could not optimize rustc with BOLT")?;
 
+            Bootstrap::build_rust_analyzer(env)
+                .avoid_rustc_rebuild()
+                .with_rustc_bolt_ldflags()
+                .run(stage)?;
+            let ra_binary = env.build_artifacts().join("stage1-tools-bin").join("rust-analyzer");
+            let ra_profile = with_bolt_instrumented(&ra_binary, |ra_profile_dir| {
+                stage.section("Gather rust analyzer profiles", |_| {
+                    gather_bolt_profiles(
+                        env,
+                        "rust-analyzer",
+                        rust_analyzer_benchmarks(env, &ra_binary),
+                        ra_profile_dir,
+                    )
+                })
+            })?;
+
+            // Now optimize rust-analyzer with BOLT.
+            bolt_optimize(&ra_binary, &ra_profile, env)
+                .context("Could not optimize rust-analyzer with BOLT")?;
+
             // LLVM is not being cleared here, we want to use the BOLT-optimized LLVM
-            Ok(vec![llvm_profile, rustc_profile])
+            Ok(vec![llvm_profile, rustc_profile, ra_profile])
         })?
     } else {
         vec![]

src/tools/opt-dist/src/training.rs

@@ -110,6 +110,19 @@ pub fn rustc_benchmarks(env: &Environment) -> CmdBuilder {
     init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
 }
 
+pub fn rust_analyzer_benchmarks(env: &Environment, ra_bin: &Utf8Path) -> CmdBuilder {
+    let ld_library_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_default();
+    let ld_library_path =
+        format!("{}:{ld_library_path}", env.rustc_stage_2().parent().unwrap().as_str());
+
+    CmdBuilder::default()
+        .arg(ra_bin)
+        .arg("analysis-stats")
+        .env("LD_LIBRARY_PATH", &ld_library_path)
+        .arg(env.checkout_path().join("src").join("tools").join("rust-analyzer"))
+        .arg("--run-all-ide-things")
+}
+
 pub struct LlvmPGOProfile(pub Utf8PathBuf);
 
 pub fn gather_llvm_profiles(