Rustify prepare.sh command

Author: GuillaumeGomez

.github/workflows/ci.yml

@@ -119,7 +119,7 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
+        ./y.sh prepare --only-libcore
         ${{ matrix.libgccjit_version.env_extra }} ./build.sh ${{ matrix.libgccjit_version.extra }}
         ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
         ./clean_all.sh
@@ -128,7 +128,7 @@ jobs:
       run: |
         git config --global user.email "[email protected]"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile

.github/workflows/release.yml

@@ -88,7 +88,7 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
+        ./y.sh prepare --only-libcore
         ./build.sh --release --release-sysroot
         cargo test
         ./clean_all.sh
@@ -97,7 +97,7 @@ jobs:
       run: |
         git config --global user.email "[email protected]"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile

.github/workflows/stdarch.yml

@@ -102,7 +102,7 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
+        ./y.sh prepare --only-libcore
         ./build.sh --release --release-sysroot
         cargo test
 
@@ -115,7 +115,7 @@ jobs:
       run: |
         git config --global user.email "[email protected]"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile

.gitignore

@@ -6,10 +6,7 @@ perf.data
 perf.data.old
 *.events
 *.string*
-/build_sysroot/sysroot
-/build_sysroot/sysroot_src
-/build_sysroot/Cargo.lock
-/build_sysroot/test_target/Cargo.lock
+build_sysroot
 /rust
 /simple-raytracer
 /regex
@@ -25,3 +22,4 @@ tools/llvmint
 tools/llvmint-2
 # The `llvm` folder is generated by the `tools/generate_intrinsics.py` script to update intrinsics.
 llvm
+build_system/target

Readme.md

@@ -65,7 +65,7 @@ $ export RUST_COMPILER_RT_ROOT="$PWD/llvm/compiler-rt"
 Then you can run commands like this:
 
 ```bash
-$ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
+$ ./y.sh prepare # download and patch sysroot src and install hyperfine for benchmarking
 $ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./build.sh --release
 ```
 

build_sysroot/prepare_sysroot_src.sh

@@ -0,0 +1,11 @@
+[package]
+name = "y"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "y"
+path = "src/main.rs"
+
+[features]
+unstable-features = [] # for rust-analyzer

build_system/Cargo.lock

@@ -0,0 +1,3 @@
+pub fn run() -> Result<(), String> {
+    Ok(())
+}

build_system/Cargo.toml

@@ -0,0 +1,49 @@
+use std::env;
+use std::process;
+
+mod build;
+mod prepare;
+mod rustc_info;
+mod utils;
+
+macro_rules! arg_error {
+    ($($err:tt)*) => {{
+        eprintln!($($err)*);
+        usage();
+        std::process::exit(1);
+    }};
+}
+
+fn usage() {
+    // println!("{}", include_str!("usage.txt"));
+}
+
+pub enum Command {
+    Prepare,
+    Build,
+}
+
+fn main() {
+    if env::var("RUST_BACKTRACE").is_err() {
+        env::set_var("RUST_BACKTRACE", "1");
+    }
+
+    let command = match env::args().nth(1).as_deref() {
+        Some("prepare") => Command::Prepare,
+        Some("build") => Command::Build,
+        Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
+        Some(command) => arg_error!("Unknown command {}", command),
+        None => {
+            usage();
+            process::exit(0);
+        }
+    };
+
+    if let Err(e) = match command {
+        Command::Prepare => prepare::run(),
+        Command::Build => build::run(),
+    } {
+        eprintln!("Command failed to run: {e:?}");
+        process::exit(1);
+    }
+}

build_system/src/build.rs

@@ -0,0 +1,157 @@
+use crate::rustc_info::get_rustc_path;
+use crate::utils::{cargo_install, git_clone, run_command, walk_dir};
+
+use std::fs;
+use std::path::Path;
+
+fn prepare_libcore() -> Result<(), String> {
+    let rustc_path = match get_rustc_path() {
+        Some(path) => path,
+        None => return Err("`rustc` path not found".to_owned()),
+    };
+
+    let parent = match rustc_path.parent() {
+        Some(path) => path,
+        None => return Err(format!("No parent for `{}`", rustc_path.display())),
+    };
+
+    let rustlib_dir = parent.join("../lib/rustlib/src/rust");
+    if !rustlib_dir.is_dir() {
+        return Err("Please install `rust-src` component".to_owned());
+    }
+
+    let sysroot_dir = Path::new("build_sysroot/sysroot_src");
+    if sysroot_dir.is_dir() {
+        if let Err(e) = fs::remove_dir_all(sysroot_dir) {
+            return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), e));
+        }
+    }
+
+    let sysroot_library_dir = sysroot_dir.join("library");
+    fs::create_dir_all(&sysroot_library_dir)
+        .map_err(|e| format!(
+            "Failed to create folder `{}`: {e:?}",
+            sysroot_library_dir.display(),
+        ))?;
+
+    run_command(&[&"cp", &"-r", &rustlib_dir, &sysroot_library_dir], None)?;
+
+    println!("[GIT] init (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"init"], Some(&sysroot_dir))?;
+    println!("[GIT] add (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"add", &"."], Some(&sysroot_dir))?;
+    println!("[GIT] commit (cwd): `{}`", sysroot_dir.display());
+
+    // This is needed on systems where nothing is configured.
+    // git really needs something here, or it will fail.
+    // Even using --author is not enough.
+    run_command(&[&"git", &"config", &"user.email", &"[email protected]"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"config", &"user.name", &"None"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"config", &"core.autocrlf=false"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"config", &"commit.gpgSign=false"], Some(&sysroot_dir))?;
+    run_command(&[&"git", &"commit", &"-m", &"Initial commit", &"-q"], Some(&sysroot_dir))?;
+
+    walk_dir("patches", |_| Ok(()), |file_path: &Path| {
+        println!("[GIT] apply `{}`", file_path.display());
+        let path = Path::new("../..").join(file_path);
+        run_command(&[&"git", &"apply", &path], Some(&sysroot_dir))?;
+        run_command(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?;
+        run_command(
+            &[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())],
+            Some(&sysroot_dir),
+        )?;
+        Ok(())
+    })?;
+    println!("Successfully prepared libcore for building");
+    Ok(())
+}
+
+// build with cg_llvm for perf comparison
+fn build_raytracer(repo_dir: &Path) -> Result<(), String> {
+    run_command(&[&"cargo", &"build"], Some(repo_dir))?;
+    run_command(&[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], Some(repo_dir))?;
+    Ok(())
+}
+
+fn clone_and_setup<F>(repo_url: &str, checkout_commit: &str, extra: Option<F>) -> Result<(), String>
+where
+    F: Fn(&Path) -> Result<(), String>,
+{
+    let clone_result = git_clone(repo_url, None)?;
+    if !clone_result.ran_clone {
+        println!("`{}` has already been cloned", clone_result.repo_name);
+    }
+    let repo_path = Path::new(&clone_result.repo_name);
+    run_command(&[&"git", &"checkout", &"--", &"."], Some(repo_path))?;
+    run_command(&[&"git", &"checkout", &checkout_commit], Some(repo_path))?;
+    let filter = format!("-{}-", clone_result.repo_name);
+    walk_dir("crate_patches", |_| Ok(()), |file_path| {
+        let s = file_path.as_os_str().to_str().unwrap();
+        if s.contains(&filter) && s.ends_with(".patch") {
+            run_command(&[&"git", &"am", &s], Some(repo_path))?;
+        }
+        Ok(())
+    })?;
+    if let Some(extra) = extra {
+        extra(repo_path)?;
+    }
+    Ok(())
+}
+
+struct PrepareArg {
+    only_libcore: bool,
+}
+
+impl PrepareArg {
+    fn new() -> Result<Option<Self>, String> {
+        let mut only_libcore = false;
+
+        for arg in std::env::args().skip(2) {
+            match arg.as_str() {
+                "--only-libcore" => only_libcore = true,
+                "--help" => {
+                    Self::usage();
+                    return Ok(None)
+                }
+                a => return Err(format!("Unknown argument `{a}`")),
+            }
+        }
+        Ok(Some(Self {
+            only_libcore,
+        }))
+    }
+
+    fn usage() {
+        println!(r#"
+`prepare` command help:
+
+    --only-libcore  : Only setup libcore and don't clone other repositories
+    --help          : Show this help
+"#)
+    }
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match PrepareArg::new()? {
+        Some(a) => a,
+        None => return Ok(()),
+    };
+    prepare_libcore()?;
+
+    if !args.only_libcore {
+        cargo_install("hyperfine")?;
+
+        let to_clone = &[
+            ("https://github.com/rust-random/rand.git", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", None),
+            ("https://github.com/rust-lang/regex.git", "341f207c1071f7290e3f228c710817c280c8dca1", None),
+            ("https://github.com/ebobby/simple-raytracer", "804a7a21b9e673a482797aa289a18ed480e4d813", Some(build_raytracer)),
+        ];
+
+        for (repo_url, checkout_commit, cb) in to_clone {
+            clone_and_setup(repo_url, checkout_commit, *cb)?;
+        }
+    }
+
+    println!("Successfully ran `prepare`");
+    Ok(())
+}

build_system/src/main.rs

@@ -0,0 +1,12 @@
+use std::path::{Path, PathBuf};
+
+use crate::utils::run_command;
+
+pub fn get_rustc_path() -> Option<PathBuf> {
+    if let Ok(rustc) = std::env::var("RUSTC") {
+        return Some(PathBuf::from(rustc));
+    }
+    run_command(&[&"rustup", &"which", &"rustc"], None)
+        .ok()
+        .map(|out| Path::new(String::from_utf8(out.stdout).unwrap().trim()).to_owned())
+}