Merge commit '11a0cceab9' into subtree-update_cg_gcc_2023-10-09

compiler/rustc_codegen_gcc/build_system/src/build.rs

@@ -0,0 +1,233 @@
+use crate::config::set_config;
+use crate::utils::{
+    get_gcc_path, run_command, run_command_with_env, run_command_with_output_and_env, walk_dir,
+};
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::Path;
+
+#[derive(Default)]
+struct BuildArg {
+    codegen_release_channel: bool,
+    sysroot_release_channel: bool,
+    features: Vec<String>,
+    gcc_path: String,
+}
+
+impl BuildArg {
+    fn new() -> Result<Option<Self>, String> {
+        let gcc_path = get_gcc_path()?;
+        let mut build_arg = Self {
+            gcc_path,
+            ..Default::default()
+        };
+        // We skip binary name and the `build` command.
+        let mut args = std::env::args().skip(2);
+
+        while let Some(arg) = args.next() {
+            match arg.as_str() {
+                "--release" => build_arg.codegen_release_channel = true,
+                "--release-sysroot" => build_arg.sysroot_release_channel = true,
+                "--no-default-features" => {
+                    build_arg.features.push("--no-default-features".to_string());
+                }
+                "--features" => {
+                    if let Some(arg) = args.next() {
+                        build_arg.features.push("--features".to_string());
+                        build_arg.features.push(arg.as_str().into());
+                    } else {
+                        return Err(
+                            "Expected a value after `--features`, found nothing".to_string()
+                        );
+                    }
+                }
+                "--help" => {
+                    Self::usage();
+                    return Ok(None);
+                }
+                arg => return Err(format!("Unknown argument `{}`", arg)),
+            }
+        }
+        Ok(Some(build_arg))
+    }
+
+    fn usage() {
+        println!(
+            r#"
+`build` command help:
+
+    --release              : Build codegen in release mode
+    --release-sysroot      : Build sysroot in release mode
+    --no-default-features  : Add `--no-default-features` flag
+    --features [arg]       : Add a new feature [arg]
+    --help                 : Show this help
+"#
+        )
+    }
+}
+
+fn build_sysroot(
+    env: &mut HashMap<String, String>,
+    release_mode: bool,
+    target_triple: &str,
+) -> Result<(), String> {
+    std::env::set_current_dir("build_sysroot")
+        .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?;
+    // Cleanup for previous run
+    // Clean target dir except for build scripts and incremental cache
+    let _ = walk_dir(
+        "target",
+        |dir: &Path| {
+            for top in &["debug", "release"] {
+                let _ = fs::remove_dir_all(dir.join(top).join("build"));
+                let _ = fs::remove_dir_all(dir.join(top).join("deps"));
+                let _ = fs::remove_dir_all(dir.join(top).join("examples"));
+                let _ = fs::remove_dir_all(dir.join(top).join("native"));
+
+                let _ = walk_dir(
+                    dir.join(top),
+                    |sub_dir: &Path| {
+                        if sub_dir
+                            .file_name()
+                            .map(|filename| filename.to_str().unwrap().starts_with("libsysroot"))
+                            .unwrap_or(false)
+                        {
+                            let _ = fs::remove_dir_all(sub_dir);
+                        }
+                        Ok(())
+                    },
+                    |file: &Path| {
+                        if file
+                            .file_name()
+                            .map(|filename| filename.to_str().unwrap().starts_with("libsysroot"))
+                            .unwrap_or(false)
+                        {
+                            let _ = fs::remove_file(file);
+                        }
+                        Ok(())
+                    },
+                );
+            }
+            Ok(())
+        },
+        |_| Ok(()),
+    );
+
+    let _ = fs::remove_file("Cargo.lock");
+    let _ = fs::remove_file("test_target/Cargo.lock");
+    let _ = fs::remove_dir_all("sysroot");
+
+    // Builds libs
+    let channel = if release_mode {
+        let rustflags = env
+            .get("RUSTFLAGS")
+            .cloned()
+            .unwrap_or_default();
+        env.insert(
+            "RUSTFLAGS".to_string(),
+            format!("{} -Zmir-opt-level=3", rustflags),
+        );
+        run_command_with_output_and_env(
+            &[
+                &"cargo",
+                &"build",
+                &"--target",
+                &target_triple,
+                &"--release",
+            ],
+            None,
+            Some(&env),
+        )?;
+        "release"
+    } else {
+        run_command_with_output_and_env(
+            &[
+                &"cargo",
+                &"build",
+                &"--target",
+                &target_triple,
+                &"--features",
+                &"compiler_builtins/c",
+            ],
+            None,
+            Some(env),
+        )?;
+        "debug"
+    };
+
+    // Copy files to sysroot
+    let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", target_triple);
+    fs::create_dir_all(&sysroot_path)
+        .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?;
+    let copier = |dir_to_copy: &Path| {
+        run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ())
+    };
+    walk_dir(
+        &format!("target/{}/{}/deps", target_triple, channel),
+        copier,
+        copier,
+    )?;
+
+    Ok(())
+}
+
+fn build_codegen(args: &BuildArg) -> Result<(), String> {
+    let mut env = HashMap::new();
+
+    let current_dir =
+        std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    if let Ok(rt_root) = std::env::var("RUST_COMPILER_RT_ROOT") {
+        env.insert("RUST_COMPILER_RT_ROOT".to_string(), rt_root);
+    } else {
+        env.insert(
+            "RUST_COMPILER_RT_ROOT".to_string(),
+            format!("{}", current_dir.join("llvm/compiler-rt").display()),
+        );
+    }
+    env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone());
+    env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone());
+
+    let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"];
+    if args.codegen_release_channel {
+        command.push(&"--release");
+        env.insert("CHANNEL".to_string(), "release".to_string());
+        env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string());
+    } else {
+        env.insert("CHANNEL".to_string(), "debug".to_string());
+    }
+    let ref_features = args.features.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    for feature in &ref_features {
+        command.push(feature);
+    }
+    run_command_with_env(&command, None, Some(&env))?;
+
+    let config = set_config(&mut env, &[], Some(&args.gcc_path))?;
+
+    // We voluntarily ignore the error.
+    let _ = fs::remove_dir_all("target/out");
+    let gccjit_target = "target/out/gccjit";
+    fs::create_dir_all(gccjit_target).map_err(|error| {
+        format!(
+            "Failed to create directory `{}`: {:?}",
+            gccjit_target, error
+        )
+    })?;
+
+    println!("[BUILD] sysroot");
+    build_sysroot(
+        &mut env,
+        args.sysroot_release_channel,
+        &config.target_triple,
+    )?;
+    Ok(())
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match BuildArg::new()? {
+        Some(args) => args,
+        None => return Ok(()),
+    };
+    build_codegen(&args)?;
+    Ok(())
+}

compiler/rustc_codegen_gcc/build_system/src/config.rs

@@ -0,0 +1,125 @@
+use crate::utils::{get_gcc_path, get_os_name, get_rustc_host_triple};
+use std::collections::HashMap;
+use std::env as std_env;
+
+pub struct ConfigInfo {
+    pub target_triple: String,
+    pub rustc_command: Vec<String>,
+    pub run_wrapper: Option<&'static str>,
+}
+
+// Returns the beginning for the command line of rustc.
+pub fn set_config(
+    env: &mut HashMap<String, String>,
+    test_flags: &[String],
+    gcc_path: Option<&str>,
+) -> Result<ConfigInfo, String> {
+    env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string());
+
+    let gcc_path = match gcc_path {
+        Some(path) => path.to_string(),
+        None => get_gcc_path()?,
+    };
+    env.insert("GCC_PATH".to_string(), gcc_path.clone());
+
+    let os_name = get_os_name()?;
+    let dylib_ext = match os_name.as_str() {
+        "Linux" => "so",
+        "Darwin" => "dylib",
+        os => return Err(format!("unsupported OS `{}`", os)),
+    };
+    let host_triple = get_rustc_host_triple()?;
+    let mut linker = None;
+    let mut target_triple = host_triple.as_str();
+    let mut run_wrapper = None;
+    // FIXME: handle this with a command line flag?
+    // let mut target_triple = "m68k-unknown-linux-gnu";
+
+    if host_triple != target_triple {
+        if target_triple == "m68k-unknown-linux-gnu" {
+            target_triple = "mips-unknown-linux-gnu";
+            linker = Some("-Clinker=m68k-linux-gcc");
+        } else if target_triple == "aarch64-unknown-linux-gnu" {
+            // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+            linker = Some("-Clinker=aarch64-linux-gnu-gcc");
+            run_wrapper = Some("qemu-aarch64 -L /usr/aarch64-linux-gnu");
+        } else {
+            return Err(format!("unknown non-native platform `{}`", target_triple));
+        }
+    }
+    let current_dir =
+        std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    let channel = if let Some(channel) = env.get("CHANNEL") {
+        channel.as_str()
+    } else {
+        "debug"
+    };
+    let cg_backend_path = current_dir
+        .join("target")
+        .join(channel)
+        .join(&format!("librustc_codegen_gcc.{}", dylib_ext));
+    let sysroot_path = current_dir.join("build_sysroot/sysroot");
+    let mut rustflags = Vec::new();
+    if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") {
+        rustflags.push(cg_rustflags.clone());
+    }
+    if let Some(linker) = linker {
+        rustflags.push(linker.to_string());
+    }
+    rustflags.extend_from_slice(&[
+        "-Csymbol-mangling-version=v0".to_string(),
+        "-Cdebuginfo=2".to_string(),
+        format!("-Zcodegen-backend={}", cg_backend_path.display()),
+        "--sysroot".to_string(),
+        sysroot_path.display().to_string(),
+    ]);
+
+    // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO.
+    // TODO(antoyo): remove when we can handle ThinLTO.
+    if !env.contains_key(&"FAT_LTO".to_string()) {
+        rustflags.push("-Clto=off".to_string());
+    }
+    rustflags.extend_from_slice(test_flags);
+    // FIXME(antoyo): remove once the atomic shim is gone
+    if os_name == "Darwin" {
+        rustflags.extend_from_slice(&[
+            "-Clink-arg=-undefined".to_string(),
+            "-Clink-arg=dynamic_lookup".to_string(),
+        ]);
+    }
+    env.insert("RUSTFLAGS".to_string(), rustflags.join(" "));
+    // display metadata load errors
+    env.insert("RUSTC_LOG".to_string(), "warn".to_string());
+
+    let sysroot = current_dir.join(&format!(
+        "build_sysroot/sysroot/lib/rustlib/{}/lib",
+        target_triple
+    ));
+    let ld_library_path = format!(
+        "{target}:{sysroot}:{gcc_path}",
+        target = current_dir.join("target/out").display(),
+        sysroot = sysroot.display(),
+    );
+    env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
+    env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+
+    // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
+    // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.
+    // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc
+    let path = std::env::var("PATH").unwrap_or_default();
+    env.insert("PATH".to_string(), format!("/opt/gcc/bin:{}", path));
+
+    let mut rustc_command = vec!["rustc".to_string()];
+    rustc_command.extend_from_slice(&rustflags);
+    rustc_command.extend_from_slice(&[
+        "-L".to_string(),
+        "crate=target/out".to_string(),
+        "--out-dir".to_string(),
+        "target/out".to_string(),
+    ]);
+    Ok(ConfigInfo {
+        target_triple: target_triple.to_string(),
+        rustc_command,
+        run_wrapper,
+    })
+}

compiler/rustc_codegen_gcc/build_system/src/main.rs

@@ -0,0 +1,62 @@
+use std::env;
+use std::process;
+
+mod build;
+mod config;
+mod prepare;
+mod rustc_info;
+mod utils;
+
+macro_rules! arg_error {
+    ($($err:tt)*) => {{
+        eprintln!($($err)*);
+        eprintln!();
+        usage();
+        std::process::exit(1);
+    }};
+}
+
+fn usage() {
+    println!(
+        "\
+Available commands for build_system:
+
+    prepare  : Run prepare command
+    build    : Run build command
+    --help   : Show this message"
+    );
+}
+
+pub enum Command {
+    Prepare,
+    Build,
+}
+
+fn main() {
+    if env::var("RUST_BACKTRACE").is_err() {
+        env::set_var("RUST_BACKTRACE", "1");
+    }
+
+    let command = match env::args().nth(1).as_deref() {
+        Some("prepare") => Command::Prepare,
+        Some("build") => Command::Build,
+        Some("--help") => {
+            usage();
+            process::exit(0);
+        }
+        Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
+        Some(command) => arg_error!("Unknown command {}", command),
+        None => {
+            usage();
+            process::exit(0);
+        }
+    };
+
+    if let Err(e) = match command {
+        Command::Prepare => prepare::run(),
+        Command::Build => build::run(),
+    } {
+        eprintln!("Command failed to run: {e:?}");
+        process::exit(1);
+    }
+}

compiler/rustc_codegen_gcc/build_system/src/prepare.rs

@@ -0,0 +1,227 @@
+use crate::rustc_info::get_rustc_path;
+use crate::utils::{cargo_install, git_clone, run_command, run_command_with_output, walk_dir};
+
+use std::fs;
+use std::path::Path;
+
+fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
+    let rustc_path = match get_rustc_path() {
+        Some(path) => path,
+        None => return Err("`rustc` path not found".to_string()),
+    };
+
+    let parent = match rustc_path.parent() {
+        Some(path) => path,
+        None => return Err(format!("No parent for `{}`", rustc_path.display())),
+    };
+
+    let rustlib_dir = parent
+        .join("../lib/rustlib/src/rust")
+        .canonicalize()
+        .map_err(|error| format!("Failed to canonicalize path: {:?}", error))?;
+    if !rustlib_dir.is_dir() {
+        return Err("Please install `rust-src` component".to_string());
+    }
+
+    let sysroot_dir = sysroot_path.join("sysroot_src");
+    if sysroot_dir.is_dir() {
+        if let Err(error) = fs::remove_dir_all(&sysroot_dir) {
+            return Err(format!(
+                "Failed to remove `{}`: {:?}",
+                sysroot_dir.display(),
+                error,
+            ));
+        }
+    }
+
+    let sysroot_library_dir = sysroot_dir.join("library");
+    fs::create_dir_all(&sysroot_library_dir).map_err(|error| {
+        format!(
+            "Failed to create folder `{}`: {:?}",
+            sysroot_library_dir.display(),
+            error,
+        )
+    })?;
+
+    run_command(
+        &[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir],
+        None,
+    )?;
+
+    println!("[GIT] init (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"init"], Some(&sysroot_dir))?;
+    println!("[GIT] add (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"add", &"."], Some(&sysroot_dir))?;
+    println!("[GIT] commit (cwd): `{}`", sysroot_dir.display());
+
+    // This is needed on systems where nothing is configured.
+    // git really needs something here, or it will fail.
+    // Even using --author is not enough.
+    run_command(
+        &[&"git", &"config", &"user.email", &"none@example.com"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"user.name", &"None"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"core.autocrlf", &"false"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"commit.gpgSign", &"false"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"commit", &"-m", &"Initial commit", &"-q"],
+        Some(&sysroot_dir),
+    )?;
+
+    let mut patches = Vec::new();
+    walk_dir(
+        "patches",
+        |_| Ok(()),
+        |file_path: &Path| {
+            patches.push(file_path.to_path_buf());
+            Ok(())
+        },
+    )?;
+    patches.sort();
+    for file_path in patches {
+        println!("[GIT] apply `{}`", file_path.display());
+        let path = Path::new("../..").join(file_path);
+        run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?;
+        run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?;
+        run_command_with_output(
+            &[
+                &"git",
+                &"commit",
+                &"--no-gpg-sign",
+                &"-m",
+                &format!("Patch {}", path.display()),
+            ],
+            Some(&sysroot_dir),
+        )?;
+    }
+    println!("Successfully prepared libcore for building");
+    Ok(())
+}
+
+// build with cg_llvm for perf comparison
+fn build_raytracer(repo_dir: &Path) -> Result<(), String> {
+    run_command(&[&"cargo", &"build"], Some(repo_dir))?;
+    let mv_target = repo_dir.join("raytracer_cg_llvm");
+    if mv_target.is_file() {
+        std::fs::remove_file(&mv_target)
+            .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?;
+    }
+    run_command(
+        &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"],
+        Some(repo_dir),
+    )?;
+    Ok(())
+}
+
+fn clone_and_setup<F>(repo_url: &str, checkout_commit: &str, extra: Option<F>) -> Result<(), String>
+where
+    F: Fn(&Path) -> Result<(), String>,
+{
+    let clone_result = git_clone(repo_url, None)?;
+    if !clone_result.ran_clone {
+        println!("`{}` has already been cloned", clone_result.repo_name);
+    }
+    let repo_path = Path::new(&clone_result.repo_name);
+    run_command(&[&"git", &"checkout", &"--", &"."], Some(&repo_path))?;
+    run_command(&[&"git", &"checkout", &checkout_commit], Some(&repo_path))?;
+    let filter = format!("-{}-", clone_result.repo_name);
+    walk_dir(
+        "crate_patches",
+        |_| Ok(()),
+        |file_path| {
+            let patch = file_path.as_os_str().to_str().unwrap();
+            if patch.contains(&filter) && patch.ends_with(".patch") {
+                run_command_with_output(
+                    &[&"git", &"am", &file_path.canonicalize().unwrap()],
+                    Some(&repo_path),
+                )?;
+            }
+            Ok(())
+        },
+    )?;
+    if let Some(extra) = extra {
+        extra(&repo_path)?;
+    }
+    Ok(())
+}
+
+struct PrepareArg {
+    only_libcore: bool,
+}
+
+impl PrepareArg {
+    fn new() -> Result<Option<Self>, String> {
+        let mut only_libcore = false;
+
+        for arg in std::env::args().skip(2) {
+            match arg.as_str() {
+                "--only-libcore" => only_libcore = true,
+                "--help" => {
+                    Self::usage();
+                    return Ok(None);
+                }
+                a => return Err(format!("Unknown argument `{a}`")),
+            }
+        }
+        Ok(Some(Self { only_libcore }))
+    }
+
+    fn usage() {
+        println!(
+            r#"
+`prepare` command help:
+
+    --only-libcore  : Only setup libcore and don't clone other repositories
+    --help          : Show this help
+"#
+        )
+    }
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match PrepareArg::new()? {
+        Some(a) => a,
+        None => return Ok(()),
+    };
+    let sysroot_path = Path::new("build_sysroot");
+    prepare_libcore(sysroot_path)?;
+
+    if !args.only_libcore {
+        cargo_install("hyperfine")?;
+
+        let to_clone = &[
+            (
+                "https://github.com/rust-random/rand.git",
+                "0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
+                None,
+            ),
+            (
+                "https://github.com/rust-lang/regex.git",
+                "341f207c1071f7290e3f228c710817c280c8dca1",
+                None,
+            ),
+            (
+                "https://github.com/ebobby/simple-raytracer",
+                "804a7a21b9e673a482797aa289a18ed480e4d813",
+                Some(build_raytracer),
+            ),
+        ];
+
+        for (repo_url, checkout_commit, cb) in to_clone {
+            clone_and_setup(repo_url, checkout_commit, *cb)?;
+        }
+    }
+
+    println!("Successfully ran `prepare`");
+    Ok(())
+}

compiler/rustc_codegen_gcc/build_system/src/rustc_info.rs

@@ -0,0 +1,12 @@
+use std::path::{Path, PathBuf};
+
+use crate::utils::run_command;
+
+pub fn get_rustc_path() -> Option<PathBuf> {
+    if let Ok(rustc) = std::env::var("RUSTC") {
+        return Some(PathBuf::from(rustc));
+    }
+    run_command(&[&"rustup", &"which", &"rustc"], None)
+        .ok()
+        .map(|out| Path::new(String::from_utf8(out.stdout).unwrap().trim()).to_path_buf())
+}

compiler/rustc_codegen_gcc/build_system/src/utils.rs

@@ -0,0 +1,240 @@
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt::Debug;
+use std::fs;
+use std::path::Path;
+use std::process::{Command, ExitStatus, Output};
+
+fn get_command_inner(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Command {
+    let (cmd, args) = match input {
+        [] => panic!("empty command"),
+        [cmd, args @ ..] => (cmd, args),
+    };
+    let mut command = Command::new(cmd);
+    command.args(args);
+    if let Some(cwd) = cwd {
+        command.current_dir(cwd);
+    }
+    if let Some(env) = env {
+        command.envs(env.iter().map(|(k, v)| (k.as_str(), v.as_str())));
+    }
+    command
+}
+
+fn check_exit_status(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    exit_status: ExitStatus,
+) -> Result<(), String> {
+    if exit_status.success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "Command `{}`{} exited with status {:?}",
+            input
+                .iter()
+                .map(|s| s.as_ref().to_str().unwrap())
+                .collect::<Vec<_>>()
+                .join(" "),
+            cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display()))
+                .unwrap_or_default(),
+            exit_status.code(),
+        ))
+    }
+}
+
+fn command_error<D: Debug>(input: &[&dyn AsRef<OsStr>], cwd: &Option<&Path>, error: D) -> String {
+    format!(
+        "Command `{}`{} failed to run: {error:?}",
+        input
+            .iter()
+            .map(|s| s.as_ref().to_str().unwrap())
+            .collect::<Vec<_>>()
+            .join(" "),
+        cwd.as_ref()
+            .map(|cwd| format!(" (running in folder `{}`)", cwd.display(),))
+            .unwrap_or_default(),
+    )
+}
+
+pub fn run_command(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Result<Output, String> {
+    run_command_with_env(input, cwd, None)
+}
+
+pub fn run_command_with_env(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Result<Output, String> {
+    let output = get_command_inner(input, cwd, env)
+        .output()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, output.status)?;
+    Ok(output)
+}
+
+pub fn run_command_with_output(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+) -> Result<(), String> {
+    let exit_status = get_command_inner(input, cwd, None)
+        .spawn()
+        .map_err(|e| command_error(input, &cwd, e))?
+        .wait()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, exit_status)?;
+    Ok(())
+}
+
+pub fn run_command_with_output_and_env(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Result<(), String> {
+    let exit_status = get_command_inner(input, cwd, env)
+        .spawn()
+        .map_err(|e| command_error(input, &cwd, e))?
+        .wait()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, exit_status)?;
+    Ok(())
+}
+
+pub fn cargo_install(to_install: &str) -> Result<(), String> {
+    let output = run_command(&[&"cargo", &"install", &"--list"], None)?;
+
+    let to_install_needle = format!("{to_install} ");
+    // cargo install --list returns something like this:
+    //
+    // mdbook-toc v0.8.0:
+    //     mdbook-toc
+    // rust-reduce v0.1.0:
+    //     rust-reduce
+    //
+    // We are only interested into the command name so we only look for lines ending with `:`.
+    if String::from_utf8(output.stdout)
+        .unwrap()
+        .lines()
+        .any(|line| line.ends_with(':') && line.starts_with(&to_install_needle))
+    {
+        return Ok(());
+    }
+    // We voluntarily ignore this error.
+    if run_command_with_output(&[&"cargo", &"install", &to_install], None).is_err() {
+        println!("Skipping installation of `{to_install}`");
+    }
+    Ok(())
+}
+
+pub fn get_os_name() -> Result<String, String> {
+    let output = run_command(&[&"uname"], None)?;
+    let name = std::str::from_utf8(&output.stdout)
+        .unwrap_or("")
+        .trim()
+        .to_string();
+    if !name.is_empty() {
+        Ok(name)
+    } else {
+        Err("Failed to retrieve the OS name".to_string())
+    }
+}
+
+pub fn get_rustc_host_triple() -> Result<String, String> {
+    let output = run_command(&[&"rustc", &"-vV"], None)?;
+    let content = std::str::from_utf8(&output.stdout).unwrap_or("");
+
+    for line in content.split('\n').map(|line| line.trim()) {
+        if !line.starts_with("host:") {
+            continue;
+        }
+        return Ok(line.split(':').nth(1).unwrap().trim().to_string());
+    }
+    Err("Cannot find host triple".to_string())
+}
+
+pub fn get_gcc_path() -> Result<String, String> {
+    let content = match fs::read_to_string("gcc_path") {
+        Ok(content) => content,
+        Err(_) => {
+            return Err(
+                "Please put the path to your custom build of libgccjit in the file \
+                   `gcc_path`, see Readme.md for details"
+                    .into(),
+            )
+        }
+    };
+    match content
+        .split('\n')
+        .map(|line| line.trim())
+        .filter(|line| !line.is_empty())
+        .next()
+    {
+        Some(gcc_path) => {
+            let path = Path::new(gcc_path);
+            if !path.exists() {
+                Err(format!(
+                    "Path `{}` contained in the `gcc_path` file doesn't exist",
+                    gcc_path,
+                ))
+            } else {
+                Ok(gcc_path.into())
+            }
+        }
+        None => Err("No path found in `gcc_path` file".into()),
+    }
+}
+
+pub struct CloneResult {
+    pub ran_clone: bool,
+    pub repo_name: String,
+}
+
+pub fn git_clone(to_clone: &str, dest: Option<&Path>) -> Result<CloneResult, String> {
+    let repo_name = to_clone.split('/').last().unwrap();
+    let repo_name = match repo_name.strip_suffix(".git") {
+        Some(n) => n.to_string(),
+        None => repo_name.to_string(),
+    };
+
+    let dest = dest
+        .map(|dest| dest.join(&repo_name))
+        .unwrap_or_else(|| Path::new(&repo_name).into());
+    if dest.is_dir() {
+        return Ok(CloneResult {
+            ran_clone: false,
+            repo_name,
+        });
+    }
+
+    run_command_with_output(&[&"git", &"clone", &to_clone, &dest], None)?;
+    Ok(CloneResult {
+        ran_clone: true,
+        repo_name,
+    })
+}
+
+pub fn walk_dir<P, D, F>(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String>
+where
+    P: AsRef<Path>,
+    D: FnMut(&Path) -> Result<(), String>,
+    F: FnMut(&Path) -> Result<(), String>,
+{
+    let dir = dir.as_ref();
+    for entry in fs::read_dir(dir)
+        .map_err(|error| format!("Failed to read dir `{}`: {:?}", dir.display(), error))?
+    {
+        let entry = entry
+            .map_err(|error| format!("Failed to read entry in `{}`: {:?}", dir.display(), error))?;
+        let entry_path = entry.path();
+        if entry_path.is_dir() {
+            dir_cb(&entry_path)?;
+        } else {
+            file_cb(&entry_path)?;
+        }
+    }
+    Ok(())
+}