From 2db4e50618faf277aa6e7b0b45b6fc1aa5389653 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 8 Jun 2021 14:33:25 +0200 Subject: [PATCH 01/14] Rewrite build.sh in rust This makes it easier to compile cg_clif on systems that don't support bash shell scripts like Windows --- .gitignore | 1 + .vscode/settings.json | 17 ++++ build.sh | 88 --------------------- build_system/build_backend.rs | 41 ++++++++++ build_system/build_sysroot.rs | 128 ++++++++++++++++++++++++++++++ build_system/rustc_info.rs | 41 ++++++++++ scripts/setup_rust_fork.sh | 2 +- test.sh | 4 +- y.rs | 141 ++++++++++++++++++++++++++++++++++ 9 files changed, 372 insertions(+), 91 deletions(-) delete mode 100755 build.sh create mode 100644 build_system/build_backend.rs create mode 100644 build_system/build_sysroot.rs create mode 100644 build_system/rustc_info.rs create mode 100755 y.rs diff --git a/.gitignore b/.gitignore index b241bef9d1e..3e010204655 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ perf.data perf.data.old *.events *.string* +/y.bin /build /build_sysroot/sysroot_src /build_sysroot/compiler-builtins diff --git a/.vscode/settings.json b/.vscode/settings.json index 9009a532c54..cec110792ce 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -49,6 +49,23 @@ "cfg": [], }, ] + }, + { + "roots": ["./y.rs"], + "crates": [ + { + "root_module": "./y.rs", + "edition": "2018", + "deps": [{ "crate": 1, "name": "std" }], + "cfg": [], + }, + { + "root_module": "./build_sysroot/sysroot_src/library/std/src/lib.rs", + "edition": "2018", + "deps": [], + "cfg": [], + }, + ] } ] } diff --git a/build.sh b/build.sh deleted file mode 100755 index 086b9617fdf..00000000000 --- a/build.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -e - -# Settings -export CHANNEL="release" -build_sysroot="clif" -target_dir='build' -while [[ $# != 0 ]]; do - case $1 in - "--debug") - export CHANNEL="debug" - ;; - "--sysroot") - build_sysroot=$2 - shift - ;; - "--target-dir") - target_dir=$2 - shift - ;; - *) - echo "Unknown flag '$1'" - echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]" - exit 1 - ;; - esac - shift -done - -# Build cg_clif -unset CARGO_TARGET_DIR -unamestr=$(uname) -if [[ "$unamestr" == 'Linux' || "$unamestr" == "FreeBSD" ]]; then - export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS -elif [[ "$unamestr" == 'Darwin' ]]; then - export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS -else - echo "Unsupported os $unamestr" - exit 1 -fi -if [[ "$CHANNEL" == "release" ]]; then - cargo build --release -else - cargo build -fi - -source scripts/ext_config.sh - -rm -rf "$target_dir" -mkdir "$target_dir" -mkdir "$target_dir"/bin "$target_dir"/lib -ln target/$CHANNEL/cg_clif{,_build_sysroot} "$target_dir"/bin -ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib -ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir" - -mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" -mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/" -if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then - cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" -fi - -case "$build_sysroot" in - "none") - ;; - "llvm") - cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/" - if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/" - fi - ;; - "clif") - echo "[BUILD] sysroot" - dir=$(pwd) - cd "$target_dir" - time "$dir/build_sysroot/build_sysroot.sh" - if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh" - fi - cp lib/rustlib/*/lib/libstd-* lib/ - ;; - *) - echo "Unknown sysroot kind \`$build_sysroot\`." - echo "The allowed values are:" - echo " none A sysroot that doesn't contain the standard library" - echo " llvm Copy the sysroot from rustc compiled by cg_llvm" - echo " clif Build a new sysroot using cg_clif" - exit 1 -esac diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs new file mode 100644 index 00000000000..cdddeae2a33 --- /dev/null +++ b/build_system/build_backend.rs @@ -0,0 +1,41 @@ +use std::env; +use std::process::{self, Command}; + +pub(crate) fn build_backend(channel: &str) -> String { + let mut cmd = Command::new("cargo"); + cmd.arg("build"); + + match channel { + "debug" => {} + "release" => { + cmd.arg("--release"); + } + _ => unreachable!(), + } + + if cfg!(unix) { + if cfg!(target_os = "macos") { + cmd.env( + "RUSTFLAGS", + "-Csplit-debuginfo=unpacked \ + -Clink-arg=-Wl,-rpath,@loader_path/../lib \ + -Zosx-rpath-install-name" + .to_string() + + env::var("RUSTFLAGS").as_deref().unwrap_or(""), + ); + } else { + cmd.env( + "RUSTFLAGS", + "-Clink-arg=-Wl,-rpath=$ORIGIN/../lib ".to_string() + + env::var("RUSTFLAGS").as_deref().unwrap_or(""), + ); + } + } + + eprintln!("[BUILD] rustc_codegen_cranelift"); + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } + + crate::rustc_info::get_dylib_name("rustc_codegen_cranelift") +} diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs new file mode 100644 index 00000000000..d6ac19dd1cc --- /dev/null +++ b/build_system/build_sysroot.rs @@ -0,0 +1,128 @@ +use crate::{try_hard_link, SysrootKind}; +use std::env; +use std::fs; +use std::path::Path; +use std::process::{self, Command}; + +pub(crate) fn build_sysroot( + channel: &str, + sysroot_kind: SysrootKind, + target_dir: &Path, + cg_clif_dylib: String, + host_triple: &str, + target_triple: &str, +) { + if target_dir.exists() { + fs::remove_dir_all(target_dir).unwrap(); + } + fs::create_dir_all(target_dir.join("bin")).unwrap(); + fs::create_dir_all(target_dir.join("lib")).unwrap(); + + // Copy the backend + for file in ["cg_clif", "cg_clif_build_sysroot"] { + try_hard_link( + Path::new("target").join(channel).join(file), + target_dir.join("bin").join(file), + ); + } + + try_hard_link( + Path::new("target").join(channel).join(&cg_clif_dylib), + target_dir.join("lib").join(cg_clif_dylib), + ); + + // Copy supporting files + try_hard_link("rust-toolchain", target_dir.join("rust-toolchain")); + try_hard_link("scripts/config.sh", target_dir.join("config.sh")); + try_hard_link("scripts/cargo.sh", target_dir.join("cargo.sh")); + + let default_sysroot = crate::rustc_info::get_default_sysroot(); + + let rustlib = target_dir.join("lib").join("rustlib"); + let host_rustlib_lib = rustlib.join(host_triple).join("lib"); + let target_rustlib_lib = rustlib.join(target_triple).join("lib"); + fs::create_dir_all(&host_rustlib_lib).unwrap(); + fs::create_dir_all(&target_rustlib_lib).unwrap(); + + if target_triple == "x86_64-pc-windows-gnu" { + if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() { + eprintln!( + "The x86_64-pc-windows-gnu target needs to be installed first before it is possible \ + to compile a sysroot for it.", + ); + process::exit(1); + } + for file in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"), + ) + .unwrap() + { + let file = file.unwrap().path(); + if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") { + continue; // only copy object files + } + try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap())); + } + } + + match sysroot_kind { + SysrootKind::None => {} // Nothing to do + SysrootKind::Llvm => { + for file in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"), + ) + .unwrap() + { + let file = file.unwrap().path(); + let file_name_str = file.file_name().unwrap().to_str().unwrap(); + if file_name_str.contains("rustc_") + || file_name_str.contains("chalk") + || file_name_str.contains("tracing") + || file_name_str.contains("regex") + { + // These are large crates that are part of the rustc-dev component and are not + // necessary to run regular programs. + continue; + } + try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap())); + } + + if target_triple != host_triple { + for file in fs::read_dir( + default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"), + ) + .unwrap() + { + let file = file.unwrap().path(); + try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap())); + } + } + } + SysrootKind::Clif => { + let cwd = env::current_dir().unwrap(); + + let mut cmd = Command::new(cwd.join("build_sysroot").join("build_sysroot.sh")); + cmd.current_dir(target_dir).env("TARGET_TRIPLE", target_triple); + eprintln!("[BUILD] sysroot"); + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } + + if host_triple != target_triple { + let mut cmd = Command::new(cwd.join("build_sysroot").join("build_sysroot.sh")); + cmd.current_dir(target_dir).env("TARGET_TRIPLE", host_triple); + eprintln!("[BUILD] sysroot"); + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } + } + + for file in fs::read_dir(host_rustlib_lib).unwrap() { + let file = file.unwrap().path(); + if file.file_name().unwrap().to_str().unwrap().contains("std-") { + try_hard_link(&file, target_dir.join("lib").join(file.file_name().unwrap())); + } + } + } + } +} diff --git a/build_system/rustc_info.rs b/build_system/rustc_info.rs new file mode 100644 index 00000000000..28222743022 --- /dev/null +++ b/build_system/rustc_info.rs @@ -0,0 +1,41 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +pub(crate) fn get_host_triple() -> String { + let version_info = + Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout; + String::from_utf8(version_info) + .unwrap() + .lines() + .to_owned() + .find(|line| line.starts_with("host")) + .unwrap() + .split(":") + .nth(1) + .unwrap() + .trim() + .to_owned() +} + +pub(crate) fn get_default_sysroot() -> PathBuf { + let default_sysroot = Command::new("rustc") + .stderr(Stdio::inherit()) + .args(&["--print", "sysroot"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned() +} + +pub(crate) fn get_dylib_name(crate_name: &str) -> String { + let dylib_name = Command::new("rustc") + .stderr(Stdio::inherit()) + .args(&["--crate-name", crate_name, "--crate-type", "dylib", "--print", "file-names", "-"]) + .output() + .unwrap() + .stdout; + let dylib_name = String::from_utf8(dylib_name).unwrap().trim().to_owned(); + assert!(!dylib_name.contains('\n')); + assert!(dylib_name.contains(crate_name)); + dylib_name +} diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index b52ca6ea2ad..c494e78050a 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -./build.sh +./y.rs build source build/config.sh echo "[SETUP] Rust fork" diff --git a/test.sh b/test.sh index e222adc7b80..a10924628bb 100755 --- a/test.sh +++ b/test.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -e -./build.sh --sysroot none "$@" +./y.rs build --sysroot none "$@" rm -r target/out || true scripts/tests.sh no_sysroot -./build.sh "$@" +./y.rs build "$@" scripts/tests.sh base_sysroot scripts/tests.sh extended_sysroot diff --git a/y.rs b/y.rs new file mode 100755 index 00000000000..7971e713082 --- /dev/null +++ b/y.rs @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +#![allow()] /*This line is ignored by bash +# This block is ignored by rustc +set -e +echo "[BUILD] y.rs" 1>&2 +rustc $0 -o ${0/.rs/.bin} -g +exec ${0/.rs/.bin} $@ +*/ + +//! The build system for cg_clif +//! +//! # Manual compilation +//! +//! If your system doesn't support shell scripts you can manually compile and run this file using +//! for example: +//! +//! ```shell +//! $ rustc y.rs -o build/y.bin +//! $ build/y.bin +//! ``` +//! +//! # Naming +//! +//! The name `y.rs` was chosen to not conflict with rustc's `x.py`. + +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process; + +#[path = "build_system/build_backend.rs"] +mod build_backend; +#[path = "build_system/build_sysroot.rs"] +mod build_sysroot; +#[path = "build_system/rustc_info.rs"] +mod rustc_info; + +fn usage() { + eprintln!("Usage:"); + eprintln!(" ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"); +} + +macro_rules! arg_error { + ($($err:tt)*) => {{ + eprintln!($($err)*); + usage(); + std::process::exit(1); + }}; +} + +enum Command { + Build, +} + +#[derive(Copy, Clone)] +enum SysrootKind { + None, + Clif, + Llvm, +} + +fn main() { + env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); + env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); + + let mut args = env::args().skip(1); + let command = match args.next().as_deref() { + Some("prepare") => { + if args.next().is_some() { + arg_error!("./x.rs prepare doesn't expect arguments"); + } + todo!(); + } + Some("build") => Command::Build, + Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), + Some(command) => arg_error!("Unknown command {}", command), + None => { + usage(); + process::exit(0); + } + }; + + let mut target_dir = PathBuf::from("build"); + let mut channel = "release"; + let mut sysroot_kind = SysrootKind::Clif; + while let Some(arg) = args.next().as_deref() { + match arg { + "--target-dir" => { + target_dir = PathBuf::from(args.next().unwrap_or_else(|| { + arg_error!("--target-dir requires argument"); + })) + } + "--debug" => channel = "debug", + "--sysroot" => { + sysroot_kind = match args.next().as_deref() { + Some("none") => SysrootKind::None, + Some("clif") => SysrootKind::Clif, + Some("llvm") => SysrootKind::Llvm, + Some(arg) => arg_error!("Unknown sysroot kind {}", arg), + None => arg_error!("--sysroot requires argument"), + } + } + flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag), + arg => arg_error!("Unexpected argument {}", arg), + } + } + + let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") { + host_triple + } else { + rustc_info::get_host_triple() + }; + let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") { + if target_triple != "" { + target_triple + } else { + host_triple.clone() // Empty target triple can happen on GHA + } + } else { + host_triple.clone() + }; + + let cg_clif_dylib = build_backend::build_backend(channel); + build_sysroot::build_sysroot( + channel, + sysroot_kind, + &target_dir, + cg_clif_dylib, + &host_triple, + &target_triple, + ); +} + +#[track_caller] +fn try_hard_link(src: impl AsRef, dst: impl AsRef) { + let src = src.as_ref(); + let dst = dst.as_ref(); + if let Err(_) = fs::hard_link(src, dst) { + fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed + } +} From fe6a2892a6dd285774c3b1c84a2de45209454ce6 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 11 Jun 2021 18:50:01 +0200 Subject: [PATCH 02/14] Rewrite prepare.sh in rust --- .cirrus.yml | 2 +- .github/workflows/main.yml | 2 +- .github/workflows/rustc.yml | 4 +- Readme.md | 8 ++-- build_system/build_backend.rs | 6 +-- build_system/build_sysroot.rs | 3 +- build_system/prepare.rs | 74 +++++++++++++++++++++++++++++++++++ build_system/utils.rs | 18 +++++++++ docs/usage.md | 2 +- prepare.sh | 29 -------------- scripts/rustup.sh | 2 +- y.rs | 20 ++++------ 12 files changed, 114 insertions(+), 56 deletions(-) create mode 100644 build_system/prepare.rs create mode 100644 build_system/utils.rs delete mode 100755 prepare.sh diff --git a/.cirrus.yml b/.cirrus.yml index e173df423a7..61da6a2491c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -14,7 +14,7 @@ task: - . $HOME/.cargo/env - git config --global user.email "user@example.com" - git config --global user.name "User" - - ./prepare.sh + - ./y.rs prepare test_script: - . $HOME/.cargo/env - # Enable backtraces for easier debugging diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4d45e36c956..6734920bdd6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -53,7 +53,7 @@ jobs: run: | git config --global user.email "user@example.com" git config --global user.name "User" - ./prepare.sh + ./y.rs prepare - name: Test env: diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml index e01a92598ba..1c08e5ece33 100644 --- a/.github/workflows/rustc.yml +++ b/.github/workflows/rustc.yml @@ -34,7 +34,7 @@ jobs: run: | git config --global user.email "user@example.com" git config --global user.name "User" - ./prepare.sh + ./y.rs prepare - name: Test run: | @@ -72,7 +72,7 @@ jobs: run: | git config --global user.email "user@example.com" git config --global user.name "User" - ./prepare.sh + ./y.rs prepare - name: Test run: | diff --git a/Readme.md b/Readme.md index 08f9373be62..37644b6382c 100644 --- a/Readme.md +++ b/Readme.md @@ -10,8 +10,8 @@ If not please open an issue. ```bash $ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git $ cd rustc_codegen_cranelift -$ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking -$ ./build.sh +$ ./y.rs prepare # download and patch sysroot src and install hyperfine for benchmarking +$ ./y.rs build ``` To run the test suite replace the last command with: @@ -20,7 +20,7 @@ To run the test suite replace the last command with: $ ./test.sh ``` -This will implicitly build cg_clif too. Both `build.sh` and `test.sh` accept a `--debug` argument to +This will implicitly build cg_clif too. Both `y.rs build` and `test.sh` accept a `--debug` argument to build in debug mode. Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section @@ -32,7 +32,7 @@ of workflow runs. Unfortunately due to GHA restrictions you need to be logged in rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`). In the directory with your project (where you can do the usual `cargo build`), run: diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index cdddeae2a33..81c47fa8358 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -1,5 +1,5 @@ use std::env; -use std::process::{self, Command}; +use std::process::Command; pub(crate) fn build_backend(channel: &str) -> String { let mut cmd = Command::new("cargo"); @@ -33,9 +33,7 @@ pub(crate) fn build_backend(channel: &str) -> String { } eprintln!("[BUILD] rustc_codegen_cranelift"); - if !cmd.spawn().unwrap().wait().unwrap().success() { - process::exit(1); - } + crate::utils::spawn_and_wait(cmd); crate::rustc_info::get_dylib_name("rustc_codegen_cranelift") } diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index d6ac19dd1cc..60906a8f19a 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -1,4 +1,5 @@ -use crate::{try_hard_link, SysrootKind}; +use crate::utils::try_hard_link; +use crate::SysrootKind; use std::env; use std::fs; use std::path::Path; diff --git a/build_system/prepare.rs b/build_system/prepare.rs new file mode 100644 index 00000000000..4aef9162594 --- /dev/null +++ b/build_system/prepare.rs @@ -0,0 +1,74 @@ +use std::ffi::OsStr; +use std::ffi::OsString; +use std::fs; +use std::process::Command; + +use crate::utils::spawn_and_wait; + +pub(crate) fn prepare() { + // FIXME implement in rust + let prepare_sysroot_cmd = Command::new("./build_sysroot/prepare_sysroot_src.sh"); + spawn_and_wait(prepare_sysroot_cmd); + + eprintln!("[INSTALL] hyperfine"); + Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap(); + + clone_repo( + "rand", + "https://github.com/rust-random/rand.git", + "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", + ); + + eprintln!("[PATCH] rand"); + for patch in get_patches("crate_patches", "rand") { + let mut patch_arg = OsString::from("../crate_patches/"); + patch_arg.push(patch); + let mut apply_patch_cmd = Command::new("git"); + apply_patch_cmd.arg("am").arg(patch_arg).current_dir("rand"); + spawn_and_wait(apply_patch_cmd); + } + + clone_repo( + "regex", + "https://github.com/rust-lang/regex.git", + "341f207c1071f7290e3f228c710817c280c8dca1", + ); + + clone_repo( + "simple-raytracer", + "https://github.com/ebobby/simple-raytracer", + "804a7a21b9e673a482797aa289a18ed480e4d813", + ); + + eprintln!("[LLVM BUILD] simple-raytracer"); + let mut build_cmd = Command::new("cargo"); + build_cmd.arg("build").env_remove("CARGO_TARGET_DIR").current_dir("simple-raytracer"); + spawn_and_wait(build_cmd); + fs::copy("simple-raytracer/target/debug/main", "simple-raytracer/raytracer_cg_llvm").unwrap(); +} + +fn clone_repo(name: &str, repo: &str, commit: &str) { + eprintln!("[CLONE] {}", repo); + // Ignore exit code as the repo may already have been checked out + Command::new("git").arg("clone").arg(repo).spawn().unwrap().wait().unwrap(); + + let mut clean_cmd = Command::new("git"); + clean_cmd.arg("checkout").arg("--").arg(".").current_dir(name); + spawn_and_wait(clean_cmd); + + let mut checkout_cmd = Command::new("git"); + checkout_cmd.arg("checkout").arg(commit).current_dir(name); + spawn_and_wait(checkout_cmd); +} + +fn get_patches(patch_dir: &str, crate_name: &str) -> Vec { + let mut patches: Vec<_> = fs::read_dir(patch_dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| path.extension() == Some(OsStr::new("patch"))) + .map(|path| path.file_name().unwrap().to_owned()) + .filter(|file_name| file_name.to_str().unwrap().split("-").nth(1).unwrap() == crate_name) + .collect(); + patches.sort(); + patches +} diff --git a/build_system/utils.rs b/build_system/utils.rs new file mode 100644 index 00000000000..3eba87ee8a4 --- /dev/null +++ b/build_system/utils.rs @@ -0,0 +1,18 @@ +use std::fs; +use std::path::Path; +use std::process::{self, Command}; + +#[track_caller] +pub(crate) fn try_hard_link(src: impl AsRef, dst: impl AsRef) { + let src = src.as_ref(); + let dst = dst.as_ref(); + if let Err(_) = fs::hard_link(src, dst) { + fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed + } +} + +pub(crate) fn spawn_and_wait(mut cmd: Command) { + if !cmd.spawn().unwrap().wait().unwrap().success() { + process::exit(1); + } +} diff --git a/docs/usage.md b/docs/usage.md index 3eee3b554e3..7e1e0a9bc9d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,7 +2,7 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`). ## Cargo diff --git a/prepare.sh b/prepare.sh deleted file mode 100755 index 64c097261c9..00000000000 --- a/prepare.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -e - -./build_sysroot/prepare_sysroot_src.sh -cargo install hyperfine || echo "Skipping hyperfine install" - -git clone https://github.com/rust-random/rand.git || echo "rust-random/rand has already been cloned" -pushd rand -git checkout -- . -git checkout 0f933f9c7176e53b2a3c7952ded484e1783f0bf1 -git am ../crate_patches/*-rand-*.patch -popd - -git clone https://github.com/rust-lang/regex.git || echo "rust-lang/regex has already been cloned" -pushd regex -git checkout -- . -git checkout 341f207c1071f7290e3f228c710817c280c8dca1 -popd - -git clone https://github.com/ebobby/simple-raytracer || echo "ebobby/simple-raytracer has already been cloned" -pushd simple-raytracer -git checkout -- . -git checkout 804a7a21b9e673a482797aa289a18ed480e4d813 - -# build with cg_llvm for perf comparison -unset CARGO_TARGET_DIR -cargo build -mv target/debug/main raytracer_cg_llvm -popd diff --git a/scripts/rustup.sh b/scripts/rustup.sh index fa7557653d8..cc34c080886 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -17,7 +17,7 @@ case $1 in done ./clean_all.sh - ./prepare.sh + ./y.rs prepare (cd build_sysroot && cargo update) diff --git a/y.rs b/y.rs index 7971e713082..62debc117c3 100755 --- a/y.rs +++ b/y.rs @@ -24,19 +24,23 @@ exec ${0/.rs/.bin} $@ //! The name `y.rs` was chosen to not conflict with rustc's `x.py`. use std::env; -use std::fs; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process; #[path = "build_system/build_backend.rs"] mod build_backend; #[path = "build_system/build_sysroot.rs"] mod build_sysroot; +#[path = "build_system/prepare.rs"] +mod prepare; #[path = "build_system/rustc_info.rs"] mod rustc_info; +#[path = "build_system/utils.rs"] +mod utils; fn usage() { eprintln!("Usage:"); + eprintln!(" ./y.rs prepare"); eprintln!(" ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"); } @@ -69,7 +73,8 @@ fn main() { if args.next().is_some() { arg_error!("./x.rs prepare doesn't expect arguments"); } - todo!(); + prepare::prepare(); + process::exit(0); } Some("build") => Command::Build, Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag), @@ -130,12 +135,3 @@ fn main() { &target_triple, ); } - -#[track_caller] -fn try_hard_link(src: impl AsRef, dst: impl AsRef) { - let src = src.as_ref(); - let dst = dst.as_ref(); - if let Err(_) = fs::hard_link(src, dst) { - fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed - } -} From 066d5f952cfb0c5d22d4ee31d6ac716b2d16668d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 16 Jun 2021 18:01:19 +0200 Subject: [PATCH 03/14] Rewrite prepare_sysroot_src.sh in rust --- build_sysroot/prepare_sysroot_src.sh | 39 --------- build_system/prepare.rs | 87 +++++++++++++++---- build_system/rustc_info.rs | 10 +++ build_system/utils.rs | 17 ++++ ...sysroot-Disable-not-compiling-tests.patch} | 0 ...> 0023-sysroot-Ignore-failing-tests.patch} | 0 ...27-sysroot-128bit-atomic-operations.patch} | 0 7 files changed, 97 insertions(+), 56 deletions(-) delete mode 100755 build_sysroot/prepare_sysroot_src.sh rename patches/{0022-core-Disable-not-compiling-tests.patch => 0022-sysroot-Disable-not-compiling-tests.patch} (100%) rename patches/{0023-core-Ignore-failing-tests.patch => 0023-sysroot-Ignore-failing-tests.patch} (100%) rename patches/{0027-Disable-128bit-atomic-operations.patch => 0027-sysroot-128bit-atomic-operations.patch} (100%) diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh deleted file mode 100755 index 7032a52a3d3..00000000000 --- a/build_sysroot/prepare_sysroot_src.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -e -cd "$(dirname "$0")" - -SRC_DIR="$(dirname "$(rustup which rustc)")/../lib/rustlib/src/rust/" -DST_DIR="sysroot_src" - -if [ ! -e "$SRC_DIR" ]; then - echo "Please install rust-src component" - exit 1 -fi - -rm -rf $DST_DIR -mkdir -p $DST_DIR/library -cp -a "$SRC_DIR/library" $DST_DIR/ - -pushd $DST_DIR -echo "[GIT] init" -git init -echo "[GIT] add" -git add . -echo "[GIT] commit" -git commit -m "Initial commit" -q -for file in $(ls ../../patches/ | grep -v patcha); do -echo "[GIT] apply" "$file" -git apply ../../patches/"$file" -git add -A -git commit --no-gpg-sign -m "Patch $file" -done -popd - -git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" -pushd compiler-builtins -git checkout -- . -git checkout 0.1.45 -git apply ../../crate_patches/000*-compiler-builtins-*.patch -popd - -echo "Successfully prepared sysroot source for building" diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 4aef9162594..b6d6457163d 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -1,14 +1,16 @@ use std::ffi::OsStr; use std::ffi::OsString; use std::fs; +use std::path::Path; +use std::path::PathBuf; use std::process::Command; +use crate::rustc_info::get_rustc_path; +use crate::utils::copy_dir_recursively; use crate::utils::spawn_and_wait; pub(crate) fn prepare() { - // FIXME implement in rust - let prepare_sysroot_cmd = Command::new("./build_sysroot/prepare_sysroot_src.sh"); - spawn_and_wait(prepare_sysroot_cmd); + prepare_sysroot(); eprintln!("[INSTALL] hyperfine"); Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap(); @@ -18,15 +20,7 @@ pub(crate) fn prepare() { "https://github.com/rust-random/rand.git", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", ); - - eprintln!("[PATCH] rand"); - for patch in get_patches("crate_patches", "rand") { - let mut patch_arg = OsString::from("../crate_patches/"); - patch_arg.push(patch); - let mut apply_patch_cmd = Command::new("git"); - apply_patch_cmd.arg("am").arg(patch_arg).current_dir("rand"); - spawn_and_wait(apply_patch_cmd); - } + apply_patches("crate_patches", "rand", Path::new("rand")); clone_repo( "regex", @@ -47,17 +41,64 @@ pub(crate) fn prepare() { fs::copy("simple-raytracer/target/debug/main", "simple-raytracer/raytracer_cg_llvm").unwrap(); } -fn clone_repo(name: &str, repo: &str, commit: &str) { +fn prepare_sysroot() { + let rustc_path = get_rustc_path(); + let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust"); + let sysroot_src = PathBuf::from("build_sysroot").canonicalize().unwrap().join("sysroot_src"); + + assert!(sysroot_src_orig.exists()); + + if sysroot_src.exists() { + fs::remove_dir_all(&sysroot_src).unwrap(); + } + fs::create_dir_all(sysroot_src.join("library")).unwrap(); + eprintln!("[COPY] sysroot src"); + copy_dir_recursively(&sysroot_src_orig.join("library"), &sysroot_src.join("library")); + + eprintln!("[GIT] init"); + let mut git_init_cmd = Command::new("git"); + git_init_cmd.arg("init").arg("-q").current_dir(&sysroot_src); + spawn_and_wait(git_init_cmd); + + let mut git_add_cmd = Command::new("git"); + git_add_cmd.arg("add").arg(".").current_dir(&sysroot_src); + spawn_and_wait(git_add_cmd); + + let mut git_commit_cmd = Command::new("git"); + git_commit_cmd + .arg("commit") + .arg("-m") + .arg("Initial commit") + .arg("-q") + .current_dir(&sysroot_src); + spawn_and_wait(git_commit_cmd); + + apply_patches("patches", "sysroot", &sysroot_src); + + clone_repo( + "build_sysroot/compiler-builtins", + "https://github.com/rust-lang/compiler-builtins.git", + "0.1.45", + ); + + apply_patches( + "crate_patches", + "compiler-builtins", + Path::new("build_sysroot/compiler-builtins"), + ); +} + +fn clone_repo(target_dir: &str, repo: &str, rev: &str) { eprintln!("[CLONE] {}", repo); // Ignore exit code as the repo may already have been checked out - Command::new("git").arg("clone").arg(repo).spawn().unwrap().wait().unwrap(); + Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap(); let mut clean_cmd = Command::new("git"); - clean_cmd.arg("checkout").arg("--").arg(".").current_dir(name); + clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir); spawn_and_wait(clean_cmd); let mut checkout_cmd = Command::new("git"); - checkout_cmd.arg("checkout").arg(commit).current_dir(name); + checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir); spawn_and_wait(checkout_cmd); } @@ -67,8 +108,20 @@ fn get_patches(patch_dir: &str, crate_name: &str) -> Vec { .map(|entry| entry.unwrap().path()) .filter(|path| path.extension() == Some(OsStr::new("patch"))) .map(|path| path.file_name().unwrap().to_owned()) - .filter(|file_name| file_name.to_str().unwrap().split("-").nth(1).unwrap() == crate_name) + .filter(|file_name| { + file_name.to_str().unwrap().split_once("-").unwrap().1.starts_with(crate_name) + }) .collect(); patches.sort(); patches } + +fn apply_patches(patch_dir: &str, crate_name: &str, target_dir: &Path) { + for patch in get_patches(patch_dir, crate_name) { + eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch); + let patch_arg = Path::new(patch_dir).join(patch).canonicalize().unwrap(); + let mut apply_patch_cmd = Command::new("git"); + apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir); + spawn_and_wait(apply_patch_cmd); + } +} diff --git a/build_system/rustc_info.rs b/build_system/rustc_info.rs index 28222743022..f7f5d42fe09 100644 --- a/build_system/rustc_info.rs +++ b/build_system/rustc_info.rs @@ -17,6 +17,16 @@ pub(crate) fn get_host_triple() -> String { .to_owned() } +pub(crate) fn get_rustc_path() -> PathBuf { + let rustc_path = Command::new("rustup") + .stderr(Stdio::inherit()) + .args(&["which", "rustc"]) + .output() + .unwrap() + .stdout; + Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned() +} + pub(crate) fn get_default_sysroot() -> PathBuf { let default_sysroot = Command::new("rustc") .stderr(Stdio::inherit()) diff --git a/build_system/utils.rs b/build_system/utils.rs index 3eba87ee8a4..12b5d70fad8 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -11,8 +11,25 @@ pub(crate) fn try_hard_link(src: impl AsRef, dst: impl AsRef) { } } +#[track_caller] pub(crate) fn spawn_and_wait(mut cmd: Command) { if !cmd.spawn().unwrap().wait().unwrap().success() { process::exit(1); } } + +pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) { + for entry in fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + let filename = entry.file_name(); + if filename == "." || filename == ".." { + continue; + } + if entry.metadata().unwrap().is_dir() { + fs::create_dir(to.join(&filename)).unwrap(); + copy_dir_recursively(&from.join(&filename), &to.join(&filename)); + } else { + fs::copy(from.join(&filename), to.join(&filename)).unwrap(); + } + } +} diff --git a/patches/0022-core-Disable-not-compiling-tests.patch b/patches/0022-sysroot-Disable-not-compiling-tests.patch similarity index 100% rename from patches/0022-core-Disable-not-compiling-tests.patch rename to patches/0022-sysroot-Disable-not-compiling-tests.patch diff --git a/patches/0023-core-Ignore-failing-tests.patch b/patches/0023-sysroot-Ignore-failing-tests.patch similarity index 100% rename from patches/0023-core-Ignore-failing-tests.patch rename to patches/0023-sysroot-Ignore-failing-tests.patch diff --git a/patches/0027-Disable-128bit-atomic-operations.patch b/patches/0027-sysroot-128bit-atomic-operations.patch similarity index 100% rename from patches/0027-Disable-128bit-atomic-operations.patch rename to patches/0027-sysroot-128bit-atomic-operations.patch From d71b12535e84a7c242cf217af49a0366b06c8691 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 16 Jun 2021 18:28:34 +0200 Subject: [PATCH 04/14] Merge the crate_patches dir into the patches dir --- build_system/prepare.rs | 21 +++++++------------ ...ins-Disable-128bit-atomic-operations.patch | 0 ...1-rand-Enable-c2-chacha-simd-feature.patch | 0 .../0002-rand-Disable-failing-test.patch | 0 4 files changed, 8 insertions(+), 13 deletions(-) rename {crate_patches => patches}/0001-compiler-builtins-Disable-128bit-atomic-operations.patch (100%) rename {crate_patches => patches}/0001-rand-Enable-c2-chacha-simd-feature.patch (100%) rename {crate_patches => patches}/0002-rand-Disable-failing-test.patch (100%) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index b6d6457163d..b5011ceb159 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -20,7 +20,7 @@ pub(crate) fn prepare() { "https://github.com/rust-random/rand.git", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", ); - apply_patches("crate_patches", "rand", Path::new("rand")); + apply_patches("rand", Path::new("rand")); clone_repo( "regex", @@ -73,19 +73,14 @@ fn prepare_sysroot() { .current_dir(&sysroot_src); spawn_and_wait(git_commit_cmd); - apply_patches("patches", "sysroot", &sysroot_src); + apply_patches("sysroot", &sysroot_src); clone_repo( "build_sysroot/compiler-builtins", "https://github.com/rust-lang/compiler-builtins.git", "0.1.45", ); - - apply_patches( - "crate_patches", - "compiler-builtins", - Path::new("build_sysroot/compiler-builtins"), - ); + apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins")); } fn clone_repo(target_dir: &str, repo: &str, rev: &str) { @@ -102,8 +97,8 @@ fn clone_repo(target_dir: &str, repo: &str, rev: &str) { spawn_and_wait(checkout_cmd); } -fn get_patches(patch_dir: &str, crate_name: &str) -> Vec { - let mut patches: Vec<_> = fs::read_dir(patch_dir) +fn get_patches(crate_name: &str) -> Vec { + let mut patches: Vec<_> = fs::read_dir("patches") .unwrap() .map(|entry| entry.unwrap().path()) .filter(|path| path.extension() == Some(OsStr::new("patch"))) @@ -116,10 +111,10 @@ fn get_patches(patch_dir: &str, crate_name: &str) -> Vec { patches } -fn apply_patches(patch_dir: &str, crate_name: &str, target_dir: &Path) { - for patch in get_patches(patch_dir, crate_name) { +fn apply_patches(crate_name: &str, target_dir: &Path) { + for patch in get_patches(crate_name) { eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch); - let patch_arg = Path::new(patch_dir).join(patch).canonicalize().unwrap(); + let patch_arg = Path::new("patches").join(patch).canonicalize().unwrap(); let mut apply_patch_cmd = Command::new("git"); apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir); spawn_and_wait(apply_patch_cmd); diff --git a/crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch b/patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch similarity index 100% rename from crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch rename to patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch diff --git a/crate_patches/0001-rand-Enable-c2-chacha-simd-feature.patch b/patches/0001-rand-Enable-c2-chacha-simd-feature.patch similarity index 100% rename from crate_patches/0001-rand-Enable-c2-chacha-simd-feature.patch rename to patches/0001-rand-Enable-c2-chacha-simd-feature.patch diff --git a/crate_patches/0002-rand-Disable-failing-test.patch b/patches/0002-rand-Disable-failing-test.patch similarity index 100% rename from crate_patches/0002-rand-Disable-failing-test.patch rename to patches/0002-rand-Disable-failing-test.patch From ad971bbed773cad7cb446b49b07c64504e9a79fb Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Jun 2021 13:27:56 +0200 Subject: [PATCH 05/14] Rewrite build_sysroot.sh in rust --- build_sysroot/build_sysroot.sh | 39 -------------------- build_system/build_sysroot.rs | 66 ++++++++++++++++++++++++++-------- y.rs | 2 ++ 3 files changed, 53 insertions(+), 54 deletions(-) delete mode 100755 build_sysroot/build_sysroot.sh diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh deleted file mode 100755 index 0354304e55b..00000000000 --- a/build_sysroot/build_sysroot.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Requires the CHANNEL env var to be set to `debug` or `release.` - -set -e - -source ./config.sh - -dir=$(pwd) - -# Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that -# build scripts are still compiled using cg_llvm. -export RUSTC=$dir"/bin/cg_clif_build_sysroot" -export RUSTFLAGS=$RUSTFLAGS" --clif" - -cd "$(dirname "$0")" - -# Cleanup for previous run -# v Clean target dir except for build scripts and incremental cache -rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true - -# We expect the target dir in the default location. Guard against the user changing it. -export CARGO_TARGET_DIR=target - -# Build libs -export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort" -export __CARGO_DEFAULT_LIB_METADATA="cg_clif" -if [[ "$1" != "--debug" ]]; then - sysroot_channel='release' - # FIXME Enable incremental again once rust-lang/rust#74946 is fixed - CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release -else - sysroot_channel='debug' - cargo build --target "$TARGET_TRIPLE" -fi - -# Copy files to sysroot -ln "target/$TARGET_TRIPLE/$sysroot_channel/deps/"* "$dir/lib/rustlib/$TARGET_TRIPLE/lib/" -rm "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"*.{rmeta,d} diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 60906a8f19a..9988b35b363 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -1,6 +1,6 @@ +use crate::utils::spawn_and_wait; use crate::utils::try_hard_link; use crate::SysrootKind; -use std::env; use std::fs; use std::path::Path; use std::process::{self, Command}; @@ -100,24 +100,14 @@ pub(crate) fn build_sysroot( } } SysrootKind::Clif => { - let cwd = env::current_dir().unwrap(); - - let mut cmd = Command::new(cwd.join("build_sysroot").join("build_sysroot.sh")); - cmd.current_dir(target_dir).env("TARGET_TRIPLE", target_triple); - eprintln!("[BUILD] sysroot"); - if !cmd.spawn().unwrap().wait().unwrap().success() { - process::exit(1); - } + build_clif_sysroot_for_triple(channel, target_dir, target_triple); if host_triple != target_triple { - let mut cmd = Command::new(cwd.join("build_sysroot").join("build_sysroot.sh")); - cmd.current_dir(target_dir).env("TARGET_TRIPLE", host_triple); - eprintln!("[BUILD] sysroot"); - if !cmd.spawn().unwrap().wait().unwrap().success() { - process::exit(1); - } + build_clif_sysroot_for_triple(channel, target_dir, host_triple); } + // Copy std for the host to the lib dir. This is necessary for the jit mode to find + // libstd. for file in fs::read_dir(host_rustlib_lib).unwrap() { let file = file.unwrap().path(); if file.file_name().unwrap().to_str().unwrap().contains("std-") { @@ -127,3 +117,49 @@ pub(crate) fn build_sysroot( } } } + +fn build_clif_sysroot_for_triple(channel: &str, target_dir: &Path, triple: &str) { + let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel); + + // FIXME add option to skip this + // Cleanup the target dir with the exception of build scripts and the incremental cache + for dir in ["build", "deps", "examples", "native"] { + if build_dir.join(dir).exists() { + fs::remove_dir_all(build_dir.join(dir)).unwrap(); + } + } + + // Build sysroot + let mut build_cmd = Command::new("cargo"); + build_cmd.arg("build").arg("--target").arg(triple).current_dir("build_sysroot"); + let mut rustflags = "--clif -Zforce-unstable-if-unmarked".to_string(); + if channel == "release" { + build_cmd.arg("--release"); + rustflags.push_str(" -Zmir-opt-level=3"); + } + build_cmd.env("RUSTFLAGS", rustflags); + build_cmd + .env("RUSTC", target_dir.join("bin").join("cg_clif_build_sysroot").canonicalize().unwrap()); + // FIXME Enable incremental again once rust-lang/rust#74946 is fixed + build_cmd.env("CARGO_INCREMENTAL", "0").env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); + spawn_and_wait(build_cmd); + + // Copy all relevant files to the sysroot + for entry in + fs::read_dir(Path::new("build_sysroot/target").join(triple).join(channel).join("deps")) + .unwrap() + { + let entry = entry.unwrap(); + if let Some(ext) = entry.path().extension() { + if ext == "rmeta" || ext == "d" || ext == "dSYM" { + continue; + } + } else { + continue; + }; + try_hard_link( + entry.path(), + target_dir.join("lib").join("rustlib").join(triple).join("lib").join(entry.file_name()), + ); + } +} diff --git a/y.rs b/y.rs index 62debc117c3..27ab44f5870 100755 --- a/y.rs +++ b/y.rs @@ -66,6 +66,8 @@ enum SysrootKind { fn main() { env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1"); env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1"); + // The target dir is expected in the default location. Guard against the user changing it. + env::set_var("CARGO_TARGET_DIR", "target"); let mut args = env::args().skip(1); let command = match args.next().as_deref() { From 0d6b3dab658d76078741cc78b0bda1682b43a7ec Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Jun 2021 13:35:58 +0200 Subject: [PATCH 06/14] Allow preserving the old sysroot --- build_system/build_sysroot.rs | 13 ++++++++----- config.txt | 7 +++++++ 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 config.txt diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 9988b35b363..fb34875f50f 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -121,11 +121,14 @@ pub(crate) fn build_sysroot( fn build_clif_sysroot_for_triple(channel: &str, target_dir: &Path, triple: &str) { let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel); - // FIXME add option to skip this - // Cleanup the target dir with the exception of build scripts and the incremental cache - for dir in ["build", "deps", "examples", "native"] { - if build_dir.join(dir).exists() { - fs::remove_dir_all(build_dir.join(dir)).unwrap(); + let keep_sysroot = + fs::read_to_string("config.txt").unwrap().lines().any(|line| line.trim() == "keep_sysroot"); + if !keep_sysroot { + // Cleanup the target dir with the exception of build scripts and the incremental cache + for dir in ["build", "deps", "examples", "native"] { + if build_dir.join(dir).exists() { + fs::remove_dir_all(build_dir.join(dir)).unwrap(); + } } } diff --git a/config.txt b/config.txt new file mode 100644 index 00000000000..f707b9322af --- /dev/null +++ b/config.txt @@ -0,0 +1,7 @@ +# This file allows configuring the build system. + +# Disables cleaning of the sysroot dir. This will cause old compiled artifacts to be re-used when +# the sysroot source hasn't changed. This is useful when the codegen backend hasn't been modified. +# This option can be changed while the build system is already running for as long as sysroot +# building hasn't started yet. +#keep_sysroot From e5563b50772e45dc2ad9111769f48d216570af71 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Jun 2021 13:38:50 +0200 Subject: [PATCH 07/14] Improve windows support --- build_system/build_backend.rs | 2 +- build_system/build_sysroot.rs | 35 ++++++++++++++++++++++++----------- build_system/prepare.rs | 18 +++++++++++------- build_system/rustc_info.rs | 22 +++++++++++++++------- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index 81c47fa8358..db046be1866 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -35,5 +35,5 @@ pub(crate) fn build_backend(channel: &str) -> String { eprintln!("[BUILD] rustc_codegen_cranelift"); crate::utils::spawn_and_wait(cmd); - crate::rustc_info::get_dylib_name("rustc_codegen_cranelift") + crate::rustc_info::get_file_name("rustc_codegen_cranelift", "dylib") } diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index fb34875f50f..547159438e9 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -1,10 +1,12 @@ -use crate::utils::spawn_and_wait; -use crate::utils::try_hard_link; -use crate::SysrootKind; +use std::env; use std::fs; use std::path::Path; use std::process::{self, Command}; +use crate::rustc_info::get_file_name; +use crate::utils::{spawn_and_wait, try_hard_link}; +use crate::SysrootKind; + pub(crate) fn build_sysroot( channel: &str, sysroot_kind: SysrootKind, @@ -22,15 +24,24 @@ pub(crate) fn build_sysroot( // Copy the backend for file in ["cg_clif", "cg_clif_build_sysroot"] { try_hard_link( - Path::new("target").join(channel).join(file), - target_dir.join("bin").join(file), + Path::new("target").join(channel).join(get_file_name(file, "bin")), + target_dir.join("bin").join(get_file_name(file, "bin")), ); } - try_hard_link( - Path::new("target").join(channel).join(&cg_clif_dylib), - target_dir.join("lib").join(cg_clif_dylib), - ); + if cfg!(windows) { + // Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the + // binaries. + try_hard_link( + Path::new("target").join(channel).join(&cg_clif_dylib), + target_dir.join("bin").join(cg_clif_dylib), + ); + } else { + try_hard_link( + Path::new("target").join(channel).join(&cg_clif_dylib), + target_dir.join("lib").join(cg_clif_dylib), + ); + } // Copy supporting files try_hard_link("rust-toolchain", target_dir.join("rust-toolchain")); @@ -141,8 +152,10 @@ fn build_clif_sysroot_for_triple(channel: &str, target_dir: &Path, triple: &str) rustflags.push_str(" -Zmir-opt-level=3"); } build_cmd.env("RUSTFLAGS", rustflags); - build_cmd - .env("RUSTC", target_dir.join("bin").join("cg_clif_build_sysroot").canonicalize().unwrap()); + build_cmd.env( + "RUSTC", + env::current_dir().unwrap().join(target_dir).join("bin").join("cg_clif_build_sysroot"), + ); // FIXME Enable incremental again once rust-lang/rust#74946 is fixed build_cmd.env("CARGO_INCREMENTAL", "0").env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); spawn_and_wait(build_cmd); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index b5011ceb159..d26f24f8856 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -1,13 +1,12 @@ +use std::env; use std::ffi::OsStr; use std::ffi::OsString; use std::fs; use std::path::Path; -use std::path::PathBuf; use std::process::Command; -use crate::rustc_info::get_rustc_path; -use crate::utils::copy_dir_recursively; -use crate::utils::spawn_and_wait; +use crate::rustc_info::{get_file_name, get_rustc_path}; +use crate::utils::{copy_dir_recursively, spawn_and_wait}; pub(crate) fn prepare() { prepare_sysroot(); @@ -38,13 +37,18 @@ pub(crate) fn prepare() { let mut build_cmd = Command::new("cargo"); build_cmd.arg("build").env_remove("CARGO_TARGET_DIR").current_dir("simple-raytracer"); spawn_and_wait(build_cmd); - fs::copy("simple-raytracer/target/debug/main", "simple-raytracer/raytracer_cg_llvm").unwrap(); + fs::copy( + Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")), + // FIXME use get_file_name here too once testing is migrated to rust + "simple-raytracer/raytracer_cg_llvm", + ) + .unwrap(); } fn prepare_sysroot() { let rustc_path = get_rustc_path(); let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust"); - let sysroot_src = PathBuf::from("build_sysroot").canonicalize().unwrap().join("sysroot_src"); + let sysroot_src = env::current_dir().unwrap().join("build_sysroot").join("sysroot_src"); assert!(sysroot_src_orig.exists()); @@ -114,7 +118,7 @@ fn get_patches(crate_name: &str) -> Vec { fn apply_patches(crate_name: &str, target_dir: &Path) { for patch in get_patches(crate_name) { eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch); - let patch_arg = Path::new("patches").join(patch).canonicalize().unwrap(); + let patch_arg = env::current_dir().unwrap().join("patches").join(patch); let mut apply_patch_cmd = Command::new("git"); apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir); spawn_and_wait(apply_patch_cmd); diff --git a/build_system/rustc_info.rs b/build_system/rustc_info.rs index f7f5d42fe09..3bf86d9a114 100644 --- a/build_system/rustc_info.rs +++ b/build_system/rustc_info.rs @@ -37,15 +37,23 @@ pub(crate) fn get_default_sysroot() -> PathBuf { Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned() } -pub(crate) fn get_dylib_name(crate_name: &str) -> String { - let dylib_name = Command::new("rustc") +pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String { + let file_name = Command::new("rustc") .stderr(Stdio::inherit()) - .args(&["--crate-name", crate_name, "--crate-type", "dylib", "--print", "file-names", "-"]) + .args(&[ + "--crate-name", + crate_name, + "--crate-type", + crate_type, + "--print", + "file-names", + "-", + ]) .output() .unwrap() .stdout; - let dylib_name = String::from_utf8(dylib_name).unwrap().trim().to_owned(); - assert!(!dylib_name.contains('\n')); - assert!(dylib_name.contains(crate_name)); - dylib_name + let file_name = String::from_utf8(file_name).unwrap().trim().to_owned(); + assert!(!file_name.contains('\n')); + assert!(file_name.contains(crate_name)); + file_name } From b0d7b526d96fc428123444d8852e6ab5674f809e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Jun 2021 13:41:20 +0200 Subject: [PATCH 08/14] [CI] Test compilation on Windows --- .github/workflows/main.yml | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6734920bdd6..4442cbe5f94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,3 +87,63 @@ jobs: with: name: cg_clif-${{ runner.os }}-cross-x86_64-mingw path: cg_clif.tar.xz + + build_windows: + runs-on: windows-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v2 + + #- name: Cache cargo installed crates + # uses: actions/cache@v2 + # with: + # path: ~/.cargo/bin + # key: ${{ runner.os }}-cargo-installed-crates + + #- name: Cache cargo registry and index + # uses: actions/cache@v2 + # with: + # path: | + # ~/.cargo/registry + # ~/.cargo/git + # key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }} + + #- name: Cache cargo target dir + # uses: actions/cache@v2 + # with: + # path: target + # key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Prepare dependencies + run: | + git config --global user.email "user@example.com" + git config --global user.name "User" + git config --global core.autocrlf false + rustup set default-host x86_64-pc-windows-gnu + rustc y.rs -o y.exe -g + ./y.exe prepare + + - name: Build + #name: Test + run: | + # Enable backtraces for easier debugging + #export RUST_BACKTRACE=1 + + # Reduce amount of benchmark runs as they are slow + #export COMPILE_RUNS=2 + #export RUN_RUNS=2 + + # Enable extra checks + #export CG_CLIF_ENABLE_VERIFIER=1 + + ./y.exe build + + #- name: Package prebuilt cg_clif + # run: tar cvfJ cg_clif.tar.xz build + + #- name: Upload prebuilt cg_clif + # uses: actions/upload-artifact@v2 + # with: + # name: cg_clif-${{ runner.os }} + # path: cg_clif.tar.xz From bc67726882cc368e6aa382ea97c616d666d9b11d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 18 Jun 2021 16:18:44 +0200 Subject: [PATCH 09/14] Update rust-analyzer import assist config --- .vscode/settings.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index cec110792ce..38ffef5ac99 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,9 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], - "rust-analyzer.assist.importMergeBehavior": "last", + "rust-analyzer.assist.importGranularity": "module", + "rust-analyzer.assist.importEnforceGranularity": true, + "rust-analyzer.assist.importPrefix": "by_crate", "rust-analyzer.cargo.runBuildScripts": true, "rust-analyzer.linkedProjects": [ "./Cargo.toml", From d571910f4dd057d26c7c2f483a4b3a813e292d00 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 19 Jun 2021 14:07:56 +0200 Subject: [PATCH 10/14] Give an error when attempting to build for MSVC --- y.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/y.rs b/y.rs index 27ab44f5870..55457745d25 100755 --- a/y.rs +++ b/y.rs @@ -127,6 +127,14 @@ fn main() { host_triple.clone() }; + if target_triple.ends_with("-msvc") { + eprintln!("The MSVC toolchain is not yet supported by rustc_codegen_cranelift."); + eprintln!("Switch to the MinGW toolchain for Windows support."); + eprintln!("Hint: You can use `rustup set default-host x86_64-pc-windows-gnu` to"); + eprintln!("set the global default target to MinGW"); + process::exit(1); + } + let cg_clif_dylib = build_backend::build_backend(channel); build_sysroot::build_sysroot( channel, From 9fd8fa21231e488c3f784e811b9f7c7548561594 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 19 Jun 2021 14:51:48 +0200 Subject: [PATCH 11/14] Move some things from config.sh to ext_config.sh --- scripts/config.sh | 5 ----- scripts/ext_config.sh | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/config.sh b/scripts/config.sh index 99b302ee1d9..1aeed266d9a 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -18,10 +18,5 @@ export RUSTC=$dir"/bin/cg_clif" export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\ '-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir -# FIXME fix `#[linkage = "extern_weak"]` without this -if [[ "$(uname)" == 'Darwin' ]]; then - export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" -fi - export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh index 3f98d77d76c..9f8331865ce 100644 --- a/scripts/ext_config.sh +++ b/scripts/ext_config.sh @@ -25,3 +25,8 @@ if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then echo "Unknown non-native platform" fi fi + +# FIXME fix `#[linkage = "extern_weak"]` without this +if [[ "$(uname)" == 'Darwin' ]]; then + export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" +fi From 80e9188fb1a404427513e080b8a27ac262769614 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 20 Jun 2021 12:41:40 +0200 Subject: [PATCH 12/14] Rewrite cargo.sh in rust --- Readme.md | 2 +- build_system/build_sysroot.rs | 12 +++-- docs/usage.md | 6 +-- scripts/cargo.rs | 85 +++++++++++++++++++++++++++++++++++ scripts/cargo.sh | 18 -------- scripts/config.sh | 17 +------ scripts/ext_config.sh | 2 +- scripts/filter_profile.rs | 5 ++- scripts/setup_rust_fork.sh | 2 +- scripts/tests.sh | 35 ++++++++------- 10 files changed, 121 insertions(+), 63 deletions(-) create mode 100644 scripts/cargo.rs delete mode 100755 scripts/cargo.sh diff --git a/Readme.md b/Readme.md index 37644b6382c..dad8ed90b53 100644 --- a/Readme.md +++ b/Readme.md @@ -37,7 +37,7 @@ Assuming `$cg_clif_dir` is the directory you cloned this repo into and you follo In the directory with your project (where you can do the usual `cargo build`), run: ```bash -$ $cg_clif_dir/build/cargo.sh build +$ $cg_clif_dir/build/cargo build ``` This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend. diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 547159438e9..507af3f9aa7 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -43,10 +43,14 @@ pub(crate) fn build_sysroot( ); } - // Copy supporting files - try_hard_link("rust-toolchain", target_dir.join("rust-toolchain")); - try_hard_link("scripts/config.sh", target_dir.join("config.sh")); - try_hard_link("scripts/cargo.sh", target_dir.join("cargo.sh")); + // Build and copy cargo wrapper + let mut build_cargo_wrapper_cmd = Command::new("rustc"); + build_cargo_wrapper_cmd + .arg("scripts/cargo.rs") + .arg("-o") + .arg(target_dir.join("cargo")) + .arg("-g"); + spawn_and_wait(build_cargo_wrapper_cmd); let default_sysroot = crate::rustc_info::get_default_sysroot(); diff --git a/docs/usage.md b/docs/usage.md index 7e1e0a9bc9d..aac44b62d01 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -9,7 +9,7 @@ Assuming `$cg_clif_dir` is the directory you cloned this repo into and you follo In the directory with your project (where you can do the usual `cargo build`), run: ```bash -$ $cg_clif_dir/build/cargo.sh build +$ $cg_clif_dir/build/cargo build ``` This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend. @@ -30,7 +30,7 @@ In jit mode cg_clif will immediately execute your code without creating an execu > The jit mode will probably need cargo integration to make this possible. ```bash -$ $cg_clif_dir/build/cargo.sh jit +$ $cg_clif_dir/build/cargo jit ``` or @@ -44,7 +44,7 @@ first called. It currently does not work with multi-threaded programs. When a no function is called from another thread than the main thread, you will get an ICE. ```bash -$ $cg_clif_dir/build/cargo.sh lazy-jit +$ $cg_clif_dir/build/cargo lazy-jit ``` ## Shell diff --git a/scripts/cargo.rs b/scripts/cargo.rs new file mode 100644 index 00000000000..543e7c19276 --- /dev/null +++ b/scripts/cargo.rs @@ -0,0 +1,85 @@ +use std::env; +use std::os::unix::process::CommandExt; +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +fn main() { + if env::var("RUSTC_WRAPPER").map_or(false, |wrapper| wrapper.contains("sccache")) { + eprintln!( + "\x1b[1;93m=== Warning: Unsetting RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m" + ); + env::remove_var("RUSTC_WRAPPER"); + } + + let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap()); + + env::set_var("RUSTC", sysroot.join("bin/cg_clif".to_string() + env::consts::EXE_SUFFIX)); + + let mut rustdoc_flags = env::var("RUSTDOCFLAGS").unwrap_or(String::new()); + rustdoc_flags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend="); + rustdoc_flags.push_str( + sysroot + .join(if cfg!(windows) { "bin" } else { "lib" }) + .join( + env::consts::DLL_PREFIX.to_string() + + "rustc_codegen_cranelift" + + env::consts::DLL_SUFFIX, + ) + .to_str() + .unwrap(), + ); + rustdoc_flags.push_str(" --sysroot "); + rustdoc_flags.push_str(sysroot.to_str().unwrap()); + env::set_var("RUSTDOCFLAGS", rustdoc_flags); + + let default_sysroot = Command::new("rustc") + .stderr(Stdio::inherit()) + .args(&["--print", "sysroot"]) + .output() + .unwrap() + .stdout; + let default_sysroot = std::str::from_utf8(&default_sysroot).unwrap().trim(); + + let extra_ld_lib_path = + default_sysroot.to_string() + ":" + sysroot.join("lib").to_str().unwrap(); + if cfg!(target_os = "macos") { + env::set_var( + "DYLD_LIBRARY_PATH", + env::var("DYLD_LIBRARY_PATH").unwrap_or(String::new()) + ":" + &extra_ld_lib_path, + ); + } else if cfg!(unix) { + env::set_var( + "LD_LIBRARY_PATH", + env::var("LD_LIBRARY_PATH").unwrap_or(String::new()) + ":" + &extra_ld_lib_path, + ); + } + + // Ensure that the right toolchain is used + env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN")); + + let args: Vec<_> = match env::args().nth(1).as_deref() { + Some("jit") => { + env::set_var( + "RUSTFLAGS", + env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic", + ); + std::array::IntoIter::new(["rustc".to_string()]) + .chain(env::args().skip(2)) + .chain(["--".to_string(), "-Cllvm-args=mode=jit".to_string()]) + .collect() + } + Some("lazy-jit") => { + env::set_var( + "RUSTFLAGS", + env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic", + ); + std::array::IntoIter::new(["rustc".to_string()]) + .chain(env::args().skip(2)) + .chain(["--".to_string(), "-Cllvm-args=mode=jit-lazy".to_string()]) + .collect() + } + _ => env::args().skip(1).collect(), + }; + + Command::new("cargo").args(args).exec(); +} diff --git a/scripts/cargo.sh b/scripts/cargo.sh deleted file mode 100755 index 267b5d99a08..00000000000 --- a/scripts/cargo.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -dir=$(dirname "$0") -source "$dir/config.sh" - -# read nightly compiler from rust-toolchain file -TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/") - -cmd=$1 -shift || true - -if [[ "$cmd" = "jit" ]]; then -RUSTFLAGS="-Cprefer-dynamic" cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -elif [[ "$cmd" = "lazy-jit" ]]; then -RUSTFLAGS="-Cprefer-dynamic" cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy -else -cargo "+${TOOLCHAIN}" "$cmd" "$@" -fi diff --git a/scripts/config.sh b/scripts/config.sh index 1aeed266d9a..cf325ab574e 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -2,21 +2,6 @@ set -e -dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -) - -if echo "$RUSTC_WRAPPER" | grep sccache; then -echo -echo -e "\x1b[1;93m=== Warning: Unset RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m" -echo -export RUSTC_WRAPPER= -fi - -dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) - -export RUSTC=$dir"/bin/cg_clif" - -export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\ -'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir - +dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../build"; pwd) export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh index 9f8331865ce..11d6c4c8318 100644 --- a/scripts/ext_config.sh +++ b/scripts/ext_config.sh @@ -1,6 +1,6 @@ # Note to people running shellcheck: this file should only be sourced, not executed directly. -# Various env vars that should only be set for the build system but not for cargo.sh +# Various env vars that should only be set for the build system set -e diff --git a/scripts/filter_profile.rs b/scripts/filter_profile.rs index 15388926ec9..9e196afbe4f 100755 --- a/scripts/filter_profile.rs +++ b/scripts/filter_profile.rs @@ -2,9 +2,10 @@ #![forbid(unsafe_code)]/* This line is ignored by bash # This block is ignored by rustc pushd $(dirname "$0")/../ -source build/config.sh +source scripts/config.sh +RUSTC="$(pwd)/build/bin/cg_clif" popd -PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0 +PROFILE=$1 OUTPUT=$2 exec $RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic $0 #*/ //! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index c494e78050a..52adaaa8de6 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -2,7 +2,7 @@ set -e ./y.rs build -source build/config.sh +source scripts/config.sh echo "[SETUP] Rust fork" git clone https://github.com/rust-lang/rust.git || true diff --git a/scripts/tests.sh b/scripts/tests.sh index 243a0ef2851..c875aad2849 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -2,9 +2,10 @@ set -e -source build/config.sh +source scripts/config.sh source scripts/ext_config.sh -MY_RUSTC="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2" +export RUSTC=false # ensure that cg_llvm isn't accidentally used +MY_RUSTC="$(pwd)/build/bin/cg_clif $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2" function no_sysroot_tests() { echo "[BUILD] mini_core" @@ -75,64 +76,64 @@ function base_sysroot_tests() { function extended_sysroot_tests() { pushd rand - cargo clean + ../build/cargo clean if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then echo "[TEST] rust-random/rand" - ../build/cargo.sh test --workspace + ../build/cargo test --workspace else echo "[AOT] rust-random/rand" - ../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests + ../build/cargo build --workspace --target $TARGET_TRIPLE --tests fi popd pushd simple-raytracer if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then echo "[BENCH COMPILE] ebobby/simple-raytracer" - hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \ + hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo clean" \ "RUSTC=rustc RUSTFLAGS='' cargo build" \ - "../build/cargo.sh build" + "../build/cargo build" echo "[BENCH RUN] ebobby/simple-raytracer" cp ./target/debug/main ./raytracer_cg_clif hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_clif else - cargo clean + ../build/cargo clean echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)" echo "[COMPILE] ebobby/simple-raytracer" - ../build/cargo.sh build --target $TARGET_TRIPLE + ../build/cargo build --target $TARGET_TRIPLE echo "[BENCH RUN] ebobby/simple-raytracer (skipped)" fi popd pushd build_sysroot/sysroot_src/library/core/tests echo "[TEST] libcore" - cargo clean + ../../../../../build/cargo clean if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then - ../../../../../build/cargo.sh test + ../../../../../build/cargo test else - ../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests + ../../../../../build/cargo build --target $TARGET_TRIPLE --tests fi popd pushd regex echo "[TEST] rust-lang/regex example shootout-regex-dna" - cargo clean + ../build/cargo clean export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning # Make sure `[codegen mono items] start` doesn't poison the diff - ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE + ../build/cargo build --example shootout-regex-dna --target $TARGET_TRIPLE if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then cat examples/regexdna-input.txt \ - | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \ + | ../build/cargo run --example shootout-regex-dna --target $TARGET_TRIPLE \ | grep -v "Spawned thread" > res.txt diff -u res.txt examples/regexdna-output.txt fi if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then echo "[TEST] rust-lang/regex tests" - ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q + ../build/cargo test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q else echo "[AOT] rust-lang/regex tests" - ../build/cargo.sh build --tests --target $TARGET_TRIPLE + ../build/cargo build --tests --target $TARGET_TRIPLE fi popd } From 62e49c5b61c196c942e644c0b81b0f4b232bd461 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 20 Jun 2021 13:20:51 +0200 Subject: [PATCH 13/14] Fix compiling cargo.rs on windows --- scripts/cargo.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/cargo.rs b/scripts/cargo.rs index 543e7c19276..fdb9bcf6dbc 100644 --- a/scripts/cargo.rs +++ b/scripts/cargo.rs @@ -1,4 +1,5 @@ use std::env; +#[cfg(unix)] use std::os::unix::process::CommandExt; use std::path::PathBuf; use std::process::{Command, Stdio}; @@ -81,5 +82,11 @@ fn main() { _ => env::args().skip(1).collect(), }; + #[cfg(unix)] Command::new("cargo").args(args).exec(); + + #[cfg(not(unix))] + std::process::exit( + Command::new("cargo").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1), + ); } From 03c9ecfb306f00fa39595c63d062f7ab98d6f19f Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 20 Jun 2021 14:40:49 +0200 Subject: [PATCH 14/14] Remove unnecessary LD_LIBRARY_PATH parts --- scripts/cargo.rs | 14 -------------- scripts/config.sh | 3 +-- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/scripts/cargo.rs b/scripts/cargo.rs index fdb9bcf6dbc..5cb352a1aff 100644 --- a/scripts/cargo.rs +++ b/scripts/cargo.rs @@ -41,20 +41,6 @@ fn main() { .stdout; let default_sysroot = std::str::from_utf8(&default_sysroot).unwrap().trim(); - let extra_ld_lib_path = - default_sysroot.to_string() + ":" + sysroot.join("lib").to_str().unwrap(); - if cfg!(target_os = "macos") { - env::set_var( - "DYLD_LIBRARY_PATH", - env::var("DYLD_LIBRARY_PATH").unwrap_or(String::new()) + ":" + &extra_ld_lib_path, - ); - } else if cfg!(unix) { - env::set_var( - "LD_LIBRARY_PATH", - env::var("LD_LIBRARY_PATH").unwrap_or(String::new()) + ":" + &extra_ld_lib_path, - ); - } - // Ensure that the right toolchain is used env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN")); diff --git a/scripts/config.sh b/scripts/config.sh index cf325ab574e..d316a7c6981 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -2,6 +2,5 @@ set -e -dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../build"; pwd) -export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib" +export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH