bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge pull request #1180 from bjorn3/rust_build_system

Browse source 
Rewrite part of the build system in rust
This commit is contained in:
bjorn3 2021-06-20 14:55:36 +02:00 • committed by GitHub
commit b33a3ec697
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 803 additions and 274 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.cirrus.yml
View file

@ -14,7 +14,7 @@ task:
- . $HOME/.cargo/env
- git config --global user.email "user@example.com"
- git config --global user.name "User"
- ./prepare.sh
- ./y.rs prepare
test_script:
- . $HOME/.cargo/env
- # Enable backtraces for easier debugging

62
.github/workflows/main.yml vendored
View file

@ -53,7 +53,7 @@ jobs:
run: |
git config --global user.email "user@example.com"
git config --global user.name "User"
./prepare.sh
./y.rs prepare
- name: Test
env:
@ -87,3 +87,63 @@ jobs:
with:
name: cg_clif-${{ runner.os }}-cross-x86_64-mingw
path: cg_clif.tar.xz
build_windows:
runs-on: windows-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
#- name: Cache cargo installed crates
# uses: actions/cache@v2
# with:
# path: ~/.cargo/bin
# key: ${{ runner.os }}-cargo-installed-crates
#- name: Cache cargo registry and index
# uses: actions/cache@v2
# with:
# path: |
# ~/.cargo/registry
# ~/.cargo/git
# key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }}
#- name: Cache cargo target dir
# uses: actions/cache@v2
# with:
# path: target
# key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
- name: Prepare dependencies
run: |
git config --global user.email "user@example.com"
git config --global user.name "User"
git config --global core.autocrlf false
rustup set default-host x86_64-pc-windows-gnu
rustc y.rs -o y.exe -g
./y.exe prepare
- name: Build
#name: Test
run: |
# Enable backtraces for easier debugging
#export RUST_BACKTRACE=1
# Reduce amount of benchmark runs as they are slow
#export COMPILE_RUNS=2
#export RUN_RUNS=2
# Enable extra checks
#export CG_CLIF_ENABLE_VERIFIER=1
./y.exe build
#- name: Package prebuilt cg_clif
# run: tar cvfJ cg_clif.tar.xz build
#- name: Upload prebuilt cg_clif
# uses: actions/upload-artifact@v2
# with:
# name: cg_clif-${{ runner.os }}
# path: cg_clif.tar.xz

4
.github/workflows/rustc.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
run: |
git config --global user.email "user@example.com"
git config --global user.name "User"
./prepare.sh
./y.rs prepare
- name: Test
run: |
@ -72,7 +72,7 @@ jobs:
run: |
git config --global user.email "user@example.com"
git config --global user.name "User"
./prepare.sh
./y.rs prepare
- name: Test
run: |

1
.gitignore vendored
View file

@ -6,6 +6,7 @@ perf.data
perf.data.old
*.events
*.string*
/y.bin
/build
/build_sysroot/sysroot_src
/build_sysroot/compiler-builtins

21
.vscode/settings.json vendored
View file

@ -1,7 +1,9 @@
{
// source for rustc_* is not included in the rust-src component; disable the errors about this
"rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"],
"rust-analyzer.assist.importMergeBehavior": "last",
"rust-analyzer.assist.importGranularity": "module",
"rust-analyzer.assist.importEnforceGranularity": true,
"rust-analyzer.assist.importPrefix": "by_crate",
"rust-analyzer.cargo.runBuildScripts": true,
"rust-analyzer.linkedProjects": [
"./Cargo.toml",
@ -49,6 +51,23 @@
"cfg": [],
},
]
},
{
"roots": ["./y.rs"],
"crates": [
{
"root_module": "./y.rs",
"edition": "2018",
"deps": [{ "crate": 1, "name": "std" }],
"cfg": [],
},
{
"root_module": "./build_sysroot/sysroot_src/library/std/src/lib.rs",
"edition": "2018",
"deps": [],
"cfg": [],
},
]
}
]
}

10
Readme.md
View file

@ -10,8 +10,8 @@ If not please open an issue.
```bash
$ git clone https://github.com/bjorn3/rustc_codegen_cranelift.git
$ cd rustc_codegen_cranelift
$ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
$ ./build.sh
$ ./y.rs prepare # download and patch sysroot src and install hyperfine for benchmarking
$ ./y.rs build
```
To run the test suite replace the last command with:
@ -20,7 +20,7 @@ To run the test suite replace the last command with:
$ ./test.sh
```
This will implicitly build cg_clif too. Both `build.sh` and `test.sh` accept a `--debug` argument to
This will implicitly build cg_clif too. Both `y.rs build` and `test.sh` accept a `--debug` argument to
build in debug mode.
Alternatively you can download a pre built version from [GHA]. It is listed in the artifacts section
@ -32,12 +32,12 @@ of workflow runs. Unfortunately due to GHA restrictions you need to be logged in
rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`).
In the directory with your project (where you can do the usual `cargo build`), run:
```bash
$ $cg_clif_dir/build/cargo.sh build
$ $cg_clif_dir/build/cargo build
```
This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.

88
build.sh
View file

@ -1,88 +0,0 @@
#!/usr/bin/env bash
set -e
# Settings
export CHANNEL="release"
build_sysroot="clif"
target_dir='build'
while [[ $# != 0 ]]; do
case $1 in
"--debug")
export CHANNEL="debug"
;;
"--sysroot")
build_sysroot=$2
shift
;;
"--target-dir")
target_dir=$2
shift
;;
*)
echo "Unknown flag '$1'"
echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"
exit 1
;;
esac
shift
done
# Build cg_clif
unset CARGO_TARGET_DIR
unamestr=$(uname)
if [[ "$unamestr" == 'Linux' || "$unamestr" == "FreeBSD" ]]; then
export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS
elif [[ "$unamestr" == 'Darwin' ]]; then
export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS
else
echo "Unsupported os $unamestr"
exit 1
fi
if [[ "$CHANNEL" == "release" ]]; then
cargo build --release
else
cargo build
fi
source scripts/ext_config.sh
rm -rf "$target_dir"
mkdir "$target_dir"
mkdir "$target_dir"/bin "$target_dir"/lib
ln target/$CHANNEL/cg_clif{,_build_sysroot} "$target_dir"/bin
ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib
ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir"
mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/"
if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then
cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/"
fi
case "$build_sysroot" in
"none")
;;
"llvm")
cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/"
if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/"
fi
;;
"clif")
echo "[BUILD] sysroot"
dir=$(pwd)
cd "$target_dir"
time "$dir/build_sysroot/build_sysroot.sh"
if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh"
fi
cp lib/rustlib/*/lib/libstd-* lib/
;;
*)
echo "Unknown sysroot kind \`$build_sysroot\`."
echo "The allowed values are:"
echo " none A sysroot that doesn't contain the standard library"
echo " llvm Copy the sysroot from rustc compiled by cg_llvm"
echo " clif Build a new sysroot using cg_clif"
exit 1
esac

39
build_sysroot/build_sysroot.sh
View file

@ -1,39 +0,0 @@
#!/usr/bin/env bash
# Requires the CHANNEL env var to be set to `debug` or `release.`
set -e
source ./config.sh
dir=$(pwd)
# Use rustc with cg_clif as hotpluggable backend instead of the custom cg_clif driver so that
# build scripts are still compiled using cg_llvm.
export RUSTC=$dir"/bin/cg_clif_build_sysroot"
export RUSTFLAGS=$RUSTFLAGS" --clif"
cd "$(dirname "$0")"
# Cleanup for previous run
# v Clean target dir except for build scripts and incremental cache
rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/null || true
# We expect the target dir in the default location. Guard against the user changing it.
export CARGO_TARGET_DIR=target
# Build libs
export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort"
export __CARGO_DEFAULT_LIB_METADATA="cg_clif"
if [[ "$1" != "--debug" ]]; then
sysroot_channel='release'
# FIXME Enable incremental again once rust-lang/rust#74946 is fixed
CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release
else
sysroot_channel='debug'
cargo build --target "$TARGET_TRIPLE"
fi
# Copy files to sysroot
ln "target/$TARGET_TRIPLE/$sysroot_channel/deps/"* "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"
rm "$dir/lib/rustlib/$TARGET_TRIPLE/lib/"*.{rmeta,d}

39
build_sysroot/prepare_sysroot_src.sh
View file

@ -1,39 +0,0 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")"
SRC_DIR="$(dirname "$(rustup which rustc)")/../lib/rustlib/src/rust/"
DST_DIR="sysroot_src"
if [ ! -e "$SRC_DIR" ]; then
echo "Please install rust-src component"
exit 1
fi
rm -rf $DST_DIR
mkdir -p $DST_DIR/library
cp -a "$SRC_DIR/library" $DST_DIR/
pushd $DST_DIR
echo "[GIT] init"
git init
echo "[GIT] add"
git add .
echo "[GIT] commit"
git commit -m "Initial commit" -q
for file in $(ls ../../patches/ | grep -v patcha); do
echo "[GIT] apply" "$file"
git apply ../../patches/"$file"
git add -A
git commit --no-gpg-sign -m "Patch $file"
done
popd
git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned"
pushd compiler-builtins
git checkout -- .
git checkout 0.1.45
git apply ../../crate_patches/000*-compiler-builtins-*.patch
popd
echo "Successfully prepared sysroot source for building"

39
build_system/build_backend.rs Normal file
View file

@ -0,0 +1,39 @@
use std::env;
use std::process::Command;
pub(crate) fn build_backend(channel: &str) -> String {
let mut cmd = Command::new("cargo");
cmd.arg("build");
match channel {
"debug" => {}
"release" => {
cmd.arg("--release");
}
_ => unreachable!(),
}
if cfg!(unix) {
if cfg!(target_os = "macos") {
cmd.env(
"RUSTFLAGS",
"-Csplit-debuginfo=unpacked \
-Clink-arg=-Wl,-rpath,@loader_path/../lib \
-Zosx-rpath-install-name"
.to_string()
+ env::var("RUSTFLAGS").as_deref().unwrap_or(""),
);
} else {
cmd.env(
"RUSTFLAGS",
"-Clink-arg=-Wl,-rpath=$ORIGIN/../lib ".to_string()
+ env::var("RUSTFLAGS").as_deref().unwrap_or(""),
);
}
}
eprintln!("[BUILD] rustc_codegen_cranelift");
crate::utils::spawn_and_wait(cmd);
crate::rustc_info::get_file_name("rustc_codegen_cranelift", "dylib")
}

185
build_system/build_sysroot.rs Normal file
View file

@ -0,0 +1,185 @@
use std::env;
use std::fs;
use std::path::Path;
use std::process::{self, Command};
use crate::rustc_info::get_file_name;
use crate::utils::{spawn_and_wait, try_hard_link};
use crate::SysrootKind;
pub(crate) fn build_sysroot(
channel: &str,
sysroot_kind: SysrootKind,
target_dir: &Path,
cg_clif_dylib: String,
host_triple: &str,
target_triple: &str,
) {
if target_dir.exists() {
fs::remove_dir_all(target_dir).unwrap();
}
fs::create_dir_all(target_dir.join("bin")).unwrap();
fs::create_dir_all(target_dir.join("lib")).unwrap();
// Copy the backend
for file in ["cg_clif", "cg_clif_build_sysroot"] {
try_hard_link(
Path::new("target").join(channel).join(get_file_name(file, "bin")),
target_dir.join("bin").join(get_file_name(file, "bin")),
);
}
if cfg!(windows) {
// Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
// binaries.
try_hard_link(
Path::new("target").join(channel).join(&cg_clif_dylib),
target_dir.join("bin").join(cg_clif_dylib),
);
} else {
try_hard_link(
Path::new("target").join(channel).join(&cg_clif_dylib),
target_dir.join("lib").join(cg_clif_dylib),
);
}
// Build and copy cargo wrapper
let mut build_cargo_wrapper_cmd = Command::new("rustc");
build_cargo_wrapper_cmd
.arg("scripts/cargo.rs")
.arg("-o")
.arg(target_dir.join("cargo"))
.arg("-g");
spawn_and_wait(build_cargo_wrapper_cmd);
let default_sysroot = crate::rustc_info::get_default_sysroot();
let rustlib = target_dir.join("lib").join("rustlib");
let host_rustlib_lib = rustlib.join(host_triple).join("lib");
let target_rustlib_lib = rustlib.join(target_triple).join("lib");
fs::create_dir_all(&host_rustlib_lib).unwrap();
fs::create_dir_all(&target_rustlib_lib).unwrap();
if target_triple == "x86_64-pc-windows-gnu" {
if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() {
eprintln!(
"The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
to compile a sysroot for it.",
);
process::exit(1);
}
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
continue; // only copy object files
}
try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
}
}
match sysroot_kind {
SysrootKind::None => {} // Nothing to do
SysrootKind::Llvm => {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
let file_name_str = file.file_name().unwrap().to_str().unwrap();
if file_name_str.contains("rustc_")
|| file_name_str.contains("chalk")
|| file_name_str.contains("tracing")
|| file_name_str.contains("regex")
{
// These are large crates that are part of the rustc-dev component and are not
// necessary to run regular programs.
continue;
}
try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
}
if target_triple != host_triple {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
}
}
}
SysrootKind::Clif => {
build_clif_sysroot_for_triple(channel, target_dir, target_triple);
if host_triple != target_triple {
build_clif_sysroot_for_triple(channel, target_dir, host_triple);
}
// Copy std for the host to the lib dir. This is necessary for the jit mode to find
// libstd.
for file in fs::read_dir(host_rustlib_lib).unwrap() {
let file = file.unwrap().path();
if file.file_name().unwrap().to_str().unwrap().contains("std-") {
try_hard_link(&file, target_dir.join("lib").join(file.file_name().unwrap()));
}
}
}
}
}
fn build_clif_sysroot_for_triple(channel: &str, target_dir: &Path, triple: &str) {
let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel);
let keep_sysroot =
fs::read_to_string("config.txt").unwrap().lines().any(|line| line.trim() == "keep_sysroot");
if !keep_sysroot {
// Cleanup the target dir with the exception of build scripts and the incremental cache
for dir in ["build", "deps", "examples", "native"] {
if build_dir.join(dir).exists() {
fs::remove_dir_all(build_dir.join(dir)).unwrap();
}
}
}
// Build sysroot
let mut build_cmd = Command::new("cargo");
build_cmd.arg("build").arg("--target").arg(triple).current_dir("build_sysroot");
let mut rustflags = "--clif -Zforce-unstable-if-unmarked".to_string();
if channel == "release" {
build_cmd.arg("--release");
rustflags.push_str(" -Zmir-opt-level=3");
}
build_cmd.env("RUSTFLAGS", rustflags);
build_cmd.env(
"RUSTC",
env::current_dir().unwrap().join(target_dir).join("bin").join("cg_clif_build_sysroot"),
);
// FIXME Enable incremental again once rust-lang/rust#74946 is fixed
build_cmd.env("CARGO_INCREMENTAL", "0").env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
spawn_and_wait(build_cmd);
// Copy all relevant files to the sysroot
for entry in
fs::read_dir(Path::new("build_sysroot/target").join(triple).join(channel).join("deps"))
.unwrap()
{
let entry = entry.unwrap();
if let Some(ext) = entry.path().extension() {
if ext == "rmeta" || ext == "d" || ext == "dSYM" {
continue;
}
} else {
continue;
};
try_hard_link(
entry.path(),
target_dir.join("lib").join("rustlib").join(triple).join("lib").join(entry.file_name()),
);
}
}

126
build_system/prepare.rs Normal file
View file

@ -0,0 +1,126 @@
use std::env;
use std::ffi::OsStr;
use std::ffi::OsString;
use std::fs;
use std::path::Path;
use std::process::Command;
use crate::rustc_info::{get_file_name, get_rustc_path};
use crate::utils::{copy_dir_recursively, spawn_and_wait};
pub(crate) fn prepare() {
prepare_sysroot();
eprintln!("[INSTALL] hyperfine");
Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap();
clone_repo(
"rand",
"https://github.com/rust-random/rand.git",
"0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
);
apply_patches("rand", Path::new("rand"));
clone_repo(
"regex",
"https://github.com/rust-lang/regex.git",
"341f207c1071f7290e3f228c710817c280c8dca1",
);
clone_repo(
"simple-raytracer",
"https://github.com/ebobby/simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
);
eprintln!("[LLVM BUILD] simple-raytracer");
let mut build_cmd = Command::new("cargo");
build_cmd.arg("build").env_remove("CARGO_TARGET_DIR").current_dir("simple-raytracer");
spawn_and_wait(build_cmd);
fs::copy(
Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
// FIXME use get_file_name here too once testing is migrated to rust
"simple-raytracer/raytracer_cg_llvm",
)
.unwrap();
}
fn prepare_sysroot() {
let rustc_path = get_rustc_path();
let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
let sysroot_src = env::current_dir().unwrap().join("build_sysroot").join("sysroot_src");
assert!(sysroot_src_orig.exists());
if sysroot_src.exists() {
fs::remove_dir_all(&sysroot_src).unwrap();
}
fs::create_dir_all(sysroot_src.join("library")).unwrap();
eprintln!("[COPY] sysroot src");
copy_dir_recursively(&sysroot_src_orig.join("library"), &sysroot_src.join("library"));
eprintln!("[GIT] init");
let mut git_init_cmd = Command::new("git");
git_init_cmd.arg("init").arg("-q").current_dir(&sysroot_src);
spawn_and_wait(git_init_cmd);
let mut git_add_cmd = Command::new("git");
git_add_cmd.arg("add").arg(".").current_dir(&sysroot_src);
spawn_and_wait(git_add_cmd);
let mut git_commit_cmd = Command::new("git");
git_commit_cmd
.arg("commit")
.arg("-m")
.arg("Initial commit")
.arg("-q")
.current_dir(&sysroot_src);
spawn_and_wait(git_commit_cmd);
apply_patches("sysroot", &sysroot_src);
clone_repo(
"build_sysroot/compiler-builtins",
"https://github.com/rust-lang/compiler-builtins.git",
"0.1.45",
);
apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
}
fn clone_repo(target_dir: &str, repo: &str, rev: &str) {
eprintln!("[CLONE] {}", repo);
// Ignore exit code as the repo may already have been checked out
Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap();
let mut clean_cmd = Command::new("git");
clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir);
spawn_and_wait(clean_cmd);
let mut checkout_cmd = Command::new("git");
checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir);
spawn_and_wait(checkout_cmd);
}
fn get_patches(crate_name: &str) -> Vec<OsString> {
let mut patches: Vec<_> = fs::read_dir("patches")
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension() == Some(OsStr::new("patch")))
.map(|path| path.file_name().unwrap().to_owned())
.filter(|file_name| {
file_name.to_str().unwrap().split_once("-").unwrap().1.starts_with(crate_name)
})
.collect();
patches.sort();
patches
}
fn apply_patches(crate_name: &str, target_dir: &Path) {
for patch in get_patches(crate_name) {
eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch);
let patch_arg = env::current_dir().unwrap().join("patches").join(patch);
let mut apply_patch_cmd = Command::new("git");
apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir);
spawn_and_wait(apply_patch_cmd);
}
}

59
build_system/rustc_info.rs Normal file
View file

@ -0,0 +1,59 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
pub(crate) fn get_host_triple() -> String {
let version_info =
Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
String::from_utf8(version_info)
.unwrap()
.lines()
.to_owned()
.find(|line| line.starts_with("host"))
.unwrap()
.split(":")
.nth(1)
.unwrap()
.trim()
.to_owned()
}
pub(crate) fn get_rustc_path() -> PathBuf {
let rustc_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "rustc"])
.output()
.unwrap()
.stdout;
Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
}
pub(crate) fn get_default_sysroot() -> PathBuf {
let default_sysroot = Command::new("rustc")
.stderr(Stdio::inherit())
.args(&["--print", "sysroot"])
.output()
.unwrap()
.stdout;
Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
}
pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
let file_name = Command::new("rustc")
.stderr(Stdio::inherit())
.args(&[
"--crate-name",
crate_name,
"--crate-type",
crate_type,
"--print",
"file-names",
"-",
])
.output()
.unwrap()
.stdout;
let file_name = String::from_utf8(file_name).unwrap().trim().to_owned();
assert!(!file_name.contains('\n'));
assert!(file_name.contains(crate_name));
file_name
}

35
build_system/utils.rs Normal file
View file

@ -0,0 +1,35 @@
use std::fs;
use std::path::Path;
use std::process::{self, Command};
#[track_caller]
pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
let src = src.as_ref();
let dst = dst.as_ref();
if let Err(_) = fs::hard_link(src, dst) {
fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed
}
}
#[track_caller]
pub(crate) fn spawn_and_wait(mut cmd: Command) {
if !cmd.spawn().unwrap().wait().unwrap().success() {
process::exit(1);
}
}
pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
for entry in fs::read_dir(from).unwrap() {
let entry = entry.unwrap();
let filename = entry.file_name();
if filename == "." || filename == ".." {
continue;
}
if entry.metadata().unwrap().is_dir() {
fs::create_dir(to.join(&filename)).unwrap();
copy_dir_recursively(&from.join(&filename), &to.join(&filename));
} else {
fs::copy(from.join(&filename), to.join(&filename)).unwrap();
}
}
}

7
config.txt Normal file
View file

@ -0,0 +1,7 @@
# This file allows configuring the build system.
# Disables cleaning of the sysroot dir. This will cause old compiled artifacts to be re-used when
# the sysroot source hasn't changed. This is useful when the codegen backend hasn't been modified.
# This option can be changed while the build system is already running for as long as sysroot
# building hasn't started yet.
#keep_sysroot

8
docs/usage.md
View file

@ -2,14 +2,14 @@
rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`).
Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`).
## Cargo
In the directory with your project (where you can do the usual `cargo build`), run:
```bash
$ $cg_clif_dir/build/cargo.sh build
$ $cg_clif_dir/build/cargo build
```
This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend.
@ -30,7 +30,7 @@ In jit mode cg_clif will immediately execute your code without creating an execu
> The jit mode will probably need cargo integration to make this possible.
```bash
$ $cg_clif_dir/build/cargo.sh jit
$ $cg_clif_dir/build/cargo jit
```
or
@ -44,7 +44,7 @@ first called. It currently does not work with multi-threaded programs. When a no
function is called from another thread than the main thread, you will get an ICE.
```bash
$ $cg_clif_dir/build/cargo.sh lazy-jit
$ $cg_clif_dir/build/cargo lazy-jit
```
## Shell

0
crate_patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch → patches/0001-compiler-builtins-Disable-128bit-atomic-operations.patch
View file

0
crate_patches/0001-rand-Enable-c2-chacha-simd-feature.patch → patches/0001-rand-Enable-c2-chacha-simd-feature.patch
View file

0
crate_patches/0002-rand-Disable-failing-test.patch → patches/0002-rand-Disable-failing-test.patch
View file

0
patches/0022-core-Disable-not-compiling-tests.patch → patches/0022-sysroot-Disable-not-compiling-tests.patch
View file

0
patches/0023-core-Ignore-failing-tests.patch → patches/0023-sysroot-Ignore-failing-tests.patch
View file

0
patches/0027-Disable-128bit-atomic-operations.patch → patches/0027-sysroot-128bit-atomic-operations.patch
View file

29
prepare.sh
View file

@ -1,29 +0,0 @@
#!/usr/bin/env bash
set -e
./build_sysroot/prepare_sysroot_src.sh
cargo install hyperfine || echo "Skipping hyperfine install"
git clone https://github.com/rust-random/rand.git || echo "rust-random/rand has already been cloned"
pushd rand
git checkout -- .
git checkout 0f933f9c7176e53b2a3c7952ded484e1783f0bf1
git am ../crate_patches/*-rand-*.patch
popd
git clone https://github.com/rust-lang/regex.git || echo "rust-lang/regex has already been cloned"
pushd regex
git checkout -- .
git checkout 341f207c1071f7290e3f228c710817c280c8dca1
popd
git clone https://github.com/ebobby/simple-raytracer || echo "ebobby/simple-raytracer has already been cloned"
pushd simple-raytracer
git checkout -- .
git checkout 804a7a21b9e673a482797aa289a18ed480e4d813
# build with cg_llvm for perf comparison
unset CARGO_TARGET_DIR
cargo build
mv target/debug/main raytracer_cg_llvm
popd

78
scripts/cargo.rs Normal file
View file

@ -0,0 +1,78 @@
use std::env;
#[cfg(unix)]
use std::os::unix::process::CommandExt;
use std::path::PathBuf;
use std::process::{Command, Stdio};
fn main() {
if env::var("RUSTC_WRAPPER").map_or(false, |wrapper| wrapper.contains("sccache")) {
eprintln!(
"\x1b[1;93m=== Warning: Unsetting RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m"
);
env::remove_var("RUSTC_WRAPPER");
}
let sysroot = PathBuf::from(env::current_exe().unwrap().parent().unwrap());
env::set_var("RUSTC", sysroot.join("bin/cg_clif".to_string() + env::consts::EXE_SUFFIX));
let mut rustdoc_flags = env::var("RUSTDOCFLAGS").unwrap_or(String::new());
rustdoc_flags.push_str(" -Cpanic=abort -Zpanic-abort-tests -Zcodegen-backend=");
rustdoc_flags.push_str(
sysroot
.join(if cfg!(windows) { "bin" } else { "lib" })
.join(
env::consts::DLL_PREFIX.to_string()
+ "rustc_codegen_cranelift"
+ env::consts::DLL_SUFFIX,
)
.to_str()
.unwrap(),
);
rustdoc_flags.push_str(" --sysroot ");
rustdoc_flags.push_str(sysroot.to_str().unwrap());
env::set_var("RUSTDOCFLAGS", rustdoc_flags);
let default_sysroot = Command::new("rustc")
.stderr(Stdio::inherit())
.args(&["--print", "sysroot"])
.output()
.unwrap()
.stdout;
let default_sysroot = std::str::from_utf8(&default_sysroot).unwrap().trim();
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
let args: Vec<_> = match env::args().nth(1).as_deref() {
Some("jit") => {
env::set_var(
"RUSTFLAGS",
env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic",
);
std::array::IntoIter::new(["rustc".to_string()])
.chain(env::args().skip(2))
.chain(["--".to_string(), "-Cllvm-args=mode=jit".to_string()])
.collect()
}
Some("lazy-jit") => {
env::set_var(
"RUSTFLAGS",
env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic",
);
std::array::IntoIter::new(["rustc".to_string()])
.chain(env::args().skip(2))
.chain(["--".to_string(), "-Cllvm-args=mode=jit-lazy".to_string()])
.collect()
}
_ => env::args().skip(1).collect(),
};
#[cfg(unix)]
Command::new("cargo").args(args).exec();
#[cfg(not(unix))]
std::process::exit(
Command::new("cargo").args(args).spawn().unwrap().wait().unwrap().code().unwrap_or(1),
);
}

18
scripts/cargo.sh
View file

@ -1,18 +0,0 @@
#!/usr/bin/env bash
dir=$(dirname "$0")
source "$dir/config.sh"
# read nightly compiler from rust-toolchain file
TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/")
cmd=$1
shift || true
if [[ "$cmd" = "jit" ]]; then
RUSTFLAGS="-Cprefer-dynamic" cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit
elif [[ "$cmd" = "lazy-jit" ]]; then
RUSTFLAGS="-Cprefer-dynamic" cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy
else
cargo "+${TOOLCHAIN}" "$cmd" "$@"
fi

23
scripts/config.sh
View file

@ -2,26 +2,5 @@
set -e
dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -)
if echo "$RUSTC_WRAPPER" | grep sccache; then
echo
echo -e "\x1b[1;93m=== Warning: Unset RUSTC_WRAPPER to prevent interference with sccache ===\x1b[0m"
echo
export RUSTC_WRAPPER=
fi
dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd)
export RUSTC=$dir"/bin/cg_clif"
export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir
# FIXME fix `#[linkage = "extern_weak"]` without this
if [[ "$(uname)" == 'Darwin' ]]; then
export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
fi
export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib:"$dir"/lib"
export LD_LIBRARY_PATH="$(rustc --print sysroot)/lib"
export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH

7
scripts/ext_config.sh
View file

@ -1,6 +1,6 @@
# Note to people running shellcheck: this file should only be sourced, not executed directly.
# Various env vars that should only be set for the build system but not for cargo.sh
# Various env vars that should only be set for the build system
set -e
@ -25,3 +25,8 @@ if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
echo "Unknown non-native platform"
fi
fi
# FIXME fix `#[linkage = "extern_weak"]` without this
if [[ "$(uname)" == 'Darwin' ]]; then
export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
fi

5
scripts/filter_profile.rs
View file

@ -2,9 +2,10 @@
#![forbid(unsafe_code)]/* This line is ignored by bash
# This block is ignored by rustc
pushd $(dirname "$0")/../
source build/config.sh
source scripts/config.sh
RUSTC="$(pwd)/build/bin/cg_clif"
popd
PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0
PROFILE=$1 OUTPUT=$2 exec $RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic $0
#*/
//! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse

2
scripts/rustup.sh
View file

@ -17,7 +17,7 @@ case $1 in
done
./clean_all.sh
./prepare.sh
./y.rs prepare
(cd build_sysroot && cargo update)

4
scripts/setup_rust_fork.sh
View file

@ -1,8 +1,8 @@
#!/bin/bash
set -e
./build.sh
source build/config.sh
./y.rs build
source scripts/config.sh
echo "[SETUP] Rust fork"
git clone https://github.com/rust-lang/rust.git || true

35
scripts/tests.sh
View file

@ -2,9 +2,10 @@
set -e
source build/config.sh
source scripts/config.sh
source scripts/ext_config.sh
MY_RUSTC="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
export RUSTC=false # ensure that cg_llvm isn't accidentally used
MY_RUSTC="$(pwd)/build/bin/cg_clif $RUSTFLAGS -L crate=target/out --out-dir target/out -Cdebuginfo=2"
function no_sysroot_tests() {
echo "[BUILD] mini_core"
@ -75,64 +76,64 @@ function base_sysroot_tests() {
function extended_sysroot_tests() {
pushd rand
cargo clean
../build/cargo clean
if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
echo "[TEST] rust-random/rand"
../build/cargo.sh test --workspace
../build/cargo test --workspace
else
echo "[AOT] rust-random/rand"
../build/cargo.sh build --workspace --target $TARGET_TRIPLE --tests
../build/cargo build --workspace --target $TARGET_TRIPLE --tests
fi
popd
pushd simple-raytracer
if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
echo "[BENCH COMPILE] ebobby/simple-raytracer"
hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \
hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "../build/cargo clean" \
"RUSTC=rustc RUSTFLAGS='' cargo build" \
"../build/cargo.sh build"
"../build/cargo build"
echo "[BENCH RUN] ebobby/simple-raytracer"
cp ./target/debug/main ./raytracer_cg_clif
hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_clif
else
cargo clean
../build/cargo clean
echo "[BENCH COMPILE] ebobby/simple-raytracer (skipped)"
echo "[COMPILE] ebobby/simple-raytracer"
../build/cargo.sh build --target $TARGET_TRIPLE
../build/cargo build --target $TARGET_TRIPLE
echo "[BENCH RUN] ebobby/simple-raytracer (skipped)"
fi
popd
pushd build_sysroot/sysroot_src/library/core/tests
echo "[TEST] libcore"
cargo clean
../../../../../build/cargo clean
if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
../../../../../build/cargo.sh test
../../../../../build/cargo test
else
../../../../../build/cargo.sh build --target $TARGET_TRIPLE --tests
../../../../../build/cargo build --target $TARGET_TRIPLE --tests
fi
popd
pushd regex
echo "[TEST] rust-lang/regex example shootout-regex-dna"
cargo clean
../build/cargo clean
export RUSTFLAGS="$RUSTFLAGS --cap-lints warn" # newer aho_corasick versions throw a deprecation warning
# Make sure `[codegen mono items] start` doesn't poison the diff
../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE
../build/cargo build --example shootout-regex-dna --target $TARGET_TRIPLE
if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
cat examples/regexdna-input.txt \
| ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \
| ../build/cargo run --example shootout-regex-dna --target $TARGET_TRIPLE \
| grep -v "Spawned thread" > res.txt
diff -u res.txt examples/regexdna-output.txt
fi
if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then
echo "[TEST] rust-lang/regex tests"
../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
../build/cargo test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q
else
echo "[AOT] rust-lang/regex tests"
../build/cargo.sh build --tests --target $TARGET_TRIPLE
../build/cargo build --tests --target $TARGET_TRIPLE
fi
popd
}

4
test.sh
View file

@ -1,13 +1,13 @@
#!/usr/bin/env bash
set -e
./build.sh --sysroot none "$@"
./y.rs build --sysroot none "$@"
rm -r target/out || true
scripts/tests.sh no_sysroot
./build.sh "$@"
./y.rs build "$@"
scripts/tests.sh base_sysroot
scripts/tests.sh extended_sysroot

147
y.rs Executable file
View file

@ -0,0 +1,147 @@
#!/usr/bin/env bash
#![allow()] /*This line is ignored by bash
# This block is ignored by rustc
set -e
echo "[BUILD] y.rs" 1>&2
rustc $0 -o ${0/.rs/.bin} -g
exec ${0/.rs/.bin} $@
*/
//! The build system for cg_clif
//!
//! # Manual compilation
//!
//! If your system doesn't support shell scripts you can manually compile and run this file using
//! for example:
//!
//! ```shell
//! $ rustc y.rs -o build/y.bin
//! $ build/y.bin
//! ```
//!
//! # Naming
//!
//! The name `y.rs` was chosen to not conflict with rustc's `x.py`.
use std::env;
use std::path::PathBuf;
use std::process;
#[path = "build_system/build_backend.rs"]
mod build_backend;
#[path = "build_system/build_sysroot.rs"]
mod build_sysroot;
#[path = "build_system/prepare.rs"]
mod prepare;
#[path = "build_system/rustc_info.rs"]
mod rustc_info;
#[path = "build_system/utils.rs"]
mod utils;
fn usage() {
eprintln!("Usage:");
eprintln!(" ./y.rs prepare");
eprintln!(" ./y.rs build [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]");
}
macro_rules! arg_error {
($($err:tt)*) => {{
eprintln!($($err)*);
usage();
std::process::exit(1);
}};
}
enum Command {
Build,
}
#[derive(Copy, Clone)]
enum SysrootKind {
None,
Clif,
Llvm,
}
fn main() {
env::set_var("CG_CLIF_DISPLAY_CG_TIME", "1");
env::set_var("CG_CLIF_DISABLE_INCR_CACHE", "1");
// The target dir is expected in the default location. Guard against the user changing it.
env::set_var("CARGO_TARGET_DIR", "target");
let mut args = env::args().skip(1);
let command = match args.next().as_deref() {
Some("prepare") => {
if args.next().is_some() {
arg_error!("./x.rs prepare doesn't expect arguments");
}
prepare::prepare();
process::exit(0);
}
Some("build") => Command::Build,
Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
Some(command) => arg_error!("Unknown command {}", command),
None => {
usage();
process::exit(0);
}
};
let mut target_dir = PathBuf::from("build");
let mut channel = "release";
let mut sysroot_kind = SysrootKind::Clif;
while let Some(arg) = args.next().as_deref() {
match arg {
"--target-dir" => {
target_dir = PathBuf::from(args.next().unwrap_or_else(|| {
arg_error!("--target-dir requires argument");
}))
}
"--debug" => channel = "debug",
"--sysroot" => {
sysroot_kind = match args.next().as_deref() {
Some("none") => SysrootKind::None,
Some("clif") => SysrootKind::Clif,
Some("llvm") => SysrootKind::Llvm,
Some(arg) => arg_error!("Unknown sysroot kind {}", arg),
None => arg_error!("--sysroot requires argument"),
}
}
flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
arg => arg_error!("Unexpected argument {}", arg),
}
}
let host_triple = if let Ok(host_triple) = std::env::var("HOST_TRIPLE") {
host_triple
} else {
rustc_info::get_host_triple()
};
let target_triple = if let Ok(target_triple) = std::env::var("TARGET_TRIPLE") {
if target_triple != "" {
target_triple
} else {
host_triple.clone() // Empty target triple can happen on GHA
}
} else {
host_triple.clone()
};
if target_triple.ends_with("-msvc") {
eprintln!("The MSVC toolchain is not yet supported by rustc_codegen_cranelift.");
eprintln!("Switch to the MinGW toolchain for Windows support.");
eprintln!("Hint: You can use `rustup set default-host x86_64-pc-windows-gnu` to");
eprintln!("set the global default target to MinGW");
process::exit(1);
}
let cg_clif_dylib = build_backend::build_backend(channel);
build_sysroot::build_sysroot(
channel,
sysroot_kind,
&target_dir,
cg_clif_dylib,
&host_triple,
&target_triple,
);
}