bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '8830dccd1d' into sync_cg_clif-2023-06-15

Browse source
This commit is contained in:
bjorn3 2023-06-15 17:56:01 +00:00
commit 82b497286d
56 changed files with 2511 additions and 592 deletions
Download patch file Download diff file Expand all files Collapse all files

7
compiler/rustc_codegen_cranelift/build_system/Cargo.lock Normal file
View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "y"
version = "0.1.0"

13
compiler/rustc_codegen_cranelift/build_system/Cargo.toml Normal file
View file

@ -0,0 +1,13 @@
[package]
name = "y"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "y"
path = "main.rs"
[features]
unstable-features = [] # for rust-analyzer
# Do not add any dependencies

30
compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
View file

@ -1,25 +1,29 @@
use std::path::Path;
use super::build_sysroot;
use super::path::Dirs;
use super::prepare::GitRepo;
use super::utils::{spawn_and_wait, CargoProject, Compiler};
use super::SysrootKind;
use super::{CodegenBackend, SysrootKind};
static ABI_CAFE_REPO: GitRepo =
GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe");
static ABI_CAFE_REPO: GitRepo = GitRepo::github(
"Gankra",
"abi-cafe",
"4c6dc8c9c687e2b3a760ff2176ce236872b37212",
"588df6d66abbe105",
"abi-cafe",
);
static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe");
static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target");
pub(crate) fn run(
channel: &str,
sysroot_kind: SysrootKind,
dirs: &Dirs,
cg_clif_dylib: &Path,
cg_clif_dylib: &CodegenBackend,
rustup_toolchain_name: Option<&str>,
bootstrap_host_compiler: &Compiler,
) {
ABI_CAFE_REPO.fetch(dirs);
spawn_and_wait(ABI_CAFE.fetch("cargo", &bootstrap_host_compiler.rustc, dirs));
ABI_CAFE_REPO.patch(dirs);
eprintln!("Building sysroot for abi-cafe");
build_sysroot::build_sysroot(
@ -28,6 +32,7 @@ pub(crate) fn run(
sysroot_kind,
cg_clif_dylib,
bootstrap_host_compiler,
rustup_toolchain_name,
bootstrap_host_compiler.triple.clone(),
);
@ -40,7 +45,14 @@ pub(crate) fn run(
cmd.arg("--pairs");
cmd.args(pairs);
cmd.arg("--add-rustc-codegen-backend");
cmd.arg(format!("cgclif:{}", cg_clif_dylib.display()));
match cg_clif_dylib {
CodegenBackend::Local(path) => {
cmd.arg(format!("cgclif:{}", path.display()));
}
CodegenBackend::Builtin(name) => {
cmd.arg(format!("cgclif:{name}"));
}
}
cmd.current_dir(ABI_CAFE.source_dir(dirs));
spawn_and_wait(cmd);

82
compiler/rustc_codegen_cranelift/build_system/bench.rs
View file

@ -1,26 +1,19 @@
use std::env;
use std::fs;
use std::path::Path;
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::get_file_name;
use super::utils::{hyperfine_command, spawn_and_wait, CargoProject, Compiler};
use super::utils::{hyperfine_command, spawn_and_wait, Compiler};
static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github(
"ebobby",
"simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
"ad6f59a2331a3f56",
"<none>",
);
// Use a separate target dir for the initial LLVM build to reduce unnecessary recompiles
static SIMPLE_RAYTRACER_LLVM: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer_llvm");
static SIMPLE_RAYTRACER: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
benchmark_simple_raytracer(dirs, bootstrap_host_compiler);
}
@ -32,35 +25,17 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
std::process::exit(1);
}
if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() {
SIMPLE_RAYTRACER_REPO.fetch(dirs);
spawn_and_wait(SIMPLE_RAYTRACER.fetch(
&bootstrap_host_compiler.cargo,
&bootstrap_host_compiler.rustc,
dirs,
));
}
eprintln!("[LLVM BUILD] simple-raytracer");
let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs);
spawn_and_wait(build_cmd);
fs::copy(
SIMPLE_RAYTRACER_LLVM
.target_dir(dirs)
.join(&bootstrap_host_compiler.triple)
.join("debug")
.join(get_file_name("main", "bin")),
RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")),
)
.unwrap();
SIMPLE_RAYTRACER_REPO.fetch(dirs);
SIMPLE_RAYTRACER_REPO.patch(dirs);
let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
let cargo_clif =
RelPath::DIST.to_path(dirs).join(get_file_name("cargo_clif", "bin").replace('_', "-"));
let manifest_path = SIMPLE_RAYTRACER.manifest_path(dirs);
let target_dir = SIMPLE_RAYTRACER.target_dir(dirs);
let cargo_clif = RelPath::DIST
.to_path(dirs)
.join(get_file_name(&bootstrap_host_compiler.rustc, "cargo_clif", "bin").replace('_', "-"));
let manifest_path = SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).join("Cargo.toml");
let target_dir = RelPath::BUILD.join("simple_raytracer").to_path(dirs);
let clean_cmd = format!(
"RUSTC=rustc cargo clean --manifest-path {manifest_path} --target-dir {target_dir}",
@ -68,35 +43,52 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
target_dir = target_dir.display(),
);
let llvm_build_cmd = format!(
"RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir}",
"RUSTC=rustc cargo build --manifest-path {manifest_path} --target-dir {target_dir} && (rm build/raytracer_cg_llvm || true) && ln build/simple_raytracer/debug/main build/raytracer_cg_llvm",
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let clif_build_cmd = format!(
"RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir}",
"RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir} && (rm build/raytracer_cg_clif || true) && ln build/simple_raytracer/debug/main build/raytracer_cg_clif",
cargo_clif = cargo_clif.display(),
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let clif_build_opt_cmd = format!(
"RUSTC=rustc {cargo_clif} build --manifest-path {manifest_path} --target-dir {target_dir} --release && (rm build/raytracer_cg_clif_opt || true) && ln build/simple_raytracer/release/main build/raytracer_cg_clif_opt",
cargo_clif = cargo_clif.display(),
manifest_path = manifest_path.display(),
target_dir = target_dir.display(),
);
let bench_compile =
hyperfine_command(1, bench_runs, Some(&clean_cmd), &llvm_build_cmd, &clif_build_cmd);
let bench_compile = hyperfine_command(
1,
bench_runs,
Some(&clean_cmd),
&[&llvm_build_cmd, &clif_build_cmd, &clif_build_opt_cmd],
);
spawn_and_wait(bench_compile);
eprintln!("[BENCH RUN] ebobby/simple-raytracer");
fs::copy(
target_dir.join("debug").join(get_file_name("main", "bin")),
RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_clif", "bin")),
)
.unwrap();
let mut bench_run = hyperfine_command(
0,
bench_runs,
None,
Path::new(".").join(get_file_name("raytracer_cg_llvm", "bin")).to_str().unwrap(),
Path::new(".").join(get_file_name("raytracer_cg_clif", "bin")).to_str().unwrap(),
&[
Path::new(".")
.join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_llvm", "bin"))
.to_str()
.unwrap(),
Path::new(".")
.join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif", "bin"))
.to_str()
.unwrap(),
Path::new(".")
.join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif_opt", "bin"))
.to_str()
.unwrap(),
],
);
bench_run.current_dir(RelPath::BUILD.to_path(dirs));
spawn_and_wait(bench_run);

11
compiler/rustc_codegen_cranelift/build_system/build_backend.rs
View file

@ -3,7 +3,7 @@ use std::path::PathBuf;
use super::path::{Dirs, RelPath};
use super::rustc_info::get_file_name;
use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler};
use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler};
pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
@ -14,8 +14,7 @@ pub(crate) fn build_backend(
use_unstable_features: bool,
) -> PathBuf {
let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
maybe_incremental(&mut cmd);
let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default();
@ -23,11 +22,9 @@ pub(crate) fn build_backend(
// Deny warnings on CI
rustflags += " -Dwarnings";
// Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
cmd.env("CARGO_BUILD_INCREMENTAL", "false");
if !is_ci_opt() {
cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true");
cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true");
}
}
@ -52,5 +49,5 @@ pub(crate) fn build_backend(
.target_dir(dirs)
.join(&bootstrap_host_compiler.triple)
.join(channel)
.join(get_file_name("rustc_codegen_cranelift", "dylib"))
.join(get_file_name(&bootstrap_host_compiler.rustc, "rustc_codegen_cranelift", "dylib"))
}

113
compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
View file

@ -1,11 +1,13 @@
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{self, Command};
use std::process::Command;
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_file_name, get_rustc_version, get_toolchain_name};
use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler};
use super::SysrootKind;
use super::rustc_info::get_file_name;
use super::utils::{
maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler,
};
use super::{CodegenBackend, SysrootKind};
static DIST_DIR: RelPath = RelPath::DIST;
static BIN_DIR: RelPath = RelPath::DIST.join("bin");
@ -15,8 +17,9 @@ pub(crate) fn build_sysroot(
dirs: &Dirs,
channel: &str,
sysroot_kind: SysrootKind,
cg_clif_dylib_src: &Path,
cg_clif_dylib_src: &CodegenBackend,
bootstrap_host_compiler: &Compiler,
rustup_toolchain_name: Option<&str>,
target_triple: String,
) -> Compiler {
eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
@ -27,32 +30,52 @@ pub(crate) fn build_sysroot(
let is_native = bootstrap_host_compiler.triple == target_triple;
// Copy the backend
let cg_clif_dylib_path = if cfg!(windows) {
// Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
// binaries.
BIN_DIR
} else {
LIB_DIR
}
.to_path(dirs)
.join(cg_clif_dylib_src.file_name().unwrap());
try_hard_link(cg_clif_dylib_src, &cg_clif_dylib_path);
let cg_clif_dylib_path = match cg_clif_dylib_src {
CodegenBackend::Local(src_path) => {
// Copy the backend
let cg_clif_dylib_path = if cfg!(windows) {
// Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
// binaries.
BIN_DIR
} else {
LIB_DIR
}
.to_path(dirs)
.join(src_path.file_name().unwrap());
try_hard_link(src_path, &cg_clif_dylib_path);
CodegenBackend::Local(cg_clif_dylib_path)
}
CodegenBackend::Builtin(name) => CodegenBackend::Builtin(name.clone()),
};
// Build and copy rustc and cargo wrappers
let wrapper_base_name = get_file_name("____", "bin");
let toolchain_name = get_toolchain_name();
let wrapper_base_name = get_file_name(&bootstrap_host_compiler.rustc, "____", "bin");
for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
let wrapper_name = wrapper_base_name.replace("____", wrapper);
let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
let wrapper_path = DIST_DIR.to_path(dirs).join(&wrapper_name);
build_cargo_wrapper_cmd
.env("TOOLCHAIN_NAME", toolchain_name.clone())
.arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
.arg("-o")
.arg(&wrapper_path)
.arg("-Cstrip=debuginfo");
if let Some(rustup_toolchain_name) = &rustup_toolchain_name {
build_cargo_wrapper_cmd
.env("TOOLCHAIN_NAME", rustup_toolchain_name)
.env_remove("CARGO")
.env_remove("RUSTC")
.env_remove("RUSTDOC");
} else {
build_cargo_wrapper_cmd
.env_remove("TOOLCHAIN_NAME")
.env("CARGO", &bootstrap_host_compiler.cargo)
.env("RUSTC", &bootstrap_host_compiler.rustc)
.env("RUSTDOC", &bootstrap_host_compiler.rustdoc);
}
if let CodegenBackend::Builtin(name) = cg_clif_dylib_src {
build_cargo_wrapper_cmd.env("BUILTIN_BACKEND", name);
}
spawn_and_wait(build_cargo_wrapper_cmd);
try_hard_link(wrapper_path, BIN_DIR.to_path(dirs).join(wrapper_name));
}
@ -134,12 +157,9 @@ impl SysrootTarget {
}
}
pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot");
pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version");
pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src");
pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib");
pub(crate) static STANDARD_LIBRARY: CargoProject =
CargoProject::new(&BUILD_SYSROOT, "build_sysroot");
CargoProject::new(&STDLIB_SRC.join("library/sysroot"), "stdlib_target");
pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup");
#[must_use]
@ -147,7 +167,7 @@ fn build_sysroot_for_triple(
dirs: &Dirs,
channel: &str,
compiler: Compiler,
cg_clif_dylib_path: &Path,
cg_clif_dylib_path: &CodegenBackend,
sysroot_kind: SysrootKind,
) -> SysrootTarget {
match sysroot_kind {
@ -155,7 +175,7 @@ fn build_sysroot_for_triple(
.unwrap_or(SysrootTarget { triple: compiler.triple, libs: vec![] }),
SysrootKind::Llvm => build_llvm_sysroot_for_triple(compiler),
SysrootKind::Clif => {
build_clif_sysroot_for_triple(dirs, channel, compiler, &cg_clif_dylib_path)
build_clif_sysroot_for_triple(dirs, channel, compiler, cg_clif_dylib_path)
}
}
}
@ -199,26 +219,8 @@ fn build_clif_sysroot_for_triple(
dirs: &Dirs,
channel: &str,
mut compiler: Compiler,
cg_clif_dylib_path: &Path,
cg_clif_dylib_path: &CodegenBackend,
) -> SysrootTarget {
match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) {
Err(e) => {
eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source");
process::exit(1);
}
Ok(source_version) => {
let rustc_version = get_rustc_version(&compiler.rustc);
if source_version != rustc_version {
eprintln!("The patched sysroot source is outdated");
eprintln!("Source version: {}", source_version.trim());
eprintln!("Rustc version: {}", rustc_version.trim());
eprintln!("Hint: Try `./y.rs prepare` to update the patched sysroot source");
process::exit(1);
}
}
}
let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) {
@ -237,19 +239,28 @@ fn build_clif_sysroot_for_triple(
// Build sysroot
let mut rustflags = " -Zforce-unstable-if-unmarked -Cpanic=abort".to_string();
rustflags.push_str(&format!(" -Zcodegen-backend={}", cg_clif_dylib_path.to_str().unwrap()));
match cg_clif_dylib_path {
CodegenBackend::Local(path) => {
rustflags.push_str(&format!(" -Zcodegen-backend={}", path.to_str().unwrap()));
}
CodegenBackend::Builtin(name) => {
rustflags.push_str(&format!(" -Zcodegen-backend={name}"));
}
};
// Necessary for MinGW to find rsbegin.o and rsend.o
rustflags
.push_str(&format!(" --sysroot={}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
.push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
if channel == "release" {
rustflags.push_str(" -Zmir-opt-level=3");
}
compiler.rustflags += &rustflags;
let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
maybe_incremental(&mut build_cmd);
if channel == "release" {
build_cmd.arg("--release");
}
build_cmd.arg("--locked");
build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind");
build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true");
build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
if compiler.triple.contains("apple") {
build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed");
@ -272,13 +283,17 @@ fn build_clif_sysroot_for_triple(
}
fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
if !super::config::get_bool("keep_sysroot") {
super::prepare::prepare_stdlib(dirs, &compiler.rustc);
}
if !compiler.triple.ends_with("windows-gnu") {
return None;
}
RTSTARTUP_SYSROOT.ensure_fresh(dirs);
let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup");
let rtstartup_src = STDLIB_SRC.to_path(dirs).join("library").join("rtstartup");
let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] };
for file in ["rsbegin", "rsend"] {

125
compiler/rustc_codegen_cranelift/build_system/mod.rs → compiler/rustc_codegen_cranelift/build_system/main.rs
View file

@ -1,3 +1,7 @@
#![warn(rust_2018_idioms)]
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
use std::env;
use std::path::PathBuf;
use std::process;
@ -37,13 +41,19 @@ enum Command {
}
#[derive(Copy, Clone, Debug)]
pub(crate) enum SysrootKind {
enum SysrootKind {
None,
Clif,
Llvm,
}
pub(crate) fn main() {
#[derive(Clone, Debug)]
enum CodegenBackend {
Local(PathBuf),
Builtin(String),
}
fn main() {
if env::var("RUST_BACKTRACE").is_err() {
env::set_var("RUST_BACKTRACE", "1");
}
@ -75,15 +85,24 @@ pub(crate) fn main() {
};
let mut out_dir = PathBuf::from(".");
let mut download_dir = None;
let mut channel = "release";
let mut sysroot_kind = SysrootKind::Clif;
let mut use_unstable_features = true;
let mut frozen = false;
let mut skip_tests = vec![];
let mut use_backend = None;
while let Some(arg) = args.next().as_deref() {
match arg {
"--out-dir" => {
out_dir = PathBuf::from(args.next().unwrap_or_else(|| {
arg_error!("--out-dir requires argument");
}))
}));
}
"--download-dir" => {
download_dir = Some(PathBuf::from(args.next().unwrap_or_else(|| {
arg_error!("--download-dir requires argument");
})));
}
"--debug" => channel = "debug",
"--sysroot" => {
@ -96,30 +115,79 @@ pub(crate) fn main() {
}
}
"--no-unstable-features" => use_unstable_features = false,
"--frozen" => frozen = true,
"--skip-test" => {
// FIXME check that all passed in tests actually exist
skip_tests.push(args.next().unwrap_or_else(|| {
arg_error!("--skip-test requires argument");
}));
}
"--use-backend" => {
use_backend = Some(match args.next() {
Some(name) => name,
None => arg_error!("--use-backend requires argument"),
});
}
flag if flag.starts_with("-") => arg_error!("Unknown flag {}", flag),
arg => arg_error!("Unexpected argument {}", arg),
}
}
let bootstrap_host_compiler = Compiler::bootstrap_with_triple(
std::env::var("HOST_TRIPLE")
let current_dir = std::env::current_dir().unwrap();
out_dir = current_dir.join(out_dir);
if command == Command::Prepare {
prepare::prepare(&path::Dirs {
source_dir: current_dir.clone(),
download_dir: download_dir
.map(|dir| current_dir.join(dir))
.unwrap_or_else(|| out_dir.join("download")),
build_dir: PathBuf::from("dummy_do_not_use"),
dist_dir: PathBuf::from("dummy_do_not_use"),
frozen,
});
process::exit(0);
}
let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) {
(Ok(_), Ok(_), Ok(_)) => None,
(Err(_), Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()),
_ => {
eprintln!("All of CARGO, RUSTC and RUSTDOC need to be set or none must be set");
process::exit(1);
}
};
let bootstrap_host_compiler = {
let cargo = rustc_info::get_cargo_path();
let rustc = rustc_info::get_rustc_path();
let rustdoc = rustc_info::get_rustdoc_path();
let triple = std::env::var("HOST_TRIPLE")
.ok()
.or_else(|| config::get_value("host"))
.unwrap_or_else(|| rustc_info::get_host_triple()),
);
.unwrap_or_else(|| rustc_info::get_host_triple(&rustc));
Compiler {
cargo,
rustc,
rustdoc,
rustflags: String::new(),
rustdocflags: String::new(),
triple,
runner: vec![],
}
};
let target_triple = std::env::var("TARGET_TRIPLE")
.ok()
.or_else(|| config::get_value("target"))
.unwrap_or_else(|| bootstrap_host_compiler.triple.clone());
// FIXME allow changing the location of these dirs using cli arguments
let current_dir = std::env::current_dir().unwrap();
out_dir = current_dir.join(out_dir);
let dirs = path::Dirs {
source_dir: current_dir.clone(),
download_dir: out_dir.join("download"),
download_dir: download_dir
.map(|dir| current_dir.join(dir))
.unwrap_or_else(|| out_dir.join("download")),
build_dir: out_dir.join("build"),
dist_dir: out_dir.join("dist"),
frozen,
};
path::RelPath::BUILD.ensure_exists(&dirs);
@ -133,20 +201,19 @@ pub(crate) fn main() {
std::fs::File::create(target).unwrap();
}
if command == Command::Prepare {
prepare::prepare(&dirs);
process::exit(0);
}
env::set_var("RUSTC", "rustc_should_be_set_explicitly");
env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly");
let cg_clif_dylib = build_backend::build_backend(
&dirs,
channel,
&bootstrap_host_compiler,
use_unstable_features,
);
let cg_clif_dylib = if let Some(name) = use_backend {
CodegenBackend::Builtin(name)
} else {
CodegenBackend::Local(build_backend::build_backend(
&dirs,
channel,
&bootstrap_host_compiler,
use_unstable_features,
))
};
match command {
Command::Prepare => {
// Handled above
@ -156,8 +223,11 @@ pub(crate) fn main() {
&dirs,
channel,
sysroot_kind,
use_unstable_features,
&skip_tests.iter().map(|test| &**test).collect::<Vec<_>>(),
&cg_clif_dylib,
&bootstrap_host_compiler,
rustup_toolchain_name.as_deref(),
target_triple.clone(),
);
}
@ -166,7 +236,14 @@ pub(crate) fn main() {
eprintln!("Abi-cafe doesn't support cross-compilation");
process::exit(1);
}
abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &bootstrap_host_compiler);
abi_cafe::run(
channel,
sysroot_kind,
&dirs,
&cg_clif_dylib,
rustup_toolchain_name.as_deref(),
&bootstrap_host_compiler,
);
}
Command::Build => {
build_sysroot::build_sysroot(
@ -175,6 +252,7 @@ pub(crate) fn main() {
sysroot_kind,
&cg_clif_dylib,
&bootstrap_host_compiler,
rustup_toolchain_name.as_deref(),
target_triple,
);
}
@ -185,6 +263,7 @@ pub(crate) fn main() {
sysroot_kind,
&cg_clif_dylib,
&bootstrap_host_compiler,
rustup_toolchain_name.as_deref(),
target_triple,
);
bench::benchmark(&dirs, &bootstrap_host_compiler);

1
compiler/rustc_codegen_cranelift/build_system/path.rs
View file

@ -9,6 +9,7 @@ pub(crate) struct Dirs {
pub(crate) download_dir: PathBuf,
pub(crate) build_dir: PathBuf,
pub(crate) dist_dir: PathBuf,
pub(crate) frozen: bool,
}
#[doc(hidden)]

200
compiler/rustc_codegen_cranelift/build_system/prepare.rs
View file

@ -3,77 +3,60 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC};
use super::build_sysroot::STDLIB_SRC;
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_default_sysroot, get_rustc_version};
use super::tests::LIBCORE_TESTS_SRC;
use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait};
use super::rustc_info::get_default_sysroot;
use super::utils::{
copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait,
};
pub(crate) fn prepare(dirs: &Dirs) {
RelPath::DOWNLOAD.ensure_fresh(dirs);
spawn_and_wait(super::build_backend::CG_CLIF.fetch("cargo", "rustc", dirs));
prepare_stdlib(dirs);
spawn_and_wait(super::build_sysroot::STANDARD_LIBRARY.fetch("cargo", "rustc", dirs));
prepare_coretests(dirs);
spawn_and_wait(super::tests::LIBCORE_TESTS.fetch("cargo", "rustc", dirs));
RelPath::DOWNLOAD.ensure_exists(dirs);
super::tests::RAND_REPO.fetch(dirs);
spawn_and_wait(super::tests::RAND.fetch("cargo", "rustc", dirs));
super::tests::REGEX_REPO.fetch(dirs);
spawn_and_wait(super::tests::REGEX.fetch("cargo", "rustc", dirs));
super::tests::PORTABLE_SIMD_REPO.fetch(dirs);
spawn_and_wait(super::tests::PORTABLE_SIMD.fetch("cargo", "rustc", dirs));
}
fn prepare_stdlib(dirs: &Dirs) {
let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {
let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust");
assert!(sysroot_src_orig.exists());
eprintln!("[COPY] stdlib src");
apply_patches(dirs, "stdlib", &sysroot_src_orig, &STDLIB_SRC.to_path(dirs));
// FIXME ensure builds error out or update the copy if any of the files copied here change
BUILD_SYSROOT.ensure_fresh(dirs);
copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs));
std::fs::write(
STDLIB_SRC.to_path(dirs).join("Cargo.toml"),
r#"
[workspace]
members = ["./library/sysroot"]
fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap();
copy_dir_recursively(
&sysroot_src_orig.join("library"),
&SYSROOT_SRC.to_path(dirs).join("library"),
);
[patch.crates-io]
rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" }
rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" }
rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" }
let rustc_version = get_rustc_version(Path::new("rustc"));
fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
# Mandatory for correctly compiling compiler-builtins
[profile.dev.package.compiler_builtins]
debug-assertions = false
overflow-checks = false
codegen-units = 10000
eprintln!("[GIT] init");
init_git_repo(&SYSROOT_SRC.to_path(dirs));
[profile.release.package.compiler_builtins]
debug-assertions = false
overflow-checks = false
codegen-units = 10000
"#,
)
.unwrap();
apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs));
}
fn prepare_coretests(dirs: &Dirs) {
let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
assert!(sysroot_src_orig.exists());
eprintln!("[COPY] coretests src");
fs::create_dir_all(LIBCORE_TESTS_SRC.to_path(dirs)).unwrap();
copy_dir_recursively(
&sysroot_src_orig.join("library/core/tests"),
&LIBCORE_TESTS_SRC.to_path(dirs),
);
eprintln!("[GIT] init");
init_git_repo(&LIBCORE_TESTS_SRC.to_path(dirs));
apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs));
let source_lockfile = RelPath::PATCHES.to_path(dirs).join("stdlib-lock.toml");
let target_lockfile = STDLIB_SRC.to_path(dirs).join("Cargo.lock");
fs::copy(source_lockfile, target_lockfile).unwrap();
}
pub(crate) struct GitRepo {
url: GitRepoUrl,
rev: &'static str,
content_hash: &'static str,
patch_name: &'static str,
}
@ -81,35 +64,107 @@ enum GitRepoUrl {
Github { user: &'static str, repo: &'static str },
}
// Note: This uses a hasher which is not cryptographically secure. This is fine as the hash is meant
// to protect against accidental modification and outdated downloads, not against manipulation.
fn hash_file(file: &std::path::Path) -> u64 {
let contents = std::fs::read(file).unwrap();
#[allow(deprecated)]
let mut hasher = std::hash::SipHasher::new();
std::hash::Hash::hash(&contents, &mut hasher);
std::hash::Hasher::finish(&hasher)
}
fn hash_dir(dir: &std::path::Path) -> u64 {
let mut sub_hashes = std::collections::BTreeMap::new();
for entry in std::fs::read_dir(dir).unwrap() {
let entry = entry.unwrap();
if entry.file_type().unwrap().is_dir() {
sub_hashes
.insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path()));
} else {
sub_hashes
.insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path()));
}
}
#[allow(deprecated)]
let mut hasher = std::hash::SipHasher::new();
std::hash::Hash::hash(&sub_hashes, &mut hasher);
std::hash::Hasher::finish(&hasher)
}
impl GitRepo {
pub(crate) const fn github(
user: &'static str,
repo: &'static str,
rev: &'static str,
content_hash: &'static str,
patch_name: &'static str,
) -> GitRepo {
GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name }
GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name }
}
fn download_dir(&self, dirs: &Dirs) -> PathBuf {
match self.url {
GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs),
}
}
pub(crate) const fn source_dir(&self) -> RelPath {
match self.url {
GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo),
GitRepoUrl::Github { user: _, repo } => RelPath::BUILD.join(repo),
}
}
pub(crate) fn fetch(&self, dirs: &Dirs) {
match self.url {
GitRepoUrl::Github { user, repo } => {
clone_repo_shallow_github(
dirs,
&self.source_dir().to_path(dirs),
user,
repo,
self.rev,
let download_dir = self.download_dir(dirs);
if download_dir.exists() {
let actual_hash = format!("{:016x}", hash_dir(&download_dir));
if actual_hash == self.content_hash {
println!("[FRESH] {}", download_dir.display());
return;
} else {
println!(
"Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.",
download_dir = download_dir.display(),
content_hash = self.content_hash,
);
}
}
apply_patches(dirs, self.patch_name, &self.source_dir().to_path(dirs));
match self.url {
GitRepoUrl::Github { user, repo } => {
clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev);
}
}
let source_lockfile =
RelPath::PATCHES.to_path(dirs).join(format!("{}-lock.toml", self.patch_name));
let target_lockfile = download_dir.join("Cargo.lock");
if source_lockfile.exists() {
fs::copy(source_lockfile, target_lockfile).unwrap();
} else {
assert!(target_lockfile.exists());
}
let actual_hash = format!("{:016x}", hash_dir(&download_dir));
if actual_hash != self.content_hash {
println!(
"Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}",
download_dir = download_dir.display(),
content_hash = self.content_hash,
);
std::process::exit(1);
}
}
pub(crate) fn patch(&self, dirs: &Dirs) {
apply_patches(
dirs,
self.patch_name,
&self.download_dir(dirs),
&self.source_dir().to_path(dirs),
);
}
}
@ -126,6 +181,8 @@ fn clone_repo(download_dir: &Path, repo: &str, rev: &str) {
let mut checkout_cmd = git_command(download_dir, "checkout");
checkout_cmd.arg("-q").arg(rev);
spawn_and_wait(checkout_cmd);
std::fs::remove_dir_all(download_dir.join(".git")).unwrap();
}
fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: &str, rev: &str) {
@ -173,8 +230,6 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo:
// Rename unpacked dir to the expected name
std::fs::rename(archive_dir, &download_dir).unwrap();
init_git_repo(&download_dir);
// Cleanup
std::fs::remove_file(archive_file).unwrap();
}
@ -213,7 +268,22 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec<PathBuf> {
patches
}
fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) {
pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) {
// FIXME avoid copy and patch if src, patches and target are unchanged
eprintln!("[COPY] {crate_name} source");
remove_dir_if_exists(target_dir);
fs::create_dir_all(target_dir).unwrap();
if crate_name == "stdlib" {
fs::create_dir(target_dir.join("library")).unwrap();
copy_dir_recursively(&source_dir.join("library"), &target_dir.join("library"));
} else {
copy_dir_recursively(source_dir, target_dir);
}
init_git_repo(target_dir);
if crate_name == "<none>" {
return;
}

24
compiler/rustc_codegen_cranelift/build_system/rustc_info.rs
View file

@ -1,15 +1,9 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
pub(crate) fn get_rustc_version(rustc: &Path) -> String {
pub(crate) fn get_host_triple(rustc: &Path) -> String {
let version_info =
Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
String::from_utf8(version_info).unwrap()
}
pub(crate) fn get_host_triple() -> String {
let version_info =
Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
Command::new(rustc).stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
String::from_utf8(version_info)
.unwrap()
.lines()
@ -34,6 +28,9 @@ pub(crate) fn get_toolchain_name() -> String {
}
pub(crate) fn get_cargo_path() -> PathBuf {
if let Ok(cargo) = std::env::var("CARGO") {
return PathBuf::from(cargo);
}
let cargo_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "cargo"])
@ -44,6 +41,9 @@ pub(crate) fn get_cargo_path() -> PathBuf {
}
pub(crate) fn get_rustc_path() -> PathBuf {
if let Ok(rustc) = std::env::var("RUSTC") {
return PathBuf::from(rustc);
}
let rustc_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "rustc"])
@ -54,6 +54,9 @@ pub(crate) fn get_rustc_path() -> PathBuf {
}
pub(crate) fn get_rustdoc_path() -> PathBuf {
if let Ok(rustdoc) = std::env::var("RUSTDOC") {
return PathBuf::from(rustdoc);
}
let rustc_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "rustdoc"])
@ -73,8 +76,9 @@ pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf {
Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
}
pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
let file_name = Command::new("rustc")
// FIXME call once for each target and pass result around in struct
pub(crate) fn get_file_name(rustc: &Path, crate_name: &str, crate_type: &str) -> String {
let file_name = Command::new(rustc)
.stderr(Stdio::inherit())
.args(&[
"--crate-name",

141
compiler/rustc_codegen_cranelift/build_system/tests.rs
View file

@ -1,13 +1,14 @@
use super::build_sysroot;
use super::config;
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::prepare::{apply_patches, GitRepo};
use super::rustc_info::get_default_sysroot;
use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
use super::SysrootKind;
use super::{CodegenBackend, SysrootKind};
use std::env;
use std::ffi::OsStr;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::process::Command;
static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example");
@ -18,7 +19,7 @@ struct TestCase {
}
enum TestCaseCmd {
Custom { func: &'static dyn Fn(&TestRunner) },
Custom { func: &'static dyn Fn(&TestRunner<'_>) },
BuildLib { source: &'static str, crate_types: &'static str },
BuildBinAndRun { source: &'static str, args: &'static [&'static str] },
JitBin { source: &'static str, args: &'static str },
@ -26,7 +27,7 @@ enum TestCaseCmd {
impl TestCase {
// FIXME reduce usage of custom test case commands
const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self {
const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner<'_>)) -> Self {
Self { config, cmd: TestCaseCmd::Custom { func } }
}
@ -95,32 +96,45 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
// FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is
// fixed.
pub(crate) static RAND_REPO: GitRepo =
GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand");
pub(crate) static RAND_REPO: GitRepo = GitRepo::github(
"rust-random",
"rand",
"50b9a447410860af8d6db9a208c3576886955874",
"446203b96054891e",
"rand",
);
pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand");
pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target");
pub(crate) static REGEX_REPO: GitRepo =
GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex");
pub(crate) static REGEX_REPO: GitRepo = GitRepo::github(
"rust-lang",
"regex",
"32fed9429eafba0ae92a64b01796a0c5a75b88c8",
"fcc4df7c5b902633",
"regex",
);
pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex");
pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir(), "regex_target");
pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
"rust-lang",
"portable-simd",
"ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b",
"800548f8000e31bd",
"portable-simd",
);
pub(crate) static PORTABLE_SIMD: CargoProject =
CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd");
CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable-simd_target");
pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::DOWNLOAD.join("coretests_src");
static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests");
pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests");
static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "coretests_target");
const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
TestCase::custom("test.rust-random/rand", &|runner| {
RAND_REPO.patch(&runner.dirs);
RAND.clean(&runner.dirs);
if runner.is_native {
@ -135,6 +149,17 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.libcore", &|runner| {
apply_patches(
&runner.dirs,
"coretests",
&runner.stdlib_source.join("library/core/tests"),
&LIBCORE_TESTS_SRC.to_path(&runner.dirs),
);
let source_lockfile = RelPath::PATCHES.to_path(&runner.dirs).join("coretests-lock.toml");
let target_lockfile = LIBCORE_TESTS_SRC.to_path(&runner.dirs).join("Cargo.lock");
fs::copy(source_lockfile, target_lockfile).unwrap();
LIBCORE_TESTS.clean(&runner.dirs);
if runner.is_native {
@ -149,6 +174,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
REGEX_REPO.patch(&runner.dirs);
REGEX.clean(&runner.dirs);
let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs);
@ -181,6 +208,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.regex", &|runner| {
REGEX_REPO.patch(&runner.dirs);
REGEX.clean(&runner.dirs);
if runner.is_native {
@ -197,6 +226,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.portable-simd", &|runner| {
PORTABLE_SIMD_REPO.patch(&runner.dirs);
PORTABLE_SIMD.clean(&runner.dirs);
let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs);
@ -215,24 +246,35 @@ pub(crate) fn run_tests(
dirs: &Dirs,
channel: &str,
sysroot_kind: SysrootKind,
cg_clif_dylib: &Path,
use_unstable_features: bool,
skip_tests: &[&str],
cg_clif_dylib: &CodegenBackend,
bootstrap_host_compiler: &Compiler,
rustup_toolchain_name: Option<&str>,
target_triple: String,
) {
if config::get_bool("testsuite.no_sysroot") {
let stdlib_source =
get_default_sysroot(&bootstrap_host_compiler.rustc).join("lib/rustlib/src/rust");
assert!(stdlib_source.exists());
if config::get_bool("testsuite.no_sysroot") && !skip_tests.contains(&"testsuite.no_sysroot") {
let target_compiler = build_sysroot::build_sysroot(
dirs,
channel,
SysrootKind::None,
cg_clif_dylib,
bootstrap_host_compiler,
rustup_toolchain_name,
target_triple.clone(),
);
let runner = TestRunner::new(
dirs.clone(),
target_compiler,
use_unstable_features,
skip_tests,
bootstrap_host_compiler.triple == target_triple,
stdlib_source.clone(),
);
BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs);
@ -241,23 +283,32 @@ pub(crate) fn run_tests(
eprintln!("[SKIP] no_sysroot tests");
}
let run_base_sysroot = config::get_bool("testsuite.base_sysroot");
let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot");
let run_base_sysroot = config::get_bool("testsuite.base_sysroot")
&& !skip_tests.contains(&"testsuite.base_sysroot");
let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot")
&& !skip_tests.contains(&"testsuite.extended_sysroot");
if run_base_sysroot || run_extended_sysroot {
let target_compiler = build_sysroot::build_sysroot(
let mut target_compiler = build_sysroot::build_sysroot(
dirs,
channel,
sysroot_kind,
cg_clif_dylib,
bootstrap_host_compiler,
rustup_toolchain_name,
target_triple.clone(),
);
// Rust's build system denies a couple of lints that trigger on several of the test
// projects. Changing the code to fix them is not worth it, so just silence all lints.
target_compiler.rustflags += " --cap-lints=allow";
let runner = TestRunner::new(
dirs.clone(),
target_compiler,
use_unstable_features,
skip_tests,
bootstrap_host_compiler.triple == target_triple,
stdlib_source,
);
if run_base_sysroot {
@ -274,15 +325,25 @@ pub(crate) fn run_tests(
}
}
struct TestRunner {
struct TestRunner<'a> {
is_native: bool,
jit_supported: bool,
use_unstable_features: bool,
skip_tests: &'a [&'a str],
dirs: Dirs,
target_compiler: Compiler,
stdlib_source: PathBuf,
}
impl TestRunner {
fn new(dirs: Dirs, mut target_compiler: Compiler, is_native: bool) -> Self {
impl<'a> TestRunner<'a> {
fn new(
dirs: Dirs,
mut target_compiler: Compiler,
use_unstable_features: bool,
skip_tests: &'a [&'a str],
is_native: bool,
stdlib_source: PathBuf,
) -> Self {
if let Ok(rustflags) = env::var("RUSTFLAGS") {
target_compiler.rustflags.push(' ');
target_compiler.rustflags.push_str(&rustflags);
@ -297,11 +358,20 @@ impl TestRunner {
target_compiler.rustflags.push_str(" -Clink-arg=-undefined -Clink-arg=dynamic_lookup");
}
let jit_supported = is_native
let jit_supported = use_unstable_features
&& is_native
&& target_compiler.triple.contains("x86_64")
&& !target_compiler.triple.contains("windows");
Self { is_native, jit_supported, dirs, target_compiler }
Self {
is_native,
jit_supported,
use_unstable_features,
skip_tests,
dirs,
target_compiler,
stdlib_source,
}
}
fn run_testsuite(&self, tests: &[TestCase]) {
@ -310,7 +380,10 @@ impl TestRunner {
let tag = tag.to_uppercase();
let is_jit_test = tag == "JIT";
if !config::get_bool(config) || (is_jit_test && !self.jit_supported) {
if !config::get_bool(config)
|| (is_jit_test && !self.jit_supported)
|| self.skip_tests.contains(&config)
{
eprintln!("[{tag}] {testname} (skipped)");
continue;
} else {
@ -320,10 +393,24 @@ impl TestRunner {
match *cmd {
TestCaseCmd::Custom { func } => func(self),
TestCaseCmd::BuildLib { source, crate_types } => {
self.run_rustc([source, "--crate-type", crate_types]);
if self.use_unstable_features {
self.run_rustc([source, "--crate-type", crate_types]);
} else {
self.run_rustc([
source,
"--crate-type",
crate_types,
"--cfg",
"no_unstable_features",
]);
}
}
TestCaseCmd::BuildBinAndRun { source, args } => {
self.run_rustc([source]);
if self.use_unstable_features {
self.run_rustc([source]);
} else {
self.run_rustc([source, "--cfg", "no_unstable_features"]);
}
self.run_out_command(
source.split('/').last().unwrap().split('.').next().unwrap(),
args,

34
compiler/rustc_codegen_cranelift/build_system/usage.txt
View file

@ -1,11 +1,11 @@
The build system of cg_clif.
USAGE:
./y.rs prepare [--out-dir DIR]
./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features]
./y.sh prepare [--out-dir DIR] [--download-dir DIR]
./y.sh build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen]
./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] [--skip-test TESTNAME]
./y.sh abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen]
./y.sh bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen]
OPTIONS:
--debug
@ -22,14 +22,28 @@ OPTIONS:
Specify the directory in which the download, build and dist directories are stored.
By default this is the working directory.
--download-dir DIR
Specify the directory in which the download directory is stored. Overrides --out-dir.
--no-unstable-features
Some features are not yet ready for production usage. This option will disable these
features. This includes the JIT mode and inline assembly support.
--frozen
Require Cargo.lock and cache are up to date
--skip-test TESTNAME
Skip testing the TESTNAME test. The test name format is the same as config.txt.
--use-backend NAME
Use the existing Cranelift (or other) backend of the rustc with which we built.
Warning: This is meant for use in rust's CI only!
REQUIREMENTS:
* Rustup: The build system has a hard coded dependency on rustup to install the right nightly
version and make sure it is used where necessary.
* Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos.
* Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for
* Rustup: By default rustup is used to install the right nightly version. If you don't want to
use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and
point the CARGO, RUSTC and RUSTDOC env vars to the right executables.
* Git: `./y.sh prepare` uses git for applying patches and on Windows for downloading test repos.
* Curl and tar (non-Windows only): Used by `./y.sh prepare` to download a single commit for
repos. Git will be used to clone the whole repo when using Windows.
* [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.rs bench`.
* [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.sh bench`.

51
compiler/rustc_codegen_cranelift/build_system/utils.rs
View file

@ -5,7 +5,6 @@ use std::path::{Path, PathBuf};
use std::process::{self, Command, Stdio};
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_cargo_path, get_rustc_path, get_rustdoc_path};
#[derive(Clone, Debug)]
pub(crate) struct Compiler {
@ -19,18 +18,6 @@ pub(crate) struct Compiler {
}
impl Compiler {
pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler {
Compiler {
cargo: get_cargo_path(),
rustc: get_rustc_path(),
rustdoc: get_rustdoc_path(),
rustflags: String::new(),
rustdocflags: String::new(),
triple,
runner: vec![],
}
}
pub(crate) fn set_cross_linker_and_runner(&mut self) {
match self.triple.as_str() {
"aarch64-unknown-linux-gnu" => {
@ -95,7 +82,11 @@ impl CargoProject {
.arg(self.manifest_path(dirs))
.arg("--target-dir")
.arg(self.target_dir(dirs))
.arg("--frozen");
.arg("--locked");
if dirs.frozen {
cmd.arg("--frozen");
}
cmd
}
@ -120,23 +111,6 @@ impl CargoProject {
cmd
}
#[must_use]
pub(crate) fn fetch(
&self,
cargo: impl AsRef<Path>,
rustc: impl AsRef<Path>,
dirs: &Dirs,
) -> Command {
let mut cmd = Command::new(cargo.as_ref());
cmd.env("RUSTC", rustc.as_ref())
.arg("fetch")
.arg("--manifest-path")
.arg(self.manifest_path(dirs));
cmd
}
pub(crate) fn clean(&self, dirs: &Dirs) {
let _ = fs::remove_dir_all(self.target_dir(dirs));
}
@ -162,8 +136,7 @@ pub(crate) fn hyperfine_command(
warmup: u64,
runs: u64,
prepare: Option<&str>,
a: &str,
b: &str,
cmds: &[&str],
) -> Command {
let mut bench = Command::new("hyperfine");
@ -179,7 +152,7 @@ pub(crate) fn hyperfine_command(
bench.arg("--prepare").arg(prepare);
}
bench.arg(a).arg(b);
bench.args(cmds);
bench
}
@ -285,3 +258,13 @@ pub(crate) fn is_ci() -> bool {
pub(crate) fn is_ci_opt() -> bool {
env::var("CI_OPT").is_ok()
}
pub(crate) fn maybe_incremental(cmd: &mut Command) {
if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") {
// Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway
cmd.env("CARGO_BUILD_INCREMENTAL", "false");
} else {
// Force incr comp even in release mode unless in CI or incremental builds are explicitly disabled
cmd.env("CARGO_BUILD_INCREMENTAL", "true");
}
}