bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '3a31c6d827' into sync_cg_clif-2021-07-07

Browse source
This commit is contained in:
bjorn3 2021-07-07 11:14:20 +02:00
commit d531f3d6ee
55 changed files with 1285 additions and 448 deletions
Download patch file Download diff file Expand all files Collapse all files

40
compiler/rustc_codegen_cranelift/build_system/build_backend.rs Normal file
View file

@ -0,0 +1,40 @@
use std::env;
use std::path::{Path, PathBuf};
use std::process::Command;
pub(crate) fn build_backend(channel: &str, host_triple: &str) -> PathBuf {
let mut cmd = Command::new("cargo");
cmd.arg("build").arg("--target").arg(host_triple);
match channel {
"debug" => {}
"release" => {
cmd.arg("--release");
}
_ => unreachable!(),
}
if cfg!(unix) {
if cfg!(target_os = "macos") {
cmd.env(
"RUSTFLAGS",
"-Csplit-debuginfo=unpacked \
-Clink-arg=-Wl,-rpath,@loader_path/../lib \
-Zosx-rpath-install-name"
.to_string()
+ env::var("RUSTFLAGS").as_deref().unwrap_or(""),
);
} else {
cmd.env(
"RUSTFLAGS",
"-Clink-arg=-Wl,-rpath=$ORIGIN/../lib ".to_string()
+ env::var("RUSTFLAGS").as_deref().unwrap_or(""),
);
}
}
eprintln!("[BUILD] rustc_codegen_cranelift");
crate::utils::spawn_and_wait(cmd);
Path::new("target").join(host_triple).join(channel)
}

216
compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs Normal file
View file

@ -0,0 +1,216 @@
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::{self, Command};
use crate::rustc_info::{get_file_name, get_rustc_version};
use crate::utils::{spawn_and_wait, try_hard_link};
use crate::SysrootKind;
pub(crate) fn build_sysroot(
channel: &str,
sysroot_kind: SysrootKind,
target_dir: &Path,
cg_clif_build_dir: PathBuf,
host_triple: &str,
target_triple: &str,
) {
if target_dir.exists() {
fs::remove_dir_all(target_dir).unwrap();
}
fs::create_dir_all(target_dir.join("bin")).unwrap();
fs::create_dir_all(target_dir.join("lib")).unwrap();
// Copy the backend
for file in ["cg_clif", "cg_clif_build_sysroot"] {
try_hard_link(
cg_clif_build_dir.join(get_file_name(file, "bin")),
target_dir.join("bin").join(get_file_name(file, "bin")),
);
}
let cg_clif_dylib = get_file_name("rustc_codegen_cranelift", "dylib");
try_hard_link(
cg_clif_build_dir.join(&cg_clif_dylib),
target_dir
.join(if cfg!(windows) {
// Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
// binaries.
"bin"
} else {
"lib"
})
.join(cg_clif_dylib),
);
// Build and copy cargo wrapper
let mut build_cargo_wrapper_cmd = Command::new("rustc");
build_cargo_wrapper_cmd
.arg("scripts/cargo.rs")
.arg("-o")
.arg(target_dir.join("cargo"))
.arg("-g");
spawn_and_wait(build_cargo_wrapper_cmd);
let default_sysroot = crate::rustc_info::get_default_sysroot();
let rustlib = target_dir.join("lib").join("rustlib");
let host_rustlib_lib = rustlib.join(host_triple).join("lib");
let target_rustlib_lib = rustlib.join(target_triple).join("lib");
fs::create_dir_all(&host_rustlib_lib).unwrap();
fs::create_dir_all(&target_rustlib_lib).unwrap();
if target_triple == "x86_64-pc-windows-gnu" {
if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() {
eprintln!(
"The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
to compile a sysroot for it.",
);
process::exit(1);
}
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
if file.extension().map_or(true, |ext| ext.to_str().unwrap() != "o") {
continue; // only copy object files
}
try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
}
}
match sysroot_kind {
SysrootKind::None => {} // Nothing to do
SysrootKind::Llvm => {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(host_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
let file_name_str = file.file_name().unwrap().to_str().unwrap();
if file_name_str.contains("rustc_")
|| file_name_str.contains("chalk")
|| file_name_str.contains("tracing")
|| file_name_str.contains("regex")
{
// These are large crates that are part of the rustc-dev component and are not
// necessary to run regular programs.
continue;
}
try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
}
if target_triple != host_triple {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
)
.unwrap()
{
let file = file.unwrap().path();
try_hard_link(&file, target_rustlib_lib.join(file.file_name().unwrap()));
}
}
}
SysrootKind::Clif => {
build_clif_sysroot_for_triple(channel, target_dir, host_triple, None);
if host_triple != target_triple {
// When cross-compiling it is often necessary to manually pick the right linker
let linker = if target_triple == "aarch64-unknown-linux-gnu" {
Some("aarch64-linux-gnu-gcc")
} else {
None
};
build_clif_sysroot_for_triple(channel, target_dir, target_triple, linker);
}
// Copy std for the host to the lib dir. This is necessary for the jit mode to find
// libstd.
for file in fs::read_dir(host_rustlib_lib).unwrap() {
let file = file.unwrap().path();
if file.file_name().unwrap().to_str().unwrap().contains("std-") {
try_hard_link(&file, target_dir.join("lib").join(file.file_name().unwrap()));
}
}
}
}
}
fn build_clif_sysroot_for_triple(
channel: &str,
target_dir: &Path,
triple: &str,
linker: Option<&str>,
) {
match fs::read_to_string(Path::new("build_sysroot").join("rustc_version")) {
Err(e) => {
eprintln!("Failed to get rustc version for patched sysroot source: {}", e);
eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source");
process::exit(1);
}
Ok(source_version) => {
let rustc_version = get_rustc_version();
if source_version != rustc_version {
eprintln!("The patched sysroot source is outdated");
eprintln!("Source version: {}", source_version.trim());
eprintln!("Rustc version: {}", rustc_version.trim());
eprintln!("Hint: Try `./y.rs prepare` to update the patched sysroot source");
process::exit(1);
}
}
}
let build_dir = Path::new("build_sysroot").join("target").join(triple).join(channel);
if !crate::config::get_bool("keep_sysroot") {
// Cleanup the target dir with the exception of build scripts and the incremental cache
for dir in ["build", "deps", "examples", "native"] {
if build_dir.join(dir).exists() {
fs::remove_dir_all(build_dir.join(dir)).unwrap();
}
}
}
// Build sysroot
let mut build_cmd = Command::new("cargo");
build_cmd.arg("build").arg("--target").arg(triple).current_dir("build_sysroot");
let mut rustflags = "--clif -Zforce-unstable-if-unmarked".to_string();
if channel == "release" {
build_cmd.arg("--release");
rustflags.push_str(" -Zmir-opt-level=3");
}
if let Some(linker) = linker {
use std::fmt::Write;
write!(rustflags, " -Clinker={}", linker).unwrap();
}
build_cmd.env("RUSTFLAGS", rustflags);
build_cmd.env(
"RUSTC",
env::current_dir().unwrap().join(target_dir).join("bin").join("cg_clif_build_sysroot"),
);
// FIXME Enable incremental again once rust-lang/rust#74946 is fixed
build_cmd.env("CARGO_INCREMENTAL", "0").env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
spawn_and_wait(build_cmd);
// Copy all relevant files to the sysroot
for entry in
fs::read_dir(Path::new("build_sysroot/target").join(triple).join(channel).join("deps"))
.unwrap()
{
let entry = entry.unwrap();
if let Some(ext) = entry.path().extension() {
if ext == "rmeta" || ext == "d" || ext == "dSYM" {
continue;
}
} else {
continue;
};
try_hard_link(
entry.path(),
target_dir.join("lib").join("rustlib").join(triple).join("lib").join(entry.file_name()),
);
}
}

55
compiler/rustc_codegen_cranelift/build_system/config.rs Normal file
View file

@ -0,0 +1,55 @@
use std::{fs, process};
fn load_config_file() -> Vec<(String, Option<String>)> {
fs::read_to_string("config.txt")
.unwrap()
.lines()
.map(|line| if let Some((line, _comment)) = line.split_once('#') { line } else { line })
.map(|line| line.trim())
.filter(|line| !line.is_empty())
.map(|line| {
if let Some((key, val)) = line.split_once('=') {
(key.trim().to_owned(), Some(val.trim().to_owned()))
} else {
(line.to_owned(), None)
}
})
.collect()
}
pub(crate) fn get_bool(name: &str) -> bool {
let values = load_config_file()
.into_iter()
.filter(|(key, _)| key == name)
.map(|(_, val)| val)
.collect::<Vec<_>>();
if values.is_empty() {
false
} else {
if values.iter().any(|val| val.is_some()) {
eprintln!("Boolean config `{}` has a value", name);
process::exit(1);
}
true
}
}
pub(crate) fn get_value(name: &str) -> Option<String> {
let values = load_config_file()
.into_iter()
.filter(|(key, _)| key == name)
.map(|(_, val)| val)
.collect::<Vec<_>>();
if values.is_empty() {
None
} else if values.len() == 1 {
if values[0].is_none() {
eprintln!("Config `{}` missing value", name);
process::exit(1);
}
values.into_iter().next().unwrap()
} else {
eprintln!("Config `{}` given multiple values: {:?}", name, values);
process::exit(1);
}
}

133
compiler/rustc_codegen_cranelift/build_system/prepare.rs Normal file
View file

@ -0,0 +1,133 @@
use std::env;
use std::ffi::OsStr;
use std::ffi::OsString;
use std::fs;
use std::path::Path;
use std::process::Command;
use crate::rustc_info::{get_file_name, get_rustc_path, get_rustc_version};
use crate::utils::{copy_dir_recursively, spawn_and_wait};
pub(crate) fn prepare() {
prepare_sysroot();
eprintln!("[INSTALL] hyperfine");
Command::new("cargo").arg("install").arg("hyperfine").spawn().unwrap().wait().unwrap();
clone_repo(
"rand",
"https://github.com/rust-random/rand.git",
"0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
);
apply_patches("rand", Path::new("rand"));
clone_repo(
"regex",
"https://github.com/rust-lang/regex.git",
"341f207c1071f7290e3f228c710817c280c8dca1",
);
clone_repo(
"simple-raytracer",
"https://github.com/ebobby/simple-raytracer",
"804a7a21b9e673a482797aa289a18ed480e4d813",
);
eprintln!("[LLVM BUILD] simple-raytracer");
let mut build_cmd = Command::new("cargo");
build_cmd.arg("build").env_remove("CARGO_TARGET_DIR").current_dir("simple-raytracer");
spawn_and_wait(build_cmd);
fs::copy(
Path::new("simple-raytracer/target/debug").join(get_file_name("main", "bin")),
// FIXME use get_file_name here too once testing is migrated to rust
"simple-raytracer/raytracer_cg_llvm",
)
.unwrap();
}
fn prepare_sysroot() {
let rustc_path = get_rustc_path();
let sysroot_src_orig = rustc_path.parent().unwrap().join("../lib/rustlib/src/rust");
let sysroot_src = env::current_dir().unwrap().join("build_sysroot").join("sysroot_src");
assert!(sysroot_src_orig.exists());
if sysroot_src.exists() {
fs::remove_dir_all(&sysroot_src).unwrap();
}
fs::create_dir_all(sysroot_src.join("library")).unwrap();
eprintln!("[COPY] sysroot src");
copy_dir_recursively(&sysroot_src_orig.join("library"), &sysroot_src.join("library"));
let rustc_version = get_rustc_version();
fs::write(
Path::new("build_sysroot").join("rustc_version"),
&rustc_version,
)
.unwrap();
eprintln!("[GIT] init");
let mut git_init_cmd = Command::new("git");
git_init_cmd.arg("init").arg("-q").current_dir(&sysroot_src);
spawn_and_wait(git_init_cmd);
let mut git_add_cmd = Command::new("git");
git_add_cmd.arg("add").arg(".").current_dir(&sysroot_src);
spawn_and_wait(git_add_cmd);
let mut git_commit_cmd = Command::new("git");
git_commit_cmd
.arg("commit")
.arg("-m")
.arg("Initial commit")
.arg("-q")
.current_dir(&sysroot_src);
spawn_and_wait(git_commit_cmd);
apply_patches("sysroot", &sysroot_src);
clone_repo(
"build_sysroot/compiler-builtins",
"https://github.com/rust-lang/compiler-builtins.git",
"0.1.46",
);
apply_patches("compiler-builtins", Path::new("build_sysroot/compiler-builtins"));
}
fn clone_repo(target_dir: &str, repo: &str, rev: &str) {
eprintln!("[CLONE] {}", repo);
// Ignore exit code as the repo may already have been checked out
Command::new("git").arg("clone").arg(repo).arg(target_dir).spawn().unwrap().wait().unwrap();
let mut clean_cmd = Command::new("git");
clean_cmd.arg("checkout").arg("--").arg(".").current_dir(target_dir);
spawn_and_wait(clean_cmd);
let mut checkout_cmd = Command::new("git");
checkout_cmd.arg("checkout").arg("-q").arg(rev).current_dir(target_dir);
spawn_and_wait(checkout_cmd);
}
fn get_patches(crate_name: &str) -> Vec<OsString> {
let mut patches: Vec<_> = fs::read_dir("patches")
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension() == Some(OsStr::new("patch")))
.map(|path| path.file_name().unwrap().to_owned())
.filter(|file_name| {
file_name.to_str().unwrap().split_once("-").unwrap().1.starts_with(crate_name)
})
.collect();
patches.sort();
patches
}
fn apply_patches(crate_name: &str, target_dir: &Path) {
for patch in get_patches(crate_name) {
eprintln!("[PATCH] {:?} <- {:?}", target_dir.file_name().unwrap(), patch);
let patch_arg = env::current_dir().unwrap().join("patches").join(patch);
let mut apply_patch_cmd = Command::new("git");
apply_patch_cmd.arg("am").arg(patch_arg).arg("-q").current_dir(target_dir);
spawn_and_wait(apply_patch_cmd);
}
}

65
compiler/rustc_codegen_cranelift/build_system/rustc_info.rs Normal file
View file

@ -0,0 +1,65 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
pub(crate) fn get_rustc_version() -> String {
let version_info =
Command::new("rustc").stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
String::from_utf8(version_info).unwrap()
}
pub(crate) fn get_host_triple() -> String {
let version_info =
Command::new("rustc").stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout;
String::from_utf8(version_info)
.unwrap()
.lines()
.to_owned()
.find(|line| line.starts_with("host"))
.unwrap()
.split(":")
.nth(1)
.unwrap()
.trim()
.to_owned()
}
pub(crate) fn get_rustc_path() -> PathBuf {
let rustc_path = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["which", "rustc"])
.output()
.unwrap()
.stdout;
Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
}
pub(crate) fn get_default_sysroot() -> PathBuf {
let default_sysroot = Command::new("rustc")
.stderr(Stdio::inherit())
.args(&["--print", "sysroot"])
.output()
.unwrap()
.stdout;
Path::new(String::from_utf8(default_sysroot).unwrap().trim()).to_owned()
}
pub(crate) fn get_file_name(crate_name: &str, crate_type: &str) -> String {
let file_name = Command::new("rustc")
.stderr(Stdio::inherit())
.args(&[
"--crate-name",
crate_name,
"--crate-type",
crate_type,
"--print",
"file-names",
"-",
])
.output()
.unwrap()
.stdout;
let file_name = String::from_utf8(file_name).unwrap().trim().to_owned();
assert!(!file_name.contains('\n'));
assert!(file_name.contains(crate_name));
file_name
}

35
compiler/rustc_codegen_cranelift/build_system/utils.rs Normal file
View file

@ -0,0 +1,35 @@
use std::fs;
use std::path::Path;
use std::process::{self, Command};
#[track_caller]
pub(crate) fn try_hard_link(src: impl AsRef<Path>, dst: impl AsRef<Path>) {
let src = src.as_ref();
let dst = dst.as_ref();
if let Err(_) = fs::hard_link(src, dst) {
fs::copy(src, dst).unwrap(); // Fallback to copying if hardlinking failed
}
}
#[track_caller]
pub(crate) fn spawn_and_wait(mut cmd: Command) {
if !cmd.spawn().unwrap().wait().unwrap().success() {
process::exit(1);
}
}
pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
for entry in fs::read_dir(from).unwrap() {
let entry = entry.unwrap();
let filename = entry.file_name();
if filename == "." || filename == ".." {
continue;
}
if entry.metadata().unwrap().is_dir() {
fs::create_dir(to.join(&filename)).unwrap();
copy_dir_recursively(&from.join(&filename), &to.join(&filename));
} else {
fs::copy(from.join(&filename), to.join(&filename)).unwrap();
}
}
}