bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Move download-ci-llvm to rustbuild

Browse source 
This attempts to keep the logic as close to the original python as possible.
`probably_large` has been removed, since it was always `True`, and UTF-8 paths are no longer supported when patching files for NixOS.
I can readd UTF-8 support if desired.

Note that this required making `llvm_link_shared` computed on-demand,
since we don't know whether it will be static or dynamic until we download LLVM from CI.
This commit is contained in:
Joshua Nelson 2022-03-21 08:59:34 -05:00
parent de1bc0008b
commit 93c1a941bb
9 changed files with 375 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Cargo.lock
View file

@ -225,8 +225,11 @@ dependencies = [
"pretty_assertions",
"serde",
"serde_json",
"tar",
"tempfile",
"toml",
"winapi",
"xz2",
]
[[package]]

3
src/bootstrap/Cargo.toml
View file

@ -42,10 +42,13 @@ cc = "1.0.69"
libc = "0.2"
serde = { version = "1.0.8", features = ["derive"] }
serde_json = "1.0.2"
tar = "0.4"
tempfile = "3"
toml = "0.5"
ignore = "0.4.10"
opener = "0.5"
once_cell = "1.7.2"
xz2 = "0.1"
[target.'cfg(windows)'.dependencies.winapi]
version = "0.3"

149
src/bootstrap/bootstrap.py
View file

@ -500,81 +500,6 @@ class RustBuild(object):
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.stage0_rustfmt.channel())
# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
# We want the most recent LLVM submodule update to avoid downloading
# LLVM more often than necessary.
#
# This git command finds that commit SHA, looking for bors-authored
# commits that modified src/llvm-project or other relevant version
# stamp files.
#
# This works even in a repository that has not yet initialized
# submodules.
top_level = subprocess.check_output([
"git", "rev-parse", "--show-toplevel",
]).decode(sys.getdefaultencoding()).strip()
llvm_sha = subprocess.check_output([
"git", "rev-list", "--author=bors@rust-lang.org", "-n1",
"--first-parent", "HEAD",
"--",
"{}/src/llvm-project".format(top_level),
"{}/src/bootstrap/download-ci-llvm-stamp".format(top_level),
# the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
"{}/src/version".format(top_level)
]).decode(sys.getdefaultencoding()).strip()
llvm_assertions = self.get_toml('assertions', 'llvm') == 'true'
llvm_root = self.llvm_root()
llvm_lib = os.path.join(llvm_root, "lib")
if self.program_out_of_date(self.llvm_stamp(), llvm_sha + str(llvm_assertions)):
self._download_ci_llvm(llvm_sha, llvm_assertions)
for binary in ["llvm-config", "FileCheck"]:
self.fix_bin_or_dylib(os.path.join(llvm_root, "bin", binary))
for lib in os.listdir(llvm_lib):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(llvm_lib, lib))
with output(self.llvm_stamp()) as llvm_stamp:
llvm_stamp.write(llvm_sha + str(llvm_assertions))
def downloading_llvm(self):
opt = self.get_toml('download-ci-llvm', 'llvm')
# This is currently all tier 1 targets and tier 2 targets with host tools
# (since others may not have CI artifacts)
# https://doc.rust-lang.org/rustc/platform-support.html#tier-1
supported_platforms = [
# tier 1
"aarch64-unknown-linux-gnu",
"i686-pc-windows-gnu",
"i686-pc-windows-msvc",
"i686-unknown-linux-gnu",
"x86_64-unknown-linux-gnu",
"x86_64-apple-darwin",
"x86_64-pc-windows-gnu",
"x86_64-pc-windows-msvc",
# tier 2 with host tools
"aarch64-apple-darwin",
"aarch64-pc-windows-msvc",
"aarch64-unknown-linux-musl",
"arm-unknown-linux-gnueabi",
"arm-unknown-linux-gnueabihf",
"armv7-unknown-linux-gnueabihf",
"mips-unknown-linux-gnu",
"mips64-unknown-linux-gnuabi64",
"mips64el-unknown-linux-gnuabi64",
"mipsel-unknown-linux-gnu",
"powerpc-unknown-linux-gnu",
"powerpc64-unknown-linux-gnu",
"powerpc64le-unknown-linux-gnu",
"riscv64gc-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"x86_64-unknown-freebsd",
"x86_64-unknown-illumos",
"x86_64-unknown-linux-musl",
"x86_64-unknown-netbsd",
]
return opt == "true" \
or (opt == "if-available" and self.build in supported_platforms)
def _download_component_helper(
self, filename, pattern, tarball_suffix, stage0=True, key=None
):
@ -606,53 +531,6 @@ class RustBuild(object):
)
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
if not llvm_sha:
print("error: could not find commit hash for downloading LLVM")
print("help: maybe your repository history is too shallow?")
print("help: consider disabling `download-ci-llvm`")
print("help: or fetch enough history to include one upstream commit")
exit(1)
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, cache_prefix)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
base = "https://ci-artifacts.rust-lang.org"
url = "rustc-builds/{}".format(llvm_sha)
if llvm_assertions:
url = url.replace('rustc-builds', 'rustc-builds-alt')
# ci-artifacts are only stored as .xz, not .gz
if not support_xz():
print("error: XZ support is required to download LLVM")
print("help: consider disabling `download-ci-llvm` or using python3")
exit(1)
tarball_suffix = '.tar.xz'
filename = "rust-dev-nightly-" + self.build + tarball_suffix
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
help_on_error = "error: failed to download llvm from ci"
help_on_error += "\nhelp: old builds get deleted after a certain time"
help_on_error += "\nhelp: if trying to compile an old commit of rustc,"
help_on_error += " disable `download-ci-llvm` in config.toml:"
help_on_error += "\n"
help_on_error += "\n[llvm]"
help_on_error += "\ndownload-ci-llvm = false"
help_on_error += "\n"
get(
base,
"{}/{}".format(url, filename),
tarball,
self.checksums_sha256,
verbose=self.verbose,
do_verify=False,
help_on_error=help_on_error,
)
unpack(tarball, tarball_suffix, self.llvm_root(),
match="rust-dev",
verbose=self.verbose)
def fix_bin_or_dylib(self, fname):
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
or the RPATH section, to fix the dynamic library search path
@ -816,17 +694,6 @@ class RustBuild(object):
"""
return os.path.join(self.bin_root(True), '.rustfmt-stamp')
def llvm_stamp(self):
"""Return the path for .llvm-stamp
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_stamp() == os.path.join("build", "ci-llvm", ".llvm-stamp")
True
"""
return os.path.join(self.llvm_root(), '.llvm-stamp')
def program_out_of_date(self, stamp_path, key):
"""Check if the given program stamp is out of date"""
if not os.path.exists(stamp_path) or self.clean:
@ -856,22 +723,6 @@ class RustBuild(object):
subdir = "ci-rustc"
return os.path.join(self.build_dir, self.build, subdir)
def llvm_root(self):
"""Return the CI LLVM root directory
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.llvm_root() == os.path.join("build", "ci-llvm")
True
When the 'build' property is given should be a nested directory:
>>> rb.build = "devel"
>>> rb.llvm_root() == os.path.join("build", "devel", "ci-llvm")
True
"""
return os.path.join(self.build_dir, self.build, "ci-llvm")
def get_toml(self, key, section=None):
"""Returns the value of the given key in config.toml, otherwise returns None

7
src/bootstrap/builder.rs
View file

@ -12,7 +12,6 @@ use std::process::Command;
use std::time::{Duration, Instant};
use crate::cache::{Cache, Interned, INTERNER};
use crate::check;
use crate::compile;
use crate::config::{SplitDebuginfo, TargetSelection};
use crate::dist;
@ -25,6 +24,7 @@ use crate::test;
use crate::tool::{self, SourceType};
use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t};
use crate::EXTRA_CHECK_CFGS;
use crate::{check, Config};
use crate::{Build, CLang, DocTests, GitRepo, Mode};
pub use crate::Compiler;
@ -960,6 +960,11 @@ impl<'a> Builder<'a> {
None
}
/// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`.
pub(crate) fn llvm_link_shared(&self) -> bool {
Config::llvm_link_shared(self)
}
/// Prepares an invocation of `cargo` to be run.
///
/// This will create a `Command` that represents a pending execution of

2
src/bootstrap/compile.rs
View file

@ -737,7 +737,7 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS
);
cargo.env("LLVM_STATIC_STDCPP", file);
}
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
cargo.env("LLVM_LINK_SHARED", "1");
}
if builder.config.llvm_use_libcxx {

70
src/bootstrap/config.rs
View file

@ -3,6 +3,7 @@
//! This module implements parsing `config.toml` configuration files to tweak
//! how the build runs.
use std::cell::Cell;
use std::cmp;
use std::collections::{HashMap, HashSet};
use std::env;
@ -11,7 +12,7 @@ use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use crate::builder::TaskPath;
use crate::builder::{Builder, TaskPath};
use crate::cache::{Interned, INTERNER};
use crate::channel::GitInfo;
pub use crate::flags::Subcommand;
@ -68,13 +69,14 @@ pub struct Config {
pub test_compare_mode: bool,
pub llvm_libunwind: LlvmLibunwind,
pub color: Color,
pub patch_binaries_for_nix: bool,
pub on_fail: Option<String>,
pub stage: u32,
pub keep_stage: Vec<u32>,
pub keep_stage_std: Vec<u32>,
pub src: PathBuf,
// defaults to `config.toml`
/// defaults to `config.toml`
pub config: PathBuf,
pub jobs: Option<u32>,
pub cmd: Subcommand,
@ -95,7 +97,11 @@ pub struct Config {
pub llvm_release_debuginfo: bool,
pub llvm_version_check: bool,
pub llvm_static_stdcpp: bool,
pub llvm_link_shared: bool,
/// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm.
#[cfg(not(test))]
llvm_link_shared: Cell<Option<bool>>,
#[cfg(test)]
pub llvm_link_shared: Cell<Option<bool>>,
pub llvm_clang_cl: Option<String>,
pub llvm_targets: Option<String>,
pub llvm_experimental_targets: Option<String>,
@ -856,6 +862,7 @@ impl Config {
set(&mut config.local_rebuild, build.local_rebuild);
set(&mut config.print_step_timings, build.print_step_timings);
set(&mut config.print_step_rusage, build.print_step_rusage);
set(&mut config.patch_binaries_for_nix, build.patch_binaries_for_nix);
config.verbose = cmp::max(config.verbose, flags.verbose);
@ -911,7 +918,9 @@ impl Config {
set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
set(&mut config.llvm_version_check, llvm.version_check);
set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
set(&mut config.llvm_link_shared, llvm.link_shared);
if let Some(v) = llvm.link_shared {
config.llvm_link_shared.set(Some(v));
}
config.llvm_targets = llvm.targets.clone();
config.llvm_experimental_targets = llvm.experimental_targets.clone();
config.llvm_link_jobs = llvm.link_jobs;
@ -981,6 +990,7 @@ impl Config {
check_ci_llvm!(llvm.optimize);
check_ci_llvm!(llvm.thin_lto);
check_ci_llvm!(llvm.release_debuginfo);
// CI-built LLVM can be either dynamic or static. We won't know until we download it.
check_ci_llvm!(llvm.link_shared);
check_ci_llvm!(llvm.static_libstdcpp);
check_ci_llvm!(llvm.targets);
@ -998,26 +1008,14 @@ impl Config {
check_ci_llvm!(llvm.clang);
check_ci_llvm!(llvm.build_config);
check_ci_llvm!(llvm.plugins);
// CI-built LLVM can be either dynamic or static.
let ci_llvm = config.out.join(&*config.build.triple).join("ci-llvm");
config.llvm_link_shared = if config.dry_run {
// just assume dynamic for now
true
} else {
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
);
link_type == "dynamic"
};
}
// NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above.
if config.llvm_thin_lto && llvm.link_shared.is_none() {
// If we're building with ThinLTO on, by default we want to link
// to LLVM shared, to avoid re-doing ThinLTO (which happens in
// the link step) with each stage.
config.llvm_link_shared = true;
config.llvm_link_shared.set(Some(true));
}
}
@ -1272,6 +1270,42 @@ impl Config {
}
}
/// The absolute path to the downloaded LLVM artifacts.
pub(crate) fn ci_llvm_root(&self) -> PathBuf {
assert!(self.llvm_from_ci);
self.out.join(&*self.build.triple).join("ci-llvm")
}
/// Determine whether llvm should be linked dynamically.
///
/// If `false`, llvm should be linked statically.
/// This is computed on demand since LLVM might have to first be downloaded from CI.
pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool {
let mut opt = builder.config.llvm_link_shared.get();
if opt.is_none() && builder.config.dry_run {
// just assume static for now - dynamic linking isn't supported on all platforms
return false;
}
let llvm_link_shared = *opt.get_or_insert_with(|| {
if builder.config.llvm_from_ci {
crate::native::maybe_download_ci_llvm(builder);
let ci_llvm = builder.config.ci_llvm_root();
let link_type = t!(
std::fs::read_to_string(ci_llvm.join("link-type.txt")),
format!("CI llvm missing: {}", ci_llvm.display())
);
link_type == "dynamic"
} else {
// unclear how thought-through this default is, but it maintains compatibility with
// previous behavior
false
}
});
builder.config.llvm_link_shared.set(opt);
llvm_link_shared
}
pub fn verbose(&self) -> bool {
self.verbose > 0
}

8
src/bootstrap/dist.rs
View file

@ -1904,7 +1904,7 @@ fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir
// clear why this is the case, though. llvm-config will emit the versioned
// paths and we don't want those in the sysroot (as we're expecting
// unversioned paths).
if target.contains("apple-darwin") && builder.config.llvm_link_shared {
if target.contains("apple-darwin") && builder.llvm_link_shared() {
let src_libdir = builder.llvm_out(target).join("lib");
let llvm_dylib_path = src_libdir.join("libLLVM.dylib");
if llvm_dylib_path.exists() {
@ -1939,7 +1939,7 @@ pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection,
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
maybe_install_llvm(builder, target, &dst_libdir);
}
}
@ -1951,7 +1951,7 @@ pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection
// We do not need to copy LLVM files into the sysroot if it is not
// dynamically linked; it is already included into librustc_llvm
// statically.
if builder.config.llvm_link_shared {
if builder.llvm_link_shared() {
maybe_install_llvm(builder, target, &dst_libdir);
}
}
@ -2077,7 +2077,7 @@ impl Step for RustDev {
// compiler libraries.
let dst_libdir = tarball.image_dir().join("lib");
maybe_install_llvm(builder, target, &dst_libdir);
let link_type = if builder.config.llvm_link_shared { "dynamic" } else { "static" };
let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" };
t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir);
Some(tarball.generate())

17
src/bootstrap/lib.rs
View file

@ -1391,6 +1391,23 @@ impl Build {
paths
}
pub fn rename(&self, src: &Path, dst: &Path) {
if self.config.dry_run {
return;
}
self.verbose_than(1, &format!("Move {:?} to {:?}", src, dst));
if src == dst {
return;
}
if let Err(e) = fs::rename(src, dst) {
if e.raw_os_error() == Some(libc::EXDEV) {
self.copy(src, dst);
return;
}
panic!("failed to rename `{}` to `{}`: {}", src.display(), dst.display(), e);
}
}
/// Copies a file from `src` to `dst`
pub fn copy(&self, src: &Path, dst: &Path) {
if self.config.dry_run {

293
src/bootstrap/native.rs
View file

@ -12,9 +12,12 @@ use std::env;
use std::env::consts::EXE_EXTENSION;
use std::ffi::{OsStr, OsString};
use std::fs::{self, File};
use std::io;
use std::io::{self, BufRead, BufReader, ErrorKind};
use std::path::{Path, PathBuf};
use std::process::Command;
use std::process::{Command, Stdio};
use once_cell::sync::OnceCell;
use xz2::bufread::XzDecoder;
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
use crate::config::TargetSelection;
@ -62,6 +65,8 @@ pub fn prebuilt_llvm_config(
builder: &Builder<'_>,
target: TargetSelection,
) -> Result<PathBuf, Meta> {
maybe_download_ci_llvm(builder);
// If we're using a custom LLVM bail out here, but we can only use a
// custom LLVM for the build triple.
if let Some(config) = builder.config.target_config.get(&target) {
@ -111,6 +116,286 @@ pub fn prebuilt_llvm_config(
Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() })
}
pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
let config = &builder.config;
if !config.llvm_from_ci {
return;
}
let mut rev_list = Command::new("git");
rev_list.args(&[
PathBuf::from("rev-list"),
"--author=bors@rust-lang.org".into(),
"-n1".into(),
"--first-parent".into(),
"HEAD".into(),
"--".into(),
builder.src.join("src/llvm-project"),
builder.src.join("src/bootstrap/download-ci-llvm-stamp"),
// the LLVM shared object file is named `LLVM-12-rust-{version}-nightly`
builder.src.join("src/version"),
]);
let llvm_sha = output(&mut rev_list);
let llvm_sha = llvm_sha.trim();
if llvm_sha == "" {
println!("error: could not find commit hash for downloading LLVM");
println!("help: maybe your repository history is too shallow?");
println!("help: consider disabling `download-ci-llvm`");
println!("help: or fetch enough history to include one upstream commit");
panic!();
}
let llvm_root = config.ci_llvm_root();
let llvm_stamp = llvm_root.join(".llvm-stamp");
let key = format!("{}{}", llvm_sha, config.llvm_assertions);
if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
download_ci_llvm(builder, &llvm_sha);
for binary in ["llvm-config", "FileCheck"] {
fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary));
}
let llvm_lib = llvm_root.join("lib");
for entry in t!(fs::read_dir(&llvm_lib)) {
let lib = t!(entry).path();
if lib.ends_with(".so") {
fix_bin_or_dylib(builder, &lib);
}
}
t!(fs::write(llvm_stamp, key));
}
}
fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
let llvm_assertions = builder.config.llvm_assertions;
let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions);
let cache_dst = builder.out.join("cache");
let rustc_cache = cache_dst.join(cache_prefix);
if !rustc_cache.exists() {
t!(fs::create_dir_all(&rustc_cache));
}
let base = "https://ci-artifacts.rust-lang.org";
let url = if llvm_assertions {
format!("rustc-builds-alt/{}", llvm_sha)
} else {
format!("rustc-builds/{}", llvm_sha)
};
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
let tarball = rustc_cache.join(&filename);
if !tarball.exists() {
download_component(builder, base, &format!("{}/{}", url, filename), &tarball);
}
let llvm_root = builder.config.ci_llvm_root();
unpack(builder, &tarball, &llvm_root);
}
/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
/// or the RPATH section, to fix the dynamic library search path
///
/// This is only required on NixOS and uses the PatchELF utility to
/// change the interpreter/RPATH of ELF executables.
///
/// Please see https://nixos.org/patchelf.html for more information
fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) {
// FIXME: cache NixOS detection?
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
Err(_) => return,
Ok(output) if !output.status.success() => return,
Ok(output) => {
let mut s = output.stdout;
if s.last() == Some(&b'\n') {
s.pop();
}
if s != b"Linux" {
return;
}
}
}
// If the user has asked binaries to be patched for Nix, then
// don't check for NixOS or `/lib`, just continue to the patching.
// FIXME: shouldn't this take precedence over the `uname` check above?
if !builder.config.patch_binaries_for_nix {
// Use `/etc/os-release` instead of `/etc/NIXOS`.
// The latter one does not exist on NixOS when using tmpfs as root.
let os_release = match File::open("/etc/os-release") {
Err(e) if e.kind() == ErrorKind::NotFound => return,
Err(e) => panic!("failed to access /etc/os-release: {}", e),
Ok(f) => f,
};
if !BufReader::new(os_release).lines().any(|l| t!(l).trim() == "ID=nixos") {
return;
}
if Path::new("/lib").exists() {
return;
}
}
// At this point we're pretty sure the user is running NixOS or using Nix
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
// Only build `.nix-deps` once.
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
let mut nix_build_succeeded = true;
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
// Run `nix-build` to "build" each dependency (which will likely reuse
// the existing `/nix/store` copy, or at most download a pre-built copy).
//
// Importantly, we create a gc-root called `.nix-deps` in the `build/`
// directory, but still reference the actual `/nix/store` path in the rpath
// as it makes it significantly more robust against changes to the location of
// the `.nix-deps` location.
//
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
// zlib: Needed as a system dependency of `libLLVM-*.so`.
// patchelf: Needed for patching ELF binaries (see doc comment above).
let nix_deps_dir = builder.out.join(".nix-deps");
const NIX_EXPR: &str = "
with (import <nixpkgs> {});
symlinkJoin {
name = \"rust-stage0-dependencies\";
paths = [
zlib
patchelf
stdenv.cc.bintools
];
}
";
nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[
Path::new("-E"),
Path::new(NIX_EXPR),
Path::new("-o"),
&nix_deps_dir,
]));
nix_deps_dir
});
if !nix_build_succeeded {
return;
}
let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
let rpath_entries = {
// ORIGIN is a relative default, all binary and dynamic libraries we ship
// appear to have this (even when `../lib` is redundant).
// NOTE: there are only two paths here, delimited by a `:`
let mut entries = OsString::from("$ORIGIN/../lib:");
entries.push(t!(fs::canonicalize(nix_deps_dir)));
entries.push("/lib");
entries
};
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
if !fname.ends_with(".so") {
// Finally, set the corret .interp for binaries
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
}
builder.try_run(patchelf.arg(fname));
}
fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) {
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
let tempfile = t!(tempfile::NamedTempFile::new());
let temppath = tempfile.path().to_owned();
drop(tempfile);
let tempfile_str = temppath.to_str().expect("tempdir must be valid unicode");
// FIXME: support `do_verify` (only really needed for nightly rustfmt)
download_with_retries(builder, tempfile_str, &format!("{}/{}", base, url));
builder.rename(&temppath, dest_path);
}
fn download_with_retries(builder: &Builder<'_>, tempdir: &str, url: &str) {
println!("downloading {}", url);
// FIXME: check if curl is installed instead of skipping straight to powershell
if builder.build.build.contains("windows-msvc") {
for _ in 0..3 {
if builder.try_run(Command::new("PowerShell.exe").args(&[
"/nologo",
"-Command",
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
&format!(
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
url, tempdir
),
])) {
return;
}
println!("\nspurious failure, trying again");
}
} else {
builder.run(Command::new("curl").args(&[
"-#",
"-y",
"30",
"-Y",
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
"--connect-timeout",
"30", // timeout if cannot connect within 30 seconds
"--retry",
"3",
"-Sf",
"-o",
tempdir,
url,
]));
}
}
fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) {
println!("extracting {} to {}", tarball.display(), dst.display());
if !dst.exists() {
t!(fs::create_dir_all(dst));
}
// FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild
const MATCH: &str = "rust-dev";
// `tarball` ends with `.tar.xz`; strip that suffix
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
let uncompressed_filename =
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
// decompress the file
let data = t!(File::open(tarball));
let decompressor = XzDecoder::new(BufReader::new(data));
let mut tar = tar::Archive::new(decompressor);
for member in t!(tar.entries()) {
let mut member = t!(member);
let original_path = t!(member.path()).into_owned();
// skip the top-level directory
if original_path == directory_prefix {
continue;
}
let mut short_path = t!(original_path.strip_prefix(directory_prefix));
if !short_path.starts_with(MATCH) {
continue;
}
short_path = t!(short_path.strip_prefix(MATCH));
let dst_path = dst.join(short_path);
builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
if !t!(member.unpack_in(dst)) {
panic!("path traversal attack ??");
}
let src_path = dst.join(original_path);
if src_path.is_dir() && dst_path.exists() {
continue;
}
t!(fs::rename(src_path, dst_path));
}
t!(fs::remove_dir_all(dst.join(directory_prefix)));
}
fn program_out_of_date(stamp: &Path, key: &str) -> bool {
if !stamp.exists() {
return true;
}
t!(fs::read_to_string(stamp)) != key
}
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Llvm {
pub target: TargetSelection,
@ -153,7 +438,7 @@ impl Step for Llvm {
};
builder.update_submodule(&Path::new("src").join("llvm-project"));
if builder.config.llvm_link_shared
if builder.llvm_link_shared()
&& (target.contains("windows") || target.contains("apple-darwin"))
{
panic!("shared linking to LLVM is not currently supported on {}", target.triple);
@ -255,7 +540,7 @@ impl Step for Llvm {
//
// If we're not linking rustc to a dynamic LLVM, though, then don't link
// tools to it.
if builder.llvm_link_tools_dynamically(target) && builder.config.llvm_link_shared {
if builder.llvm_link_tools_dynamically(target) && builder.llvm_link_shared() {
cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
}