bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Compile run-make recipes using the stage0 compiler

Browse source
This commit is contained in:
Jakub Beránek 2025-02-24 09:52:53 +01:00
parent 4af94f842d
commit 0034d6c928
11 changed files with 78 additions and 123 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/bootstrap/src/core/build_steps/test.rs
View file

@ -1679,6 +1679,11 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
};
cmd.arg("--cargo-path").arg(cargo_path);
// We need to pass the compiler that was used to compile run-make-support,
// because we have to use the same compiler to compile rmake.rs recipes.
let stage0_rustc_path = builder.compiler(0, compiler.host);
cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path));
}
// Avoid depending on rustdoc when we don't need it.

3
src/tools/compiletest/src/common.rs
View file

@ -190,6 +190,9 @@ pub struct Config {
/// The cargo executable.
pub cargo_path: Option<PathBuf>,
/// Rustc executable used to compile run-make recipes.
pub stage0_rustc_path: Option<PathBuf>,
/// The rustdoc executable.
pub rustdoc_path: Option<PathBuf>,

7
src/tools/compiletest/src/lib.rs
View file

@ -54,6 +54,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
.reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
.reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
.optopt("", "cargo-path", "path to cargo to use for compiling", "PATH")
.optopt(
"",
"stage0-rustc-path",
"path to rustc to use for compiling run-make recipes",
"PATH",
)
.optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
.optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
.reqopt("", "python", "path to python to use for doc tests", "PATH")
@ -320,6 +326,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
rustc_path: opt_path(matches, "rustc-path"),
cargo_path: matches.opt_str("cargo-path").map(PathBuf::from),
stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(PathBuf::from),
rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
python: matches.opt_str("python").unwrap(),

119
src/tools/compiletest/src/runtest/run_make.rs
View file

@ -173,7 +173,8 @@ impl TestCx<'_> {
fn run_rmake_v2_test(&self) {
// For `run-make` V2, we need to perform 2 steps to build and run a `run-make` V2 recipe
// (`rmake.rs`) to run the actual tests. The support library is already built as a tool rust
// library and is available under `build/$TARGET/stageN-tools-bin/librun_make_support.rlib`.
// library and is available under
// `build/$HOST/stage0-bootstrap-tools/$TARGET/release/librun_make_support.rlib`.
//
// 1. We need to build the recipe `rmake.rs` as a binary and link in the `run_make_support`
// library.
@ -224,25 +225,21 @@ impl TestCx<'_> {
//
// ```
// build/<target_triple>/
// ├── stageN-tools-bin/
// │ └── librun_make_support.rlib // <- support rlib itself
// ├── stageN-tools/
// │ ├── release/deps/ // <- deps of deps
// │ └── <host_triple>/release/deps/ // <- deps
// ├── stage0-bootstrap-tools/
// │ ├── <host_triple>/release/librun_make_support.rlib // <- support rlib itself
// │ ├── <host_triple>/release/deps/ // <- deps
// │ └── release/deps/ // <- deps of deps
// ```
//
// FIXME(jieyouxu): there almost certainly is a better way to do this (specifically how the
// support lib and its deps are organized, can't we copy them to the tools-bin dir as
// well?), but this seems to work for now.
// support lib and its deps are organized), but this seems to work for now.
let stage_number = self.config.stage;
let tools_bin = host_build_root.join("stage0-bootstrap-tools");
let support_host_path = tools_bin.join(&self.config.host).join("release");
let support_lib_path = support_host_path.join("librun_make_support.rlib");
let stage_tools_bin = host_build_root.join(format!("stage{stage_number}-tools-bin"));
let support_lib_path = stage_tools_bin.join("librun_make_support.rlib");
let stage_tools = host_build_root.join(format!("stage{stage_number}-tools"));
let support_lib_deps = stage_tools.join(&self.config.host).join("release").join("deps");
let support_lib_deps_deps = stage_tools.join("release").join("deps");
let support_lib_deps = support_host_path.join("deps");
let support_lib_deps_deps = tools_bin.join("release").join("deps");
// To compile the recipe with rustc, we need to provide suitable dynamic library search
// paths to rustc. This includes both:
@ -253,12 +250,6 @@ impl TestCx<'_> {
let base_dylib_search_paths =
Vec::from_iter(env::split_paths(&env::var(dylib_env_var()).unwrap()));
let host_dylib_search_paths = {
let mut paths = vec![self.config.compile_lib_path.clone()];
paths.extend(base_dylib_search_paths.iter().cloned());
paths
};
// Calculate the paths of the recipe binary. As previously discussed, this is placed at
// `<base_dir>/<bin_name>` with `bin_name` being `rmake` or `rmake.exe` depending on
// platform.
@ -268,7 +259,15 @@ impl TestCx<'_> {
p
};
let mut rustc = Command::new(&self.config.rustc_path);
// run-make-support and run-make tests are compiled using the stage0 compiler
// If the stage is 0, then the compiler that we test (either bootstrap or an explicitly
// set compiler) is the one that actually compiled run-make-support.
let stage0_rustc = self
.config
.stage0_rustc_path
.as_ref()
.expect("stage0 rustc is required to run run-make tests");
let mut rustc = Command::new(&stage0_rustc);
rustc
.arg("-o")
.arg(&recipe_bin)
@ -282,35 +281,12 @@ impl TestCx<'_> {
.arg(format!("run_make_support={}", &support_lib_path.to_string_lossy()))
.arg("--edition=2021")
.arg(&self.testpaths.file.join("rmake.rs"))
.arg("-Cprefer-dynamic")
// Provide necessary library search paths for rustc.
.env(dylib_env_var(), &env::join_paths(host_dylib_search_paths).unwrap());
.arg("-Cprefer-dynamic");
// In test code we want to be very pedantic about values being silently discarded that are
// annotated with `#[must_use]`.
rustc.arg("-Dunused_must_use");
// > `cg_clif` uses `COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0` for running the rustc
// > test suite. With the introduction of rmake.rs this broke. `librun_make_support.rlib` is
// > compiled using the bootstrap rustc wrapper which sets `--sysroot
// > build/aarch64-unknown-linux-gnu/stage0-sysroot`, but then compiletest will compile
// > `rmake.rs` using the sysroot of the bootstrap compiler causing it to not find the
// > `libstd.rlib` against which `librun_make_support.rlib` is compiled.
//
// The gist here is that we have to pass the proper stage0 sysroot if we want
//
// ```
// $ COMPILETEST_FORCE_STAGE0=1 ./x test run-make --stage 0
// ```
//
// to work correctly.
//
// See <https://github.com/rust-lang/rust/pull/122248> for more background.
let stage0_sysroot = host_build_root.join("stage0-sysroot");
if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() {
rustc.arg("--sysroot").arg(&stage0_sysroot);
}
// Now run rustc to build the recipe.
let res = self.run_command_to_procres(&mut rustc);
if !res.status.success() {
@ -320,35 +296,24 @@ impl TestCx<'_> {
// To actually run the recipe, we have to provide the recipe with a bunch of information
// provided through env vars.
// Compute stage-specific standard library paths.
let stage_std_path = host_build_root.join(format!("stage{stage_number}")).join("lib");
// Compute dynamic library search paths for recipes.
// These dylib directories are needed to **execute the recipe**.
let recipe_dylib_search_paths = {
let mut paths = base_dylib_search_paths.clone();
// For stage 0, we need to explicitly include the stage0-sysroot libstd dylib.
// See <https://github.com/rust-lang/rust/issues/135373>.
if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() {
paths.push(
stage0_sysroot.join("lib").join("rustlib").join(&self.config.host).join("lib"),
);
}
paths.push(support_lib_path.parent().unwrap().to_path_buf());
paths.push(stage_std_path.join("rustlib").join(&self.config.host).join("lib"));
paths.push(
stage0_rustc
.parent()
.unwrap()
.parent()
.unwrap()
.join("lib")
.join("rustlib")
.join(&self.config.host)
.join("lib"),
);
paths
};
// Compute runtime library search paths for recipes. This is target-specific.
let target_runtime_dylib_search_paths = {
let mut paths = vec![rmake_out_dir.clone()];
paths.extend(base_dylib_search_paths.iter().cloned());
paths
};
// FIXME(jieyouxu): please rename `TARGET_RPATH_ENV`, `HOST_RPATH_DIR` and
// `TARGET_RPATH_DIR`, it is **extremely** confusing!
let mut cmd = Command::new(&recipe_bin);
cmd.current_dir(&rmake_out_dir)
.stdout(Stdio::piped())
@ -357,9 +322,14 @@ impl TestCx<'_> {
// example, this could be `LD_LIBRARY_PATH` on some linux distros but `PATH` on Windows.
.env("LD_LIB_PATH_ENVVAR", dylib_env_var())
// Provide the dylib search paths.
// This is required to run the **recipe** itself.
.env(dylib_env_var(), &env::join_paths(recipe_dylib_search_paths).unwrap())
// Provide runtime dylib search paths.
.env("TARGET_RPATH_ENV", &env::join_paths(target_runtime_dylib_search_paths).unwrap())
// Provide the directory to libraries that are needed to run the *compiler* invoked
// by the recipe.
.env("HOST_RUSTC_DYLIB_PATH", &self.config.compile_lib_path)
// Provide the directory to libraries that might be needed to run binaries created
// by a compiler invoked by the recipe.
.env("TARGET_EXE_DYLIB_PATH", &self.config.run_lib_path)
// Provide the target.
.env("TARGET", &self.config.target)
// Some tests unfortunately still need Python, so provide path to a Python interpreter.
@ -370,13 +340,6 @@ impl TestCx<'_> {
.env("BUILD_ROOT", &host_build_root)
// Provide path to stage-corresponding rustc.
.env("RUSTC", &self.config.rustc_path)
// Provide the directory to libraries that are needed to run the *compiler*. This is not
// to be confused with `TARGET_RPATH_ENV` or `TARGET_RPATH_DIR`. This is needed if the
// recipe wants to invoke rustc.
.env("HOST_RPATH_DIR", &self.config.compile_lib_path)
// Provide the directory to libraries that might be needed to run compiled binaries
// (further compiled by the recipe!).
.env("TARGET_RPATH_DIR", &self.config.run_lib_path)
// Provide which LLVM components are available (e.g. which LLVM components are provided
// through a specific CI runner).
.env("LLVM_COMPONENTS", &self.config.llvm_components);

8
src/tools/run-make-support/src/external_deps/rustc.rs
View file

@ -5,7 +5,7 @@ use std::str::FromStr as _;
use crate::command::Command;
use crate::env::env_var;
use crate::path_helpers::cwd;
use crate::util::set_host_rpath;
use crate::util::set_host_compiler_dylib_path;
use crate::{is_aix, is_darwin, is_msvc, is_windows, uname};
/// Construct a new `rustc` invocation. This will automatically set the library
@ -15,8 +15,8 @@ pub fn rustc() -> Rustc {
Rustc::new()
}
/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_rpath`]
/// still presets the environment variable `HOST_RPATH_DIR` by default.
/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_compiler_dylib_path`]
/// still presets the environment variable `HOST_RUSTC_DYLIB_PATH` by default.
#[track_caller]
pub fn bare_rustc() -> Rustc {
Rustc::bare()
@ -44,7 +44,7 @@ pub fn rustc_path() -> String {
#[track_caller]
fn setup_common() -> Command {
let mut cmd = Command::new(rustc_path());
set_host_rpath(&mut cmd);
set_host_compiler_dylib_path(&mut cmd);
cmd
}

24
src/tools/run-make-support/src/external_deps/rustdoc.rs
View file

@ -2,16 +2,10 @@ use std::ffi::OsStr;
use std::path::Path;
use crate::command::Command;
use crate::env::{env_var, env_var_os};
use crate::util::set_host_rpath;
use crate::env::env_var;
use crate::util::set_host_compiler_dylib_path;
/// Construct a plain `rustdoc` invocation with no flags set.
#[track_caller]
pub fn bare_rustdoc() -> Rustdoc {
Rustdoc::bare()
}
/// Construct a new `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set.
/// Construct a new `rustdoc` invocation.
#[track_caller]
pub fn rustdoc() -> Rustdoc {
Rustdoc::new()
@ -29,23 +23,15 @@ crate::macros::impl_common_helpers!(Rustdoc);
fn setup_common() -> Command {
let rustdoc = env_var("RUSTDOC");
let mut cmd = Command::new(rustdoc);
set_host_rpath(&mut cmd);
set_host_compiler_dylib_path(&mut cmd);
cmd
}
impl Rustdoc {
/// Construct a bare `rustdoc` invocation.
#[track_caller]
pub fn bare() -> Self {
let cmd = setup_common();
Self { cmd }
}
/// Construct a `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set.
#[track_caller]
pub fn new() -> Self {
let mut cmd = setup_common();
cmd.arg("-L").arg(env_var_os("TARGET_RPATH_DIR"));
let cmd = setup_common();
Self { cmd }
}

2
src/tools/run-make-support/src/lib.rs
View file

@ -67,7 +67,7 @@ pub use llvm::{
};
pub use python::python_command;
pub use rustc::{aux_build, bare_rustc, rustc, rustc_path, Rustc};
pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc};
pub use rustdoc::{rustdoc, Rustdoc};
/// [`diff`][mod@diff] is implemented in terms of the [similar] library.
///

19
src/tools/run-make-support/src/run.rs
View file

@ -1,10 +1,10 @@
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::{env, panic};
use crate::command::{Command, CompletedProcess};
use crate::util::{handle_failed_output, set_host_rpath};
use crate::{cwd, env_var, is_windows};
use crate::util::handle_failed_output;
use crate::{cwd, env_var};
#[track_caller]
fn run_common(name: &str, args: Option<&[&str]>) -> Command {
@ -18,10 +18,11 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command {
cmd.arg(arg);
}
}
cmd.env(&ld_lib_path_envvar, {
let mut paths = vec![];
paths.push(cwd());
for p in env::split_paths(&env_var("TARGET_RPATH_ENV")) {
for p in env::split_paths(&env_var("TARGET_EXE_DYLIB_PATH")) {
paths.push(p.to_path_buf());
}
for p in env::split_paths(&env_var(&ld_lib_path_envvar)) {
@ -31,15 +32,6 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command {
});
cmd.env("LC_ALL", "C"); // force english locale
if is_windows() {
let mut paths = vec![];
for p in env::split_paths(&std::env::var("PATH").unwrap_or(String::new())) {
paths.push(p.to_path_buf());
}
paths.push(Path::new(&env_var("TARGET_RPATH_DIR")).to_path_buf());
cmd.env("PATH", env::join_paths(paths.iter()).unwrap());
}
cmd
}
@ -84,7 +76,6 @@ pub fn run_fail(name: &str) -> CompletedProcess {
#[track_caller]
pub fn cmd<S: AsRef<OsStr>>(program: S) -> Command {
let mut command = Command::new(program);
set_host_rpath(&mut command);
command.env("LC_ALL", "C"); // force english locale
command
}

6
src/tools/run-make-support/src/util.rs
View file

@ -24,13 +24,13 @@ pub(crate) fn handle_failed_output(
std::process::exit(1)
}
/// Set the runtime library path as needed for running the host rustc/rustdoc/etc.
pub(crate) fn set_host_rpath(cmd: &mut Command) {
/// Set the runtime library paths as needed for running the host compilers (rustc/rustdoc/etc).
pub(crate) fn set_host_compiler_dylib_path(cmd: &mut Command) {
let ld_lib_path_envvar = env_var("LD_LIB_PATH_ENVVAR");
cmd.env(&ld_lib_path_envvar, {
let mut paths = vec![];
paths.push(cwd());
paths.push(PathBuf::from(env_var("HOST_RPATH_DIR")));
paths.push(PathBuf::from(env_var("HOST_RUSTC_DYLIB_PATH")));
for p in std::env::split_paths(&env_var(&ld_lib_path_envvar)) {
paths.push(p.to_path_buf());
}

4
tests/run-make/rustdoc-default-output/rmake.rs
View file

@ -3,10 +3,10 @@
// ensures the output of rustdoc's help menu is as expected.
// See https://github.com/rust-lang/rust/issues/88756
use run_make_support::{bare_rustdoc, diff};
use run_make_support::{diff, rustdoc};
fn main() {
let out = bare_rustdoc().run().stdout_utf8();
let out = rustdoc().run().stdout_utf8();
diff()
.expected_file("output-default.stdout")
.actual_text("actual", out)

4
tests/run-make/version-verbose-commit-hash/rmake.rs
View file

@ -5,13 +5,13 @@
//@ needs-git-hash
use run_make_support::{bare_rustc, bare_rustdoc, regex};
use run_make_support::{bare_rustc, regex, rustdoc};
fn main() {
let out_rustc =
bare_rustc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase();
let out_rustdoc =
bare_rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase();
rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase();
let re =
regex::Regex::new(r#"commit-hash: [0-9a-f]{40}\ncommit-date: [0-9]{4}-[0-9]{2}-[0-9]{2}"#)
.unwrap();