diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 79be42cdd2f..99ab876ccfa 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1679,6 +1679,11 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the }; cmd.arg("--cargo-path").arg(cargo_path); + + // We need to pass the compiler that was used to compile run-make-support, + // because we have to use the same compiler to compile rmake.rs recipes. + let stage0_rustc_path = builder.compiler(0, compiler.host); + cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path)); } // Avoid depending on rustdoc when we don't need it. diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index b302c6a49f5..978836cb663 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -190,6 +190,9 @@ pub struct Config { /// The cargo executable. pub cargo_path: Option, + /// Rustc executable used to compile run-make recipes. + pub stage0_rustc_path: Option, + /// The rustdoc executable. pub rustdoc_path: Option, diff --git a/src/tools/compiletest/src/lib.rs b/src/tools/compiletest/src/lib.rs index 9dff7047bc4..e93f32c86bc 100644 --- a/src/tools/compiletest/src/lib.rs +++ b/src/tools/compiletest/src/lib.rs @@ -54,6 +54,12 @@ pub fn parse_config(args: Vec) -> Config { .reqopt("", "run-lib-path", "path to target shared libraries", "PATH") .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH") .optopt("", "cargo-path", "path to cargo to use for compiling", "PATH") + .optopt( + "", + "stage0-rustc-path", + "path to rustc to use for compiling run-make recipes", + "PATH", + ) .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH") .optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH") .reqopt("", "python", "path to python to use for doc tests", "PATH") @@ -320,6 +326,7 @@ pub fn parse_config(args: Vec) -> Config { run_lib_path: make_absolute(opt_path(matches, "run-lib-path")), rustc_path: opt_path(matches, "rustc-path"), cargo_path: matches.opt_str("cargo-path").map(PathBuf::from), + stage0_rustc_path: matches.opt_str("stage0-rustc-path").map(PathBuf::from), rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from), coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from), python: matches.opt_str("python").unwrap(), diff --git a/src/tools/compiletest/src/runtest/run_make.rs b/src/tools/compiletest/src/runtest/run_make.rs index 9bb3993223e..07f4d7c4942 100644 --- a/src/tools/compiletest/src/runtest/run_make.rs +++ b/src/tools/compiletest/src/runtest/run_make.rs @@ -173,7 +173,8 @@ impl TestCx<'_> { fn run_rmake_v2_test(&self) { // For `run-make` V2, we need to perform 2 steps to build and run a `run-make` V2 recipe // (`rmake.rs`) to run the actual tests. The support library is already built as a tool rust - // library and is available under `build/$TARGET/stageN-tools-bin/librun_make_support.rlib`. + // library and is available under + // `build/$HOST/stage0-bootstrap-tools/$TARGET/release/librun_make_support.rlib`. // // 1. We need to build the recipe `rmake.rs` as a binary and link in the `run_make_support` // library. @@ -224,25 +225,21 @@ impl TestCx<'_> { // // ``` // build// - // ├── stageN-tools-bin/ - // │ └── librun_make_support.rlib // <- support rlib itself - // ├── stageN-tools/ - // │ ├── release/deps/ // <- deps of deps - // │ └── /release/deps/ // <- deps + // ├── stage0-bootstrap-tools/ + // │ ├── /release/librun_make_support.rlib // <- support rlib itself + // │ ├── /release/deps/ // <- deps + // │ └── release/deps/ // <- deps of deps // ``` // // FIXME(jieyouxu): there almost certainly is a better way to do this (specifically how the - // support lib and its deps are organized, can't we copy them to the tools-bin dir as - // well?), but this seems to work for now. + // support lib and its deps are organized), but this seems to work for now. - let stage_number = self.config.stage; + let tools_bin = host_build_root.join("stage0-bootstrap-tools"); + let support_host_path = tools_bin.join(&self.config.host).join("release"); + let support_lib_path = support_host_path.join("librun_make_support.rlib"); - let stage_tools_bin = host_build_root.join(format!("stage{stage_number}-tools-bin")); - let support_lib_path = stage_tools_bin.join("librun_make_support.rlib"); - - let stage_tools = host_build_root.join(format!("stage{stage_number}-tools")); - let support_lib_deps = stage_tools.join(&self.config.host).join("release").join("deps"); - let support_lib_deps_deps = stage_tools.join("release").join("deps"); + let support_lib_deps = support_host_path.join("deps"); + let support_lib_deps_deps = tools_bin.join("release").join("deps"); // To compile the recipe with rustc, we need to provide suitable dynamic library search // paths to rustc. This includes both: @@ -253,12 +250,6 @@ impl TestCx<'_> { let base_dylib_search_paths = Vec::from_iter(env::split_paths(&env::var(dylib_env_var()).unwrap())); - let host_dylib_search_paths = { - let mut paths = vec![self.config.compile_lib_path.clone()]; - paths.extend(base_dylib_search_paths.iter().cloned()); - paths - }; - // Calculate the paths of the recipe binary. As previously discussed, this is placed at // `/` with `bin_name` being `rmake` or `rmake.exe` depending on // platform. @@ -268,7 +259,15 @@ impl TestCx<'_> { p }; - let mut rustc = Command::new(&self.config.rustc_path); + // run-make-support and run-make tests are compiled using the stage0 compiler + // If the stage is 0, then the compiler that we test (either bootstrap or an explicitly + // set compiler) is the one that actually compiled run-make-support. + let stage0_rustc = self + .config + .stage0_rustc_path + .as_ref() + .expect("stage0 rustc is required to run run-make tests"); + let mut rustc = Command::new(&stage0_rustc); rustc .arg("-o") .arg(&recipe_bin) @@ -282,35 +281,12 @@ impl TestCx<'_> { .arg(format!("run_make_support={}", &support_lib_path.to_string_lossy())) .arg("--edition=2021") .arg(&self.testpaths.file.join("rmake.rs")) - .arg("-Cprefer-dynamic") - // Provide necessary library search paths for rustc. - .env(dylib_env_var(), &env::join_paths(host_dylib_search_paths).unwrap()); + .arg("-Cprefer-dynamic"); // In test code we want to be very pedantic about values being silently discarded that are // annotated with `#[must_use]`. rustc.arg("-Dunused_must_use"); - // > `cg_clif` uses `COMPILETEST_FORCE_STAGE0=1 ./x.py test --stage 0` for running the rustc - // > test suite. With the introduction of rmake.rs this broke. `librun_make_support.rlib` is - // > compiled using the bootstrap rustc wrapper which sets `--sysroot - // > build/aarch64-unknown-linux-gnu/stage0-sysroot`, but then compiletest will compile - // > `rmake.rs` using the sysroot of the bootstrap compiler causing it to not find the - // > `libstd.rlib` against which `librun_make_support.rlib` is compiled. - // - // The gist here is that we have to pass the proper stage0 sysroot if we want - // - // ``` - // $ COMPILETEST_FORCE_STAGE0=1 ./x test run-make --stage 0 - // ``` - // - // to work correctly. - // - // See for more background. - let stage0_sysroot = host_build_root.join("stage0-sysroot"); - if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { - rustc.arg("--sysroot").arg(&stage0_sysroot); - } - // Now run rustc to build the recipe. let res = self.run_command_to_procres(&mut rustc); if !res.status.success() { @@ -320,35 +296,24 @@ impl TestCx<'_> { // To actually run the recipe, we have to provide the recipe with a bunch of information // provided through env vars. - // Compute stage-specific standard library paths. - let stage_std_path = host_build_root.join(format!("stage{stage_number}")).join("lib"); - // Compute dynamic library search paths for recipes. + // These dylib directories are needed to **execute the recipe**. let recipe_dylib_search_paths = { let mut paths = base_dylib_search_paths.clone(); - - // For stage 0, we need to explicitly include the stage0-sysroot libstd dylib. - // See . - if std::env::var_os("COMPILETEST_FORCE_STAGE0").is_some() { - paths.push( - stage0_sysroot.join("lib").join("rustlib").join(&self.config.host).join("lib"), - ); - } - - paths.push(support_lib_path.parent().unwrap().to_path_buf()); - paths.push(stage_std_path.join("rustlib").join(&self.config.host).join("lib")); + paths.push( + stage0_rustc + .parent() + .unwrap() + .parent() + .unwrap() + .join("lib") + .join("rustlib") + .join(&self.config.host) + .join("lib"), + ); paths }; - // Compute runtime library search paths for recipes. This is target-specific. - let target_runtime_dylib_search_paths = { - let mut paths = vec![rmake_out_dir.clone()]; - paths.extend(base_dylib_search_paths.iter().cloned()); - paths - }; - - // FIXME(jieyouxu): please rename `TARGET_RPATH_ENV`, `HOST_RPATH_DIR` and - // `TARGET_RPATH_DIR`, it is **extremely** confusing! let mut cmd = Command::new(&recipe_bin); cmd.current_dir(&rmake_out_dir) .stdout(Stdio::piped()) @@ -357,9 +322,14 @@ impl TestCx<'_> { // example, this could be `LD_LIBRARY_PATH` on some linux distros but `PATH` on Windows. .env("LD_LIB_PATH_ENVVAR", dylib_env_var()) // Provide the dylib search paths. + // This is required to run the **recipe** itself. .env(dylib_env_var(), &env::join_paths(recipe_dylib_search_paths).unwrap()) - // Provide runtime dylib search paths. - .env("TARGET_RPATH_ENV", &env::join_paths(target_runtime_dylib_search_paths).unwrap()) + // Provide the directory to libraries that are needed to run the *compiler* invoked + // by the recipe. + .env("HOST_RUSTC_DYLIB_PATH", &self.config.compile_lib_path) + // Provide the directory to libraries that might be needed to run binaries created + // by a compiler invoked by the recipe. + .env("TARGET_EXE_DYLIB_PATH", &self.config.run_lib_path) // Provide the target. .env("TARGET", &self.config.target) // Some tests unfortunately still need Python, so provide path to a Python interpreter. @@ -370,13 +340,6 @@ impl TestCx<'_> { .env("BUILD_ROOT", &host_build_root) // Provide path to stage-corresponding rustc. .env("RUSTC", &self.config.rustc_path) - // Provide the directory to libraries that are needed to run the *compiler*. This is not - // to be confused with `TARGET_RPATH_ENV` or `TARGET_RPATH_DIR`. This is needed if the - // recipe wants to invoke rustc. - .env("HOST_RPATH_DIR", &self.config.compile_lib_path) - // Provide the directory to libraries that might be needed to run compiled binaries - // (further compiled by the recipe!). - .env("TARGET_RPATH_DIR", &self.config.run_lib_path) // Provide which LLVM components are available (e.g. which LLVM components are provided // through a specific CI runner). .env("LLVM_COMPONENTS", &self.config.llvm_components); diff --git a/src/tools/run-make-support/src/external_deps/rustc.rs b/src/tools/run-make-support/src/external_deps/rustc.rs index 710ba025830..cb2bd83815a 100644 --- a/src/tools/run-make-support/src/external_deps/rustc.rs +++ b/src/tools/run-make-support/src/external_deps/rustc.rs @@ -5,7 +5,7 @@ use std::str::FromStr as _; use crate::command::Command; use crate::env::env_var; use crate::path_helpers::cwd; -use crate::util::set_host_rpath; +use crate::util::set_host_compiler_dylib_path; use crate::{is_aix, is_darwin, is_msvc, is_windows, uname}; /// Construct a new `rustc` invocation. This will automatically set the library @@ -15,8 +15,8 @@ pub fn rustc() -> Rustc { Rustc::new() } -/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_rpath`] -/// still presets the environment variable `HOST_RPATH_DIR` by default. +/// Construct a plain `rustc` invocation with no flags set. Note that [`set_host_compiler_dylib_path`] +/// still presets the environment variable `HOST_RUSTC_DYLIB_PATH` by default. #[track_caller] pub fn bare_rustc() -> Rustc { Rustc::bare() @@ -44,7 +44,7 @@ pub fn rustc_path() -> String { #[track_caller] fn setup_common() -> Command { let mut cmd = Command::new(rustc_path()); - set_host_rpath(&mut cmd); + set_host_compiler_dylib_path(&mut cmd); cmd } diff --git a/src/tools/run-make-support/src/external_deps/rustdoc.rs b/src/tools/run-make-support/src/external_deps/rustdoc.rs index 3c0e9c82f0b..61351ace12c 100644 --- a/src/tools/run-make-support/src/external_deps/rustdoc.rs +++ b/src/tools/run-make-support/src/external_deps/rustdoc.rs @@ -2,16 +2,10 @@ use std::ffi::OsStr; use std::path::Path; use crate::command::Command; -use crate::env::{env_var, env_var_os}; -use crate::util::set_host_rpath; +use crate::env::env_var; +use crate::util::set_host_compiler_dylib_path; -/// Construct a plain `rustdoc` invocation with no flags set. -#[track_caller] -pub fn bare_rustdoc() -> Rustdoc { - Rustdoc::bare() -} - -/// Construct a new `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set. +/// Construct a new `rustdoc` invocation. #[track_caller] pub fn rustdoc() -> Rustdoc { Rustdoc::new() @@ -29,23 +23,15 @@ crate::macros::impl_common_helpers!(Rustdoc); fn setup_common() -> Command { let rustdoc = env_var("RUSTDOC"); let mut cmd = Command::new(rustdoc); - set_host_rpath(&mut cmd); + set_host_compiler_dylib_path(&mut cmd); cmd } impl Rustdoc { /// Construct a bare `rustdoc` invocation. #[track_caller] - pub fn bare() -> Self { - let cmd = setup_common(); - Self { cmd } - } - - /// Construct a `rustdoc` invocation with `-L $(TARGET_RPATH_DIR)` set. - #[track_caller] pub fn new() -> Self { - let mut cmd = setup_common(); - cmd.arg("-L").arg(env_var_os("TARGET_RPATH_DIR")); + let cmd = setup_common(); Self { cmd } } diff --git a/src/tools/run-make-support/src/lib.rs b/src/tools/run-make-support/src/lib.rs index 7e63ab3159a..d40ec9c4116 100644 --- a/src/tools/run-make-support/src/lib.rs +++ b/src/tools/run-make-support/src/lib.rs @@ -67,7 +67,7 @@ pub use llvm::{ }; pub use python::python_command; pub use rustc::{aux_build, bare_rustc, rustc, rustc_path, Rustc}; -pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc}; +pub use rustdoc::{rustdoc, Rustdoc}; /// [`diff`][mod@diff] is implemented in terms of the [similar] library. /// diff --git a/src/tools/run-make-support/src/run.rs b/src/tools/run-make-support/src/run.rs index 3eeba6fd526..7812863ccc2 100644 --- a/src/tools/run-make-support/src/run.rs +++ b/src/tools/run-make-support/src/run.rs @@ -1,10 +1,10 @@ use std::ffi::OsStr; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::{env, panic}; use crate::command::{Command, CompletedProcess}; -use crate::util::{handle_failed_output, set_host_rpath}; -use crate::{cwd, env_var, is_windows}; +use crate::util::handle_failed_output; +use crate::{cwd, env_var}; #[track_caller] fn run_common(name: &str, args: Option<&[&str]>) -> Command { @@ -18,10 +18,11 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command { cmd.arg(arg); } } + cmd.env(&ld_lib_path_envvar, { let mut paths = vec![]; paths.push(cwd()); - for p in env::split_paths(&env_var("TARGET_RPATH_ENV")) { + for p in env::split_paths(&env_var("TARGET_EXE_DYLIB_PATH")) { paths.push(p.to_path_buf()); } for p in env::split_paths(&env_var(&ld_lib_path_envvar)) { @@ -31,15 +32,6 @@ fn run_common(name: &str, args: Option<&[&str]>) -> Command { }); cmd.env("LC_ALL", "C"); // force english locale - if is_windows() { - let mut paths = vec![]; - for p in env::split_paths(&std::env::var("PATH").unwrap_or(String::new())) { - paths.push(p.to_path_buf()); - } - paths.push(Path::new(&env_var("TARGET_RPATH_DIR")).to_path_buf()); - cmd.env("PATH", env::join_paths(paths.iter()).unwrap()); - } - cmd } @@ -84,7 +76,6 @@ pub fn run_fail(name: &str) -> CompletedProcess { #[track_caller] pub fn cmd>(program: S) -> Command { let mut command = Command::new(program); - set_host_rpath(&mut command); command.env("LC_ALL", "C"); // force english locale command } diff --git a/src/tools/run-make-support/src/util.rs b/src/tools/run-make-support/src/util.rs index 703e3ad1c6c..af01758447b 100644 --- a/src/tools/run-make-support/src/util.rs +++ b/src/tools/run-make-support/src/util.rs @@ -24,13 +24,13 @@ pub(crate) fn handle_failed_output( std::process::exit(1) } -/// Set the runtime library path as needed for running the host rustc/rustdoc/etc. -pub(crate) fn set_host_rpath(cmd: &mut Command) { +/// Set the runtime library paths as needed for running the host compilers (rustc/rustdoc/etc). +pub(crate) fn set_host_compiler_dylib_path(cmd: &mut Command) { let ld_lib_path_envvar = env_var("LD_LIB_PATH_ENVVAR"); cmd.env(&ld_lib_path_envvar, { let mut paths = vec![]; paths.push(cwd()); - paths.push(PathBuf::from(env_var("HOST_RPATH_DIR"))); + paths.push(PathBuf::from(env_var("HOST_RUSTC_DYLIB_PATH"))); for p in std::env::split_paths(&env_var(&ld_lib_path_envvar)) { paths.push(p.to_path_buf()); } diff --git a/tests/run-make/rustdoc-default-output/rmake.rs b/tests/run-make/rustdoc-default-output/rmake.rs index 06720445a35..5f9c501e528 100644 --- a/tests/run-make/rustdoc-default-output/rmake.rs +++ b/tests/run-make/rustdoc-default-output/rmake.rs @@ -3,10 +3,10 @@ // ensures the output of rustdoc's help menu is as expected. // See https://github.com/rust-lang/rust/issues/88756 -use run_make_support::{bare_rustdoc, diff}; +use run_make_support::{diff, rustdoc}; fn main() { - let out = bare_rustdoc().run().stdout_utf8(); + let out = rustdoc().run().stdout_utf8(); diff() .expected_file("output-default.stdout") .actual_text("actual", out) diff --git a/tests/run-make/version-verbose-commit-hash/rmake.rs b/tests/run-make/version-verbose-commit-hash/rmake.rs index 733c0e2cdb1..ae8dda38ab4 100644 --- a/tests/run-make/version-verbose-commit-hash/rmake.rs +++ b/tests/run-make/version-verbose-commit-hash/rmake.rs @@ -5,13 +5,13 @@ //@ needs-git-hash -use run_make_support::{bare_rustc, bare_rustdoc, regex}; +use run_make_support::{bare_rustc, regex, rustdoc}; fn main() { let out_rustc = bare_rustc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); let out_rustdoc = - bare_rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); + rustdoc().arg("--version").arg("--verbose").run().stdout_utf8().to_lowercase(); let re = regex::Regex::new(r#"commit-hash: [0-9a-f]{40}\ncommit-date: [0-9]{4}-[0-9]{2}-[0-9]{2}"#) .unwrap();