bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #137215 - onur-ozkan:rustc-tool-build-stages, r=jieyouxu,Kobzol

Browse source 
stabilize stage management for rustc tools

https://github.com/rust-lang/rust/pull/135990 got out of control due to excessive complexity. This PR aims to achieve the same goal with a simpler approach, likely through multiple smaller PRs. I will keep the other one read-only and open as a reference for future work.

This work stabilizes the staging logic for `ToolRustc` programs, so you no longer need to handle build and target compilers separately in steps. Previously, most tools didn't do this correctly, which was causing the compiler to be built twice (e.g., `x test cargo --stage 1` would compile the stage 2 compiler before, but now it only compiles the stage 1 compiler).

I also tried to document how we should write `ToolRustc` steps as they are quite different and require more attention than other tools.

Next goal is to stabilize how stages are handled for the rustc itself. Currently, `x build --stage 1` builds the stage 1 compiler which is fine, but `x build compiler --stage 1` builds stage 2 compiler.

~~for now, r? ghost~~
This commit is contained in:
bors 2025-02-23 05:03:26 +00:00
commit bb2cc59a21
12 changed files with 340 additions and 263 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/bootstrap/defaults/config.tools.toml
View file

@ -8,6 +8,8 @@ incremental = true
download-rustc = "if-unchanged" download-rustc = "if-unchanged"
[build] [build]
# cargo and clippy tests don't pass on stage 1
test-stage = 2
# Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile. # Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile.
doc-stage = 2 doc-stage = 2
# Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API. # Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API.

26
src/bootstrap/src/core/build_steps/compile.rs
View file

@ -1983,13 +1983,14 @@ impl Step for Assemble {
let maybe_install_llvm_bitcode_linker = |compiler| { let maybe_install_llvm_bitcode_linker = |compiler| {
if builder.config.llvm_bitcode_linker_enabled { if builder.config.llvm_bitcode_linker_enabled {
trace!("llvm-bitcode-linker enabled, installing"); trace!("llvm-bitcode-linker enabled, installing");
let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker { let llvm_bitcode_linker =
compiler, builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
target: target_compiler.host, compiler,
extra_features: vec![], target: target_compiler.host,
}); extra_features: vec![],
});
let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); let tool_exe = exe("llvm-bitcode-linker", target_compiler.host);
builder.copy_link(&src_path, &libdir_bin.join(tool_exe)); builder.copy_link(&llvm_bitcode_linker.tool_path, &libdir_bin.join(tool_exe));
} }
}; };
@ -2181,14 +2182,13 @@ impl Step for Assemble {
// logic to create the final binary. This is used by the // logic to create the final binary. This is used by the
// `wasm32-wasip2` target of Rust. // `wasm32-wasip2` target of Rust.
if builder.tool_enabled("wasm-component-ld") { if builder.tool_enabled("wasm-component-ld") {
let wasm_component_ld_exe = let wasm_component = builder.ensure(crate::core::build_steps::tool::WasmComponentLd {
builder.ensure(crate::core::build_steps::tool::WasmComponentLd { compiler: build_compiler,
compiler: build_compiler, target: target_compiler.host,
target: target_compiler.host, });
});
builder.copy_link( builder.copy_link(
&wasm_component_ld_exe, &wasm_component.tool_path,
&libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()), &libdir_bin.join(wasm_component.tool_path.file_name().unwrap()),
); );
} }

22
src/bootstrap/src/core/build_steps/dist.rs
View file

@ -430,7 +430,7 @@ impl Step for Rustc {
}, },
builder.kind, builder.kind,
) { ) {
builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); builder.install(&ra_proc_macro_srv.tool_path, &image.join("libexec"), 0o755);
} }
let libdir_relative = builder.libdir_relative(compiler); let libdir_relative = builder.libdir_relative(compiler);
@ -1145,7 +1145,7 @@ impl Step for Cargo {
let mut tarball = Tarball::new(builder, "cargo", &target.triple); let mut tarball = Tarball::new(builder, "cargo", &target.triple);
tarball.set_overlay(OverlayKind::Cargo); tarball.set_overlay(OverlayKind::Cargo);
tarball.add_file(cargo, "bin", 0o755); tarball.add_file(cargo.tool_path, "bin", 0o755);
tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
tarball.add_dir(etc.join("man"), "share/man/man1"); tarball.add_dir(etc.join("man"), "share/man/man1");
@ -1191,7 +1191,7 @@ impl Step for Rls {
let mut tarball = Tarball::new(builder, "rls", &target.triple); let mut tarball = Tarball::new(builder, "rls", &target.triple);
tarball.set_overlay(OverlayKind::Rls); tarball.set_overlay(OverlayKind::Rls);
tarball.is_preview(true); tarball.is_preview(true);
tarball.add_file(rls, "bin", 0o755); tarball.add_file(rls.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rls"); tarball.add_legal_and_readme_to("share/doc/rls");
Some(tarball.generate()) Some(tarball.generate())
} }
@ -1233,7 +1233,7 @@ impl Step for RustAnalyzer {
let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
tarball.set_overlay(OverlayKind::RustAnalyzer); tarball.set_overlay(OverlayKind::RustAnalyzer);
tarball.is_preview(true); tarball.is_preview(true);
tarball.add_file(rust_analyzer, "bin", 0o755); tarball.add_file(rust_analyzer.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
Some(tarball.generate()) Some(tarball.generate())
} }
@ -1279,8 +1279,8 @@ impl Step for Clippy {
let mut tarball = Tarball::new(builder, "clippy", &target.triple); let mut tarball = Tarball::new(builder, "clippy", &target.triple);
tarball.set_overlay(OverlayKind::Clippy); tarball.set_overlay(OverlayKind::Clippy);
tarball.is_preview(true); tarball.is_preview(true);
tarball.add_file(clippy, "bin", 0o755); tarball.add_file(clippy.tool_path, "bin", 0o755);
tarball.add_file(cargoclippy, "bin", 0o755); tarball.add_file(cargoclippy.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/clippy"); tarball.add_legal_and_readme_to("share/doc/clippy");
Some(tarball.generate()) Some(tarball.generate())
} }
@ -1329,8 +1329,8 @@ impl Step for Miri {
let mut tarball = Tarball::new(builder, "miri", &target.triple); let mut tarball = Tarball::new(builder, "miri", &target.triple);
tarball.set_overlay(OverlayKind::Miri); tarball.set_overlay(OverlayKind::Miri);
tarball.is_preview(true); tarball.is_preview(true);
tarball.add_file(miri, "bin", 0o755); tarball.add_file(miri.tool_path, "bin", 0o755);
tarball.add_file(cargomiri, "bin", 0o755); tarball.add_file(cargomiri.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/miri"); tarball.add_legal_and_readme_to("share/doc/miri");
Some(tarball.generate()) Some(tarball.generate())
} }
@ -1460,8 +1460,8 @@ impl Step for Rustfmt {
let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
tarball.set_overlay(OverlayKind::Rustfmt); tarball.set_overlay(OverlayKind::Rustfmt);
tarball.is_preview(true); tarball.is_preview(true);
tarball.add_file(rustfmt, "bin", 0o755); tarball.add_file(rustfmt.tool_path, "bin", 0o755);
tarball.add_file(cargofmt, "bin", 0o755); tarball.add_file(cargofmt.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rustfmt"); tarball.add_legal_and_readme_to("share/doc/rustfmt");
Some(tarball.generate()) Some(tarball.generate())
} }
@ -2283,7 +2283,7 @@ impl Step for LlvmBitcodeLinker {
tarball.set_overlay(OverlayKind::LlvmBitcodeLinker); tarball.set_overlay(OverlayKind::LlvmBitcodeLinker);
tarball.is_preview(true); tarball.is_preview(true);
tarball.add_file(llbc_linker, self_contained_bin_dir, 0o755); tarball.add_file(llbc_linker.tool_path, self_contained_bin_dir, 0o755);
Some(tarball.generate()) Some(tarball.generate())
} }

2
src/bootstrap/src/core/build_steps/perf.rs
View file

@ -166,7 +166,7 @@ Consider setting `rust.debuginfo-level = 1` in `config.toml`."#);
let results_dir = rustc_perf_dir.join("results"); let results_dir = rustc_perf_dir.join("results");
builder.create_dir(&results_dir); builder.create_dir(&results_dir);
let mut cmd = command(collector); let mut cmd = command(collector.tool_path);
// We need to set the working directory to `src/tools/rustc-perf`, so that it can find the directory // We need to set the working directory to `src/tools/rustc-perf`, so that it can find the directory
// with compile-time benchmarks. // with compile-time benchmarks.

6
src/bootstrap/src/core/build_steps/run.rs
View file

@ -126,11 +126,7 @@ impl Step for Miri {
// This compiler runs on the host, we'll just use it for the target. // This compiler runs on the host, we'll just use it for the target.
let target_compiler = builder.compiler(stage, host); let target_compiler = builder.compiler(stage, host);
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise let host_compiler = tool::get_tool_rustc_compiler(builder, target_compiler);
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let host_compiler = builder.compiler(stage - 1, host);
// Get a target sysroot for Miri. // Get a target sysroot for Miri.
let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target); let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target);

66
src/bootstrap/src/core/build_steps/test.rs
View file

@ -263,7 +263,7 @@ impl Step for Cargotest {
let _time = helpers::timeit(builder); let _time = helpers::timeit(builder);
let mut cmd = builder.tool_cmd(Tool::CargoTest); let mut cmd = builder.tool_cmd(Tool::CargoTest);
cmd.arg(&cargo) cmd.arg(&cargo.tool_path)
.arg(&out_dir) .arg(&out_dir)
.args(builder.config.test_args()) .args(builder.config.test_args())
.env("RUSTC", builder.rustc(compiler)) .env("RUSTC", builder.rustc(compiler))
@ -298,9 +298,16 @@ impl Step for Cargo {
/// Runs `cargo test` for `cargo` packaged with Rust. /// Runs `cargo test` for `cargo` packaged with Rust.
fn run(self, builder: &Builder<'_>) { fn run(self, builder: &Builder<'_>) {
if self.stage < 2 {
eprintln!("WARNING: cargo tests on stage {} may not behave well.", self.stage);
eprintln!("HELP: consider using stage 2");
}
let compiler = builder.compiler(self.stage, self.host); let compiler = builder.compiler(self.stage, self.host);
builder.ensure(tool::Cargo { compiler, target: self.host }); let cargo = builder.ensure(tool::Cargo { compiler, target: self.host });
let compiler = cargo.build_compiler;
let cargo = tool::prepare_tool_cargo( let cargo = tool::prepare_tool_cargo(
builder, builder,
compiler, compiler,
@ -367,6 +374,7 @@ impl Step for RustAnalyzer {
let stage = self.stage; let stage = self.stage;
let host = self.host; let host = self.host;
let compiler = builder.compiler(stage, host); let compiler = builder.compiler(stage, host);
let compiler = tool::get_tool_rustc_compiler(builder, compiler);
// We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite,
// but we do need the standard library to be present. // but we do need the standard library to be present.
@ -427,7 +435,8 @@ impl Step for Rustfmt {
let host = self.host; let host = self.host;
let compiler = builder.compiler(stage, host); let compiler = builder.compiler(stage, host);
builder.ensure(tool::Rustfmt { compiler, target: self.host }); let tool_result = builder.ensure(tool::Rustfmt { compiler, target: self.host });
let compiler = tool_result.build_compiler;
let mut cargo = tool::prepare_tool_cargo( let mut cargo = tool::prepare_tool_cargo(
builder, builder,
@ -522,16 +531,11 @@ impl Step for Miri {
// This compiler runs on the host, we'll just use it for the target. // This compiler runs on the host, we'll just use it for the target.
let target_compiler = builder.compiler(stage, host); let target_compiler = builder.compiler(stage, host);
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let host_compiler = builder.compiler(stage - 1, host);
// Build our tools. // Build our tools.
let miri = builder.ensure(tool::Miri { compiler: host_compiler, target: host }); let miri = builder.ensure(tool::Miri { compiler: target_compiler, target: host });
// the ui tests also assume cargo-miri has been built // the ui tests also assume cargo-miri has been built
builder.ensure(tool::CargoMiri { compiler: host_compiler, target: host }); builder.ensure(tool::CargoMiri { compiler: target_compiler, target: host });
// We also need sysroots, for Miri and for the host (the latter for build scripts). // We also need sysroots, for Miri and for the host (the latter for build scripts).
// This is for the tests so everything is done with the target compiler. // This is for the tests so everything is done with the target compiler.
@ -542,7 +546,8 @@ impl Step for Miri {
// Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when // Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when
// the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors. // the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors.
if !builder.config.dry_run() { if !builder.config.dry_run() {
let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui"); let ui_test_dep_dir =
builder.stage_out(miri.build_compiler, Mode::ToolStd).join("miri_ui");
// The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see
// <https://github.com/RalfJung/rustc-build-sysroot/commit/10ebcf60b80fe2c3dc765af0ff19fdc0da4b7466>). // <https://github.com/RalfJung/rustc-build-sysroot/commit/10ebcf60b80fe2c3dc765af0ff19fdc0da4b7466>).
// We can hence use that directly as a signal to clear the ui test dir. // We can hence use that directly as a signal to clear the ui test dir.
@ -553,7 +558,7 @@ impl Step for Miri {
// This is with the Miri crate, so it uses the host compiler. // This is with the Miri crate, so it uses the host compiler.
let mut cargo = tool::prepare_tool_cargo( let mut cargo = tool::prepare_tool_cargo(
builder, builder,
host_compiler, miri.build_compiler,
Mode::ToolRustc, Mode::ToolRustc,
host, host,
Kind::Test, Kind::Test,
@ -571,7 +576,7 @@ impl Step for Miri {
// miri tests need to know about the stage sysroot // miri tests need to know about the stage sysroot
cargo.env("MIRI_SYSROOT", &miri_sysroot); cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", &host_sysroot); cargo.env("MIRI_HOST_SYSROOT", &host_sysroot);
cargo.env("MIRI", &miri); cargo.env("MIRI", &miri.tool_path);
// Set the target. // Set the target.
cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg());
@ -743,7 +748,13 @@ impl Step for Clippy {
let host = self.host; let host = self.host;
let compiler = builder.compiler(stage, host); let compiler = builder.compiler(stage, host);
builder.ensure(tool::Clippy { compiler, target: self.host }); if stage < 2 {
eprintln!("WARNING: clippy tests on stage {stage} may not behave well.");
eprintln!("HELP: consider using stage 2");
}
let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host });
let compiler = tool_result.build_compiler;
let mut cargo = tool::prepare_tool_cargo( let mut cargo = tool::prepare_tool_cargo(
builder, builder,
compiler, compiler,
@ -1728,18 +1739,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
// If we're using `--stage 0`, we should provide the bootstrap cargo. // If we're using `--stage 0`, we should provide the bootstrap cargo.
builder.initial_cargo.clone() builder.initial_cargo.clone()
} else { } else {
// We need to properly build cargo using the suitable stage compiler. builder.ensure(tool::Cargo { compiler, target: compiler.host }).tool_path
let compiler = builder.download_rustc().then_some(compiler).unwrap_or_else(||
// HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if
// you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built
// and produce a cargo built with stage 2 rustc. To fix this, we need to chop off
// the compiler stage by 1 to align with expected `./x test run-make --stage N`
// behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri
// which does a similar hack.
builder.compiler(builder.top_stage - 1, compiler.host));
builder.ensure(tool::Cargo { compiler, target: compiler.host })
}; };
cmd.arg("--cargo-path").arg(cargo_path); cmd.arg("--cargo-path").arg(cargo_path);
@ -1760,9 +1760,10 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
// Use the beta compiler for jsondocck // Use the beta compiler for jsondocck
let json_compiler = compiler.with_stage(0); let json_compiler = compiler.with_stage(0);
cmd.arg("--jsondocck-path") cmd.arg("--jsondocck-path")
.arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }).tool_path);
cmd.arg("--jsondoclint-path") cmd.arg("--jsondoclint-path").arg(
.arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }).tool_path,
);
} }
if matches!(mode, "coverage-map" | "coverage-run") { if matches!(mode, "coverage-map" | "coverage-run") {
@ -2999,12 +3000,15 @@ impl Step for RemoteCopyLibs {
builder.info(&format!("REMOTE copy libs to emulator ({target})")); builder.info(&format!("REMOTE copy libs to emulator ({target})"));
let server = builder.ensure(tool::RemoteTestServer { compiler, target }); let remote_test_server = builder.ensure(tool::RemoteTestServer { compiler, target });
// Spawn the emulator and wait for it to come online // Spawn the emulator and wait for it to come online
let tool = builder.tool_exe(Tool::RemoteTestClient); let tool = builder.tool_exe(Tool::RemoteTestClient);
let mut cmd = command(&tool); let mut cmd = command(&tool);
cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); cmd.arg("spawn-emulator")
.arg(target.triple)
.arg(&remote_test_server.tool_path)
.arg(builder.tempdir());
if let Some(rootfs) = builder.qemu_rootfs(target) { if let Some(rootfs) = builder.qemu_rootfs(target) {
cmd.arg(rootfs); cmd.arg(rootfs);
} }

386
src/bootstrap/src/core/build_steps/tool.rs
View file

@ -1,3 +1,14 @@
//! This module handles building and managing various tools in bootstrap
//! build system.
//!
//! **What It Does**
//! - Defines how tools are built, configured and installed.
//! - Manages tool dependencies and build steps.
//! - Copies built tool binaries to the correct locations.
//!
//! Each Rust tool **MUST** utilize `ToolBuild` inside their `Step` logic,
//! return `ToolBuildResult` and should never prepare `cargo` invocations manually.
use std::path::PathBuf; use std::path::PathBuf;
use std::{env, fs}; use std::{env, fs};
@ -64,8 +75,21 @@ impl Builder<'_> {
} }
} }
/// Result of the tool build process. Each `Step` in this module is responsible
/// for using this type as `type Output = ToolBuildResult;`
#[derive(Clone)]
pub struct ToolBuildResult {
/// Executable path of the corresponding tool that was built.
pub tool_path: PathBuf,
/// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`.
/// For `ToolRustc` this is one stage before of the `target_compiler`.
pub build_compiler: Compiler,
/// Target compiler passed to `Step`.
pub target_compiler: Compiler,
}
impl Step for ToolBuild { impl Step for ToolBuild {
type Output = PathBuf; type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.never() run.never()
@ -75,25 +99,31 @@ impl Step for ToolBuild {
/// ///
/// This will build the specified tool with the specified `host` compiler in /// This will build the specified tool with the specified `host` compiler in
/// `stage` into the normal cargo output directory. /// `stage` into the normal cargo output directory.
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(mut self, builder: &Builder<'_>) -> ToolBuildResult {
let compiler = self.compiler;
let target = self.target; let target = self.target;
let mut tool = self.tool; let mut tool = self.tool;
let path = self.path; let path = self.path;
let target_compiler = self.compiler;
self.compiler = if self.mode == Mode::ToolRustc {
get_tool_rustc_compiler(builder, self.compiler)
} else {
self.compiler
};
match self.mode { match self.mode {
Mode::ToolRustc => { Mode::ToolRustc => {
builder.ensure(compile::Std::new(compiler, compiler.host)); builder.ensure(compile::Std::new(self.compiler, self.compiler.host));
builder.ensure(compile::Rustc::new(compiler, target)); builder.ensure(compile::Rustc::new(self.compiler, target));
} }
Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), Mode::ToolStd => builder.ensure(compile::Std::new(self.compiler, target)),
Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs
_ => panic!("unexpected Mode for tool build"), _ => panic!("unexpected Mode for tool build"),
} }
let mut cargo = prepare_tool_cargo( let mut cargo = prepare_tool_cargo(
builder, builder,
compiler, self.compiler,
self.mode, self.mode,
target, target,
Kind::Build, Kind::Build,
@ -101,10 +131,28 @@ impl Step for ToolBuild {
self.source_type, self.source_type,
&self.extra_features, &self.extra_features,
); );
if path.ends_with("/rustdoc") &&
// rustdoc is performance sensitive, so apply LTO to it.
is_lto_stage(&self.compiler)
{
let lto = match builder.config.rust_lto {
RustcLto::Off => Some("off"),
RustcLto::Thin => Some("thin"),
RustcLto::Fat => Some("fat"),
RustcLto::ThinLocal => None,
};
if let Some(lto) = lto {
cargo.env(cargo_profile_var("LTO", &builder.config), lto);
}
}
if !self.allow_features.is_empty() { if !self.allow_features.is_empty() {
cargo.allow_features(self.allow_features); cargo.allow_features(self.allow_features);
} }
cargo.args(self.cargo_args); cargo.args(self.cargo_args);
let _guard = builder.msg_tool( let _guard = builder.msg_tool(
Kind::Build, Kind::Build,
self.mode, self.mode,
@ -131,7 +179,10 @@ impl Step for ToolBuild {
if tool == "tidy" { if tool == "tidy" {
tool = "rust-tidy"; tool = "rust-tidy";
} }
copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool) let tool_path =
copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool);
ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler }
} }
} }
} }
@ -240,6 +291,23 @@ pub fn prepare_tool_cargo(
cargo cargo
} }
/// Handle stage-off logic for `ToolRustc` tools when necessary.
pub(crate) fn get_tool_rustc_compiler(
builder: &Builder<'_>,
target_compiler: Compiler,
) -> Compiler {
if builder.download_rustc() && target_compiler.stage == 1 {
// We already have the stage 1 compiler, we don't need to cut the stage.
builder.compiler(target_compiler.stage, builder.config.build)
} else {
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/$rustc_tool be effectively different stage
// compilers, which isn't what we want. Rustc tools should be linked in the same way as the
// compiler it's paired with, so it must be built with the previous stage compiler.
builder.compiler(target_compiler.stage.saturating_sub(1), builder.config.build)
}
}
/// Links a built tool binary with the given `name` from the build directory to the /// Links a built tool binary with the given `name` from the build directory to the
/// tools directory. /// tools directory.
fn copy_link_tool_bin( fn copy_link_tool_bin(
@ -279,7 +347,7 @@ macro_rules! bootstrap_tool {
self.ensure($name { self.ensure($name {
compiler: self.compiler(0, self.config.build), compiler: self.compiler(0, self.config.build),
target: self.config.build, target: self.config.build,
}), }).tool_path,
)+ )+
} }
} }
@ -293,7 +361,7 @@ macro_rules! bootstrap_tool {
} }
impl Step for $name { impl Step for $name {
type Output = PathBuf; type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path($path) run.path($path)
@ -315,7 +383,7 @@ macro_rules! bootstrap_tool {
skip_all, skip_all,
), ),
)] )]
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
$( $(
for submodule in $submodules { for submodule in $submodules {
builder.require_submodule(submodule, None); builder.require_submodule(submodule, None);
@ -390,7 +458,7 @@ pub struct OptimizedDist {
} }
impl Step for OptimizedDist { impl Step for OptimizedDist {
type Output = PathBuf; type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/opt-dist") run.path("src/tools/opt-dist")
@ -403,7 +471,7 @@ impl Step for OptimizedDist {
}); });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
// We need to ensure the rustc-perf submodule is initialized when building opt-dist since // We need to ensure the rustc-perf submodule is initialized when building opt-dist since
// the tool requires it to be in place to run. // the tool requires it to be in place to run.
builder.require_submodule("src/tools/rustc-perf", None); builder.require_submodule("src/tools/rustc-perf", None);
@ -432,7 +500,7 @@ pub struct RustcPerf {
impl Step for RustcPerf { impl Step for RustcPerf {
/// Path to the built `collector` binary. /// Path to the built `collector` binary.
type Output = PathBuf; type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/rustc-perf") run.path("src/tools/rustc-perf")
@ -445,7 +513,7 @@ impl Step for RustcPerf {
}); });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
// We need to ensure the rustc-perf submodule is initialized. // We need to ensure the rustc-perf submodule is initialized.
builder.require_submodule("src/tools/rustc-perf", None); builder.require_submodule("src/tools/rustc-perf", None);
@ -462,12 +530,12 @@ impl Step for RustcPerf {
// a CLI. // a CLI.
cargo_args: vec!["-p".to_string(), "collector".to_string()], cargo_args: vec!["-p".to_string(), "collector".to_string()],
}; };
let collector_bin = builder.ensure(tool.clone()); let res = builder.ensure(tool.clone());
// We also need to symlink the `rustc-fake` binary to the corresponding directory, // We also need to symlink the `rustc-fake` binary to the corresponding directory,
// because `collector` expects it in the same directory. // because `collector` expects it in the same directory.
copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake"); copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake");
collector_bin res
} }
} }
@ -482,7 +550,7 @@ impl ErrorIndex {
// for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc.
let host = builder.config.build; let host = builder.config.build;
let compiler = builder.compiler_for(builder.top_stage, host, host); let compiler = builder.compiler_for(builder.top_stage, host, host);
let mut cmd = command(builder.ensure(ErrorIndex { compiler })); let mut cmd = command(builder.ensure(ErrorIndex { compiler }).tool_path);
let mut dylib_paths = builder.rustc_lib_paths(compiler); let mut dylib_paths = builder.rustc_lib_paths(compiler);
dylib_paths.push(PathBuf::from(&builder.sysroot_target_libdir(compiler, compiler.host))); dylib_paths.push(PathBuf::from(&builder.sysroot_target_libdir(compiler, compiler.host)));
add_dylib_path(dylib_paths, &mut cmd); add_dylib_path(dylib_paths, &mut cmd);
@ -491,27 +559,23 @@ impl ErrorIndex {
} }
impl Step for ErrorIndex { impl Step for ErrorIndex {
type Output = PathBuf; type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/error_index_generator") run.path("src/tools/error_index_generator")
} }
fn make_run(run: RunConfig<'_>) { fn make_run(run: RunConfig<'_>) {
// Compile the error-index in the same stage as rustdoc to avoid
// recompiling rustdoc twice if we can.
//
// NOTE: This `make_run` isn't used in normal situations, only if you // NOTE: This `make_run` isn't used in normal situations, only if you
// manually build the tool with `x.py build // manually build the tool with `x.py build
// src/tools/error-index-generator` which almost nobody does. // src/tools/error-index-generator` which almost nobody does.
// Normally, `x.py test` or `x.py doc` will use the // Normally, `x.py test` or `x.py doc` will use the
// `ErrorIndex::command` function instead. // `ErrorIndex::command` function instead.
let compiler = let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build);
run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build);
run.builder.ensure(ErrorIndex { compiler }); run.builder.ensure(ErrorIndex { compiler });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.ensure(ToolBuild { builder.ensure(ToolBuild {
compiler: self.compiler, compiler: self.compiler,
target: self.compiler.host, target: self.compiler.host,
@ -533,7 +597,7 @@ pub struct RemoteTestServer {
} }
impl Step for RemoteTestServer { impl Step for RemoteTestServer {
type Output = PathBuf; type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/remote-test-server") run.path("src/tools/remote-test-server")
@ -546,7 +610,7 @@ impl Step for RemoteTestServer {
}); });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.ensure(ToolBuild { builder.ensure(ToolBuild {
compiler: self.compiler, compiler: self.compiler,
target: self.target, target: self.target,
@ -569,7 +633,7 @@ pub struct Rustdoc {
} }
impl Step for Rustdoc { impl Step for Rustdoc {
type Output = PathBuf; type Output = ToolBuildResult;
const DEFAULT: bool = true; const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
@ -578,24 +642,25 @@ impl Step for Rustdoc {
} }
fn make_run(run: RunConfig<'_>) { fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rustdoc { run.builder
// NOTE: this is somewhat unique in that we actually want a *target* .ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
// compiler here, because rustdoc *is* a compiler. We won't be using
// this as the compiler to build with, but rather this is "what
// compiler are we producing"?
compiler: run.builder.compiler(run.builder.top_stage, run.target),
});
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let target_compiler = self.compiler; let target_compiler = self.compiler;
let target = target_compiler.host;
if target_compiler.stage == 0 { if target_compiler.stage == 0 {
if !target_compiler.is_snapshot(builder) { if !target_compiler.is_snapshot(builder) {
panic!("rustdoc in stage 0 must be snapshot rustdoc"); panic!("rustdoc in stage 0 must be snapshot rustdoc");
} }
return builder.initial_rustdoc.clone();
return ToolBuildResult {
tool_path: builder.initial_rustdoc.clone(),
build_compiler: target_compiler,
target_compiler,
};
} }
let target = target_compiler.host;
let bin_rustdoc = || { let bin_rustdoc = || {
let sysroot = builder.sysroot(target_compiler); let sysroot = builder.sysroot(target_compiler);
@ -625,27 +690,15 @@ impl Step for Rustdoc {
let bin_rustdoc = bin_rustdoc(); let bin_rustdoc = bin_rustdoc();
builder.copy_link(&precompiled_rustdoc, &bin_rustdoc); builder.copy_link(&precompiled_rustdoc, &bin_rustdoc);
return bin_rustdoc;
return ToolBuildResult {
tool_path: bin_rustdoc,
build_compiler: target_compiler,
target_compiler,
};
} }
} }
let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 {
// We already have the stage 1 compiler, we don't need to cut the stage.
builder.compiler(target_compiler.stage, builder.config.build)
} else {
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
builder.compiler(target_compiler.stage - 1, builder.config.build)
};
// When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually
// build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build
// it.
builder.ensure(compile::Std::new(build_compiler, target_compiler.host));
builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host));
// The presence of `target_compiler` ensures that the necessary libraries (codegen backends, // The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
// compiler libraries, ...) are built. Rustdoc does not require the presence of any // compiler libraries, ...) are built. Rustdoc does not require the presence of any
// libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since
@ -653,65 +706,39 @@ impl Step for Rustdoc {
// libraries here. The intuition here is that If we've built a compiler, we should be able // libraries here. The intuition here is that If we've built a compiler, we should be able
// to build rustdoc. // to build rustdoc.
// //
let mut features = Vec::new(); let mut extra_features = Vec::new();
if builder.config.jemalloc(target) { if builder.config.jemalloc(target) {
features.push("jemalloc".to_string()); extra_features.push("jemalloc".to_string());
} }
// NOTE: Never modify the rustflags here, it breaks the build cache for other tools! let ToolBuildResult { tool_path, build_compiler, target_compiler } =
let mut cargo = prepare_tool_cargo( builder.ensure(ToolBuild {
builder, compiler: target_compiler,
build_compiler, target,
Mode::ToolRustc, // Cargo adds a number of paths to the dylib search path on windows, which results in
target, // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
Kind::Build, // rustdoc a different name.
"src/tools/rustdoc", tool: "rustdoc_tool_binary",
SourceType::InTree, mode: Mode::ToolRustc,
features.as_slice(), path: "src/tools/rustdoc",
); source_type: SourceType::InTree,
extra_features,
// rustdoc is performance sensitive, so apply LTO to it. allow_features: "",
if is_lto_stage(&build_compiler) { cargo_args: Vec::new(),
let lto = match builder.config.rust_lto { });
RustcLto::Off => Some("off"),
RustcLto::Thin => Some("thin"),
RustcLto::Fat => Some("fat"),
RustcLto::ThinLocal => None,
};
if let Some(lto) = lto {
cargo.env(cargo_profile_var("LTO", &builder.config), lto);
}
}
let _guard = builder.msg_tool(
Kind::Build,
Mode::ToolRustc,
"rustdoc",
build_compiler.stage,
&self.compiler.host,
&target,
);
cargo.into_cmd().run(builder);
// Cargo adds a number of paths to the dylib search path on windows, which results in
// the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
// rustdoc a different name.
let tool_rustdoc = builder
.cargo_out(build_compiler, Mode::ToolRustc, target)
.join(exe("rustdoc_tool_binary", target_compiler.host));
// don't create a stage0-sysroot/bin directory. // don't create a stage0-sysroot/bin directory.
if target_compiler.stage > 0 { if target_compiler.stage > 0 {
if builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None { if builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None {
// Due to LTO a lot of debug info from C++ dependencies such as jemalloc can make it into // Due to LTO a lot of debug info from C++ dependencies such as jemalloc can make it into
// our final binaries // our final binaries
compile::strip_debug(builder, target, &tool_rustdoc); compile::strip_debug(builder, target, &tool_path);
} }
let bin_rustdoc = bin_rustdoc(); let bin_rustdoc = bin_rustdoc();
builder.copy_link(&tool_rustdoc, &bin_rustdoc); builder.copy_link(&tool_path, &bin_rustdoc);
bin_rustdoc ToolBuildResult { tool_path: bin_rustdoc, build_compiler, target_compiler }
} else { } else {
tool_rustdoc ToolBuildResult { tool_path, build_compiler, target_compiler }
} }
} }
} }
@ -723,7 +750,7 @@ pub struct Cargo {
} }
impl Step for Cargo { impl Step for Cargo {
type Output = PathBuf; type Output = ToolBuildResult;
const DEFAULT: bool = true; const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
@ -739,7 +766,7 @@ impl Step for Cargo {
}); });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.build.require_submodule("src/tools/cargo", None); builder.build.require_submodule("src/tools/cargo", None);
builder.ensure(ToolBuild { builder.ensure(ToolBuild {
@ -763,7 +790,7 @@ pub struct LldWrapper {
} }
impl Step for LldWrapper { impl Step for LldWrapper {
type Output = (); type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.never() run.never()
@ -778,14 +805,19 @@ impl Step for LldWrapper {
fields(build_compiler = ?self.build_compiler, target_compiler = ?self.target_compiler), fields(build_compiler = ?self.build_compiler, target_compiler = ?self.target_compiler),
), ),
)] )]
fn run(self, builder: &Builder<'_>) {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
if builder.config.dry_run() { if builder.config.dry_run() {
return; return ToolBuildResult {
tool_path: Default::default(),
build_compiler: self.build_compiler,
target_compiler: self.target_compiler,
};
} }
let target = self.target_compiler.host; let target = self.target_compiler.host;
let executable = builder.ensure(ToolBuild { let tool_result = builder.ensure(ToolBuild {
compiler: self.build_compiler, compiler: self.build_compiler,
target, target,
tool: "lld-wrapper", tool: "lld-wrapper",
@ -809,8 +841,11 @@ impl Step for LldWrapper {
t!(fs::create_dir_all(&self_contained_lld_dir)); t!(fs::create_dir_all(&self_contained_lld_dir));
for name in crate::LLD_FILE_NAMES { for name in crate::LLD_FILE_NAMES {
builder.copy_link(&executable, &self_contained_lld_dir.join(exe(name, target))); builder
.copy_link(&tool_result.tool_path, &self_contained_lld_dir.join(exe(name, target)));
} }
tool_result
} }
} }
@ -825,7 +860,7 @@ impl RustAnalyzer {
} }
impl Step for RustAnalyzer { impl Step for RustAnalyzer {
type Output = PathBuf; type Output = ToolBuildResult;
const DEFAULT: bool = true; const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
@ -841,7 +876,7 @@ impl Step for RustAnalyzer {
}); });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.ensure(ToolBuild { builder.ensure(ToolBuild {
compiler: self.compiler, compiler: self.compiler,
target: self.target, target: self.target,
@ -863,7 +898,7 @@ pub struct RustAnalyzerProcMacroSrv {
} }
impl Step for RustAnalyzerProcMacroSrv { impl Step for RustAnalyzerProcMacroSrv {
type Output = Option<PathBuf>; type Output = Option<ToolBuildResult>;
const DEFAULT: bool = true; const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
@ -885,8 +920,8 @@ impl Step for RustAnalyzerProcMacroSrv {
}); });
} }
fn run(self, builder: &Builder<'_>) -> Option<PathBuf> { fn run(self, builder: &Builder<'_>) -> Option<ToolBuildResult> {
let path = builder.ensure(ToolBuild { let tool_result = builder.ensure(ToolBuild {
compiler: self.compiler, compiler: self.compiler,
target: self.target, target: self.target,
tool: "rust-analyzer-proc-macro-srv", tool: "rust-analyzer-proc-macro-srv",
@ -902,9 +937,10 @@ impl Step for RustAnalyzerProcMacroSrv {
// so that r-a can use it. // so that r-a can use it.
let libexec_path = builder.sysroot(self.compiler).join("libexec"); let libexec_path = builder.sysroot(self.compiler).join("libexec");
t!(fs::create_dir_all(&libexec_path)); t!(fs::create_dir_all(&libexec_path));
builder.copy_link(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); builder
.copy_link(&tool_result.tool_path, &libexec_path.join("rust-analyzer-proc-macro-srv"));
Some(path) Some(tool_result)
} }
} }
@ -916,7 +952,7 @@ pub struct LlvmBitcodeLinker {
} }
impl Step for LlvmBitcodeLinker { impl Step for LlvmBitcodeLinker {
type Output = PathBuf; type Output = ToolBuildResult;
const DEFAULT: bool = true; const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
@ -938,51 +974,34 @@ impl Step for LlvmBitcodeLinker {
feature = "tracing", feature = "tracing",
instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all) instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all)
)] )]
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let bin_name = "llvm-bitcode-linker"; let tool_result = builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.target,
tool: "llvm-bitcode-linker",
mode: Mode::ToolRustc,
path: "src/tools/llvm-bitcode-linker",
source_type: SourceType::InTree,
extra_features: self.extra_features,
allow_features: "",
cargo_args: Vec::new(),
});
// If enabled, use ci-rustc and skip building the in-tree compiler. if tool_result.target_compiler.stage > 0 {
if !builder.download_rustc() {
builder.ensure(compile::Std::new(self.compiler, self.compiler.host));
builder.ensure(compile::Rustc::new(self.compiler, self.target));
}
let cargo = prepare_tool_cargo(
builder,
self.compiler,
Mode::ToolRustc,
self.target,
Kind::Build,
"src/tools/llvm-bitcode-linker",
SourceType::InTree,
&self.extra_features,
);
let _guard = builder.msg_tool(
Kind::Build,
Mode::ToolRustc,
bin_name,
self.compiler.stage,
&self.compiler.host,
&self.target,
);
cargo.into_cmd().run(builder);
let tool_out = builder
.cargo_out(self.compiler, Mode::ToolRustc, self.target)
.join(exe(bin_name, self.compiler.host));
if self.compiler.stage > 0 {
let bindir_self_contained = builder let bindir_self_contained = builder
.sysroot(self.compiler) .sysroot(tool_result.target_compiler)
.join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple)); .join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple));
t!(fs::create_dir_all(&bindir_self_contained)); t!(fs::create_dir_all(&bindir_self_contained));
let bin_destination = bindir_self_contained.join(exe(bin_name, self.compiler.host)); let bin_destination = bindir_self_contained
builder.copy_link(&tool_out, &bin_destination); .join(exe("llvm-bitcode-linker", tool_result.target_compiler.host));
bin_destination builder.copy_link(&tool_result.tool_path, &bin_destination);
ToolBuildResult {
tool_path: bin_destination,
build_compiler: tool_result.build_compiler,
target_compiler: tool_result.target_compiler,
}
} else { } else {
tool_out tool_result
} }
} }
} }
@ -1067,7 +1086,7 @@ macro_rules! tool_extended {
} }
impl Step for $name { impl Step for $name {
type Output = PathBuf; type Output = ToolBuildResult;
const DEFAULT: bool = true; // Overridden by `should_run_tool_build_step` const DEFAULT: bool = true; // Overridden by `should_run_tool_build_step`
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
@ -1087,7 +1106,7 @@ macro_rules! tool_extended {
}); });
} }
fn run(self, builder: &Builder<'_>) -> PathBuf { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let Self { compiler, target } = self; let Self { compiler, target } = self;
run_tool_build_step( run_tool_build_step(
builder, builder,
@ -1133,38 +1152,37 @@ fn run_tool_build_step(
tool_name: &'static str, tool_name: &'static str,
path: &'static str, path: &'static str,
add_bins_to_sysroot: Option<&[&str]>, add_bins_to_sysroot: Option<&[&str]>,
) -> PathBuf { ) -> ToolBuildResult {
let tool = builder.ensure(ToolBuild { let ToolBuildResult { tool_path, build_compiler, target_compiler } =
compiler, builder.ensure(ToolBuild {
target, compiler,
tool: tool_name, target,
mode: Mode::ToolRustc, tool: tool_name,
path, mode: Mode::ToolRustc,
extra_features: vec![], path,
source_type: SourceType::InTree, extra_features: vec![],
allow_features: "", source_type: SourceType::InTree,
cargo_args: vec![], allow_features: "",
}); cargo_args: vec![],
});
// FIXME: This should just be an if-let-chain, but those are unstable. // FIXME: This should just be an if-let-chain, but those are unstable.
if let Some(add_bins_to_sysroot) = if let Some(add_bins_to_sysroot) =
add_bins_to_sysroot.filter(|bins| !bins.is_empty() && compiler.stage > 0) add_bins_to_sysroot.filter(|bins| !bins.is_empty() && target_compiler.stage > 0)
{ {
let bindir = builder.sysroot(compiler).join("bin"); let bindir = builder.sysroot(target_compiler).join("bin");
t!(fs::create_dir_all(&bindir)); t!(fs::create_dir_all(&bindir));
let tools_out = builder.cargo_out(compiler, Mode::ToolRustc, target);
for add_bin in add_bins_to_sysroot { for add_bin in add_bins_to_sysroot {
let bin_source = tools_out.join(exe(add_bin, target)); let bin_destination = bindir.join(exe(add_bin, target_compiler.host));
let bin_destination = bindir.join(exe(add_bin, compiler.host)); builder.copy_link(&tool_path, &bin_destination);
builder.copy_link(&bin_source, &bin_destination);
} }
// Return a path into the bin dir. // Return a path into the bin dir.
bindir.join(exe(tool_name, compiler.host)) let path = bindir.join(exe(tool_name, target_compiler.host));
ToolBuildResult { tool_path: path, build_compiler, target_compiler }
} else { } else {
tool ToolBuildResult { tool_path, build_compiler, target_compiler }
} }
} }
@ -1202,7 +1220,7 @@ pub struct TestFloatParse {
} }
impl Step for TestFloatParse { impl Step for TestFloatParse {
type Output = (); type Output = ToolBuildResult;
const ONLY_HOSTS: bool = true; const ONLY_HOSTS: bool = true;
const DEFAULT: bool = false; const DEFAULT: bool = false;
@ -1210,7 +1228,7 @@ impl Step for TestFloatParse {
run.path("src/etc/test-float-parse") run.path("src/etc/test-float-parse")
} }
fn run(self, builder: &Builder<'_>) { fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let bootstrap_host = builder.config.build; let bootstrap_host = builder.config.build;
let compiler = builder.compiler(builder.top_stage, bootstrap_host); let compiler = builder.compiler(builder.top_stage, bootstrap_host);
@ -1224,7 +1242,7 @@ impl Step for TestFloatParse {
extra_features: Vec::new(), extra_features: Vec::new(),
allow_features: "", allow_features: "",
cargo_args: Vec::new(), cargo_args: Vec::new(),
}); })
} }
} }

33
src/bootstrap/src/core/builder/mod.rs
View file

@ -1392,7 +1392,7 @@ impl<'a> Builder<'a> {
} }
pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
self.ensure(tool::Rustdoc { compiler }) self.ensure(tool::Rustdoc { compiler }).tool_path
} }
pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand {
@ -1408,14 +1408,13 @@ impl<'a> Builder<'a> {
return cmd; return cmd;
} }
let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build); let _ = self.ensure(tool::Clippy { compiler: run_compiler, target: self.build.build });
self.ensure(tool::Clippy { compiler: build_compiler, target: self.build.build });
let cargo_clippy = let cargo_clippy =
self.ensure(tool::CargoClippy { compiler: build_compiler, target: self.build.build }); self.ensure(tool::CargoClippy { compiler: run_compiler, target: self.build.build });
let mut dylib_path = helpers::dylib_path(); let mut dylib_path = helpers::dylib_path();
dylib_path.insert(0, self.sysroot(run_compiler).join("lib")); dylib_path.insert(0, self.sysroot(run_compiler).join("lib"));
let mut cmd = command(cargo_clippy); let mut cmd = command(cargo_clippy.tool_path);
cmd.env(helpers::dylib_path_var(), env::join_paths(&dylib_path).unwrap()); cmd.env(helpers::dylib_path_var(), env::join_paths(&dylib_path).unwrap());
cmd.env("CARGO", &self.initial_cargo); cmd.env("CARGO", &self.initial_cargo);
cmd cmd
@ -1423,23 +1422,21 @@ impl<'a> Builder<'a> {
pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> BootstrapCommand { pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> BootstrapCommand {
assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0"); assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0");
let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build);
// Prepare the tools // Prepare the tools
let miri = self.ensure(tool::Miri { compiler: build_compiler, target: self.build.build }); let miri = self.ensure(tool::Miri { compiler: run_compiler, target: self.build.build });
let cargo_miri = let cargo_miri =
self.ensure(tool::CargoMiri { compiler: build_compiler, target: self.build.build }); self.ensure(tool::CargoMiri { compiler: run_compiler, target: self.build.build });
// Invoke cargo-miri, make sure it can find miri and cargo. // Invoke cargo-miri, make sure it can find miri and cargo.
let mut cmd = command(cargo_miri); let mut cmd = command(cargo_miri.tool_path);
cmd.env("MIRI", &miri); cmd.env("MIRI", &miri.tool_path);
cmd.env("CARGO", &self.initial_cargo); cmd.env("CARGO", &self.initial_cargo);
// Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`, // Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`
// so they match the Miri we just built. However this means they are actually living one // in `tool::ToolBuild` step, so they match the Miri we just built. However this means they
// stage up, i.e. we are running `stage0-tools-bin/miri` with the libraries in `stage1/lib`. // are actually living one stage up, i.e. we are running `stage0-tools-bin/miri` with the
// This is an unfortunate off-by-1 caused (possibly) by the fact that Miri doesn't have an // libraries in `stage1/lib`. This is an unfortunate off-by-1 caused (possibly) by the fact
// "assemble" step like rustc does that would cross the stage boundary. We can't use // that Miri doesn't have an "assemble" step like rustc does that would cross the stage boundary.
// `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries added to // We can't use `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries
// the PATH due to the stage mismatch. // added to the PATH due to the stage mismatch.
// Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503. // Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503.
add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd); add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd);
cmd cmd

31
src/bootstrap/src/core/builder/tests.rs
View file

@ -525,6 +525,7 @@ mod dist {
first(cache.all::<compile::Rustc>()), first(cache.all::<compile::Rustc>()),
&[ &[
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0), rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0),
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 0),
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1), rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1),
] ]
); );
@ -1084,3 +1085,33 @@ fn test_is_builder_target() {
assert!(!builder.is_builder_target(target2)); assert!(!builder.is_builder_target(target2));
} }
} }
#[test]
fn test_get_tool_rustc_compiler() {
let mut config = configure("build", &[], &[]);
config.download_rustc_commit = None;
let build = Build::new(config);
let builder = Builder::new(&build);
let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1);
let compiler = Compiler { stage: 2, host: target_triple_1 };
let expected = Compiler { stage: 1, host: target_triple_1 };
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let expected = Compiler { stage: 0, host: target_triple_1 };
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
let mut config = configure("build", &[], &[]);
config.download_rustc_commit = Some("".to_owned());
let build = Build::new(config);
let builder = Builder::new(&build);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let expected = Compiler { stage: 1, host: target_triple_1 };
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
}

5
src/bootstrap/src/utils/change_tracker.rs
View file

@ -355,4 +355,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
severity: ChangeSeverity::Info, severity: ChangeSeverity::Info,
summary: "It is now possible to configure `jemalloc` for each target", summary: "It is now possible to configure `jemalloc` for each target",
}, },
ChangeInfo {
change_id: 137215,
severity: ChangeSeverity::Info,
summary: "Added `build.test-stage = 2` to 'tools' profile defaults",
},
]; ];

1
src/doc/rustc-dev-guide/src/SUMMARY.md
View file

@ -75,6 +75,7 @@
- [Prologue](./building/bootstrapping/intro.md) - [Prologue](./building/bootstrapping/intro.md)
- [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md) - [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md)
- [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md) - [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md)
- [Writing tools in Bootstrap](./building/bootstrapping/writing-tools-in-bootstrap.md)
- [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md) - [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md)
# High-level Compiler Architecture # High-level Compiler Architecture

23
src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md Normal file
View file

@ -0,0 +1,23 @@
# Writing tools in Bootstrap
There are three types of tools you can write in bootstrap:
- **`Mode::ToolBootstrap`**
Use this for tools that don’t need anything from the in-tree compiler and can run with the stage0 `rustc`.
The output is placed in the "stage0-bootstrap-tools" directory. This mode is for general-purpose tools built
entirely with the stage0 compiler, including target libraries and only works for stage 0.
- **`Mode::ToolStd`**
Use this for tools that rely on the locally built std. The output goes into the "stageN-tools" directory.
This mode is rarely used, mainly for `compiletest` which requires `libtest`.
- **`Mode::ToolRustc`**
Use this for tools that depend on both the locally built `rustc` and the target `std`. This is more complex than
the other modes because the tool must be built with the same compiler used for `rustc` and placed in the "stageN-tools"
directory. When you choose `Mode::ToolRustc`, `ToolBuild` implementation takes care of this automatically.
If you need to use the builder’s compiler for something specific, you can get it from `ToolBuildResult`, which is
returned by the tool's [`Step`].
Regardless of the tool type you must return `ToolBuildResult` from the tool’s [`Step`] implementation and use `ToolBuild` inside it.
[`Step`]: https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html