bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #137215 - onur-ozkan:rustc-tool-build-stages, r=jieyouxu,Kobzol

Browse source 
stabilize stage management for rustc tools

https://github.com/rust-lang/rust/pull/135990 got out of control due to excessive complexity. This PR aims to achieve the same goal with a simpler approach, likely through multiple smaller PRs. I will keep the other one read-only and open as a reference for future work.

This work stabilizes the staging logic for `ToolRustc` programs, so you no longer need to handle build and target compilers separately in steps. Previously, most tools didn't do this correctly, which was causing the compiler to be built twice (e.g., `x test cargo --stage 1` would compile the stage 2 compiler before, but now it only compiles the stage 1 compiler).

I also tried to document how we should write `ToolRustc` steps as they are quite different and require more attention than other tools.

Next goal is to stabilize how stages are handled for the rustc itself. Currently, `x build --stage 1` builds the stage 1 compiler which is fine, but `x build compiler --stage 1` builds stage 2 compiler.

~~for now, r? ghost~~
This commit is contained in:
bors 2025-02-23 05:03:26 +00:00
commit bb2cc59a21
12 changed files with 340 additions and 263 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/bootstrap/defaults/config.tools.toml
View file

@ -8,6 +8,8 @@ incremental = true
download-rustc = "if-unchanged"
[build]
# cargo and clippy tests don't pass on stage 1
test-stage = 2
# Document with the in-tree rustdoc by default, since `download-rustc` makes it quick to compile.
doc-stage = 2
# Contributors working on tools will probably expect compiler docs to be generated, so they can figure out how to use the API.

26
src/bootstrap/src/core/build_steps/compile.rs
View file

@ -1983,13 +1983,14 @@ impl Step for Assemble {
let maybe_install_llvm_bitcode_linker = |compiler| {
if builder.config.llvm_bitcode_linker_enabled {
trace!("llvm-bitcode-linker enabled, installing");
let src_path = builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
compiler,
target: target_compiler.host,
extra_features: vec![],
});
let llvm_bitcode_linker =
builder.ensure(crate::core::build_steps::tool::LlvmBitcodeLinker {
compiler,
target: target_compiler.host,
extra_features: vec![],
});
let tool_exe = exe("llvm-bitcode-linker", target_compiler.host);
builder.copy_link(&src_path, &libdir_bin.join(tool_exe));
builder.copy_link(&llvm_bitcode_linker.tool_path, &libdir_bin.join(tool_exe));
}
};
@ -2181,14 +2182,13 @@ impl Step for Assemble {
// logic to create the final binary. This is used by the
// `wasm32-wasip2` target of Rust.
if builder.tool_enabled("wasm-component-ld") {
let wasm_component_ld_exe =
builder.ensure(crate::core::build_steps::tool::WasmComponentLd {
compiler: build_compiler,
target: target_compiler.host,
});
let wasm_component = builder.ensure(crate::core::build_steps::tool::WasmComponentLd {
compiler: build_compiler,
target: target_compiler.host,
});
builder.copy_link(
&wasm_component_ld_exe,
&libdir_bin.join(wasm_component_ld_exe.file_name().unwrap()),
&wasm_component.tool_path,
&libdir_bin.join(wasm_component.tool_path.file_name().unwrap()),
);
}

22
src/bootstrap/src/core/build_steps/dist.rs
View file

@ -430,7 +430,7 @@ impl Step for Rustc {
},
builder.kind,
) {
builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755);
builder.install(&ra_proc_macro_srv.tool_path, &image.join("libexec"), 0o755);
}
let libdir_relative = builder.libdir_relative(compiler);
@ -1145,7 +1145,7 @@ impl Step for Cargo {
let mut tarball = Tarball::new(builder, "cargo", &target.triple);
tarball.set_overlay(OverlayKind::Cargo);
tarball.add_file(cargo, "bin", 0o755);
tarball.add_file(cargo.tool_path, "bin", 0o755);
tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644);
tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo");
tarball.add_dir(etc.join("man"), "share/man/man1");
@ -1191,7 +1191,7 @@ impl Step for Rls {
let mut tarball = Tarball::new(builder, "rls", &target.triple);
tarball.set_overlay(OverlayKind::Rls);
tarball.is_preview(true);
tarball.add_file(rls, "bin", 0o755);
tarball.add_file(rls.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rls");
Some(tarball.generate())
}
@ -1233,7 +1233,7 @@ impl Step for RustAnalyzer {
let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);
tarball.set_overlay(OverlayKind::RustAnalyzer);
tarball.is_preview(true);
tarball.add_file(rust_analyzer, "bin", 0o755);
tarball.add_file(rust_analyzer.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rust-analyzer");
Some(tarball.generate())
}
@ -1279,8 +1279,8 @@ impl Step for Clippy {
let mut tarball = Tarball::new(builder, "clippy", &target.triple);
tarball.set_overlay(OverlayKind::Clippy);
tarball.is_preview(true);
tarball.add_file(clippy, "bin", 0o755);
tarball.add_file(cargoclippy, "bin", 0o755);
tarball.add_file(clippy.tool_path, "bin", 0o755);
tarball.add_file(cargoclippy.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/clippy");
Some(tarball.generate())
}
@ -1329,8 +1329,8 @@ impl Step for Miri {
let mut tarball = Tarball::new(builder, "miri", &target.triple);
tarball.set_overlay(OverlayKind::Miri);
tarball.is_preview(true);
tarball.add_file(miri, "bin", 0o755);
tarball.add_file(cargomiri, "bin", 0o755);
tarball.add_file(miri.tool_path, "bin", 0o755);
tarball.add_file(cargomiri.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/miri");
Some(tarball.generate())
}
@ -1460,8 +1460,8 @@ impl Step for Rustfmt {
let mut tarball = Tarball::new(builder, "rustfmt", &target.triple);
tarball.set_overlay(OverlayKind::Rustfmt);
tarball.is_preview(true);
tarball.add_file(rustfmt, "bin", 0o755);
tarball.add_file(cargofmt, "bin", 0o755);
tarball.add_file(rustfmt.tool_path, "bin", 0o755);
tarball.add_file(cargofmt.tool_path, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rustfmt");
Some(tarball.generate())
}
@ -2283,7 +2283,7 @@ impl Step for LlvmBitcodeLinker {
tarball.set_overlay(OverlayKind::LlvmBitcodeLinker);
tarball.is_preview(true);
tarball.add_file(llbc_linker, self_contained_bin_dir, 0o755);
tarball.add_file(llbc_linker.tool_path, self_contained_bin_dir, 0o755);
Some(tarball.generate())
}

2
src/bootstrap/src/core/build_steps/perf.rs
View file

@ -166,7 +166,7 @@ Consider setting `rust.debuginfo-level = 1` in `config.toml`."#);
let results_dir = rustc_perf_dir.join("results");
builder.create_dir(&results_dir);
let mut cmd = command(collector);
let mut cmd = command(collector.tool_path);
// We need to set the working directory to `src/tools/rustc-perf`, so that it can find the directory
// with compile-time benchmarks.

6
src/bootstrap/src/core/build_steps/run.rs
View file

@ -126,11 +126,7 @@ impl Step for Miri {
// This compiler runs on the host, we'll just use it for the target.
let target_compiler = builder.compiler(stage, host);
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let host_compiler = builder.compiler(stage - 1, host);
let host_compiler = tool::get_tool_rustc_compiler(builder, target_compiler);
// Get a target sysroot for Miri.
let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target);

66
src/bootstrap/src/core/build_steps/test.rs
View file

@ -263,7 +263,7 @@ impl Step for Cargotest {
let _time = helpers::timeit(builder);
let mut cmd = builder.tool_cmd(Tool::CargoTest);
cmd.arg(&cargo)
cmd.arg(&cargo.tool_path)
.arg(&out_dir)
.args(builder.config.test_args())
.env("RUSTC", builder.rustc(compiler))
@ -298,9 +298,16 @@ impl Step for Cargo {
/// Runs `cargo test` for `cargo` packaged with Rust.
fn run(self, builder: &Builder<'_>) {
if self.stage < 2 {
eprintln!("WARNING: cargo tests on stage {} may not behave well.", self.stage);
eprintln!("HELP: consider using stage 2");
}
let compiler = builder.compiler(self.stage, self.host);
builder.ensure(tool::Cargo { compiler, target: self.host });
let cargo = builder.ensure(tool::Cargo { compiler, target: self.host });
let compiler = cargo.build_compiler;
let cargo = tool::prepare_tool_cargo(
builder,
compiler,
@ -367,6 +374,7 @@ impl Step for RustAnalyzer {
let stage = self.stage;
let host = self.host;
let compiler = builder.compiler(stage, host);
let compiler = tool::get_tool_rustc_compiler(builder, compiler);
// We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite,
// but we do need the standard library to be present.
@ -427,7 +435,8 @@ impl Step for Rustfmt {
let host = self.host;
let compiler = builder.compiler(stage, host);
builder.ensure(tool::Rustfmt { compiler, target: self.host });
let tool_result = builder.ensure(tool::Rustfmt { compiler, target: self.host });
let compiler = tool_result.build_compiler;
let mut cargo = tool::prepare_tool_cargo(
builder,
@ -522,16 +531,11 @@ impl Step for Miri {
// This compiler runs on the host, we'll just use it for the target.
let target_compiler = builder.compiler(stage, host);
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let host_compiler = builder.compiler(stage - 1, host);
// Build our tools.
let miri = builder.ensure(tool::Miri { compiler: host_compiler, target: host });
let miri = builder.ensure(tool::Miri { compiler: target_compiler, target: host });
// the ui tests also assume cargo-miri has been built
builder.ensure(tool::CargoMiri { compiler: host_compiler, target: host });
builder.ensure(tool::CargoMiri { compiler: target_compiler, target: host });
// We also need sysroots, for Miri and for the host (the latter for build scripts).
// This is for the tests so everything is done with the target compiler.
@ -542,7 +546,8 @@ impl Step for Miri {
// Miri has its own "target dir" for ui test dependencies. Make sure it gets cleared when
// the sysroot gets rebuilt, to avoid "found possibly newer version of crate `std`" errors.
if !builder.config.dry_run() {
let ui_test_dep_dir = builder.stage_out(host_compiler, Mode::ToolStd).join("miri_ui");
let ui_test_dep_dir =
builder.stage_out(miri.build_compiler, Mode::ToolStd).join("miri_ui");
// The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see
// <https://github.com/RalfJung/rustc-build-sysroot/commit/10ebcf60b80fe2c3dc765af0ff19fdc0da4b7466>).
// We can hence use that directly as a signal to clear the ui test dir.
@ -553,7 +558,7 @@ impl Step for Miri {
// This is with the Miri crate, so it uses the host compiler.
let mut cargo = tool::prepare_tool_cargo(
builder,
host_compiler,
miri.build_compiler,
Mode::ToolRustc,
host,
Kind::Test,
@ -571,7 +576,7 @@ impl Step for Miri {
// miri tests need to know about the stage sysroot
cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", &host_sysroot);
cargo.env("MIRI", &miri);
cargo.env("MIRI", &miri.tool_path);
// Set the target.
cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg());
@ -743,7 +748,13 @@ impl Step for Clippy {
let host = self.host;
let compiler = builder.compiler(stage, host);
builder.ensure(tool::Clippy { compiler, target: self.host });
if stage < 2 {
eprintln!("WARNING: clippy tests on stage {stage} may not behave well.");
eprintln!("HELP: consider using stage 2");
}
let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host });
let compiler = tool_result.build_compiler;
let mut cargo = tool::prepare_tool_cargo(
builder,
compiler,
@ -1728,18 +1739,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
// If we're using `--stage 0`, we should provide the bootstrap cargo.
builder.initial_cargo.clone()
} else {
// We need to properly build cargo using the suitable stage compiler.
let compiler = builder.download_rustc().then_some(compiler).unwrap_or_else(||
// HACK: currently tool stages are off-by-one compared to compiler stages, i.e. if
// you give `tool::Cargo` a stage 1 rustc, it will cause stage 2 rustc to be built
// and produce a cargo built with stage 2 rustc. To fix this, we need to chop off
// the compiler stage by 1 to align with expected `./x test run-make --stage N`
// behavior, i.e. we need to pass `N - 1` compiler stage to cargo. See also Miri
// which does a similar hack.
builder.compiler(builder.top_stage - 1, compiler.host));
builder.ensure(tool::Cargo { compiler, target: compiler.host })
builder.ensure(tool::Cargo { compiler, target: compiler.host }).tool_path
};
cmd.arg("--cargo-path").arg(cargo_path);
@ -1760,9 +1760,10 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
// Use the beta compiler for jsondocck
let json_compiler = compiler.with_stage(0);
cmd.arg("--jsondocck-path")
.arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }));
cmd.arg("--jsondoclint-path")
.arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
.arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }).tool_path);
cmd.arg("--jsondoclint-path").arg(
builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }).tool_path,
);
}
if matches!(mode, "coverage-map" | "coverage-run") {
@ -2999,12 +3000,15 @@ impl Step for RemoteCopyLibs {
builder.info(&format!("REMOTE copy libs to emulator ({target})"));
let server = builder.ensure(tool::RemoteTestServer { compiler, target });
let remote_test_server = builder.ensure(tool::RemoteTestServer { compiler, target });
// Spawn the emulator and wait for it to come online
let tool = builder.tool_exe(Tool::RemoteTestClient);
let mut cmd = command(&tool);
cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir());
cmd.arg("spawn-emulator")
.arg(target.triple)
.arg(&remote_test_server.tool_path)
.arg(builder.tempdir());
if let Some(rootfs) = builder.qemu_rootfs(target) {
cmd.arg(rootfs);
}

386
src/bootstrap/src/core/build_steps/tool.rs
View file

@ -1,3 +1,14 @@
//! This module handles building and managing various tools in bootstrap
//! build system.
//!
//! **What It Does**
//! - Defines how tools are built, configured and installed.
//! - Manages tool dependencies and build steps.
//! - Copies built tool binaries to the correct locations.
//!
//! Each Rust tool **MUST** utilize `ToolBuild` inside their `Step` logic,
//! return `ToolBuildResult` and should never prepare `cargo` invocations manually.
use std::path::PathBuf;
use std::{env, fs};
@ -64,8 +75,21 @@ impl Builder<'_> {
}
}
/// Result of the tool build process. Each `Step` in this module is responsible
/// for using this type as `type Output = ToolBuildResult;`
#[derive(Clone)]
pub struct ToolBuildResult {
/// Executable path of the corresponding tool that was built.
pub tool_path: PathBuf,
/// Compiler used to build the tool. For non-`ToolRustc` tools this is equal to `target_compiler`.
/// For `ToolRustc` this is one stage before of the `target_compiler`.
pub build_compiler: Compiler,
/// Target compiler passed to `Step`.
pub target_compiler: Compiler,
}
impl Step for ToolBuild {
type Output = PathBuf;
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.never()
@ -75,25 +99,31 @@ impl Step for ToolBuild {
///
/// This will build the specified tool with the specified `host` compiler in
/// `stage` into the normal cargo output directory.
fn run(self, builder: &Builder<'_>) -> PathBuf {
let compiler = self.compiler;
fn run(mut self, builder: &Builder<'_>) -> ToolBuildResult {
let target = self.target;
let mut tool = self.tool;
let path = self.path;
let target_compiler = self.compiler;
self.compiler = if self.mode == Mode::ToolRustc {
get_tool_rustc_compiler(builder, self.compiler)
} else {
self.compiler
};
match self.mode {
Mode::ToolRustc => {
builder.ensure(compile::Std::new(compiler, compiler.host));
builder.ensure(compile::Rustc::new(compiler, target));
builder.ensure(compile::Std::new(self.compiler, self.compiler.host));
builder.ensure(compile::Rustc::new(self.compiler, target));
}
Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)),
Mode::ToolStd => builder.ensure(compile::Std::new(self.compiler, target)),
Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs
_ => panic!("unexpected Mode for tool build"),
}
let mut cargo = prepare_tool_cargo(
builder,
compiler,
self.compiler,
self.mode,
target,
Kind::Build,
@ -101,10 +131,28 @@ impl Step for ToolBuild {
self.source_type,
&self.extra_features,
);
if path.ends_with("/rustdoc") &&
// rustdoc is performance sensitive, so apply LTO to it.
is_lto_stage(&self.compiler)
{
let lto = match builder.config.rust_lto {
RustcLto::Off => Some("off"),
RustcLto::Thin => Some("thin"),
RustcLto::Fat => Some("fat"),
RustcLto::ThinLocal => None,
};
if let Some(lto) = lto {
cargo.env(cargo_profile_var("LTO", &builder.config), lto);
}
}
if !self.allow_features.is_empty() {
cargo.allow_features(self.allow_features);
}
cargo.args(self.cargo_args);
let _guard = builder.msg_tool(
Kind::Build,
self.mode,
@ -131,7 +179,10 @@ impl Step for ToolBuild {
if tool == "tidy" {
tool = "rust-tidy";
}
copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool)
let tool_path =
copy_link_tool_bin(builder, self.compiler, self.target, self.mode, tool);
ToolBuildResult { tool_path, build_compiler: self.compiler, target_compiler }
}
}
}
@ -240,6 +291,23 @@ pub fn prepare_tool_cargo(
cargo
}
/// Handle stage-off logic for `ToolRustc` tools when necessary.
pub(crate) fn get_tool_rustc_compiler(
builder: &Builder<'_>,
target_compiler: Compiler,
) -> Compiler {
if builder.download_rustc() && target_compiler.stage == 1 {
// We already have the stage 1 compiler, we don't need to cut the stage.
builder.compiler(target_compiler.stage, builder.config.build)
} else {
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/$rustc_tool be effectively different stage
// compilers, which isn't what we want. Rustc tools should be linked in the same way as the
// compiler it's paired with, so it must be built with the previous stage compiler.
builder.compiler(target_compiler.stage.saturating_sub(1), builder.config.build)
}
}
/// Links a built tool binary with the given `name` from the build directory to the
/// tools directory.
fn copy_link_tool_bin(
@ -279,7 +347,7 @@ macro_rules! bootstrap_tool {
self.ensure($name {
compiler: self.compiler(0, self.config.build),
target: self.config.build,
}),
}).tool_path,
)+
}
}
@ -293,7 +361,7 @@ macro_rules! bootstrap_tool {
}
impl Step for $name {
type Output = PathBuf;
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path($path)
@ -315,7 +383,7 @@ macro_rules! bootstrap_tool {
skip_all,
),
)]
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
$(
for submodule in $submodules {
builder.require_submodule(submodule, None);
@ -390,7 +458,7 @@ pub struct OptimizedDist {
}
impl Step for OptimizedDist {
type Output = PathBuf;
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/opt-dist")
@ -403,7 +471,7 @@ impl Step for OptimizedDist {
});
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
// We need to ensure the rustc-perf submodule is initialized when building opt-dist since
// the tool requires it to be in place to run.
builder.require_submodule("src/tools/rustc-perf", None);
@ -432,7 +500,7 @@ pub struct RustcPerf {
impl Step for RustcPerf {
/// Path to the built `collector` binary.
type Output = PathBuf;
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/rustc-perf")
@ -445,7 +513,7 @@ impl Step for RustcPerf {
});
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
// We need to ensure the rustc-perf submodule is initialized.
builder.require_submodule("src/tools/rustc-perf", None);
@ -462,12 +530,12 @@ impl Step for RustcPerf {
// a CLI.
cargo_args: vec!["-p".to_string(), "collector".to_string()],
};
let collector_bin = builder.ensure(tool.clone());
let res = builder.ensure(tool.clone());
// We also need to symlink the `rustc-fake` binary to the corresponding directory,
// because `collector` expects it in the same directory.
copy_link_tool_bin(builder, tool.compiler, tool.target, tool.mode, "rustc-fake");
collector_bin
res
}
}
@ -482,7 +550,7 @@ impl ErrorIndex {
// for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc.
let host = builder.config.build;
let compiler = builder.compiler_for(builder.top_stage, host, host);
let mut cmd = command(builder.ensure(ErrorIndex { compiler }));
let mut cmd = command(builder.ensure(ErrorIndex { compiler }).tool_path);
let mut dylib_paths = builder.rustc_lib_paths(compiler);
dylib_paths.push(PathBuf::from(&builder.sysroot_target_libdir(compiler, compiler.host)));
add_dylib_path(dylib_paths, &mut cmd);
@ -491,27 +559,23 @@ impl ErrorIndex {
}
impl Step for ErrorIndex {
type Output = PathBuf;
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/error_index_generator")
}
fn make_run(run: RunConfig<'_>) {
// Compile the error-index in the same stage as rustdoc to avoid
// recompiling rustdoc twice if we can.
//
// NOTE: This `make_run` isn't used in normal situations, only if you
// manually build the tool with `x.py build
// src/tools/error-index-generator` which almost nobody does.
// Normally, `x.py test` or `x.py doc` will use the
// `ErrorIndex::command` function instead.
let compiler =
run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build);
let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build);
run.builder.ensure(ErrorIndex { compiler });
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.compiler.host,
@ -533,7 +597,7 @@ pub struct RemoteTestServer {
}
impl Step for RemoteTestServer {
type Output = PathBuf;
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/remote-test-server")
@ -546,7 +610,7 @@ impl Step for RemoteTestServer {
});
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.target,
@ -569,7 +633,7 @@ pub struct Rustdoc {
}
impl Step for Rustdoc {
type Output = PathBuf;
type Output = ToolBuildResult;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
@ -578,24 +642,25 @@ impl Step for Rustdoc {
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Rustdoc {
// NOTE: this is somewhat unique in that we actually want a *target*
// compiler here, because rustdoc *is* a compiler. We won't be using
// this as the compiler to build with, but rather this is "what
// compiler are we producing"?
compiler: run.builder.compiler(run.builder.top_stage, run.target),
});
run.builder
.ensure(Rustdoc { compiler: run.builder.compiler(run.builder.top_stage, run.target) });
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let target_compiler = self.compiler;
let target = target_compiler.host;
if target_compiler.stage == 0 {
if !target_compiler.is_snapshot(builder) {
panic!("rustdoc in stage 0 must be snapshot rustdoc");
}
return builder.initial_rustdoc.clone();
return ToolBuildResult {
tool_path: builder.initial_rustdoc.clone(),
build_compiler: target_compiler,
target_compiler,
};
}
let target = target_compiler.host;
let bin_rustdoc = || {
let sysroot = builder.sysroot(target_compiler);
@ -625,27 +690,15 @@ impl Step for Rustdoc {
let bin_rustdoc = bin_rustdoc();
builder.copy_link(&precompiled_rustdoc, &bin_rustdoc);
return bin_rustdoc;
return ToolBuildResult {
tool_path: bin_rustdoc,
build_compiler: target_compiler,
target_compiler,
};
}
}
let build_compiler = if builder.download_rustc() && target_compiler.stage == 1 {
// We already have the stage 1 compiler, we don't need to cut the stage.
builder.compiler(target_compiler.stage, builder.config.build)
} else {
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
builder.compiler(target_compiler.stage - 1, builder.config.build)
};
// When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually
// build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build
// it.
builder.ensure(compile::Std::new(build_compiler, target_compiler.host));
builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host));
// The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
// compiler libraries, ...) are built. Rustdoc does not require the presence of any
// libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since
@ -653,65 +706,39 @@ impl Step for Rustdoc {
// libraries here. The intuition here is that If we've built a compiler, we should be able
// to build rustdoc.
//
let mut features = Vec::new();
let mut extra_features = Vec::new();
if builder.config.jemalloc(target) {
features.push("jemalloc".to_string());
extra_features.push("jemalloc".to_string());
}
// NOTE: Never modify the rustflags here, it breaks the build cache for other tools!
let mut cargo = prepare_tool_cargo(
builder,
build_compiler,
Mode::ToolRustc,
target,
Kind::Build,
"src/tools/rustdoc",
SourceType::InTree,
features.as_slice(),
);
// rustdoc is performance sensitive, so apply LTO to it.
if is_lto_stage(&build_compiler) {
let lto = match builder.config.rust_lto {
RustcLto::Off => Some("off"),
RustcLto::Thin => Some("thin"),
RustcLto::Fat => Some("fat"),
RustcLto::ThinLocal => None,
};
if let Some(lto) = lto {
cargo.env(cargo_profile_var("LTO", &builder.config), lto);
}
}
let _guard = builder.msg_tool(
Kind::Build,
Mode::ToolRustc,
"rustdoc",
build_compiler.stage,
&self.compiler.host,
&target,
);
cargo.into_cmd().run(builder);
// Cargo adds a number of paths to the dylib search path on windows, which results in
// the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
// rustdoc a different name.
let tool_rustdoc = builder
.cargo_out(build_compiler, Mode::ToolRustc, target)
.join(exe("rustdoc_tool_binary", target_compiler.host));
let ToolBuildResult { tool_path, build_compiler, target_compiler } =
builder.ensure(ToolBuild {
compiler: target_compiler,
target,
// Cargo adds a number of paths to the dylib search path on windows, which results in
// the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool"
// rustdoc a different name.
tool: "rustdoc_tool_binary",
mode: Mode::ToolRustc,
path: "src/tools/rustdoc",
source_type: SourceType::InTree,
extra_features,
allow_features: "",
cargo_args: Vec::new(),
});
// don't create a stage0-sysroot/bin directory.
if target_compiler.stage > 0 {
if builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None {
// Due to LTO a lot of debug info from C++ dependencies such as jemalloc can make it into
// our final binaries
compile::strip_debug(builder, target, &tool_rustdoc);
compile::strip_debug(builder, target, &tool_path);
}
let bin_rustdoc = bin_rustdoc();
builder.copy_link(&tool_rustdoc, &bin_rustdoc);
bin_rustdoc
builder.copy_link(&tool_path, &bin_rustdoc);
ToolBuildResult { tool_path: bin_rustdoc, build_compiler, target_compiler }
} else {
tool_rustdoc
ToolBuildResult { tool_path, build_compiler, target_compiler }
}
}
}
@ -723,7 +750,7 @@ pub struct Cargo {
}
impl Step for Cargo {
type Output = PathBuf;
type Output = ToolBuildResult;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
@ -739,7 +766,7 @@ impl Step for Cargo {
});
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.build.require_submodule("src/tools/cargo", None);
builder.ensure(ToolBuild {
@ -763,7 +790,7 @@ pub struct LldWrapper {
}
impl Step for LldWrapper {
type Output = ();
type Output = ToolBuildResult;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.never()
@ -778,14 +805,19 @@ impl Step for LldWrapper {
fields(build_compiler = ?self.build_compiler, target_compiler = ?self.target_compiler),
),
)]
fn run(self, builder: &Builder<'_>) {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
if builder.config.dry_run() {
return;
return ToolBuildResult {
tool_path: Default::default(),
build_compiler: self.build_compiler,
target_compiler: self.target_compiler,
};
}
let target = self.target_compiler.host;
let executable = builder.ensure(ToolBuild {
let tool_result = builder.ensure(ToolBuild {
compiler: self.build_compiler,
target,
tool: "lld-wrapper",
@ -809,8 +841,11 @@ impl Step for LldWrapper {
t!(fs::create_dir_all(&self_contained_lld_dir));
for name in crate::LLD_FILE_NAMES {
builder.copy_link(&executable, &self_contained_lld_dir.join(exe(name, target)));
builder
.copy_link(&tool_result.tool_path, &self_contained_lld_dir.join(exe(name, target)));
}
tool_result
}
}
@ -825,7 +860,7 @@ impl RustAnalyzer {
}
impl Step for RustAnalyzer {
type Output = PathBuf;
type Output = ToolBuildResult;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
@ -841,7 +876,7 @@ impl Step for RustAnalyzer {
});
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.target,
@ -863,7 +898,7 @@ pub struct RustAnalyzerProcMacroSrv {
}
impl Step for RustAnalyzerProcMacroSrv {
type Output = Option<PathBuf>;
type Output = Option<ToolBuildResult>;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
@ -885,8 +920,8 @@ impl Step for RustAnalyzerProcMacroSrv {
});
}
fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
let path = builder.ensure(ToolBuild {
fn run(self, builder: &Builder<'_>) -> Option<ToolBuildResult> {
let tool_result = builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.target,
tool: "rust-analyzer-proc-macro-srv",
@ -902,9 +937,10 @@ impl Step for RustAnalyzerProcMacroSrv {
// so that r-a can use it.
let libexec_path = builder.sysroot(self.compiler).join("libexec");
t!(fs::create_dir_all(&libexec_path));
builder.copy_link(&path, &libexec_path.join("rust-analyzer-proc-macro-srv"));
builder
.copy_link(&tool_result.tool_path, &libexec_path.join("rust-analyzer-proc-macro-srv"));
Some(path)
Some(tool_result)
}
}
@ -916,7 +952,7 @@ pub struct LlvmBitcodeLinker {
}
impl Step for LlvmBitcodeLinker {
type Output = PathBuf;
type Output = ToolBuildResult;
const DEFAULT: bool = true;
const ONLY_HOSTS: bool = true;
@ -938,51 +974,34 @@ impl Step for LlvmBitcodeLinker {
feature = "tracing",
instrument(level = "debug", name = "LlvmBitcodeLinker::run", skip_all)
)]
fn run(self, builder: &Builder<'_>) -> PathBuf {
let bin_name = "llvm-bitcode-linker";
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let tool_result = builder.ensure(ToolBuild {
compiler: self.compiler,
target: self.target,
tool: "llvm-bitcode-linker",
mode: Mode::ToolRustc,
path: "src/tools/llvm-bitcode-linker",
source_type: SourceType::InTree,
extra_features: self.extra_features,
allow_features: "",
cargo_args: Vec::new(),
});
// If enabled, use ci-rustc and skip building the in-tree compiler.
if !builder.download_rustc() {
builder.ensure(compile::Std::new(self.compiler, self.compiler.host));
builder.ensure(compile::Rustc::new(self.compiler, self.target));
}
let cargo = prepare_tool_cargo(
builder,
self.compiler,
Mode::ToolRustc,
self.target,
Kind::Build,
"src/tools/llvm-bitcode-linker",
SourceType::InTree,
&self.extra_features,
);
let _guard = builder.msg_tool(
Kind::Build,
Mode::ToolRustc,
bin_name,
self.compiler.stage,
&self.compiler.host,
&self.target,
);
cargo.into_cmd().run(builder);
let tool_out = builder
.cargo_out(self.compiler, Mode::ToolRustc, self.target)
.join(exe(bin_name, self.compiler.host));
if self.compiler.stage > 0 {
if tool_result.target_compiler.stage > 0 {
let bindir_self_contained = builder
.sysroot(self.compiler)
.sysroot(tool_result.target_compiler)
.join(format!("lib/rustlib/{}/bin/self-contained", self.target.triple));
t!(fs::create_dir_all(&bindir_self_contained));
let bin_destination = bindir_self_contained.join(exe(bin_name, self.compiler.host));
builder.copy_link(&tool_out, &bin_destination);
bin_destination
let bin_destination = bindir_self_contained
.join(exe("llvm-bitcode-linker", tool_result.target_compiler.host));
builder.copy_link(&tool_result.tool_path, &bin_destination);
ToolBuildResult {
tool_path: bin_destination,
build_compiler: tool_result.build_compiler,
target_compiler: tool_result.target_compiler,
}
} else {
tool_out
tool_result
}
}
}
@ -1067,7 +1086,7 @@ macro_rules! tool_extended {
}
impl Step for $name {
type Output = PathBuf;
type Output = ToolBuildResult;
const DEFAULT: bool = true; // Overridden by `should_run_tool_build_step`
const ONLY_HOSTS: bool = true;
@ -1087,7 +1106,7 @@ macro_rules! tool_extended {
});
}
fn run(self, builder: &Builder<'_>) -> PathBuf {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let Self { compiler, target } = self;
run_tool_build_step(
builder,
@ -1133,38 +1152,37 @@ fn run_tool_build_step(
tool_name: &'static str,
path: &'static str,
add_bins_to_sysroot: Option<&[&str]>,
) -> PathBuf {
let tool = builder.ensure(ToolBuild {
compiler,
target,
tool: tool_name,
mode: Mode::ToolRustc,
path,
extra_features: vec![],
source_type: SourceType::InTree,
allow_features: "",
cargo_args: vec![],
});
) -> ToolBuildResult {
let ToolBuildResult { tool_path, build_compiler, target_compiler } =
builder.ensure(ToolBuild {
compiler,
target,
tool: tool_name,
mode: Mode::ToolRustc,
path,
extra_features: vec![],
source_type: SourceType::InTree,
allow_features: "",
cargo_args: vec![],
});
// FIXME: This should just be an if-let-chain, but those are unstable.
if let Some(add_bins_to_sysroot) =
add_bins_to_sysroot.filter(|bins| !bins.is_empty() && compiler.stage > 0)
add_bins_to_sysroot.filter(|bins| !bins.is_empty() && target_compiler.stage > 0)
{
let bindir = builder.sysroot(compiler).join("bin");
let bindir = builder.sysroot(target_compiler).join("bin");
t!(fs::create_dir_all(&bindir));
let tools_out = builder.cargo_out(compiler, Mode::ToolRustc, target);
for add_bin in add_bins_to_sysroot {
let bin_source = tools_out.join(exe(add_bin, target));
let bin_destination = bindir.join(exe(add_bin, compiler.host));
builder.copy_link(&bin_source, &bin_destination);
let bin_destination = bindir.join(exe(add_bin, target_compiler.host));
builder.copy_link(&tool_path, &bin_destination);
}
// Return a path into the bin dir.
bindir.join(exe(tool_name, compiler.host))
let path = bindir.join(exe(tool_name, target_compiler.host));
ToolBuildResult { tool_path: path, build_compiler, target_compiler }
} else {
tool
ToolBuildResult { tool_path, build_compiler, target_compiler }
}
}
@ -1202,7 +1220,7 @@ pub struct TestFloatParse {
}
impl Step for TestFloatParse {
type Output = ();
type Output = ToolBuildResult;
const ONLY_HOSTS: bool = true;
const DEFAULT: bool = false;
@ -1210,7 +1228,7 @@ impl Step for TestFloatParse {
run.path("src/etc/test-float-parse")
}
fn run(self, builder: &Builder<'_>) {
fn run(self, builder: &Builder<'_>) -> ToolBuildResult {
let bootstrap_host = builder.config.build;
let compiler = builder.compiler(builder.top_stage, bootstrap_host);
@ -1224,7 +1242,7 @@ impl Step for TestFloatParse {
extra_features: Vec::new(),
allow_features: "",
cargo_args: Vec::new(),
});
})
}
}

33
src/bootstrap/src/core/builder/mod.rs
View file

@ -1392,7 +1392,7 @@ impl<'a> Builder<'a> {
}
pub fn rustdoc(&self, compiler: Compiler) -> PathBuf {
self.ensure(tool::Rustdoc { compiler })
self.ensure(tool::Rustdoc { compiler }).tool_path
}
pub fn cargo_clippy_cmd(&self, run_compiler: Compiler) -> BootstrapCommand {
@ -1408,14 +1408,13 @@ impl<'a> Builder<'a> {
return cmd;
}
let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build);
self.ensure(tool::Clippy { compiler: build_compiler, target: self.build.build });
let _ = self.ensure(tool::Clippy { compiler: run_compiler, target: self.build.build });
let cargo_clippy =
self.ensure(tool::CargoClippy { compiler: build_compiler, target: self.build.build });
self.ensure(tool::CargoClippy { compiler: run_compiler, target: self.build.build });
let mut dylib_path = helpers::dylib_path();
dylib_path.insert(0, self.sysroot(run_compiler).join("lib"));
let mut cmd = command(cargo_clippy);
let mut cmd = command(cargo_clippy.tool_path);
cmd.env(helpers::dylib_path_var(), env::join_paths(&dylib_path).unwrap());
cmd.env("CARGO", &self.initial_cargo);
cmd
@ -1423,23 +1422,21 @@ impl<'a> Builder<'a> {
pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> BootstrapCommand {
assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0");
let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build);
// Prepare the tools
let miri = self.ensure(tool::Miri { compiler: build_compiler, target: self.build.build });
let miri = self.ensure(tool::Miri { compiler: run_compiler, target: self.build.build });
let cargo_miri =
self.ensure(tool::CargoMiri { compiler: build_compiler, target: self.build.build });
self.ensure(tool::CargoMiri { compiler: run_compiler, target: self.build.build });
// Invoke cargo-miri, make sure it can find miri and cargo.
let mut cmd = command(cargo_miri);
cmd.env("MIRI", &miri);
let mut cmd = command(cargo_miri.tool_path);
cmd.env("MIRI", &miri.tool_path);
cmd.env("CARGO", &self.initial_cargo);
// Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`,
// so they match the Miri we just built. However this means they are actually living one
// stage up, i.e. we are running `stage0-tools-bin/miri` with the libraries in `stage1/lib`.
// This is an unfortunate off-by-1 caused (possibly) by the fact that Miri doesn't have an
// "assemble" step like rustc does that would cross the stage boundary. We can't use
// `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries added to
// the PATH due to the stage mismatch.
// Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`
// in `tool::ToolBuild` step, so they match the Miri we just built. However this means they
// are actually living one stage up, i.e. we are running `stage0-tools-bin/miri` with the
// libraries in `stage1/lib`. This is an unfortunate off-by-1 caused (possibly) by the fact
// that Miri doesn't have an "assemble" step like rustc does that would cross the stage boundary.
// We can't use `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries
// added to the PATH due to the stage mismatch.
// Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503.
add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd);
cmd

31
src/bootstrap/src/core/builder/tests.rs
View file

@ -525,6 +525,7 @@ mod dist {
first(cache.all::<compile::Rustc>()),
&[
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_1, stage = 0),
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 0),
rustc!(TEST_TRIPLE_1 => TEST_TRIPLE_2, stage = 1),
]
);
@ -1084,3 +1085,33 @@ fn test_is_builder_target() {
assert!(!builder.is_builder_target(target2));
}
}
#[test]
fn test_get_tool_rustc_compiler() {
let mut config = configure("build", &[], &[]);
config.download_rustc_commit = None;
let build = Build::new(config);
let builder = Builder::new(&build);
let target_triple_1 = TargetSelection::from_user(TEST_TRIPLE_1);
let compiler = Compiler { stage: 2, host: target_triple_1 };
let expected = Compiler { stage: 1, host: target_triple_1 };
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let expected = Compiler { stage: 0, host: target_triple_1 };
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
let mut config = configure("build", &[], &[]);
config.download_rustc_commit = Some("".to_owned());
let build = Build::new(config);
let builder = Builder::new(&build);
let compiler = Compiler { stage: 1, host: target_triple_1 };
let expected = Compiler { stage: 1, host: target_triple_1 };
let actual = tool::get_tool_rustc_compiler(&builder, compiler);
assert_eq!(expected, actual);
}

5
src/bootstrap/src/utils/change_tracker.rs
View file

@ -355,4 +355,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
severity: ChangeSeverity::Info,
summary: "It is now possible to configure `jemalloc` for each target",
},
ChangeInfo {
change_id: 137215,
severity: ChangeSeverity::Info,
summary: "Added `build.test-stage = 2` to 'tools' profile defaults",
},
];

1
src/doc/rustc-dev-guide/src/SUMMARY.md
View file

@ -75,6 +75,7 @@
- [Prologue](./building/bootstrapping/intro.md)
- [What Bootstrapping does](./building/bootstrapping/what-bootstrapping-does.md)
- [How Bootstrap does it](./building/bootstrapping/how-bootstrap-does-it.md)
- [Writing tools in Bootstrap](./building/bootstrapping/writing-tools-in-bootstrap.md)
- [Debugging bootstrap](./building/bootstrapping/debugging-bootstrap.md)
# High-level Compiler Architecture

23
src/doc/rustc-dev-guide/src/building/bootstrapping/writing-tools-in-bootstrap.md Normal file
View file

@ -0,0 +1,23 @@
# Writing tools in Bootstrap
There are three types of tools you can write in bootstrap:
- **`Mode::ToolBootstrap`**
Use this for tools that don’t need anything from the in-tree compiler and can run with the stage0 `rustc`.
The output is placed in the "stage0-bootstrap-tools" directory. This mode is for general-purpose tools built
entirely with the stage0 compiler, including target libraries and only works for stage 0.
- **`Mode::ToolStd`**
Use this for tools that rely on the locally built std. The output goes into the "stageN-tools" directory.
This mode is rarely used, mainly for `compiletest` which requires `libtest`.
- **`Mode::ToolRustc`**
Use this for tools that depend on both the locally built `rustc` and the target `std`. This is more complex than
the other modes because the tool must be built with the same compiler used for `rustc` and placed in the "stageN-tools"
directory. When you choose `Mode::ToolRustc`, `ToolBuild` implementation takes care of this automatically.
If you need to use the builder’s compiler for something specific, you can get it from `ToolBuildResult`, which is
returned by the tool's [`Step`].
Regardless of the tool type you must return `ToolBuildResult` from the tool’s [`Step`] implementation and use `ToolBuild` inside it.
[`Step`]: https://doc.rust-lang.org/nightly/nightly-rustc/bootstrap/core/builder/trait.Step.html