bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Rollup merge of #82739 - jyn514:separate-stage0-stage1, r=Mark-Simulacrum

Browse source 
Use the beta compiler for building bootstrap tools when `download-rustc` is set

 ## Motivation

This avoids having to rebuild bootstrap and tidy each time you rebase
over master. In particular, it makes rebasing and running `x.py fmt` on
each commit in a branch significantly faster. It also avoids having to
rebuild bootstrap after setting `download-rustc = true`.

 ## Implementation

Instead of extracting the CI artifacts directly to `stage0/`, extract
them to `ci-rustc/` instead. Continue to copy them to the proper
sysroots as necessary for all stages except stage 0.

This also requires `bootstrap.py` to download both stage0 and CI
artifacts and distinguish between the two when checking stamp files.

Note that since tools have to be built by the same compiler that built
`rustc-dev` and the standard library, the downloaded artifacts can't be
reused when building with the beta compiler. To make sure this is still
a good user experience, warn when building with the beta compiler, and
default to building with stage 2.

I tested this by rebasing this PR from edeee915b1 over 1c77a1fa3c and confirming that only the bootstrap library itself had to be rebuilt, not any dependencies and not `tidy`. I also tested that a clean build with `x.py build` builds rustdoc exactly once and does no other work, and that `touch src/librustdoc/lib.rs && x.py build` works. `x.py check` still behaves as before (checks using the beta compiler, even if there are changes to `compiler/`).

Helps with https://github.com/rust-lang/rust/issues/81930.

r? `@Mark-Simulacrum`
This commit is contained in:
Dylan DPC 2021-04-05 13:03:38 +02:00 • committed by GitHub
commit ca9cbea27a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 158 additions and 117 deletions
Download patch file Download diff file Expand all files Collapse all files

147
src/bootstrap/bootstrap.py
View file

@ -383,7 +383,7 @@ class RustBuild(object):
self.nix_deps_dir = None
self.rustc_commit = None
def download_stage0(self):
def download_toolchain(self, stage0=True, rustc_channel=None):
"""Fetch the build system for Rust, written in Rust
This method will build a cache directory, then it will fetch the
@ -393,43 +393,47 @@ class RustBuild(object):
Each downloaded tarball is extracted, after that, the script
will move all the content to the right place.
"""
if rustc_channel is None:
rustc_channel = self.rustc_channel
rustfmt_channel = self.rustfmt_channel
bin_root = self.bin_root(stage0)
if self.rustc().startswith(self.bin_root()) and \
(not os.path.exists(self.rustc()) or
self.program_out_of_date(self.rustc_stamp(), self.date + str(self.rustc_commit))):
if os.path.exists(self.bin_root()):
shutil.rmtree(self.bin_root())
download_rustc = self.rustc_commit is not None
key = self.date
if not stage0:
key += str(self.rustc_commit)
if self.rustc(stage0).startswith(bin_root) and \
(not os.path.exists(self.rustc(stage0)) or
self.program_out_of_date(self.rustc_stamp(stage0), key)):
if os.path.exists(bin_root):
shutil.rmtree(bin_root)
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
filename = "rust-std-{}-{}{}".format(
rustc_channel, self.build, tarball_suffix)
pattern = "rust-std-{}".format(self.build)
self._download_component_helper(filename, pattern, tarball_suffix, download_rustc)
self._download_component_helper(filename, pattern, tarball_suffix, stage0)
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "rustc", tarball_suffix, download_rustc)
self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
tarball_suffix)
self._download_component_helper(filename, "cargo", tarball_suffix)
if self.rustc_commit is not None:
if not stage0:
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
self._download_component_helper(
filename, "rustc-dev", tarball_suffix, download_rustc
filename, "rustc-dev", tarball_suffix, stage0
)
self.fix_bin_or_dylib("{}/bin/rustc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo".format(self.bin_root()))
lib_dir = "{}/lib".format(self.bin_root())
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
lib_dir = "{}/lib".format(bin_root)
for lib in os.listdir(lib_dir):
if lib.endswith(".so"):
self.fix_bin_or_dylib(os.path.join(lib_dir, lib), rpath_libz=True)
with output(self.rustc_stamp()) as rust_stamp:
rust_stamp.write(self.date + str(self.rustc_commit))
with output(self.rustc_stamp(stage0)) as rust_stamp:
rust_stamp.write(key)
if self.rustfmt() and self.rustfmt().startswith(self.bin_root()) and (
if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
not os.path.exists(self.rustfmt())
or self.program_out_of_date(self.rustfmt_stamp(), self.rustfmt_channel)
):
@ -440,12 +444,13 @@ class RustBuild(object):
self._download_component_helper(
filename, "rustfmt-preview", tarball_suffix, key=date
)
self.fix_bin_or_dylib("{}/bin/rustfmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(self.bin_root()))
self.fix_bin_or_dylib("{}/bin/rustfmt".format(bin_root))
self.fix_bin_or_dylib("{}/bin/cargo-fmt".format(bin_root))
with output(self.rustfmt_stamp()) as rustfmt_stamp:
rustfmt_stamp.write(self.rustfmt_channel)
if self.downloading_llvm():
# Avoid downloading LLVM twice (once for stage0 and once for the master rustc)
if self.downloading_llvm() and stage0:
# We want the most recent LLVM submodule update to avoid downloading
# LLVM more often than necessary.
#
@ -498,27 +503,26 @@ class RustBuild(object):
or (opt == "if-available" and self.build in supported_platforms)
def _download_component_helper(
self, filename, pattern, tarball_suffix, download_rustc=False, key=None
self, filename, pattern, tarball_suffix, stage0=True, key=None
):
if key is None:
if download_rustc:
key = self.rustc_commit
else:
if stage0:
key = self.date
else:
key = self.rustc_commit
cache_dst = os.path.join(self.build_dir, "cache")
rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache)
if download_rustc:
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
else:
if stage0:
url = "{}/dist/{}".format(self._download_url, key)
else:
url = "https://ci-artifacts.rust-lang.org/rustc-builds/{}".format(self.rustc_commit)
tarball = os.path.join(rustc_cache, filename)
if not os.path.exists(tarball):
do_verify = not download_rustc
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=do_verify)
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
get("{}/{}".format(url, filename), tarball, verbose=self.verbose, do_verify=stage0)
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
def _download_ci_llvm(self, llvm_sha, llvm_assertions):
cache_prefix = "llvm-{}-{}".format(llvm_sha, llvm_assertions)
@ -576,10 +580,10 @@ class RustBuild(object):
nix_os_msg = "info: you seem to be running NixOS. Attempting to patch"
print(nix_os_msg, fname)
# Only build `stage0/.nix-deps` once.
# Only build `.nix-deps` once.
nix_deps_dir = self.nix_deps_dir
if not nix_deps_dir:
nix_deps_dir = "{}/.nix-deps".format(self.bin_root())
nix_deps_dir = ".nix-deps"
if not os.path.exists(nix_deps_dir):
os.makedirs(nix_deps_dir)
@ -637,8 +641,8 @@ class RustBuild(object):
print("warning: failed to call patchelf:", reason)
return
# Return the stage1 compiler to download, if any.
def maybe_download_rustc(self):
# If `download-rustc` is set, download the most recent commit with CI artifacts
def maybe_download_ci_toolchain(self):
# If `download-rustc` is not set, default to rebuilding.
if self.get_toml("download-rustc", section="rust") != "true":
return None
@ -658,17 +662,23 @@ class RustBuild(object):
if status != 0:
print("warning: `download-rustc` is enabled, but there are changes to compiler/")
return commit
if self.verbose:
print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
self.rustc_commit = commit
# FIXME: support downloading artifacts from the beta channel
self.download_toolchain(False, "nightly")
def rustc_stamp(self):
"""Return the path for .rustc-stamp
def rustc_stamp(self, stage0):
"""Return the path for .rustc-stamp at the given stage
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
>>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
True
>>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
True
"""
return os.path.join(self.bin_root(), '.rustc-stamp')
return os.path.join(self.bin_root(stage0), '.rustc-stamp')
def rustfmt_stamp(self):
"""Return the path for .rustfmt-stamp
@ -678,7 +688,7 @@ class RustBuild(object):
>>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
True
"""
return os.path.join(self.bin_root(), '.rustfmt-stamp')
return os.path.join(self.bin_root(True), '.rustfmt-stamp')
def llvm_stamp(self):
"""Return the path for .rustfmt-stamp
@ -698,21 +708,27 @@ class RustBuild(object):
with open(stamp_path, 'r') as stamp:
return key != stamp.read()
def bin_root(self):
"""Return the binary root directory
def bin_root(self, stage0):
"""Return the binary root directory for the given stage
>>> rb = RustBuild()
>>> rb.build_dir = "build"
>>> rb.bin_root() == os.path.join("build", "stage0")
>>> rb.bin_root(True) == os.path.join("build", "stage0")
True
>>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
True
When the 'build' property is given should be a nested directory:
>>> rb.build = "devel"
>>> rb.bin_root() == os.path.join("build", "devel", "stage0")
>>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
True
"""
return os.path.join(self.build_dir, self.build, "stage0")
if stage0:
subdir = "stage0"
else:
subdir = "ci-rustc"
return os.path.join(self.build_dir, self.build, subdir)
def llvm_root(self):
"""Return the CI LLVM root directory
@ -775,9 +791,9 @@ class RustBuild(object):
"""Return config path for cargo"""
return self.program_config('cargo')
def rustc(self):
def rustc(self, stage0):
"""Return config path for rustc"""
return self.program_config('rustc')
return self.program_config('rustc', stage0)
def rustfmt(self):
"""Return config path for rustfmt"""
@ -785,23 +801,27 @@ class RustBuild(object):
return None
return self.program_config('rustfmt')
def program_config(self, program):
"""Return config path for the given program
def program_config(self, program, stage0=True):
"""Return config path for the given program at the given stage
>>> rb = RustBuild()
>>> rb.config_toml = 'rustc = "rustc"\\n'
>>> rb.program_config('rustc')
'rustc'
>>> rb.config_toml = ''
>>> cargo_path = rb.program_config('cargo')
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
>>> cargo_path = rb.program_config('cargo', True)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
... "bin", "cargo")
True
>>> cargo_path = rb.program_config('cargo', False)
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
... "bin", "cargo")
True
"""
config = self.get_toml(program)
if config:
return os.path.expanduser(config)
return os.path.join(self.bin_root(), "bin", "{}{}".format(
return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
program, self.exe_suffix()))
@staticmethod
@ -856,14 +876,14 @@ class RustBuild(object):
if "CARGO_BUILD_TARGET" in env:
del env["CARGO_BUILD_TARGET"]
env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
env["RUSTC"] = self.rustc(True)
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
(os.pathsep + env["LD_LIBRARY_PATH"]) \
if "LD_LIBRARY_PATH" in env else ""
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
if "DYLD_LIBRARY_PATH" in env else ""
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
(os.pathsep + env["LIBRARY_PATH"]) \
if "LIBRARY_PATH" in env else ""
# preserve existing RUSTFLAGS
@ -886,7 +906,7 @@ class RustBuild(object):
if self.get_toml("deny-warnings", "rust") != "false":
env["RUSTFLAGS"] += " -Dwarnings"
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `{}`".format(
@ -1137,14 +1157,9 @@ def bootstrap(help_triggered):
build.update_submodules()
# Fetch/build the bootstrap
build.rustc_commit = build.maybe_download_rustc()
if build.rustc_commit is not None:
if build.verbose:
commit = build.rustc_commit
print("using downloaded stage1 artifacts from CI (commit {})".format(commit))
# FIXME: support downloading artifacts from the beta channel
build.rustc_channel = "nightly"
build.download_stage0()
build.download_toolchain()
# Download the master compiler if `download-rustc` is set
build.maybe_download_ci_toolchain()
sys.stdout.flush()
build.ensure_vendored()
build.build_bootstrap()

15
src/bootstrap/compile.rs
View file

@ -65,7 +65,9 @@ impl Step for Std {
// These artifacts were already copied (in `impl Step for Sysroot`).
// Don't recompile them.
if builder.config.download_rustc {
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
// so its artifacts can't be reused.
if builder.config.download_rustc && compiler.stage != 0 {
return;
}
@ -513,7 +515,9 @@ impl Step for Rustc {
let compiler = self.compiler;
let target = self.target;
if builder.config.download_rustc {
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
// so its artifacts can't be reused.
if builder.config.download_rustc && compiler.stage != 0 {
// Copy the existing artifacts instead of rebuilding them.
// NOTE: this path is only taken for tools linking to rustc-dev.
builder.ensure(Sysroot { compiler });
@ -934,14 +938,15 @@ impl Step for Sysroot {
t!(fs::create_dir_all(&sysroot));
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
if builder.config.download_rustc {
if builder.config.download_rustc && compiler.stage != 0 {
assert_eq!(
builder.config.build, compiler.host,
"Cross-compiling is not yet supported with `download-rustc`",
);
// Copy the compiler into the correct sysroot.
let stage0_dir = builder.config.out.join(&*builder.config.build.triple).join("stage0");
builder.cp_r(&stage0_dir, &sysroot);
let ci_rustc_dir =
builder.config.out.join(&*builder.config.build.triple).join("ci-rustc");
builder.cp_r(&ci_rustc_dir, &sysroot);
return INTERNER.intern_path(sysroot);
}

98
src/bootstrap/config.rs
View file

@ -687,51 +687,6 @@ impl Config {
set(&mut config.print_step_timings, build.print_step_timings);
set(&mut config.print_step_rusage, build.print_step_rusage);
// See https://github.com/rust-lang/compiler-team/issues/326
config.stage = match config.cmd {
Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0),
Subcommand::Doc { .. } => flags.stage.or(build.doc_stage).unwrap_or(0),
Subcommand::Build { .. } => flags.stage.or(build.build_stage).unwrap_or(1),
Subcommand::Test { .. } => flags.stage.or(build.test_stage).unwrap_or(1),
Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2),
Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2),
Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2),
// These are all bootstrap tools, which don't depend on the compiler.
// The stage we pass shouldn't matter, but use 0 just in case.
Subcommand::Clean { .. }
| Subcommand::Clippy { .. }
| Subcommand::Fix { .. }
| Subcommand::Run { .. }
| Subcommand::Setup { .. }
| Subcommand::Format { .. } => flags.stage.unwrap_or(0),
};
// CI should always run stage 2 builds, unless it specifically states otherwise
#[cfg(not(test))]
if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None {
match config.cmd {
Subcommand::Test { .. }
| Subcommand::Doc { .. }
| Subcommand::Build { .. }
| Subcommand::Bench { .. }
| Subcommand::Dist { .. }
| Subcommand::Install { .. } => {
assert_eq!(
config.stage, 2,
"x.py should be run with `--stage 2` on CI, but was run with `--stage {}`",
config.stage,
);
}
Subcommand::Clean { .. }
| Subcommand::Check { .. }
| Subcommand::Clippy { .. }
| Subcommand::Fix { .. }
| Subcommand::Run { .. }
| Subcommand::Setup { .. }
| Subcommand::Format { .. } => {}
}
}
config.verbose = cmp::max(config.verbose, flags.verbose);
if let Some(install) = toml.install {
@ -1005,6 +960,59 @@ impl Config {
let default = config.channel == "dev";
config.ignore_git = ignore_git.unwrap_or(default);
let download_rustc = config.download_rustc;
// See https://github.com/rust-lang/compiler-team/issues/326
config.stage = match config.cmd {
Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0),
// `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden.
Subcommand::Doc { .. } => {
flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 })
}
Subcommand::Build { .. } => {
flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 })
}
Subcommand::Test { .. } => {
flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 })
}
Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2),
Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2),
Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2),
// These are all bootstrap tools, which don't depend on the compiler.
// The stage we pass shouldn't matter, but use 0 just in case.
Subcommand::Clean { .. }
| Subcommand::Clippy { .. }
| Subcommand::Fix { .. }
| Subcommand::Run { .. }
| Subcommand::Setup { .. }
| Subcommand::Format { .. } => flags.stage.unwrap_or(0),
};
// CI should always run stage 2 builds, unless it specifically states otherwise
#[cfg(not(test))]
if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None {
match config.cmd {
Subcommand::Test { .. }
| Subcommand::Doc { .. }
| Subcommand::Build { .. }
| Subcommand::Bench { .. }
| Subcommand::Dist { .. }
| Subcommand::Install { .. } => {
assert_eq!(
config.stage, 2,
"x.py should be run with `--stage 2` on CI, but was run with `--stage {}`",
config.stage,
);
}
Subcommand::Clean { .. }
| Subcommand::Check { .. }
| Subcommand::Clippy { .. }
| Subcommand::Fix { .. }
| Subcommand::Run { .. }
| Subcommand::Setup { .. }
| Subcommand::Format { .. } => {}
}
}
config
}

13
src/bootstrap/tool.rs
View file

@ -513,6 +513,19 @@ impl Step for Rustdoc {
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
// When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually
// build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build
// it.
builder.ensure(compile::Std { compiler: build_compiler, target: target_compiler.host });
builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host });
// NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0
// compiler, since you do just as much work.
if !builder.config.dry_run && builder.config.download_rustc && build_compiler.stage == 0 {
println!(
"warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead"
);
}
// The presence of `target_compiler` ensures that the necessary libraries (codegen backends,
// compiler libraries, ...) are built. Rustdoc does not require the presence of any
// libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since