From 28af00c611164bfbb242b2184bd3168ac1e57d2b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 8 Jun 2023 21:22:23 +0000 Subject: [PATCH] Support configuring the set of codegen backends to build per host triple This allows building the compiler itself with one backend while using another backend at runtime. For example this allows compiling rustc to wasm using LLVM, while using Cranelift at runtime to produce actual code. Cranelift can't compile to wasm, but is perfectly capable of running on wasm. LLVM can compile to wasm, but can't run on wasm. [^1] [^1]: The prototype of this still requires a couple of other patches. --- config.example.toml | 5 +++ src/bootstrap/src/core/build_steps/compile.rs | 16 ++++----- src/bootstrap/src/core/build_steps/dist.rs | 4 +-- src/bootstrap/src/core/build_steps/test.rs | 7 ++-- src/bootstrap/src/core/builder.rs | 5 +-- src/bootstrap/src/core/config/config.rs | 35 ++++++++++++++++--- src/bootstrap/src/core/sanity.rs | 32 +++++++++-------- src/bootstrap/src/lib.rs | 7 ++-- 8 files changed, 74 insertions(+), 37 deletions(-) diff --git a/config.example.toml b/config.example.toml index a5ef4022d39..098811195d7 100644 --- a/config.example.toml +++ b/config.example.toml @@ -829,6 +829,11 @@ # target triples containing `-none`, `nvptx`, `switch`, or `-uefi`. #no-std = (bool) +# This is an array of the codegen backends that will be compiled a rustc +# compiled for this target, overriding the global rust.codegen-backends option. +# See that option for more info. +#codegen-backends = rust.codegen-backends (array) + # ============================================================================= # Distribution options # diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index e06bc3fb09a..b2a9a89fff6 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1003,7 +1003,7 @@ impl Step for Rustc { pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { cargo .arg("--features") - .arg(builder.rustc_features(builder.kind)) + .arg(builder.rustc_features(builder.kind, target)) .arg("--manifest-path") .arg(builder.src.join("compiler/rustc/Cargo.toml")); @@ -1060,7 +1060,7 @@ pub fn rustc_cargo_env( cargo.env("CFG_OMIT_GIT_HASH", "1"); } - if let Some(backend) = builder.config.default_codegen_backend() { + if let Some(backend) = builder.config.default_codegen_backend(target) { cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); } @@ -1101,7 +1101,7 @@ pub fn rustc_cargo_env( // build. If we are in a check build we still go ahead here presuming we've // detected that LLVM is already built and good to go which helps prevent // busting caches (e.g. like #71152). - if builder.config.llvm_enabled() { + if builder.config.llvm_enabled(target) { let building_is_expensive = crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err(); // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler @@ -1250,7 +1250,7 @@ pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { let mut needs_codegen_backend_config = true; - for &backend in &run.builder.config.rust_codegen_backends { + for &backend in run.builder.config.codegen_backends(run.target) { if path .path .to_str() @@ -1287,7 +1287,7 @@ impl Step for CodegenBackend { return; } - for &backend in &run.builder.config.rust_codegen_backends { + for &backend in run.builder.config.codegen_backends(run.target) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1387,7 +1387,7 @@ fn copy_codegen_backends_to_sysroot( return; } - for backend in builder.config.rust_codegen_backends.iter() { + for backend in builder.config.codegen_backends(target) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1694,7 +1694,7 @@ impl Step for Assemble { // to not fail while linking the artifacts. build_compiler.stage = actual_stage; - for &backend in builder.config.rust_codegen_backends.iter() { + for &backend in builder.config.codegen_backends(target_compiler.host) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1779,7 +1779,7 @@ impl Step for Assemble { } } - if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + if builder.config.llvm_enabled(target_compiler.host) { let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target: target_compiler.host }); if !builder.config.dry_run() && builder.config.llvm_tools_enabled { diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index f50026368da..6d56492e41b 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1278,7 +1278,7 @@ impl Step for CodegenBackend { } fn make_run(run: RunConfig<'_>) { - for &backend in &run.builder.config.rust_codegen_backends { + for &backend in run.builder.config.codegen_backends(run.target) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1302,7 +1302,7 @@ impl Step for CodegenBackend { return None; } - if !builder.config.rust_codegen_backends.contains(&self.backend) { + if !builder.config.codegen_backends(self.compiler.host).contains(&self.backend) { return None; } diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 04728e2e00d..e0e94f194e0 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1841,7 +1841,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the let mut llvm_components_passed = false; let mut copts_passed = false; - if builder.config.llvm_enabled() { + if builder.config.llvm_enabled(compiler.host) { let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target: builder.config.build }); if !builder.config.dry_run() { @@ -3155,7 +3155,8 @@ impl Step for CodegenCranelift { return; } - if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) { + if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("cranelift")) + { builder.info("cranelift not in rust.codegen-backends. skipping"); return; } @@ -3277,7 +3278,7 @@ impl Step for CodegenGCC { return; } - if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) { + if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("gcc")) { builder.info("gcc not in rust.codegen-backends. skipping"); return; } diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 3770d0687b2..0ececcdcacc 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -1230,7 +1230,7 @@ impl<'a> Builder<'a> { /// Note that this returns `None` if LLVM is disabled, or if we're in a /// check build or dry-run, where there's no need to build all of LLVM. fn llvm_config(&self, target: TargetSelection) -> Option { - if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { + if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() { let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); if llvm_config.is_file() { return Some(llvm_config); @@ -2113,7 +2113,8 @@ impl<'a> Builder<'a> { }; if let Some(limit) = limit { - if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" + if stage == 0 + || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm" { rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); } diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index fcdd742e69c..5969ddfdf90 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -581,6 +581,7 @@ pub struct Target { pub wasi_root: Option, pub qemu_rootfs: Option, pub no_std: bool, + pub codegen_backends: Option>>, } impl Target { @@ -1139,6 +1140,7 @@ define_config! { wasi_root: Option = "wasi-root", qemu_rootfs: Option = "qemu-rootfs", no_std: Option = "no-std", + codegen_backends: Option> = "codegen-backends", } } @@ -1845,6 +1847,24 @@ impl Config { target.profiler = cfg.profiler; target.rpath = cfg.rpath; + if let Some(ref backends) = cfg.codegen_backends { + let available_backends = vec!["llvm", "cranelift", "gcc"]; + + target.codegen_backends = Some(backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + INTERNER.intern_str(s) + }).collect()); + } + config.target_config.insert(TargetSelection::from_user(&triple), target); } } @@ -2227,8 +2247,8 @@ impl Config { self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath) } - pub fn llvm_enabled(&self) -> bool { - self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + pub fn llvm_enabled(&self, target: TargetSelection) -> bool { + self.codegen_backends(target).contains(&INTERNER.intern_str("llvm")) } pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { @@ -2247,8 +2267,15 @@ impl Config { self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) } - pub fn default_codegen_backend(&self) -> Option> { - self.rust_codegen_backends.get(0).cloned() + pub fn codegen_backends(&self, target: TargetSelection) -> &[Interned] { + self.target_config + .get(&target) + .and_then(|cfg| cfg.codegen_backends.as_deref()) + .unwrap_or(&self.rust_codegen_backends) + } + + pub fn default_codegen_backend(&self, target: TargetSelection) -> Option> { + self.codegen_backends(target).get(0).cloned() } pub fn git_config(&self) -> GitConfig<'_> { diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index 5f1ca5de74a..1dce8d8ac71 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -16,7 +16,6 @@ use std::path::PathBuf; use std::process::Command; use crate::core::config::Target; -use crate::utils::cache::INTERNER; use crate::utils::helpers::output; use crate::Build; @@ -88,19 +87,19 @@ pub fn check(build: &mut Build) { } // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) - && build - .hosts - .iter() - .map(|host| { - build + let building_llvm = build + .hosts + .iter() + .map(|host| { + build.config.llvm_enabled(*host) + && build .config .target_config .get(host) .map(|config| config.llvm_config.is_none()) .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); let need_cmake = building_llvm || build.config.any_sanitizers_to_build(); if need_cmake && cmd_finder.maybe_have("cmake").is_none() { @@ -190,13 +189,16 @@ than building it. if !build.config.dry_run() { cmd_finder.must_have(build.cxx(*host).unwrap()); } - } - if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - // Externally configured LLVM requires FileCheck to exist - let filecheck = build.llvm_filecheck(build.build); - if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { - panic!("FileCheck executable {filecheck:?} does not exist"); + if build.config.llvm_enabled(*host) { + // Externally configured LLVM requires FileCheck to exist + let filecheck = build.llvm_filecheck(build.build); + if !filecheck.starts_with(&build.out) + && !filecheck.exists() + && build.config.codegen_tests + { + panic!("FileCheck executable {filecheck:?} does not exist"); + } } } diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 1726e7aacbc..6a7bba2463d 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -734,12 +734,12 @@ impl Build { } /// Gets the space-separated set of activated features for the compiler. - fn rustc_features(&self, kind: Kind) -> String { + fn rustc_features(&self, kind: Kind, target: TargetSelection) -> String { let mut features = vec![]; if self.config.jemalloc { features.push("jemalloc"); } - if self.config.llvm_enabled() || kind == Kind::Check { + if self.config.llvm_enabled(target) || kind == Kind::Check { features.push("llvm"); } // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` @@ -1560,7 +1560,8 @@ impl Build { || target .map(|t| self.config.profiler_enabled(t)) .unwrap_or_else(|| self.config.any_profiler_enabled())) - && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) + && (dep != "rustc_codegen_llvm" + || self.config.hosts.iter().any(|host| self.config.llvm_enabled(*host))) { list.push(*dep); }