1
Fork 0

Rollup merge of #138934 - onur-ozkan:extended-config-profiles, r=Kobzol

support config extensions

_Copied from the `rustc-dev-guide` addition:_

>When working on different tasks, you might need to switch between different bootstrap >configurations.
>Sometimes you may want to keep an old configuration for future use. But saving raw config >values in
>random files and manually copying and pasting them can quickly become messy, especially if >you have a
>long history of different configurations.
>
>To simplify managing multiple configurations, you can create config extensions.
>
>For example, you can create a simple config file named `cross.toml`:
>
>```toml
>[build]
>build = "x86_64-unknown-linux-gnu"
>host = ["i686-unknown-linux-gnu"]
>target = ["i686-unknown-linux-gnu"]
>
>
>[llvm]
>download-ci-llvm = false
>
>[target.x86_64-unknown-linux-gnu]
>llvm-config = "/path/to/llvm-19/bin/llvm-config"
>```
>
>Then, include this in your `bootstrap.toml`:
>
>```toml
>include = ["cross.toml"]
>```
>
>You can also include extensions within extensions recursively.
>
>**Note:** In the `include` field, the overriding logic follows a right-to-left order. For example,
in `include = ["a.toml", "b.toml"]`, extension `b.toml` overrides `a.toml`. Also, parent extensions
always overrides the inner ones.

try-job: x86_64-mingw-2
This commit is contained in:
Chris Denton 2025-04-19 15:09:32 +00:00 committed by GitHub
commit 237064a0c4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 374 additions and 20 deletions

View file

@ -19,6 +19,14 @@
# Note that this has no default value (x.py uses the defaults in `bootstrap.example.toml`).
#profile = <none>
# Inherits configuration values from different configuration files (a.k.a. config extensions).
# Supports absolute paths, and uses the current directory (where the bootstrap was invoked)
# as the base if the given path is not absolute.
#
# The overriding logic follows a right-to-left order. For example, in `include = ["a.toml", "b.toml"]`,
# extension `b.toml` overrides `a.toml`. Also, parent extensions always overrides the inner ones.
#include = []
# Keeps track of major changes made to this configuration.
#
# This value also represents ID of the PR that caused major changes. Meaning,

View file

@ -6,6 +6,7 @@
use std::cell::{Cell, RefCell};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::fmt::{self, Display};
use std::hash::Hash;
use std::io::IsTerminal;
use std::path::{Path, PathBuf, absolute};
use std::process::Command;
@ -701,6 +702,7 @@ pub(crate) struct TomlConfig {
target: Option<HashMap<String, TomlTarget>>,
dist: Option<Dist>,
profile: Option<String>,
include: Option<Vec<PathBuf>>,
}
/// This enum is used for deserializing change IDs from TOML, allowing both numeric values and the string `"ignore"`.
@ -747,27 +749,35 @@ enum ReplaceOpt {
}
trait Merge {
fn merge(&mut self, other: Self, replace: ReplaceOpt);
fn merge(
&mut self,
parent_config_path: Option<PathBuf>,
included_extensions: &mut HashSet<PathBuf>,
other: Self,
replace: ReplaceOpt,
);
}
impl Merge for TomlConfig {
fn merge(
&mut self,
TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id }: Self,
parent_config_path: Option<PathBuf>,
included_extensions: &mut HashSet<PathBuf>,
TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self,
replace: ReplaceOpt,
) {
fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
if let Some(new) = y {
if let Some(original) = x {
original.merge(new, replace);
original.merge(None, &mut Default::default(), new, replace);
} else {
*x = Some(new);
}
}
}
self.change_id.inner.merge(change_id.inner, replace);
self.profile.merge(profile, replace);
self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace);
self.profile.merge(None, &mut Default::default(), profile, replace);
do_merge(&mut self.build, build, replace);
do_merge(&mut self.install, install, replace);
@ -782,13 +792,50 @@ impl Merge for TomlConfig {
(Some(original_target), Some(new_target)) => {
for (triple, new) in new_target {
if let Some(original) = original_target.get_mut(&triple) {
original.merge(new, replace);
original.merge(None, &mut Default::default(), new, replace);
} else {
original_target.insert(triple, new);
}
}
}
}
let parent_dir = parent_config_path
.as_ref()
.and_then(|p| p.parent().map(ToOwned::to_owned))
.unwrap_or_default();
// `include` handled later since we ignore duplicates using `ReplaceOpt::IgnoreDuplicate` to
// keep the upper-level configuration to take precedence.
for include_path in include.clone().unwrap_or_default().iter().rev() {
let include_path = parent_dir.join(include_path);
let include_path = include_path.canonicalize().unwrap_or_else(|e| {
eprintln!("ERROR: Failed to canonicalize '{}' path: {e}", include_path.display());
exit!(2);
});
let included_toml = Config::get_toml_inner(&include_path).unwrap_or_else(|e| {
eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display());
exit!(2);
});
assert!(
included_extensions.insert(include_path.clone()),
"Cyclic inclusion detected: '{}' is being included again before its previous inclusion was fully processed.",
include_path.display()
);
self.merge(
Some(include_path.clone()),
included_extensions,
included_toml,
// Ensures that parent configuration always takes precedence
// over child configurations.
ReplaceOpt::IgnoreDuplicate,
);
included_extensions.remove(&include_path);
}
}
}
@ -803,7 +850,13 @@ macro_rules! define_config {
}
impl Merge for $name {
fn merge(&mut self, other: Self, replace: ReplaceOpt) {
fn merge(
&mut self,
_parent_config_path: Option<PathBuf>,
_included_extensions: &mut HashSet<PathBuf>,
other: Self,
replace: ReplaceOpt
) {
$(
match replace {
ReplaceOpt::IgnoreDuplicate => {
@ -903,7 +956,13 @@ macro_rules! define_config {
}
impl<T> Merge for Option<T> {
fn merge(&mut self, other: Self, replace: ReplaceOpt) {
fn merge(
&mut self,
_parent_config_path: Option<PathBuf>,
_included_extensions: &mut HashSet<PathBuf>,
other: Self,
replace: ReplaceOpt,
) {
match replace {
ReplaceOpt::IgnoreDuplicate => {
if self.is_none() {
@ -1363,13 +1422,15 @@ impl Config {
Self::get_toml(&builder_config_path)
}
#[cfg(test)]
pub(crate) fn get_toml(_: &Path) -> Result<TomlConfig, toml::de::Error> {
Ok(TomlConfig::default())
pub(crate) fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> {
#[cfg(test)]
return Ok(TomlConfig::default());
#[cfg(not(test))]
Self::get_toml_inner(file)
}
#[cfg(not(test))]
pub(crate) fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> {
fn get_toml_inner(file: &Path) -> Result<TomlConfig, toml::de::Error> {
let contents =
t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
// Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
@ -1548,7 +1609,8 @@ impl Config {
// but not if `bootstrap.toml` hasn't been created.
let mut toml = if !using_default_path || toml_path.exists() {
config.config = Some(if cfg!(not(test)) {
toml_path.canonicalize().unwrap()
toml_path = toml_path.canonicalize().unwrap();
toml_path.clone()
} else {
toml_path.clone()
});
@ -1576,6 +1638,26 @@ impl Config {
toml.profile = Some("dist".into());
}
// Reverse the list to ensure the last added config extension remains the most dominant.
// For example, given ["a.toml", "b.toml"], "b.toml" should take precedence over "a.toml".
//
// This must be handled before applying the `profile` since `include`s should always take
// precedence over `profile`s.
for include_path in toml.include.clone().unwrap_or_default().iter().rev() {
let include_path = toml_path.parent().unwrap().join(include_path);
let included_toml = get_toml(&include_path).unwrap_or_else(|e| {
eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display());
exit!(2);
});
toml.merge(
Some(include_path),
&mut Default::default(),
included_toml,
ReplaceOpt::IgnoreDuplicate,
);
}
if let Some(include) = &toml.profile {
// Allows creating alias for profile names, allowing
// profiles to be renamed while maintaining back compatibility
@ -1597,7 +1679,12 @@ impl Config {
);
exit!(2);
});
toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate);
toml.merge(
Some(include_path),
&mut Default::default(),
included_toml,
ReplaceOpt::IgnoreDuplicate,
);
}
let mut override_toml = TomlConfig::default();
@ -1608,7 +1695,12 @@ impl Config {
let mut err = match get_table(option) {
Ok(v) => {
override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate);
override_toml.merge(
None,
&mut Default::default(),
v,
ReplaceOpt::ErrorOnDuplicate,
);
continue;
}
Err(e) => e,
@ -1619,7 +1711,12 @@ impl Config {
if !value.contains('"') {
match get_table(&format!(r#"{key}="{value}""#)) {
Ok(v) => {
override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate);
override_toml.merge(
None,
&mut Default::default(),
v,
ReplaceOpt::ErrorOnDuplicate,
);
continue;
}
Err(e) => err = e,
@ -1629,7 +1726,7 @@ impl Config {
eprintln!("failed to parse override `{option}`: `{err}");
exit!(2)
}
toml.merge(override_toml, ReplaceOpt::Override);
toml.merge(None, &mut Default::default(), override_toml, ReplaceOpt::Override);
config.change_id = toml.change_id.inner;

View file

@ -1,8 +1,8 @@
use std::collections::BTreeSet;
use std::env;
use std::fs::{File, remove_file};
use std::io::Write;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::{env, fs};
use build_helper::ci::CiEnv;
use clap::CommandFactory;
@ -23,6 +23,27 @@ pub(crate) fn parse(config: &str) -> Config {
)
}
fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> {
let contents = std::fs::read_to_string(file).unwrap();
toml::from_str(&contents).and_then(|table: toml::Value| TomlConfig::deserialize(table))
}
/// Helps with debugging by using consistent test-specific directories instead of
/// random temporary directories.
fn prepare_test_specific_dir() -> PathBuf {
let current = std::thread::current();
// Replace "::" with "_" to make it safe for directory names on Windows systems
let test_path = current.name().unwrap().replace("::", "_");
let testdir = parse("").tempdir().join(test_path);
// clean up any old test files
let _ = fs::remove_dir_all(&testdir);
let _ = fs::create_dir_all(&testdir);
testdir
}
#[test]
fn download_ci_llvm() {
let config = parse("llvm.download-ci-llvm = false");
@ -539,3 +560,189 @@ fn test_ci_flag() {
let config = Config::parse_inner(Flags::parse(&["check".into()]), |&_| toml::from_str(""));
assert_eq!(config.is_running_on_ci, CiEnv::is_ci());
}
#[test]
fn test_precedence_of_includes() {
let testdir = prepare_test_specific_dir();
let root_config = testdir.join("config.toml");
let root_config_content = br#"
include = ["./extension.toml"]
[llvm]
link-jobs = 2
"#;
File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
let extension = testdir.join("extension.toml");
let extension_content = br#"
change-id=543
include = ["./extension2.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let extension = testdir.join("extension2.toml");
let extension_content = br#"
change-id=742
[llvm]
link-jobs = 10
[build]
description = "Some creative description"
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let config = Config::parse_inner(
Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
get_toml,
);
assert_eq!(config.change_id.unwrap(), ChangeId::Id(543));
assert_eq!(config.llvm_link_jobs.unwrap(), 2);
assert_eq!(config.description.unwrap(), "Some creative description");
}
#[test]
#[should_panic(expected = "Cyclic inclusion detected")]
fn test_cyclic_include_direct() {
let testdir = prepare_test_specific_dir();
let root_config = testdir.join("config.toml");
let root_config_content = br#"
include = ["./extension.toml"]
"#;
File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
let extension = testdir.join("extension.toml");
let extension_content = br#"
include = ["./config.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let config = Config::parse_inner(
Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
get_toml,
);
}
#[test]
#[should_panic(expected = "Cyclic inclusion detected")]
fn test_cyclic_include_indirect() {
let testdir = prepare_test_specific_dir();
let root_config = testdir.join("config.toml");
let root_config_content = br#"
include = ["./extension.toml"]
"#;
File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
let extension = testdir.join("extension.toml");
let extension_content = br#"
include = ["./extension2.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let extension = testdir.join("extension2.toml");
let extension_content = br#"
include = ["./extension3.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let extension = testdir.join("extension3.toml");
let extension_content = br#"
include = ["./extension.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let config = Config::parse_inner(
Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
get_toml,
);
}
#[test]
fn test_include_absolute_paths() {
let testdir = prepare_test_specific_dir();
let extension = testdir.join("extension.toml");
File::create(&extension).unwrap().write_all(&[]).unwrap();
let root_config = testdir.join("config.toml");
let extension_absolute_path =
extension.canonicalize().unwrap().to_str().unwrap().replace('\\', r"\\");
let root_config_content = format!(r#"include = ["{}"]"#, extension_absolute_path);
File::create(&root_config).unwrap().write_all(root_config_content.as_bytes()).unwrap();
let config = Config::parse_inner(
Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
get_toml,
);
}
#[test]
fn test_include_relative_paths() {
let testdir = prepare_test_specific_dir();
let _ = fs::create_dir_all(&testdir.join("subdir/another_subdir"));
let root_config = testdir.join("config.toml");
let root_config_content = br#"
include = ["./subdir/extension.toml"]
"#;
File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
let extension = testdir.join("subdir/extension.toml");
let extension_content = br#"
include = ["../extension2.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let extension = testdir.join("extension2.toml");
let extension_content = br#"
include = ["./subdir/another_subdir/extension3.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let extension = testdir.join("subdir/another_subdir/extension3.toml");
let extension_content = br#"
include = ["../../extension4.toml"]
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let extension = testdir.join("extension4.toml");
File::create(extension).unwrap().write_all(&[]).unwrap();
let config = Config::parse_inner(
Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
get_toml,
);
}
#[test]
fn test_include_precedence_over_profile() {
let testdir = prepare_test_specific_dir();
let root_config = testdir.join("config.toml");
let root_config_content = br#"
profile = "dist"
include = ["./extension.toml"]
"#;
File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
let extension = testdir.join("extension.toml");
let extension_content = br#"
[rust]
channel = "dev"
"#;
File::create(extension).unwrap().write_all(extension_content).unwrap();
let config = Config::parse_inner(
Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
get_toml,
);
// "dist" profile would normally set the channel to "auto-detect", but includes should
// override profile settings, so we expect this to be "dev" here.
assert_eq!(config.channel, "dev");
}

View file

@ -396,4 +396,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
severity: ChangeSeverity::Info,
summary: "Added a new option `build.compiletest-use-stage0-libtest` to force `compiletest` to use the stage 0 libtest.",
},
ChangeInfo {
change_id: 138934,
severity: ChangeSeverity::Info,
summary: "Added new option `include` to create config extensions.",
},
];

View file

@ -20,6 +20,43 @@ your `.git/hooks` folder as `pre-push` (without the `.sh` extension!).
You can also install the hook as a step of running `./x setup`!
## Config extensions
When working on different tasks, you might need to switch between different bootstrap configurations.
Sometimes you may want to keep an old configuration for future use. But saving raw config values in
random files and manually copying and pasting them can quickly become messy, especially if you have a
long history of different configurations.
To simplify managing multiple configurations, you can create config extensions.
For example, you can create a simple config file named `cross.toml`:
```toml
[build]
build = "x86_64-unknown-linux-gnu"
host = ["i686-unknown-linux-gnu"]
target = ["i686-unknown-linux-gnu"]
[llvm]
download-ci-llvm = false
[target.x86_64-unknown-linux-gnu]
llvm-config = "/path/to/llvm-19/bin/llvm-config"
```
Then, include this in your `bootstrap.toml`:
```toml
include = ["cross.toml"]
```
You can also include extensions within extensions recursively.
**Note:** In the `include` field, the overriding logic follows a right-to-left order. For example,
in `include = ["a.toml", "b.toml"]`, extension `b.toml` overrides `a.toml`. Also, parent extensions
always overrides the inner ones.
## Configuring `rust-analyzer` for `rustc`
### Project-local rust-analyzer setup