Rollup merge of #138934 - onur-ozkan:extended-config-profiles, r=Kobzol

support config extensions

_Copied from the `rustc-dev-guide` addition:_

>When working on different tasks, you might need to switch between different bootstrap >configurations.
>Sometimes you may want to keep an old configuration for future use. But saving raw config >values in
>random files and manually copying and pasting them can quickly become messy, especially if >you have a
>long history of different configurations.
>
>To simplify managing multiple configurations, you can create config extensions.
>
>For example, you can create a simple config file named `cross.toml`:
>
>```toml
>[build]
>build = "x86_64-unknown-linux-gnu"
>host = ["i686-unknown-linux-gnu"]
>target = ["i686-unknown-linux-gnu"]
>
>
>[llvm]
>download-ci-llvm = false
>
>[target.x86_64-unknown-linux-gnu]
>llvm-config = "/path/to/llvm-19/bin/llvm-config"
>```
>
>Then, include this in your `bootstrap.toml`:
>
>```toml
>include = ["cross.toml"]
>```
>
>You can also include extensions within extensions recursively.
>
>**Note:** In the `include` field, the overriding logic follows a right-to-left order. For example,
in `include = ["a.toml", "b.toml"]`, extension `b.toml` overrides `a.toml`. Also, parent extensions
always overrides the inner ones.

try-job: x86_64-mingw-2

bootstrap.example.toml

@@ -19,6 +19,14 @@
 # Note that this has no default value (x.py uses the defaults in `bootstrap.example.toml`).
 #profile = <none>
 
+# Inherits configuration values from different configuration files (a.k.a. config extensions).
+# Supports absolute paths, and uses the current directory (where the bootstrap was invoked)
+# as the base if the given path is not absolute.
+#
+# The overriding logic follows a right-to-left order. For example, in `include = ["a.toml", "b.toml"]`,
+# extension `b.toml` overrides `a.toml`. Also, parent extensions always overrides the inner ones.
+#include = []
+
 # Keeps track of major changes made to this configuration.
 #
 # This value also represents ID of the PR that caused major changes. Meaning,

src/bootstrap/src/core/config/config.rs

@@ -6,6 +6,7 @@
 use std::cell::{Cell, RefCell};
 use std::collections::{BTreeSet, HashMap, HashSet};
 use std::fmt::{self, Display};
+use std::hash::Hash;
 use std::io::IsTerminal;
 use std::path::{Path, PathBuf, absolute};
 use std::process::Command;
@@ -701,6 +702,7 @@ pub(crate) struct TomlConfig {
     target: Option<HashMap<String, TomlTarget>>,
     dist: Option<Dist>,
     profile: Option<String>,
+    include: Option<Vec<PathBuf>>,
 }
 
 /// This enum is used for deserializing change IDs from TOML, allowing both numeric values and the string `"ignore"`.
@@ -747,27 +749,35 @@ enum ReplaceOpt {
 }
 
 trait Merge {
-    fn merge(&mut self, other: Self, replace: ReplaceOpt);
+    fn merge(
+        &mut self,
+        parent_config_path: Option<PathBuf>,
+        included_extensions: &mut HashSet<PathBuf>,
+        other: Self,
+        replace: ReplaceOpt,
+    );
 }
 
 impl Merge for TomlConfig {
     fn merge(
         &mut self,
-        TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id }: Self,
+        parent_config_path: Option<PathBuf>,
+        included_extensions: &mut HashSet<PathBuf>,
+        TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id, include }: Self,
         replace: ReplaceOpt,
     ) {
         fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
             if let Some(new) = y {
                 if let Some(original) = x {
-                    original.merge(new, replace);
+                    original.merge(None, &mut Default::default(), new, replace);
                 } else {
                     *x = Some(new);
                 }
             }
         }
 
-        self.change_id.inner.merge(change_id.inner, replace);
+        self.change_id.inner.merge(None, &mut Default::default(), change_id.inner, replace);
-        self.profile.merge(profile, replace);
+        self.profile.merge(None, &mut Default::default(), profile, replace);
 
         do_merge(&mut self.build, build, replace);
         do_merge(&mut self.install, install, replace);
@@ -782,13 +792,50 @@ impl Merge for TomlConfig {
             (Some(original_target), Some(new_target)) => {
                 for (triple, new) in new_target {
                     if let Some(original) = original_target.get_mut(&triple) {
-                        original.merge(new, replace);
+                        original.merge(None, &mut Default::default(), new, replace);
                     } else {
                         original_target.insert(triple, new);
                     }
                 }
             }
         }
+
+        let parent_dir = parent_config_path
+            .as_ref()
+            .and_then(|p| p.parent().map(ToOwned::to_owned))
+            .unwrap_or_default();
+
+        // `include` handled later since we ignore duplicates using `ReplaceOpt::IgnoreDuplicate` to
+        // keep the upper-level configuration to take precedence.
+        for include_path in include.clone().unwrap_or_default().iter().rev() {
+            let include_path = parent_dir.join(include_path);
+            let include_path = include_path.canonicalize().unwrap_or_else(|e| {
+                eprintln!("ERROR: Failed to canonicalize '{}' path: {e}", include_path.display());
+                exit!(2);
+            });
+
+            let included_toml = Config::get_toml_inner(&include_path).unwrap_or_else(|e| {
+                eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display());
+                exit!(2);
+            });
+
+            assert!(
+                included_extensions.insert(include_path.clone()),
+                "Cyclic inclusion detected: '{}' is being included again before its previous inclusion was fully processed.",
+                include_path.display()
+            );
+
+            self.merge(
+                Some(include_path.clone()),
+                included_extensions,
+                included_toml,
+                // Ensures that parent configuration always takes precedence
+                // over child configurations.
+                ReplaceOpt::IgnoreDuplicate,
+            );
+
+            included_extensions.remove(&include_path);
+        }
     }
 }
 
@@ -803,7 +850,13 @@ macro_rules! define_config {
         }
 
         impl Merge for $name {
-            fn merge(&mut self, other: Self, replace: ReplaceOpt) {
+            fn merge(
+                &mut self,
+                _parent_config_path: Option<PathBuf>,
+                _included_extensions: &mut HashSet<PathBuf>,
+                other: Self,
+                replace: ReplaceOpt
+            ) {
                 $(
                     match replace {
                         ReplaceOpt::IgnoreDuplicate => {
@@ -903,7 +956,13 @@ macro_rules! define_config {
 }
 
 impl<T> Merge for Option<T> {
-    fn merge(&mut self, other: Self, replace: ReplaceOpt) {
+    fn merge(
+        &mut self,
+        _parent_config_path: Option<PathBuf>,
+        _included_extensions: &mut HashSet<PathBuf>,
+        other: Self,
+        replace: ReplaceOpt,
+    ) {
         match replace {
             ReplaceOpt::IgnoreDuplicate => {
                 if self.is_none() {
@@ -1363,13 +1422,15 @@ impl Config {
         Self::get_toml(&builder_config_path)
     }
 
-    #[cfg(test)]
+    pub(crate) fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> {
-    pub(crate) fn get_toml(_: &Path) -> Result<TomlConfig, toml::de::Error> {
+        #[cfg(test)]
-        Ok(TomlConfig::default())
+        return Ok(TomlConfig::default());
+
+        #[cfg(not(test))]
+        Self::get_toml_inner(file)
     }
 
-    #[cfg(not(test))]
+    fn get_toml_inner(file: &Path) -> Result<TomlConfig, toml::de::Error> {
-    pub(crate) fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> {
         let contents =
             t!(fs::read_to_string(file), format!("config file {} not found", file.display()));
         // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of
@@ -1548,7 +1609,8 @@ impl Config {
         // but not if `bootstrap.toml` hasn't been created.
         let mut toml = if !using_default_path || toml_path.exists() {
             config.config = Some(if cfg!(not(test)) {
-                toml_path.canonicalize().unwrap()
+                toml_path = toml_path.canonicalize().unwrap();
+                toml_path.clone()
             } else {
                 toml_path.clone()
             });
@@ -1576,6 +1638,26 @@ impl Config {
             toml.profile = Some("dist".into());
         }
 
+        // Reverse the list to ensure the last added config extension remains the most dominant.
+        // For example, given ["a.toml", "b.toml"], "b.toml" should take precedence over "a.toml".
+        //
+        // This must be handled before applying the `profile` since `include`s should always take
+        // precedence over `profile`s.
+        for include_path in toml.include.clone().unwrap_or_default().iter().rev() {
+            let include_path = toml_path.parent().unwrap().join(include_path);
+
+            let included_toml = get_toml(&include_path).unwrap_or_else(|e| {
+                eprintln!("ERROR: Failed to parse '{}': {e}", include_path.display());
+                exit!(2);
+            });
+            toml.merge(
+                Some(include_path),
+                &mut Default::default(),
+                included_toml,
+                ReplaceOpt::IgnoreDuplicate,
+            );
+        }
+
         if let Some(include) = &toml.profile {
             // Allows creating alias for profile names, allowing
             // profiles to be renamed while maintaining back compatibility
@@ -1597,7 +1679,12 @@ impl Config {
                 );
                 exit!(2);
             });
-            toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate);
+            toml.merge(
+                Some(include_path),
+                &mut Default::default(),
+                included_toml,
+                ReplaceOpt::IgnoreDuplicate,
+            );
         }
 
         let mut override_toml = TomlConfig::default();
@@ -1608,7 +1695,12 @@ impl Config {
 
             let mut err = match get_table(option) {
                 Ok(v) => {
-                    override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate);
+                    override_toml.merge(
+                        None,
+                        &mut Default::default(),
+                        v,
+                        ReplaceOpt::ErrorOnDuplicate,
+                    );
                     continue;
                 }
                 Err(e) => e,
@@ -1619,7 +1711,12 @@ impl Config {
                 if !value.contains('"') {
                     match get_table(&format!(r#"{key}="{value}""#)) {
                         Ok(v) => {
-                            override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate);
+                            override_toml.merge(
+                                None,
+                                &mut Default::default(),
+                                v,
+                                ReplaceOpt::ErrorOnDuplicate,
+                            );
                             continue;
                         }
                         Err(e) => err = e,
@@ -1629,7 +1726,7 @@ impl Config {
             eprintln!("failed to parse override `{option}`: `{err}");
             exit!(2)
         }
-        toml.merge(override_toml, ReplaceOpt::Override);
+        toml.merge(None, &mut Default::default(), override_toml, ReplaceOpt::Override);
 
         config.change_id = toml.change_id.inner;
 

src/bootstrap/src/core/config/tests.rs

@@ -1,8 +1,8 @@
 use std::collections::BTreeSet;
-use std::env;
 use std::fs::{File, remove_file};
 use std::io::Write;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::{env, fs};
 
 use build_helper::ci::CiEnv;
 use clap::CommandFactory;
@@ -23,6 +23,27 @@ pub(crate) fn parse(config: &str) -> Config {
     )
 }
 
+fn get_toml(file: &Path) -> Result<TomlConfig, toml::de::Error> {
+    let contents = std::fs::read_to_string(file).unwrap();
+    toml::from_str(&contents).and_then(|table: toml::Value| TomlConfig::deserialize(table))
+}
+
+/// Helps with debugging by using consistent test-specific directories instead of
+/// random temporary directories.
+fn prepare_test_specific_dir() -> PathBuf {
+    let current = std::thread::current();
+    // Replace "::" with "_" to make it safe for directory names on Windows systems
+    let test_path = current.name().unwrap().replace("::", "_");
+
+    let testdir = parse("").tempdir().join(test_path);
+
+    // clean up any old test files
+    let _ = fs::remove_dir_all(&testdir);
+    let _ = fs::create_dir_all(&testdir);
+
+    testdir
+}
+
 #[test]
 fn download_ci_llvm() {
     let config = parse("llvm.download-ci-llvm = false");
@@ -539,3 +560,189 @@ fn test_ci_flag() {
     let config = Config::parse_inner(Flags::parse(&["check".into()]), |&_| toml::from_str(""));
     assert_eq!(config.is_running_on_ci, CiEnv::is_ci());
 }
+
+#[test]
+fn test_precedence_of_includes() {
+    let testdir = prepare_test_specific_dir();
+
+    let root_config = testdir.join("config.toml");
+    let root_config_content = br#"
+        include = ["./extension.toml"]
+
+        [llvm]
+        link-jobs = 2
+    "#;
+    File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
+
+    let extension = testdir.join("extension.toml");
+    let extension_content = br#"
+        change-id=543
+        include = ["./extension2.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let extension = testdir.join("extension2.toml");
+    let extension_content = br#"
+        change-id=742
+
+        [llvm]
+        link-jobs = 10
+
+        [build]
+        description = "Some creative description"
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let config = Config::parse_inner(
+        Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
+        get_toml,
+    );
+
+    assert_eq!(config.change_id.unwrap(), ChangeId::Id(543));
+    assert_eq!(config.llvm_link_jobs.unwrap(), 2);
+    assert_eq!(config.description.unwrap(), "Some creative description");
+}
+
+#[test]
+#[should_panic(expected = "Cyclic inclusion detected")]
+fn test_cyclic_include_direct() {
+    let testdir = prepare_test_specific_dir();
+
+    let root_config = testdir.join("config.toml");
+    let root_config_content = br#"
+        include = ["./extension.toml"]
+    "#;
+    File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
+
+    let extension = testdir.join("extension.toml");
+    let extension_content = br#"
+        include = ["./config.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let config = Config::parse_inner(
+        Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
+        get_toml,
+    );
+}
+
+#[test]
+#[should_panic(expected = "Cyclic inclusion detected")]
+fn test_cyclic_include_indirect() {
+    let testdir = prepare_test_specific_dir();
+
+    let root_config = testdir.join("config.toml");
+    let root_config_content = br#"
+        include = ["./extension.toml"]
+    "#;
+    File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
+
+    let extension = testdir.join("extension.toml");
+    let extension_content = br#"
+        include = ["./extension2.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let extension = testdir.join("extension2.toml");
+    let extension_content = br#"
+        include = ["./extension3.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let extension = testdir.join("extension3.toml");
+    let extension_content = br#"
+        include = ["./extension.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let config = Config::parse_inner(
+        Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
+        get_toml,
+    );
+}
+
+#[test]
+fn test_include_absolute_paths() {
+    let testdir = prepare_test_specific_dir();
+
+    let extension = testdir.join("extension.toml");
+    File::create(&extension).unwrap().write_all(&[]).unwrap();
+
+    let root_config = testdir.join("config.toml");
+    let extension_absolute_path =
+        extension.canonicalize().unwrap().to_str().unwrap().replace('\\', r"\\");
+    let root_config_content = format!(r#"include = ["{}"]"#, extension_absolute_path);
+    File::create(&root_config).unwrap().write_all(root_config_content.as_bytes()).unwrap();
+
+    let config = Config::parse_inner(
+        Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
+        get_toml,
+    );
+}
+
+#[test]
+fn test_include_relative_paths() {
+    let testdir = prepare_test_specific_dir();
+
+    let _ = fs::create_dir_all(&testdir.join("subdir/another_subdir"));
+
+    let root_config = testdir.join("config.toml");
+    let root_config_content = br#"
+        include = ["./subdir/extension.toml"]
+    "#;
+    File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
+
+    let extension = testdir.join("subdir/extension.toml");
+    let extension_content = br#"
+        include = ["../extension2.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let extension = testdir.join("extension2.toml");
+    let extension_content = br#"
+        include = ["./subdir/another_subdir/extension3.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let extension = testdir.join("subdir/another_subdir/extension3.toml");
+    let extension_content = br#"
+        include = ["../../extension4.toml"]
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let extension = testdir.join("extension4.toml");
+    File::create(extension).unwrap().write_all(&[]).unwrap();
+
+    let config = Config::parse_inner(
+        Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
+        get_toml,
+    );
+}
+
+#[test]
+fn test_include_precedence_over_profile() {
+    let testdir = prepare_test_specific_dir();
+
+    let root_config = testdir.join("config.toml");
+    let root_config_content = br#"
+        profile = "dist"
+        include = ["./extension.toml"]
+    "#;
+    File::create(&root_config).unwrap().write_all(root_config_content).unwrap();
+
+    let extension = testdir.join("extension.toml");
+    let extension_content = br#"
+        [rust]
+        channel = "dev"
+    "#;
+    File::create(extension).unwrap().write_all(extension_content).unwrap();
+
+    let config = Config::parse_inner(
+        Flags::parse(&["check".to_owned(), format!("--config={}", root_config.to_str().unwrap())]),
+        get_toml,
+    );
+
+    // "dist" profile would normally set the channel to "auto-detect", but includes should
+    // override profile settings, so we expect this to be "dev" here.
+    assert_eq!(config.channel, "dev");
+}

src/bootstrap/src/utils/change_tracker.rs

@@ -396,4 +396,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
         severity: ChangeSeverity::Info,
         summary: "Added a new option `build.compiletest-use-stage0-libtest` to force `compiletest` to use the stage 0 libtest.",
     },
+    ChangeInfo {
+        change_id: 138934,
+        severity: ChangeSeverity::Info,
+        summary: "Added new option `include` to create config extensions.",
+    },
 ];

src/doc/rustc-dev-guide/src/building/suggested.md

@@ -20,6 +20,43 @@ your `.git/hooks` folder as `pre-push` (without the `.sh` extension!).
 
 You can also install the hook as a step of running `./x setup`!
 
+## Config extensions
+
+When working on different tasks, you might need to switch between different bootstrap configurations.
+Sometimes you may want to keep an old configuration for future use. But saving raw config values in
+random files and manually copying and pasting them can quickly become messy, especially if you have a
+long history of different configurations.
+
+To simplify managing multiple configurations, you can create config extensions.
+
+For example, you can create a simple config file named `cross.toml`:
+
+```toml
+[build]
+build = "x86_64-unknown-linux-gnu"
+host = ["i686-unknown-linux-gnu"]
+target = ["i686-unknown-linux-gnu"]
+
+
+[llvm]
+download-ci-llvm = false
+
+[target.x86_64-unknown-linux-gnu]
+llvm-config = "/path/to/llvm-19/bin/llvm-config"
+```
+
+Then, include this in your `bootstrap.toml`:
+
+```toml
+include = ["cross.toml"]
+```
+
+You can also include extensions within extensions recursively.
+
+**Note:** In the `include` field, the overriding logic follows a right-to-left order. For example,
+in `include = ["a.toml", "b.toml"]`, extension `b.toml` overrides `a.toml`. Also, parent extensions
+always overrides the inner ones.
+
 ## Configuring `rust-analyzer` for `rustc`
 
 ### Project-local rust-analyzer setup