Rollup merge of #131831 - onur-ozkan:improve-rustc-if-unchanged-logic, r=Mark-Simulacrum

extend the "if-unchanged" logic for compiler builds

Implements the first item from [this tracking issue](https://github.com/rust-lang/rust/issues/131744).

In short, we want to make "if-unchanged" logic to check for changes outside of certain allowed directories, and this PR implements that.

See https://github.com/rust-lang/rust/issues/131658 for more context.

Cargo.toml

@@ -2,12 +2,12 @@
 resolver = "2"
 members = [
   "compiler/rustc",
+  "src/build_helper",
   "src/etc/test-float-parse",
   "src/rustc-std-workspace/rustc-std-workspace-core",
   "src/rustc-std-workspace/rustc-std-workspace-alloc",
   "src/rustc-std-workspace/rustc-std-workspace-std",
   "src/rustdoc-json-types",
-  "src/tools/build_helper",
   "src/tools/cargotest",
   "src/tools/clippy",
   "src/tools/clippy/clippy_dev",

config.example.toml

@@ -500,8 +500,9 @@
 # This is useful if you are working on tools, doc-comments, or library (you will be able to build
 # the standard library without needing to build the compiler).
 #
-# Set this to "if-unchanged" to only download if the compiler (and library if running on CI) have
+# Set this to "if-unchanged" if you are working on `src/tools`, `tests` or `library` (on CI, `library`
-# not been modified.
+# changes triggers in-tree compiler build) to speed up the build process.
+#
 # Set this to `true` to download unconditionally.
 #download-rustc = false
 

src/bootstrap/Cargo.toml

@@ -40,7 +40,7 @@ test = false
 cc = "=1.1.22"
 cmake = "=0.1.48"
 
-build_helper = { path = "../tools/build_helper" }
+build_helper = { path = "../build_helper" }
 clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] }
 clap_complete = "4.4"
 fd-lock = "4.0"

src/bootstrap/src/core/build_steps/clippy.rs

@@ -295,7 +295,7 @@ macro_rules! lint_any {
 
 lint_any!(
     Bootstrap, "src/bootstrap", "bootstrap";
-    BuildHelper, "src/tools/build_helper", "build_helper";
+    BuildHelper, "src/build_helper", "build_helper";
     BuildManifest, "src/tools/build-manifest", "build-manifest";
     CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri";
     Clippy, "src/tools/clippy", "clippy";

src/bootstrap/src/core/build_steps/doc.rs

@@ -1030,7 +1030,7 @@ macro_rules! tool_doc {
 // NOTE: make sure to register these in `Builder::get_step_description`.
 tool_doc!(
     BuildHelper,
-    "src/tools/build_helper",
+    "src/build_helper",
     rustc_tool = false,
     is_library = true,
     crates = ["build_helper"]

src/bootstrap/src/core/build_steps/test.rs

@@ -1354,7 +1354,7 @@ impl Step for CrateBuildHelper {
     const ONLY_HOSTS: bool = true;
 
     fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
-        run.path("src/tools/build_helper")
+        run.path("src/build_helper")
     }
 
     fn make_run(run: RunConfig<'_>) {
@@ -1372,7 +1372,7 @@ impl Step for CrateBuildHelper {
             Mode::ToolBootstrap,
             host,
             Kind::Test,
-            "src/tools/build_helper",
+            "src/build_helper",
             SourceType::InTree,
             &[],
         );

src/bootstrap/src/core/config/config.rs

@@ -28,6 +28,24 @@ use crate::utils::cache::{INTERNER, Interned};
 use crate::utils::channel::{self, GitInfo};
 use crate::utils::helpers::{self, exe, output, t};
 
+/// Each path in this list is considered "allowed" in the `download-rustc="if-unchanged"` logic.
+/// This means they can be modified and changes to these paths should never trigger a compiler build
+/// when "if-unchanged" is set.
+///
+/// NOTE: Paths must have the ":!" prefix to tell git to ignore changes in those paths during
+/// the diff check.
+///
+/// WARNING: Be cautious when adding paths to this list. If a path that influences the compiler build
+/// is added here, it will cause bootstrap to skip necessary rebuilds, which may lead to risky results.
+/// For example, "src/bootstrap" should never be included in this list as it plays a crucial role in the
+/// final output/compiler, which can be significantly affected by changes made to the bootstrap sources.
+#[rustfmt::skip] // We don't want rustfmt to oneline this list
+pub(crate) const RUSTC_IF_UNCHANGED_ALLOWED_PATHS: &[&str] = &[
+    ":!src/tools",
+    ":!tests",
+    ":!triagebot.toml",
+];
+
 macro_rules! check_ci_llvm {
     ($name:expr) => {
         assert!(
@@ -2768,32 +2786,33 @@ impl Config {
             }
         };
 
-        let mut files_to_track = vec!["compiler", "src/version", "src/stage0", "src/ci/channel"];
+        // RUSTC_IF_UNCHANGED_ALLOWED_PATHS
+        let mut allowed_paths = RUSTC_IF_UNCHANGED_ALLOWED_PATHS.to_vec();
 
-        // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, ignore
+        // In CI, disable ci-rustc if there are changes in the library tree. But for non-CI, allow
         // these changes to speed up the build process for library developers. This provides consistent
         // functionality for library developers between `download-rustc=true` and `download-rustc="if-unchanged"`
         // options.
-        if CiEnv::is_ci() {
+        if !CiEnv::is_ci() {
-            files_to_track.push("library");
+            allowed_paths.push(":!library");
         }
 
         // Look for a version to compare to based on the current commit.
         // Only commits merged by bors will have CI artifacts.
-        let commit =
+        let commit = match self.last_modified_commit(&allowed_paths, "download-rustc", if_unchanged)
-            match self.last_modified_commit(&files_to_track, "download-rustc", if_unchanged) {
+        {
-                Some(commit) => commit,
+            Some(commit) => commit,
-                None => {
+            None => {
-                    if if_unchanged {
+                if if_unchanged {
-                        return None;
+                    return None;
-                    }
-                    println!("ERROR: could not find commit hash for downloading rustc");
-                    println!("HELP: maybe your repository history is too shallow?");
-                    println!("HELP: consider disabling `download-rustc`");
-                    println!("HELP: or fetch enough history to include one upstream commit");
-                    crate::exit!(1);
                 }
-            };
+                println!("ERROR: could not find commit hash for downloading rustc");
+                println!("HELP: maybe your repository history is too shallow?");
+                println!("HELP: consider setting `rust.download-rustc=false` in config.toml");
+                println!("HELP: or fetch enough history to include one upstream commit");
+                crate::exit!(1);
+            }
+        };
 
         if CiEnv::is_ci() && {
             let head_sha =

src/bootstrap/src/core/config/tests.rs

@@ -8,7 +8,7 @@ use clap::CommandFactory;
 use serde::Deserialize;
 
 use super::flags::Flags;
-use super::{ChangeIdWrapper, Config};
+use super::{ChangeIdWrapper, Config, RUSTC_IF_UNCHANGED_ALLOWED_PATHS};
 use crate::core::build_steps::clippy::get_clippy_rules_in_order;
 use crate::core::build_steps::llvm;
 use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig};
@@ -427,3 +427,18 @@ fn jobs_precedence() {
     );
     assert_eq!(config.jobs, Some(123));
 }
+
+#[test]
+fn check_rustc_if_unchanged_paths() {
+    let config = parse("");
+    let normalised_allowed_paths: Vec<_> = RUSTC_IF_UNCHANGED_ALLOWED_PATHS
+        .iter()
+        .map(|t| {
+            t.strip_prefix(":!").expect(&format!("{t} doesn't have ':!' prefix, but it should."))
+        })
+        .collect();
+
+    for p in normalised_allowed_paths {
+        assert!(config.src.join(p).exists(), "{p} doesn't exist.");
+    }
+}

src/build_helper/src/stage0_parser.rs

@@ -25,7 +25,7 @@ pub struct Stage0Config {
 }
 
 pub fn parse_stage0_file() -> Stage0 {
-    let stage0_content = include_str!("../../../stage0");
+    let stage0_content = include_str!("../../stage0");
 
     let mut stage0 = Stage0::default();
     for line in stage0_content.lines() {

src/ci/run.sh

@@ -179,9 +179,7 @@ else
   fi
 
   if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then
-    # disabled for now, see https://github.com/rust-lang/rust/issues/131658
+    RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
-    #RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
-    true
   fi
 fi
 

src/tools/bump-stage0/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.34"
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 curl = "0.4.38"
 indexmap = { version = "2.0.0", features = ["serde"] }
 serde = { version = "1.0.125", features = ["derive"] }

src/tools/compiletest/Cargo.toml

@@ -14,7 +14,7 @@ unified-diff = "0.2.1"
 getopts = "0.2"
 indexmap = "2.0.0"
 miropt-test-tools = { path = "../miropt-test-tools" }
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.3", default-features = false, features = ["fmt", "env-filter", "smallvec", "parking_lot", "ansi"] }
 regex = "1.0"

src/tools/opt-dist/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 env_logger = "0.11"
 log = "0.4"
 anyhow = { version = "1", features = ["backtrace"] }

src/tools/run-make-support/Cargo.toml

@@ -10,7 +10,7 @@ similar = "2.5.0"
 wasmparser = { version = "0.216", default-features = false, features = ["std"] }
 regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace
 gimli = "0.31.0"
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 serde_json = "1.0"
 libc = "0.2"
 

src/tools/rustdoc-gui-test/Cargo.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 compiletest = { path = "../compiletest" }
 getopts = "0.2"
 walkdir = "2"

src/tools/suggest-tests/Cargo.toml

@@ -5,4 +5,4 @@ edition = "2021"
 
 [dependencies]
 glob = "0.3.0"
-build_helper = { version = "0.1.0", path = "../build_helper" }
+build_helper = { version = "0.1.0", path = "../../build_helper" }

src/tools/tidy/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 autobins = false
 
 [dependencies]
-build_helper = { path = "../build_helper" }
+build_helper = { path = "../../build_helper" }
 cargo_metadata = "0.18"
 regex = "1"
 miropt-test-tools = { path = "../miropt-test-tools" }