Auto merge of #136921 - Kobzol:gcc-build, r=onur-ozkan

Build GCC on CI

Previously, we have downloaded a specific commit of GCC and prebuilt it inside Docker using the `build-gccjit.sh` script. This PR removes that scripts and uses the bootstrap GCC step. This allows us to use the `src/gcc` submodule for determining which GCC should be built, and it also moves the logic closer to LLVM, which is also built by bootstrap.

A few things to note:
- The `sccache` option is currently in the `llvm` block, so the GCC build uses `llvm.ccache`, which is a bit weird :) We could either add `gcc.ccache`, or (what I think would be better) to just move `ccache` to the `build` section, as I don't think that it will be necessary to use ccache for LLVM, but not for GCC.
- When the GCC codegen backend is built, it needs to depend on a step that first builds GCC. This is currently done in a hacky way. The proper solution is to create a separate step for the GCC codegen backend, but that is a larger change. Let me know what you think.

r? `@onur-ozkan`

try-job: i686-msvc-1
try-job: x86_64-mingw-1

license-metadata.json

@@ -13,7 +13,8 @@
                         "directories": [],
                         "files": [
                           "analyzer-decls.h",
-                          "malloc-macro.h"
+                          "malloc-macro.h",
+                          "sarif-path-role.h"
                         ],
                         "license": {
                           "copyright": [

src/bootstrap/src/core/build_steps/compile.rs

@@ -19,6 +19,7 @@ use serde_derive::Deserialize;
 #[cfg(feature = "tracing")]
 use tracing::{instrument, span};
 
+use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags};
 use crate::core::build_steps::tool::SourceType;
 use crate::core::build_steps::{dist, llvm};
 use crate::core::builder;
@@ -1614,6 +1615,14 @@ impl Step for CodegenBackend {
             .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml")));
         rustc_cargo_env(builder, &mut cargo, target, compiler.stage);
 
+        // Ideally, we'd have a separate step for the individual codegen backends,
+        // like we have in tests (test::CodegenGCC) but that would require a lot of restructuring.
+        // If the logic gets more complicated, it should probably be done.
+        if backend == "gcc" {
+            let gcc = builder.ensure(Gcc { target });
+            add_cg_gcc_cargo_flags(&mut cargo, &gcc);
+        }
+
         let tmp_stamp = BuildStamp::new(&out_dir).with_prefix("tmp");
 
         let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target);

src/bootstrap/src/core/build_steps/gcc.rs

@@ -9,13 +9,13 @@
 //! ensure that they're always in place if needed.
 
 use std::fs;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::sync::OnceLock;
 
 use build_helper::ci::CiEnv;
 
 use crate::Kind;
-use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
+use crate::core::builder::{Builder, Cargo, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
 use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash};
 use crate::utils::exec::command;
@@ -29,7 +29,8 @@ pub struct Meta {
 }
 
 pub enum GccBuildStatus {
-    AlreadyBuilt,
+    /// libgccjit is already built at this path
+    AlreadyBuilt(PathBuf),
     ShouldBuild(Meta),
 }
 
@@ -41,9 +42,6 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
     // Initialize the gcc submodule if not initialized already.
     builder.config.update_submodule("src/gcc");
 
-    // FIXME (GuillaumeGomez): To be done once gccjit has been built in the CI.
-    // builder.config.maybe_download_ci_gcc();
-
     let root = builder.src.join("src/gcc");
     let out_dir = builder.gcc_out(target).join("build");
     let install_dir = builder.gcc_out(target).join("install");
@@ -70,19 +68,37 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc
                 stamp.path().display()
             ));
         }
-        return GccBuildStatus::AlreadyBuilt;
+        let path = libgccjit_built_path(&install_dir);
+        if path.is_file() {
+            return GccBuildStatus::AlreadyBuilt(path);
+        } else {
+            builder.info(&format!(
+                "GCC stamp is up-to-date, but the libgccjit.so file was not found at `{}`",
+                path.display(),
+            ));
+        }
     }
 
     GccBuildStatus::ShouldBuild(Meta { stamp, out_dir, install_dir, root })
 }
 
+/// Returns the path to a libgccjit.so file in the install directory of GCC.
+fn libgccjit_built_path(install_dir: &Path) -> PathBuf {
+    install_dir.join("lib/libgccjit.so")
+}
+
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Gcc {
     pub target: TargetSelection,
 }
 
+#[derive(Clone)]
+pub struct GccOutput {
+    pub libgccjit: PathBuf,
+}
+
 impl Step for Gcc {
-    type Output = bool;
+    type Output = GccOutput;
 
     const ONLY_HOSTS: bool = true;
 
@@ -94,14 +110,14 @@ impl Step for Gcc {
         run.builder.ensure(Gcc { target: run.target });
     }
 
-    /// Compile GCC for `target`.
-    fn run(self, builder: &Builder<'_>) -> bool {
+    /// Compile GCC (specifically `libgccjit`) for `target`.
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
         let target = self.target;
 
         // If GCC has already been built, we avoid building it again.
         let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target)
         {
-            GccBuildStatus::AlreadyBuilt => return true,
+            GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path },
             GccBuildStatus::ShouldBuild(m) => m,
         };
 
@@ -110,8 +126,9 @@ impl Step for Gcc {
         let _time = helpers::timeit(builder);
         t!(fs::create_dir_all(&out_dir));
 
+        let libgccjit_path = libgccjit_built_path(&install_dir);
         if builder.config.dry_run() {
-            return true;
+            return GccOutput { libgccjit: libgccjit_path };
         }
 
         // GCC creates files (e.g. symlinks to the downloaded dependencies)
@@ -173,11 +190,17 @@ impl Step for Gcc {
 
         let lib_alias = install_dir.join("lib/libgccjit.so.0");
         if !lib_alias.exists() {
-            t!(builder.symlink_file(install_dir.join("lib/libgccjit.so"), lib_alias,));
+            t!(builder.symlink_file(&libgccjit_path, lib_alias));
         }
 
         t!(stamp.write());
 
-        true
+        GccOutput { libgccjit: libgccjit_path }
     }
 }
+
+/// Configures a Cargo invocation so that it can build the GCC codegen backend.
+pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) {
+    // Add the path to libgccjit.so to the linker search paths.
+    cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().to_str().unwrap()));
+}

src/bootstrap/src/core/build_steps/test.rs

@@ -12,6 +12,7 @@ use clap_complete::shells;
 
 use crate::core::build_steps::compile::run_cargo;
 use crate::core::build_steps::doc::DocumentationFormat;
+use crate::core::build_steps::gcc::{Gcc, add_cg_gcc_cargo_flags};
 use crate::core::build_steps::llvm::get_llvm_version;
 use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget;
 use crate::core::build_steps::tool::{self, SourceType, Tool};
@@ -3549,6 +3550,8 @@ impl Step for CodegenGCC {
         let compiler = self.compiler;
         let target = self.target;
 
+        let gcc = builder.ensure(Gcc { target });
+
         builder.ensure(
             compile::Std::new(compiler, target)
                 .extra_rust_args(&["-Csymbol-mangling-version=v0", "-Cpanic=abort"]),
@@ -3575,6 +3578,7 @@ impl Step for CodegenGCC {
                 .arg("--manifest-path")
                 .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml"));
             compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage);
+            add_cg_gcc_cargo_flags(&mut cargo, &gcc);
 
             // Avoid incremental cache issues when changing rustc
             cargo.env("CARGO_BUILD_INCREMENTAL", "false");
@@ -3607,9 +3611,10 @@ impl Step for CodegenGCC {
             .env("CG_RUSTFLAGS", "-Alinker-messages")
             .arg("--")
             .arg("test")
-            .arg("--use-system-gcc")
             .arg("--use-backend")
             .arg("gcc")
+            .arg("--gcc-path")
+            .arg(gcc.libgccjit.parent().unwrap())
             .arg("--out-dir")
             .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc"))
             .arg("--release")

src/ci/docker/host-x86_64/x86_64-gnu-llvm-18/Dockerfile

@@ -3,6 +3,7 @@ FROM ubuntu:24.04
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+  bzip2 \
   g++ \
   gcc-multilib \
   make \
@@ -55,9 +56,6 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.thin-lto-import-instr-limit=10
 
 COPY scripts/shared.sh /scripts/
-COPY scripts/build-gccjit.sh /scripts/
-
-RUN /scripts/build-gccjit.sh /scripts
 
 ARG SCRIPT_ARG
 

src/ci/docker/host-x86_64/x86_64-gnu-llvm-19/Dockerfile

@@ -55,9 +55,6 @@ ENV RUST_CONFIGURE_ARGS \
       --set rust.thin-lto-import-instr-limit=10
 
 COPY scripts/shared.sh /scripts/
-COPY scripts/build-gccjit.sh /scripts/
-
-RUN /scripts/build-gccjit.sh /scripts
 
 ARG SCRIPT_ARG
 

src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

@@ -90,9 +90,6 @@ ENV HOST_TARGET x86_64-unknown-linux-gnu
 #ENV FORCE_CI_RUSTC 1
 
 COPY scripts/shared.sh /scripts/
-COPY scripts/build-gccjit.sh /scripts/
-
-RUN /scripts/build-gccjit.sh /scripts
 
 # For now, we need to use `--unsafe-perm=true` to go around an issue when npm tries
 # to create a new folder. For reference:

src/ci/docker/scripts/build-gccjit.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-GIT_REPO="https://github.com/rust-lang/gcc"
-
-# This commit hash needs to be updated to use a more recent gcc fork version.
-GIT_COMMIT="45648c2edd4ecd862d9f08196d3d6c6ccba79f07"
-
-set -ex
-
-cd $1
-
-source shared.sh
-
-# Setting up folders for GCC
-curl -L "$GIT_REPO/archive/$GIT_COMMIT.tar.gz" |
-    tar -xz --transform "s/gcc-$GIT_COMMIT/gcc-src/"
-
-mkdir gcc-build gcc-install
-pushd gcc-build
-
-# Building GCC.
-hide_output \
-  ../gcc-src/configure \
-    --enable-host-shared \
-    --enable-languages=jit \
-    --enable-checking=release \
-    --disable-bootstrap \
-    --disable-multilib \
-    --prefix=$(pwd)/../gcc-install \
-
-hide_output make -j$(nproc)
-hide_output make install
-
-popd
-rm -rf gcc-src gcc-build
-ln -s /scripts/gcc-install/lib/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so
-ln -s /scripts/gcc-install/lib/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so.0

src/gcc

@@ -1 +1 @@
-Subproject commit fd3498bff0b939dda91d56960acc33d55f2f9cdf
+Subproject commit e607be166673a8de9fc07f6f02c60426e556c5f2