Merge pull request #355 from rust-lang/refactor-ci

Refactor CI

.github/workflows/ci.yml

@@ -19,9 +19,8 @@ jobs:
       fail-fast: false
       matrix:
         libgccjit_version:
-          - { gcc: "libgccjit.so", extra: "", env_extra: "", artifacts_branch: "master" }
-          - { gcc: "libgccjit_without_int128.so", extra: "", env_extra: "", artifacts_branch: "master-without-128bit-integers" }
-          - { gcc: "libgccjit12.so", extra: "--no-default-features", env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests'", artifacts_branch: "gcc12" }
+          - { gcc: "libgccjit.so", artifacts_branch: "master" }
+          - { gcc: "libgccjit_without_int128.so", artifacts_branch: "master-without-128bit-integers" }
         commands: [
           "--mini-tests",
           "--std-tests",
@@ -33,7 +32,6 @@ jobs:
           "--extended-regex-tests",
           "--test-successful-rustc --nb-parts 2 --current-part 0",
           "--test-successful-rustc --nb-parts 2 --current-part 1",
-          "--test-failing-rustc",
         ]
 
     steps:
@@ -48,12 +46,7 @@ jobs:
       # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
       run: sudo apt-get install ninja-build ripgrep llvm-14-tools
 
-    - name: Install libgccjit12
-      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
-      run: sudo apt-get install libgccjit-12-dev
-
     - name: Download artifact
-      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
@@ -65,11 +58,6 @@ jobs:
           search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
 
     - name: Setup path to libgccjit
-      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
-      run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
-
-    - name: Setup path to libgccjit
-      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       run: |
           sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
           echo /usr/lib/ > gcc_path
@@ -119,8 +107,8 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        ${{ matrix.libgccjit_version.env_extra }} ./y.sh build ${{ matrix.libgccjit_version.extra }}
-        ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
+        ./y.sh build
+        cargo test
         ./clean_all.sh
 
     - name: Prepare dependencies
@@ -136,16 +124,12 @@ jobs:
         command: build
         args: --release
 
-    - name: Add more failing tests for GCC 12
-      if: ${{ matrix.libgccjit_version.gcc == 'libgccjit12.so' }}
-      run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
-
     - name: Add more failing tests because the sysroot is not compiled with LTO
       run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
 
     - name: Run tests
       run: |
-        ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} ${{ matrix.libgccjit_version.extra }}
+        ./test.sh --release --clean --build-sysroot ${{ matrix.commands }}
 
   duplicates:
     runs-on: ubuntu-latest

.github/workflows/failures.yml

@@ -0,0 +1,119 @@
+# TODO: refactor to avoid duplication with the ci.yml file.
+name: Failures
+
+on:
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        libgccjit_version:
+          - { gcc: "libgccjit.so", artifacts_branch: "master" }
+          - { gcc: "libgccjit_without_int128.so", artifacts_branch: "master-without-128bit-integers" }
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/checkout@v3
+      with:
+        repository: llvm/llvm-project
+        path: llvm
+
+    - name: Install packages
+      run: sudo apt-get install ninja-build ripgrep
+
+    - name: Download artifact
+      uses: dawidd6/action-download-artifact@v2
+      with:
+          workflow: main.yml
+          name: gcc-13
+          path: gcc-13
+          repo: antoyo/gcc
+          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          event: push
+          search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
+
+    - name: Setup path to libgccjit
+      run: |
+          sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
+          echo /usr/lib/ > gcc_path
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Set RUST_COMPILER_RT_ROOT
+      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    - name: Cache cargo registry
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/registry
+        key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/git
+        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    #- name: Cache rust repository
+      #uses: actions/cache@v3
+      #id: cache-rust-repository
+      #with:
+        #path: rust
+        #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
+
+    - name: Build
+      run: |
+        ./y.sh prepare --only-libcore
+        ./y.sh build
+        cargo test
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./y.sh prepare
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Add more failing tests because the sysroot is not compiled with LTO
+      run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
+
+    - name: Run tests
+      id: tests
+      run: |
+        ./test.sh --release --clean --build-sysroot --test-failing-rustc | tee output_log
+        rg "test result" output_log >> $GITHUB_STEP_SUMMARY

.github/workflows/gcc12.yml

@@ -0,0 +1,124 @@
+name: CI libgccjit 12
+
+on:
+  - push
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+  TEST_FLAGS: "-Cpanic=abort -Zpanic-abort-tests"
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        commands: [
+          "--mini-tests",
+          # TODO(antoyo): re-enable those commands when the build with libgccjit 12 is fixed.
+          #"--std-tests",
+          # FIXME: re-enable asm tests when GCC can emit in the right syntax.
+          # "--asm-tests",
+          #"--test-libcore",
+          #"--extended-rand-tests",
+          #"--extended-regex-example-tests",
+          #"--extended-regex-tests",
+          #"--test-successful-rustc --nb-parts 2 --current-part 0",
+          #"--test-successful-rustc --nb-parts 2 --current-part 1",
+          #"--test-failing-rustc",
+        ]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/checkout@v3
+      with:
+        repository: llvm/llvm-project
+        path: llvm
+
+    - name: Install packages
+      # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
+      run: sudo apt-get install ninja-build ripgrep llvm-14-tools libgccjit-12-dev
+
+    - name: Setup path to libgccjit
+      run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Set RUST_COMPILER_RT_ROOT
+      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    - name: Cache cargo registry
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/registry
+        key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/git
+        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    #- name: Cache rust repository
+      ## We only clone the rust repository for rustc tests
+      #if: ${{ contains(matrix.commands, 'rustc') }}
+      #uses: actions/cache@v3
+      #id: cache-rust-repository
+      #with:
+        #path: rust
+        #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
+
+    - name: Build
+      run: |
+        ./y.sh prepare --only-libcore
+        # TODO(antoyo): build the sysroot when the build with libgccjit 12 is fixed.
+        #./y.sh build --no-default-features
+        # TODO(antoyo): run the tests when we can build the sysroot with libgccjit 12.
+        #cargo test --no-default-features
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./y.sh prepare
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Add more failing tests for GCC 12
+      run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
+
+    - name: Run tests
+      run: |
+        # TODO(antoyo): add --build-sysroot when the build with libgccjit 12 is fixed.
+        # FIXME(antoyo): we need to set GCC_EXEC_PREFIX so that the linker can find the linker plugin.
+        # Not sure why it's not found otherwise.
+        GCC_EXEC_PREFIX=/usr/lib/gcc/ ./test.sh --release --clean ${{ matrix.commands }} --no-default-features

.ignore

@@ -0,0 +1,10 @@
+!/build_sysroot/sysroot_src
+!/simple-raytracer
+!/regex
+!/rand
+!/test-backend
+!/gcc_path
+!/benchmarks
+!*gimple*
+!*asm*
+!.github

build_system/src/build.rs

@@ -1,6 +1,6 @@
 use crate::config::set_config;
 use crate::utils::{
-    get_gcc_path, run_command, run_command_with_env, run_command_with_output_and_env, walk_dir,
+    get_gcc_path, run_command, run_command_with_output_and_env, walk_dir,
 };
 use std::collections::HashMap;
 use std::ffi::OsStr;
@@ -200,7 +200,7 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> {
     for feature in &ref_features {
         command.push(feature);
     }
-    run_command_with_env(&command, None, Some(&env))?;
+    run_command_with_output_and_env(&command, None, Some(&env))?;
 
     let config = set_config(&mut env, &[], Some(&args.gcc_path))?;