307 lines
12 KiB
YAML
307 lines
12 KiB
YAML
# This file defines our primary CI workflow that runs on pull requests
|
|
# and also on pushes to special branches (auto, try).
|
|
#
|
|
# The actual definition of the executed jobs is calculated by the
|
|
# `src/ci/citool` crate, which
|
|
# uses job definition data from src/ci/github-actions/jobs.yml.
|
|
# You should primarily modify the `jobs.yml` file if you want to modify
|
|
# what jobs are executed in CI.
|
|
|
|
name: CI
|
|
on:
|
|
push:
|
|
branches:
|
|
- auto
|
|
- try
|
|
- try-perf
|
|
- automation/bors/try
|
|
pull_request:
|
|
branches:
|
|
- "**"
|
|
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
|
|
defaults:
|
|
run:
|
|
# On Linux, macOS, and Windows, use the system-provided bash as the default
|
|
# shell. (This should only make a difference on Windows, where the default
|
|
# shell is PowerShell.)
|
|
shell: bash
|
|
|
|
concurrency:
|
|
# For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
|
|
# We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
|
|
# are all triggered on the same branch, but which should be able to run concurrently.
|
|
group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
|
|
cancel-in-progress: true
|
|
env:
|
|
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
|
|
# This will be empty in PR jobs.
|
|
TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
|
|
jobs:
|
|
# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
|
|
# It calculates which jobs should be executed, based on the data of the ${{ github }} context.
|
|
# If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
|
|
calculate_matrix:
|
|
name: Calculate job matrix
|
|
runs-on: ubuntu-24.04
|
|
outputs:
|
|
jobs: ${{ steps.jobs.outputs.jobs }}
|
|
run_type: ${{ steps.jobs.outputs.run_type }}
|
|
steps:
|
|
- name: Checkout the source code
|
|
uses: actions/checkout@v4
|
|
# Cache citool to make its build faster, as it's in the critical path.
|
|
# The rust-cache doesn't bleed into the main `job`, so it should not affect any other
|
|
# Rust compilation.
|
|
- name: Cache citool
|
|
uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
|
|
with:
|
|
workspaces: src/ci/citool
|
|
- name: Calculate the CI job matrix
|
|
env:
|
|
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
|
|
run: |
|
|
cd src/ci/citool
|
|
CARGO_INCREMENTAL=0 cargo test
|
|
CARGO_INCREMENTAL=0 cargo run calculate-job-matrix >> $GITHUB_OUTPUT
|
|
id: jobs
|
|
job:
|
|
name: ${{ matrix.full_name }}
|
|
needs: [ calculate_matrix ]
|
|
runs-on: "${{ matrix.os }}"
|
|
timeout-minutes: 360
|
|
env:
|
|
CI_JOB_NAME: ${{ matrix.name }}
|
|
CI_JOB_DOC_URL: ${{ matrix.doc_url }}
|
|
GITHUB_WORKFLOW_RUN_ID: ${{ github.run_id }}
|
|
GITHUB_REPOSITORY: ${{ github.repository }}
|
|
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
|
|
# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
|
|
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
|
|
DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
SCCACHE_BUCKET: rust-lang-ci-sccache2
|
|
SCCACHE_REGION: us-west-1
|
|
CACHE_DOMAIN: ci-caches.rust-lang.org
|
|
continue-on-error: ${{ matrix.continue_on_error || false }}
|
|
strategy:
|
|
matrix:
|
|
# Check the `calculate_matrix` job to see how is the matrix defined.
|
|
include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
|
|
steps:
|
|
- name: disable git crlf conversion
|
|
run: git config --global core.autocrlf false
|
|
|
|
- name: checkout the source code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 2
|
|
|
|
# Free up disk space on Linux by removing preinstalled components that
|
|
# we do not need. We do this to enable some of the less resource
|
|
# intensive jobs to run on free runners, which however also have
|
|
# less disk space.
|
|
- name: free up disk space
|
|
run: src/ci/scripts/free-disk-space.sh
|
|
if: matrix.free_disk
|
|
|
|
# Rust Log Analyzer can't currently detect the PR number of a GitHub
|
|
# Actions build on its own, so a hint in the log message is needed to
|
|
# point it in the right direction.
|
|
- name: configure the PR in which the error message will be posted
|
|
run: echo "[CI_PR_NUMBER=$num]"
|
|
env:
|
|
num: ${{ github.event.number }}
|
|
if: needs.calculate_matrix.outputs.run_type == 'pr'
|
|
|
|
- name: add extra environment variables
|
|
run: src/ci/scripts/setup-environment.sh
|
|
env:
|
|
# Since it's not possible to merge `${{ matrix.env }}` with the other
|
|
# variables in `job.<name>.env`, the variables defined in the matrix
|
|
# are passed to the `setup-environment.sh` script encoded in JSON,
|
|
# which then uses log commands to actually set them.
|
|
EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
|
|
|
|
- name: setup upstream remote
|
|
run: src/ci/scripts/setup-upstream-remote.sh
|
|
|
|
- name: ensure the channel matches the target branch
|
|
run: src/ci/scripts/verify-channel.sh
|
|
|
|
- name: collect CPU statistics
|
|
run: src/ci/scripts/collect-cpu-stats.sh
|
|
|
|
- name: show the current environment
|
|
run: src/ci/scripts/dump-environment.sh
|
|
|
|
- name: install awscli
|
|
run: src/ci/scripts/install-awscli.sh
|
|
|
|
- name: install sccache
|
|
run: src/ci/scripts/install-sccache.sh
|
|
|
|
- name: select Xcode
|
|
run: src/ci/scripts/select-xcode.sh
|
|
|
|
- name: install clang
|
|
run: src/ci/scripts/install-clang.sh
|
|
|
|
- name: install tidy
|
|
run: src/ci/scripts/install-tidy.sh
|
|
|
|
- name: install WIX
|
|
run: src/ci/scripts/install-wix.sh
|
|
|
|
- name: disable git crlf conversion
|
|
run: src/ci/scripts/disable-git-crlf-conversion.sh
|
|
|
|
- name: checkout submodules
|
|
run: src/ci/scripts/checkout-submodules.sh
|
|
|
|
- name: install MinGW
|
|
run: src/ci/scripts/install-mingw.sh
|
|
|
|
- name: install ninja
|
|
run: src/ci/scripts/install-ninja.sh
|
|
|
|
- name: enable ipv6 on Docker
|
|
run: src/ci/scripts/enable-docker-ipv6.sh
|
|
|
|
# Disable automatic line ending conversion (again). On Windows, when we're
|
|
# installing dependencies, something switches the git configuration directory or
|
|
# re-enables autocrlf. We've not tracked down the exact cause -- and there may
|
|
# be multiple -- but this should ensure submodules are checked out with the
|
|
# appropriate line endings.
|
|
- name: disable git crlf conversion
|
|
run: src/ci/scripts/disable-git-crlf-conversion.sh
|
|
|
|
- name: ensure line endings are correct
|
|
run: src/ci/scripts/verify-line-endings.sh
|
|
|
|
- name: ensure backported commits are in upstream branches
|
|
run: src/ci/scripts/verify-backported-commits.sh
|
|
|
|
- name: ensure the stable version number is correct
|
|
run: src/ci/scripts/verify-stable-version-number.sh
|
|
|
|
# Show the environment just before we run the build
|
|
# This makes it easier to diagnose problems with the above install scripts.
|
|
- name: show the current environment
|
|
run: src/ci/scripts/dump-environment.sh
|
|
|
|
# Pre-build citool before the following step uninstalls rustup
|
|
# Build it into the build directory, to avoid modifying sources
|
|
- name: build citool
|
|
run: |
|
|
cd src/ci/citool
|
|
CARGO_INCREMENTAL=0 CARGO_TARGET_DIR=../../../build/citool cargo build
|
|
|
|
- name: run the build
|
|
run: |
|
|
set +e
|
|
# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
|
|
src/ci/scripts/run-build-from-ci.sh 2>&1
|
|
STATUS=$?
|
|
set -e
|
|
|
|
if [[ "$STATUS" -ne 0 && -n "$CI_JOB_DOC_URL" ]]; then
|
|
echo "****************************************************************************"
|
|
echo "To find more information about this job, visit the following URL:"
|
|
echo "$CI_JOB_DOC_URL"
|
|
echo "****************************************************************************"
|
|
fi
|
|
exit ${STATUS}
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
|
|
|
|
- name: create github artifacts
|
|
run: src/ci/scripts/create-doc-artifacts.sh
|
|
|
|
- name: print disk usage
|
|
run: |
|
|
echo "disk usage:"
|
|
df -h
|
|
|
|
- name: upload artifacts to github
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
# name is set in previous step
|
|
name: ${{ env.DOC_ARTIFACT_NAME }}
|
|
path: obj/artifacts/doc
|
|
if-no-files-found: ignore
|
|
retention-days: 5
|
|
|
|
- name: upload artifacts to S3
|
|
run: src/ci/scripts/upload-artifacts.sh
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
|
|
# Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
|
|
# builders *should* have the AWS credentials available. Still, explicitly
|
|
# adding the condition is helpful as this way CI will not silently skip
|
|
# deploying artifacts from a dist builder if the variables are misconfigured,
|
|
# erroring about invalid credentials instead.
|
|
if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
|
|
|
|
- name: postprocess metrics into the summary
|
|
# This step is not critical, and if some I/O problem happens, we don't want
|
|
# to cancel the build.
|
|
continue-on-error: true
|
|
run: |
|
|
if [ -f build/metrics.json ]; then
|
|
METRICS=build/metrics.json
|
|
elif [ -f obj/build/metrics.json ]; then
|
|
METRICS=obj/build/metrics.json
|
|
else
|
|
echo "No metrics.json found"
|
|
exit 0
|
|
fi
|
|
|
|
# Get closest bors merge commit
|
|
PARENT_COMMIT=`git rev-list --author='bors <bors@rust-lang.org>' -n1 --first-parent HEAD^1`
|
|
|
|
./build/citool/debug/citool postprocess-metrics \
|
|
--job-name ${CI_JOB_NAME} \
|
|
--parent ${PARENT_COMMIT} \
|
|
${METRICS} >> ${GITHUB_STEP_SUMMARY}
|
|
|
|
- name: upload job metrics to DataDog
|
|
# This step is not critical, and if some I/O problem happens, we don't want
|
|
# to cancel the build.
|
|
continue-on-error: true
|
|
if: needs.calculate_matrix.outputs.run_type != 'pr'
|
|
env:
|
|
DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
|
DD_GITHUB_JOB_NAME: ${{ matrix.full_name }}
|
|
run: ./build/citool/debug/citool upload-build-metrics build/cpu-usage.csv
|
|
|
|
# This job isused to tell bors the final status of the build, as there is no practical way to detect
|
|
# when a workflow is successful listening to webhooks only in our current bors implementation (homu).
|
|
outcome:
|
|
name: bors build finished
|
|
runs-on: ubuntu-24.04
|
|
needs: [ calculate_matrix, job ]
|
|
# !cancelled() executes the job regardless of whether the previous jobs passed or failed
|
|
if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
|
|
steps:
|
|
- name: checkout the source code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 2
|
|
# Calculate the exit status of the whole CI workflow.
|
|
# If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
|
|
# If a some dependent job has failed, this exits with 1.
|
|
- name: calculate the correct exit status
|
|
run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
|
|
# Publish the toolstate if an auto build succeeds (just before push to master)
|
|
- name: publish toolstate
|
|
run: src/ci/publish_toolstate.sh
|
|
shell: bash
|
|
if: needs.calculate_matrix.outputs.run_type == 'auto'
|
|
env:
|
|
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
|
|
TOOLSTATE_PUBLISH: 1
|