diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0e151d2577..d8d7560ffcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ # and also on pushes to special branches (auto, try). # # The actual definition of the executed jobs is calculated by a Python -# script located at src/ci/github-actions/calculate-job-matrix.py, which +# script located at src/ci/github-actions/ci.py, which # uses job definition data from src/ci/github-actions/jobs.yml. # You should primarily modify the `jobs.yml` file if you want to modify # what jobs are executed in CI. @@ -56,7 +56,7 @@ jobs: - name: Calculate the CI job matrix env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT + run: python3 src/ci/github-actions/ci.py calculate-job-matrix >> $GITHUB_OUTPUT id: jobs job: name: ${{ matrix.name }} diff --git a/src/ci/docker/README.md b/src/ci/docker/README.md index 508b7b40c01..2f52ff5a99a 100644 --- a/src/ci/docker/README.md +++ b/src/ci/docker/README.md @@ -1,29 +1,30 @@ # Docker images for CI This folder contains a bunch of docker images used by the continuous integration -(CI) of Rust. An script is accompanied (`run.sh`) with these images to actually -execute them. To test out an image execute: +(CI) of Rust. A script is accompanied (`run.sh`) with these images to actually +execute them. + +Note that a single Docker image can be used by multiple CI jobs, so the job name +is the important thing that you should know. You can examine the existing CI jobs in +the [`jobs.yml`](../github-actions/jobs.yml) file. + +To run a specific CI job locally, you can use the following script: ``` -./src/ci/docker/run.sh $image_name +python3 ./src/ci/github-actions/ci.py run-local ``` -for example: - +For example, to run the `x86_64-gnu-llvm-18-1` job: ``` -./src/ci/docker/run.sh x86_64-gnu +python3 ./src/ci/github-actions/ci.py run-local x86_64-gnu-llvm-18-1 ``` -Images will output artifacts in an `obj/$image_name` dir at the root of a repository. Note -that the script will overwrite the contents of this directory. - -To match conditions in rusts CI, also set the environment variable `DEPLOY=1`, e.g.: -``` -DEPLOY=1 ./src/ci/docker/run.sh x86_64-gnu -``` +The job will output artifacts in an `obj/` dir at the root of a repository. Note +that the script will overwrite the contents of this directory. `` is set based on the +Docker image executed in the given CI job. **NOTE**: In CI, the script outputs the artifacts to the `obj` directory, -while locally, to the `obj/$image_name` directory. This is primarily to prevent +while locally, to the `obj/` directory. This is primarily to prevent strange linker errors when using multiple Docker images. For some Linux workflows (for example `x86_64-gnu-llvm-18-N`), the process is more involved. You will need to see which script is executed for the given workflow inside the [`jobs.yml`](../github-actions/jobs.yml) file and pass it through the `DOCKER_SCRIPT` environment variable. For example, to reproduce the `x86_64-gnu-llvm-18-3` workflow, you can run the following script: diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/ci.py similarity index 52% rename from src/ci/github-actions/calculate-job-matrix.py rename to src/ci/github-actions/ci.py index 1f994f0ffd2..97539cf1cdb 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/ci.py @@ -1,18 +1,20 @@ #!/usr/bin/env python3 """ -This script serves for generating a matrix of jobs that should -be executed on CI. +This script contains CI functionality. +It can be used to generate a matrix of jobs that should +be executed on CI, or run a specific CI job locally. -It reads job definitions from `src/ci/github-actions/jobs.yml` -and filters them based on the event that happened on CI. +It reads job definitions from `src/ci/github-actions/jobs.yml`. """ +import argparse import dataclasses import json import logging import os import re +import subprocess import typing from pathlib import Path from typing import List, Dict, Any, Optional @@ -25,13 +27,19 @@ JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml" Job = Dict[str, Any] -def name_jobs(jobs: List[Dict], prefix: str) -> List[Job]: +def add_job_properties(jobs: List[Dict], prefix: str) -> List[Job]: """ - Add a `name` attribute to each job, based on its image and the given `prefix`. + Modify the `name` attribute of each job, based on its base name and the given `prefix`. + Add an `image` attribute to each job, based on its image. """ + modified_jobs = [] for job in jobs: - job["name"] = f"{prefix} - {job['image']}" - return jobs + # Create a copy of the `job` dictionary to avoid modifying `jobs` + new_job = dict(job) + new_job["image"] = get_job_image(new_job) + new_job["name"] = f"{prefix} - {new_job['name']}" + modified_jobs.append(new_job) + return modified_jobs def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: @@ -39,11 +47,15 @@ def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: Prepends `environment` to the `env` attribute of each job. The `env` of each job has higher precedence than `environment`. """ + modified_jobs = [] for job in jobs: env = environment.copy() env.update(job.get("env", {})) - job["env"] = env - return jobs + + new_job = dict(job) + new_job["env"] = env + modified_jobs.append(new_job) + return modified_jobs @dataclasses.dataclass @@ -116,7 +128,9 @@ def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: if isinstance(run_type, PRRunType): - return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) + return add_base_env( + add_job_properties(job_data["pr"], "PR"), job_data["envs"]["pr"] + ) elif isinstance(run_type, TryRunType): jobs = job_data["try"] custom_jobs = run_type.custom_jobs @@ -130,7 +144,7 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ jobs = [] unknown_jobs = [] for custom_job in custom_jobs: - job = [j for j in job_data["auto"] if j["image"] == custom_job] + job = [j for j in job_data["auto"] if j["name"] == custom_job] if not job: unknown_jobs.append(custom_job) continue @@ -140,10 +154,10 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ f"Custom job(s) `{unknown_jobs}` not found in auto jobs" ) - return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) + return add_base_env(add_job_properties(jobs, "try"), job_data["envs"]["try"]) elif isinstance(run_type, AutoRunType): return add_base_env( - name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"] + add_job_properties(job_data["auto"], "auto"), job_data["envs"]["auto"] ) return [] @@ -181,12 +195,64 @@ def format_run_type(run_type: WorkflowRunType) -> str: raise AssertionError() -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) +def get_job_image(job: Job) -> str: + """ + By default, the Docker image of a job is based on its name. + However, it can be overridden by its IMAGE environment variable. + """ + env = job.get("env", {}) + # Return the IMAGE environment variable if it exists, otherwise return the job name + return env.get("IMAGE", job["name"]) - with open(JOBS_YAML_PATH) as f: - data = yaml.safe_load(f) +def is_linux_job(job: Job) -> bool: + return "ubuntu" in job["os"] + + +def find_linux_job(job_data: Dict[str, Any], job_name: str, pr_jobs: bool) -> Job: + candidates = job_data["pr"] if pr_jobs else job_data["auto"] + jobs = [job for job in candidates if job.get("name") == job_name] + if len(jobs) == 0: + available_jobs = "\n".join( + sorted(job["name"] for job in candidates if is_linux_job(job)) + ) + raise Exception(f"""Job `{job_name}` not found in {'pr' if pr_jobs else 'auto'} jobs. +The following jobs are available: +{available_jobs}""") + assert len(jobs) == 1 + + job = jobs[0] + if not is_linux_job(job): + raise Exception("Only Linux jobs can be executed locally") + return job + + +def run_workflow_locally(job_data: Dict[str, Any], job_name: str, pr_jobs: bool): + DOCKER_DIR = Path(__file__).absolute().parent.parent / "docker" + + job = find_linux_job(job_data, job_name=job_name, pr_jobs=pr_jobs) + + custom_env = {} + # Replicate src/ci/scripts/setup-environment.sh + # Adds custom environment variables to the job + if job_name.startswith("dist-"): + if job_name.endswith("-alt"): + custom_env["DEPLOY_ALT"] = "1" + else: + custom_env["DEPLOY"] = "1" + custom_env.update({k: str(v) for (k, v) in job.get("env", {}).items()}) + + args = [str(DOCKER_DIR / "run.sh"), get_job_image(job)] + env_formatted = [f"{k}={v}" for (k, v) in sorted(custom_env.items())] + print(f"Executing `{' '.join(env_formatted)} {' '.join(args)}`") + + env = os.environ.copy() + env.update(custom_env) + + subprocess.run(args, env=env) + + +def calculate_job_matrix(job_data: Dict[str, Any]): github_ctx = get_github_ctx() run_type = find_run_type(github_ctx) @@ -197,7 +263,7 @@ if __name__ == "__main__": jobs = [] if run_type is not None: - jobs = calculate_jobs(run_type, data) + jobs = calculate_jobs(run_type, job_data) jobs = skip_jobs(jobs, channel) if not jobs: @@ -208,3 +274,45 @@ if __name__ == "__main__": logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}") print(f"jobs={json.dumps(jobs)}") print(f"run_type={run_type}") + + +def create_cli_parser(): + parser = argparse.ArgumentParser( + prog="ci.py", description="Generate or run CI workflows" + ) + subparsers = parser.add_subparsers( + help="Command to execute", dest="command", required=True + ) + subparsers.add_parser( + "calculate-job-matrix", + help="Generate a matrix of jobs that should be executed in CI", + ) + run_parser = subparsers.add_parser( + "run-local", help="Run a CI jobs locally (on Linux)" + ) + run_parser.add_argument( + "job_name", + help="CI job that should be executed. By default, a merge (auto) " + "job with the given name will be executed", + ) + run_parser.add_argument( + "--pr", action="store_true", help="Run a PR job instead of an auto job" + ) + return parser + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + with open(JOBS_YAML_PATH) as f: + data = yaml.safe_load(f) + + parser = create_cli_parser() + args = parser.parse_args() + + if args.command == "calculate-job-matrix": + calculate_job_matrix(data) + elif args.command == "run-local": + run_workflow_locally(data, args.job_name, args.pr) + else: + raise Exception(f"Unknown command {args.command}") diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml index 876a7793592..d88be6543a3 100644 --- a/src/ci/github-actions/jobs.yml +++ b/src/ci/github-actions/jobs.yml @@ -91,26 +91,26 @@ envs: # These jobs automatically inherit envs.pr, to avoid repeating # it in each job definition. pr: - - image: mingw-check + - name: mingw-check <<: *job-linux-4c - - image: mingw-check-tidy + - name: mingw-check-tidy continue_on_error: true <<: *job-linux-4c - - image: x86_64-gnu-llvm-18 + - name: x86_64-gnu-llvm-18 env: ENABLE_GCC_CODEGEN: "1" # We are adding (temporarily) a dummy commit on the compiler READ_ONLY_SRC: "0" DOCKER_SCRIPT: x86_64-gnu-llvm.sh <<: *job-linux-16c - - image: x86_64-gnu-tools + - name: x86_64-gnu-tools <<: *job-linux-16c # Jobs that run when you perform a try build (@bors try) # These jobs automatically inherit envs.try, to avoid repeating # it in each job definition. try: - - image: dist-x86_64-linux + - name: dist-x86_64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c @@ -123,106 +123,106 @@ auto: # Linux/Docker builders # ############################# - - image: aarch64-gnu + - name: aarch64-gnu <<: *job-aarch64-linux - - image: aarch64-gnu-debug + - name: aarch64-gnu-debug <<: *job-aarch64-linux - - image: arm-android + - name: arm-android <<: *job-linux-4c - - image: armhf-gnu + - name: armhf-gnu <<: *job-linux-4c - - image: dist-aarch64-linux + - name: dist-aarch64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-4c - - image: dist-android + - name: dist-android <<: *job-linux-4c - - image: dist-arm-linux + - name: dist-arm-linux <<: *job-linux-8c - - image: dist-armhf-linux + - name: dist-armhf-linux <<: *job-linux-4c - - image: dist-armv7-linux + - name: dist-armv7-linux <<: *job-linux-4c - - image: dist-i586-gnu-i586-i686-musl + - name: dist-i586-gnu-i586-i686-musl <<: *job-linux-4c - - image: dist-i686-linux + - name: dist-i686-linux <<: *job-linux-4c - - image: dist-loongarch64-linux + - name: dist-loongarch64-linux <<: *job-linux-4c - - image: dist-loongarch64-musl + - name: dist-loongarch64-musl <<: *job-linux-4c - - image: dist-ohos + - name: dist-ohos <<: *job-linux-4c - - image: dist-powerpc-linux + - name: dist-powerpc-linux <<: *job-linux-4c - - image: dist-powerpc64-linux + - name: dist-powerpc64-linux <<: *job-linux-4c - - image: dist-powerpc64le-linux + - name: dist-powerpc64le-linux <<: *job-linux-4c-largedisk - - image: dist-riscv64-linux + - name: dist-riscv64-linux <<: *job-linux-4c - - image: dist-s390x-linux + - name: dist-s390x-linux <<: *job-linux-4c - - image: dist-various-1 + - name: dist-various-1 <<: *job-linux-4c - - image: dist-various-2 + - name: dist-various-2 <<: *job-linux-4c - - image: dist-x86_64-freebsd + - name: dist-x86_64-freebsd <<: *job-linux-4c - - image: dist-x86_64-illumos + - name: dist-x86_64-illumos <<: *job-linux-4c - - image: dist-x86_64-linux + - name: dist-x86_64-linux env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - - image: dist-x86_64-linux-alt + - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - - image: dist-x86_64-musl + - name: dist-x86_64-musl env: CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-4c - - image: dist-x86_64-netbsd + - name: dist-x86_64-netbsd <<: *job-linux-4c # The i686-gnu job is split into multiple jobs to run tests in parallel. # i686-gnu-1 skips tests that run in i686-gnu-2. - - image: i686-gnu-1 + - name: i686-gnu-1 env: IMAGE: i686-gnu DOCKER_SCRIPT: stage_2_test_set1.sh <<: *job-linux-4c # Skip tests that run in i686-gnu-1 - - image: i686-gnu-2 + - name: i686-gnu-2 env: IMAGE: i686-gnu DOCKER_SCRIPT: stage_2_test_set2.sh @@ -230,14 +230,14 @@ auto: # The i686-gnu-nopt job is split into multiple jobs to run tests in parallel. # i686-gnu-nopt-1 skips tests that run in i686-gnu-nopt-2 - - image: i686-gnu-nopt-1 + - name: i686-gnu-nopt-1 env: IMAGE: i686-gnu-nopt DOCKER_SCRIPT: /scripts/stage_2_test_set1.sh <<: *job-linux-4c # Skip tests that run in i686-gnu-nopt-1 - - image: i686-gnu-nopt-2 + - name: i686-gnu-nopt-2 env: IMAGE: i686-gnu-nopt DOCKER_SCRIPT: >- @@ -245,13 +245,13 @@ auto: /scripts/stage_2_test_set2.sh <<: *job-linux-4c - - image: mingw-check + - name: mingw-check <<: *job-linux-4c - - image: test-various + - name: test-various <<: *job-linux-4c - - image: x86_64-fuchsia + - name: x86_64-fuchsia # Only run this job on the nightly channel. Fuchsia requires # nightly features to compile, and this job would fail if # executed on beta and stable. @@ -260,10 +260,10 @@ auto: # Tests integration with Rust for Linux. # Builds stage 1 compiler and tries to compile a few RfL examples with it. - - image: x86_64-rust-for-linux + - name: x86_64-rust-for-linux <<: *job-linux-4c - - image: x86_64-gnu + - name: x86_64-gnu <<: *job-linux-4c # This job ensures commits landing on nightly still pass the full @@ -271,7 +271,7 @@ auto: # depend on the channel being built (for example if they include the # channel name on the output), and this builder prevents landing # changes that would result in broken builds after a promotion. - - image: x86_64-gnu-stable + - name: x86_64-gnu-stable # Only run this job on the nightly channel. Running this on beta # could cause failures when `dev: 1` in `stage0.txt`, and running # this on stable is useless. @@ -281,20 +281,20 @@ auto: RUST_CI_OVERRIDE_RELEASE_CHANNEL: stable <<: *job-linux-4c - - image: x86_64-gnu-aux + - name: x86_64-gnu-aux <<: *job-linux-4c - - image: x86_64-gnu-debug + - name: x86_64-gnu-debug # This seems to be needed because a full stage 2 build + run-make tests # overwhelms the storage capacity of the standard 4c runner. <<: *job-linux-4c-largedisk - - image: x86_64-gnu-distcheck + - name: x86_64-gnu-distcheck <<: *job-linux-8c # The x86_64-gnu-llvm-19 job is split into multiple jobs to run tests in parallel. # x86_64-gnu-llvm-19-1 skips tests that run in x86_64-gnu-llvm-19-{2,3}. - - image: x86_64-gnu-llvm-19-1 + - name: x86_64-gnu-llvm-19-1 env: RUST_BACKTRACE: 1 IMAGE: x86_64-gnu-llvm-19 @@ -302,7 +302,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-19-{1,3} - - image: x86_64-gnu-llvm-19-2 + - name: x86_64-gnu-llvm-19-2 env: RUST_BACKTRACE: 1 IMAGE: x86_64-gnu-llvm-19 @@ -310,7 +310,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-19-{1,2} - - image: x86_64-gnu-llvm-19-3 + - name: x86_64-gnu-llvm-19-3 env: RUST_BACKTRACE: 1 IMAGE: x86_64-gnu-llvm-19 @@ -319,7 +319,7 @@ auto: # The x86_64-gnu-llvm-18 job is split into multiple jobs to run tests in parallel. # x86_64-gnu-llvm-18-1 skips tests that run in x86_64-gnu-llvm-18-{2,3}. - - image: x86_64-gnu-llvm-18-1 + - name: x86_64-gnu-llvm-18-1 env: RUST_BACKTRACE: 1 READ_ONLY_SRC: "0" @@ -328,7 +328,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-18-{1,3} - - image: x86_64-gnu-llvm-18-2 + - name: x86_64-gnu-llvm-18-2 env: RUST_BACKTRACE: 1 READ_ONLY_SRC: "0" @@ -337,7 +337,7 @@ auto: <<: *job-linux-4c # Skip tests that run in x86_64-gnu-llvm-18-{1,2} - - image: x86_64-gnu-llvm-18-3 + - name: x86_64-gnu-llvm-18-3 env: RUST_BACKTRACE: 1 READ_ONLY_SRC: "0" @@ -345,10 +345,10 @@ auto: DOCKER_SCRIPT: x86_64-gnu-llvm3.sh <<: *job-linux-4c - - image: x86_64-gnu-nopt + - name: x86_64-gnu-nopt <<: *job-linux-4c - - image: x86_64-gnu-tools + - name: x86_64-gnu-tools env: DEPLOY_TOOLSTATES_JSON: toolstates-linux.json <<: *job-linux-4c @@ -357,7 +357,7 @@ auto: # macOS Builders # #################### - - image: dist-x86_64-apple + - name: dist-x86_64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set rust.lto=thin --set rust.codegen-units=1 @@ -371,7 +371,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-macos-xl - - image: dist-apple-various + - name: dist-apple-various env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host='' --target=aarch64-apple-ios,x86_64-apple-ios,aarch64-apple-ios-sim,aarch64-apple-ios-macabi,x86_64-apple-ios-macabi # Mac Catalyst cannot currently compile the sanitizer: @@ -385,19 +385,19 @@ auto: NO_OVERFLOW_CHECKS: 1 <<: *job-macos-xl - - image: x86_64-apple-1 + - name: x86_64-apple-1 env: <<: *env-x86_64-apple-tests <<: *job-macos-xl - - image: x86_64-apple-2 + - name: x86_64-apple-2 env: SCRIPT: ./x.py --stage 2 test tests/ui tests/rustdoc <<: *env-x86_64-apple-tests <<: *job-macos-xl # This target only needs to support 11.0 and up as nothing else supports the hardware - - image: dist-aarch64-apple + - name: dist-aarch64-apple env: SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- @@ -421,7 +421,7 @@ auto: <<: *job-macos-m1 # This target only needs to support 11.0 and up as nothing else supports the hardware - - image: aarch64-apple + - name: aarch64-apple env: SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- @@ -442,20 +442,20 @@ auto: # Windows Builders # ###################### - - image: x86_64-msvc + - name: x86_64-msvc env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-msvc <<: *job-windows-8c - - image: i686-msvc + - name: i686-msvc env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc SCRIPT: make ci-msvc <<: *job-windows-8c # x86_64-msvc-ext is split into multiple jobs to run tests in parallel. - - image: x86_64-msvc-ext1 + - name: x86_64-msvc-ext1 env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld @@ -464,7 +464,7 @@ auto: # Temporary builder to workaround CI issues # See #FIXME: Remove this, and re-enable the same tests in `checktools.sh`, once CI issues are fixed. - - image: x86_64-msvc-ext2 + - name: x86_64-msvc-ext2 env: SCRIPT: > python x.py test --stage 2 src/tools/miri --target aarch64-apple-darwin --test-args pass && @@ -476,7 +476,7 @@ auto: <<: *job-windows # Run `checktools.sh` and upload the toolstate file. - - image: x86_64-msvc-ext3 + - name: x86_64-msvc-ext3 env: SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows HOST_TARGET: x86_64-pc-windows-msvc @@ -500,7 +500,7 @@ auto: # came from the mingw-w64 SourceForge download site. Unfortunately # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - - image: i686-mingw + - name: i686-mingw env: RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu SCRIPT: make ci-mingw @@ -510,7 +510,7 @@ auto: <<: *job-windows-8c # x86_64-mingw is split into two jobs to run tests in parallel. - - image: x86_64-mingw-1 + - name: x86_64-mingw-1 env: SCRIPT: make ci-mingw-x RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu @@ -519,7 +519,7 @@ auto: NO_DOWNLOAD_CI_LLVM: 1 <<: *job-windows - - image: x86_64-mingw-2 + - name: x86_64-mingw-2 env: SCRIPT: make ci-mingw-bootstrap RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu @@ -528,7 +528,7 @@ auto: NO_DOWNLOAD_CI_LLVM: 1 <<: *job-windows - - image: dist-x86_64-msvc + - name: dist-x86_64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc @@ -542,7 +542,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows-8c - - image: dist-i686-msvc + - name: dist-i686-msvc env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-msvc @@ -555,7 +555,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows - - image: dist-aarch64-msvc + - name: dist-aarch64-msvc env: RUST_CONFIGURE_ARGS: >- --build=x86_64-pc-windows-msvc @@ -567,7 +567,7 @@ auto: DIST_REQUIRE_ALL_TOOLS: 1 <<: *job-windows - - image: dist-i686-mingw + - name: dist-i686-mingw env: RUST_CONFIGURE_ARGS: >- --build=i686-pc-windows-gnu @@ -580,7 +580,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows - - image: dist-x86_64-mingw + - name: dist-x86_64-mingw env: SCRIPT: python x.py dist bootstrap --include-default-paths RUST_CONFIGURE_ARGS: >- @@ -593,7 +593,7 @@ auto: CODEGEN_BACKENDS: llvm,cranelift <<: *job-windows - - image: dist-x86_64-msvc-alt + - name: dist-x86_64-msvc-alt env: RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler SCRIPT: python x.py dist bootstrap --include-default-paths diff --git a/src/doc/rustc-dev-guide/src/building/optimized-build.md b/src/doc/rustc-dev-guide/src/building/optimized-build.md index 3080dc6bf5d..8feda59829b 100644 --- a/src/doc/rustc-dev-guide/src/building/optimized-build.md +++ b/src/doc/rustc-dev-guide/src/building/optimized-build.md @@ -126,4 +126,4 @@ Here is an example of how can `opt-dist` be used locally (outside of CI): [`Environment`]: https://github.com/rust-lang/rust/blob/ee451f8faccf3050c76cdcd82543c917b40c7962/src/tools/opt-dist/src/environment.rs#L5 > Note: if you want to run the actual CI pipeline, instead of running `opt-dist` locally, -> you can execute `DEPLOY=1 src/ci/docker/run.sh dist-x86_64-linux`. +> you can execute `python3 src/ci/github-actions/ci.py run-local dist-x86_64-linux`. diff --git a/src/doc/rustc-dev-guide/src/tests/ci.md b/src/doc/rustc-dev-guide/src/tests/ci.md index 5e27a2fd770..0ad1d2a28ed 100644 --- a/src/doc/rustc-dev-guide/src/tests/ci.md +++ b/src/doc/rustc-dev-guide/src/tests/ci.md @@ -28,7 +28,7 @@ Our CI is primarily executed on [GitHub Actions], with a single workflow defined in [`.github/workflows/ci.yml`], which contains a bunch of steps that are unified for all CI jobs that we execute. When a commit is pushed to a corresponding branch or a PR, the workflow executes the -[`calculate-job-matrix.py`] script, which dynamically generates the specific CI +[`ci.py`] script, which dynamically generates the specific CI jobs that should be executed. This script uses the [`jobs.yml`] file as an input, which contains a declarative configuration of all our CI jobs. @@ -299,8 +299,7 @@ platform’s custom [Docker container]. This has a lot of advantages for us: - We can avoid reinstalling tools (like QEMU or the Android emulator) every time thanks to Docker image caching. - Users can run the same tests in the same environment locally by just running - `src/ci/docker/run.sh image-name`, which is awesome to debug failures. Note - that there are only linux docker images available locally due to licensing and + `python3 src/ci/github-actions/ci.py run-local `, which is awesome to debug failures. Note that there are only linux docker images available locally due to licensing and other restrictions. The docker images prefixed with `dist-` are used for building artifacts while @@ -413,7 +412,7 @@ To learn more about the dashboard, see the [Datadog CI docs]. [GitHub Actions]: https://github.com/rust-lang/rust/actions [`jobs.yml`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/jobs.yml [`.github/workflows/ci.yml`]: https://github.com/rust-lang/rust/blob/master/.github/workflows/ci.yml -[`calculate-job-matrix.py`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/calculate-job-matrix.py +[`ci.py`]: https://github.com/rust-lang/rust/blob/master/src/ci/github-actions/ci.py [rust-lang-ci]: https://github.com/rust-lang-ci/rust/actions [bors]: https://github.com/bors [homu]: https://github.com/rust-lang/homu diff --git a/src/doc/rustc-dev-guide/src/tests/docker.md b/src/doc/rustc-dev-guide/src/tests/docker.md index 31e3825f567..a0aa8bd3e77 100644 --- a/src/doc/rustc-dev-guide/src/tests/docker.md +++ b/src/doc/rustc-dev-guide/src/tests/docker.md @@ -45,6 +45,15 @@ Some additional notes about using the Docker images: containers. With the container name, run `docker exec -it /bin/bash` where `` is the container name like `4ba195e95cef`. +The approach described above is a relatively low-level interface for running the Docker images +directly. If you want to run a full CI Linux job locally with Docker, in a way that is as close to CI as possible, you can use the following command: + +```bash +python3 src/ci/github-actions/ci.py run-local +# For example: +python3 src/ci/github-actions/ci.py run-local dist-x86_64-linux-alt +``` + [Docker]: https://www.docker.com/ [`src/ci/docker`]: https://github.com/rust-lang/rust/tree/master/src/ci/docker [`src/ci/docker/run.sh`]: https://github.com/rust-lang/rust/blob/master/src/ci/docker/run.sh