From ce3f300e40146480d3f508e9657c454bf6881b10 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 13 Feb 2023 10:38:25 +0000 Subject: [PATCH 01/20] Add --download-dir option to specify download dir separate from --out-dir --- build_system/mod.rs | 12 ++++++++++-- build_system/usage.txt | 13 ++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/build_system/mod.rs b/build_system/mod.rs index 0110011169d..e1b46c3120e 100644 --- a/build_system/mod.rs +++ b/build_system/mod.rs @@ -81,6 +81,7 @@ pub(crate) fn main() { }; let mut out_dir = PathBuf::from("."); + let mut download_dir = None; let mut channel = "release"; let mut sysroot_kind = SysrootKind::Clif; let mut use_unstable_features = true; @@ -91,7 +92,12 @@ pub(crate) fn main() { "--out-dir" => { out_dir = PathBuf::from(args.next().unwrap_or_else(|| { arg_error!("--out-dir requires argument"); - })) + })); + } + "--download-dir" => { + download_dir = Some(PathBuf::from(args.next().unwrap_or_else(|| { + arg_error!("--download-dir requires argument"); + }))); } "--debug" => channel = "debug", "--sysroot" => { @@ -152,7 +158,9 @@ pub(crate) fn main() { out_dir = current_dir.join(out_dir); let dirs = path::Dirs { source_dir: current_dir.clone(), - download_dir: out_dir.join("download"), + download_dir: download_dir + .map(|dir| current_dir.join(dir)) + .unwrap_or_else(|| out_dir.join("download")), build_dir: out_dir.join("build"), dist_dir: out_dir.join("dist"), frozen, diff --git a/build_system/usage.txt b/build_system/usage.txt index 3bc18a93331..98726ae5af1 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -1,11 +1,11 @@ The build system of cg_clif. USAGE: - ./y.rs prepare [--out-dir DIR] - ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs prepare [--out-dir DIR] [--download-dir DIR] + ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] OPTIONS: --debug @@ -22,6 +22,9 @@ OPTIONS: Specify the directory in which the download, build and dist directories are stored. By default this is the working directory. + --download-dir DIR + Specify the directory in which the download directory is stored. Overrides --out-dir. + --no-unstable-features Some features are not yet ready for production usage. This option will disable these features. This includes the JIT mode and inline assembly support. From d0b8896189f5b2fb472d54ed389681dff97d907b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 15 Feb 2023 18:23:11 +0000 Subject: [PATCH 02/20] Allow building the build system using cargo Rust's build system only handles cargo, not rustc. --- .cirrus.yml | 4 ++-- .github/workflows/abi-cafe.yml | 6 +++--- .github/workflows/main.yml | 22 ++++++++++----------- .github/workflows/rustc.yml | 4 ++-- .gitignore | 1 + .vscode/settings.json | 3 ++- Cargo.toml | 8 -------- Readme.md | 8 ++++---- build_system/Cargo.lock | 7 +++++++ build_system/Cargo.toml | 13 +++++++++++++ build_system/build_sysroot.rs | 4 ++-- build_system/{mod.rs => main.rs} | 10 +++++++--- build_system/usage.txt | 16 ++++++++-------- clean_all.sh | 2 +- docs/usage.md | 2 +- scripts/rustup.sh | 2 +- scripts/setup_rust_fork.sh | 2 +- test.sh | 2 +- y.rs | 33 ++------------------------------ y.sh | 6 ++++++ 20 files changed, 75 insertions(+), 80 deletions(-) create mode 100644 build_system/Cargo.lock create mode 100644 build_system/Cargo.toml rename build_system/{mod.rs => main.rs} (98%) create mode 100755 y.sh diff --git a/.cirrus.yml b/.cirrus.yml index 1405ea74903..8b4efd4e394 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,7 +10,7 @@ task: folder: target prepare_script: - . $HOME/.cargo/env - - ./y.rs prepare + - ./y.sh prepare test_script: - . $HOME/.cargo/env - - ./y.rs test + - ./y.sh test diff --git a/.github/workflows/abi-cafe.yml b/.github/workflows/abi-cafe.yml index 3c40555669c..12aa69d3c79 100644 --- a/.github/workflows/abi-cafe.yml +++ b/.github/workflows/abi-cafe.yml @@ -46,12 +46,12 @@ jobs: run: rustup set default-host x86_64-pc-windows-gnu - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build - run: ./y.rs build --sysroot none + run: ./y.sh build --sysroot none - name: Test abi-cafe env: TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} - run: ./y.rs abi-cafe + run: ./y.sh abi-cafe diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index abcd1affdee..8e6c1e8ade0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ jobs: - name: Rustfmt run: | cargo fmt --check - rustfmt --check build_system/mod.rs + rustfmt --check build_system/main.rs rustfmt --check example/* @@ -91,15 +91,15 @@ jobs: sudo apt-get install -y gcc-s390x-linux-gnu qemu-user - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build - run: ./y.rs build --sysroot none + run: ./y.sh build --sysroot none - name: Test env: TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} - run: ./y.rs test + run: ./y.sh test - name: Install LLVM standard library run: rustup target add ${{ matrix.env.TARGET_TRIPLE }} @@ -111,7 +111,7 @@ jobs: if: matrix.os != 'windows-latest' || matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu' env: TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} - run: ./y.rs test --sysroot llvm --no-unstable-features + run: ./y.sh test --sysroot llvm --no-unstable-features # This job doesn't use cg_clif in any way. It checks that all cg_clif tests work with cg_llvm too. @@ -165,13 +165,13 @@ jobs: run: cargo install hyperfine || true - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build - run: CI_OPT=1 ./y.rs build --sysroot none + run: CI_OPT=1 ./y.sh build --sysroot none - name: Benchmark - run: CI_OPT=1 ./y.rs bench + run: CI_OPT=1 ./y.sh bench dist: @@ -224,13 +224,13 @@ jobs: sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Build backend - run: CI_OPT=1 ./y.rs build --sysroot none + run: CI_OPT=1 ./y.sh build --sysroot none - name: Build sysroot - run: CI_OPT=1 ./y.rs build + run: CI_OPT=1 ./y.sh build - name: Package prebuilt cg_clif run: tar cvfJ cg_clif.tar.xz dist diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml index b2f772c4fc4..b49dc3aff7a 100644 --- a/.github/workflows/rustc.yml +++ b/.github/workflows/rustc.yml @@ -18,7 +18,7 @@ jobs: key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Test run: ./scripts/test_bootstrap.sh @@ -38,7 +38,7 @@ jobs: key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} - name: Prepare dependencies - run: ./y.rs prepare + run: ./y.sh prepare - name: Test run: ./scripts/test_rustc_tests.sh diff --git a/.gitignore b/.gitignore index e5d10a937ae..e6ac8c8408d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +/build_system/target **/*.rs.bk *.rlib *.o diff --git a/.vscode/settings.json b/.vscode/settings.json index 7c8703cba50..60cb51d5663 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,9 +6,10 @@ "rust-analyzer.imports.granularity.enforce": true, "rust-analyzer.imports.granularity.group": "module", "rust-analyzer.imports.prefix": "crate", - "rust-analyzer.cargo.features": ["unstable-features", "__check_build_system_using_ra"], + "rust-analyzer.cargo.features": ["unstable-features"], "rust-analyzer.linkedProjects": [ "./Cargo.toml", + "./build_system/Cargo.toml", { "crates": [ { diff --git a/Cargo.toml b/Cargo.toml index 145b1150423..1c1f2d8577b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,13 +3,6 @@ name = "rustc_codegen_cranelift" version = "0.1.0" edition = "2021" -[[bin]] -# This is used just to teach rust-analyzer how to check the build system. required-features is used -# to disable it for regular builds. -name = "y" -path = "./y.rs" -required-features = ["__check_build_system_using_ra"] - [lib] crate-type = ["dylib"] @@ -45,7 +38,6 @@ smallvec = "1.8.1" unstable-features = ["jit", "inline_asm"] jit = ["cranelift-jit", "libloading"] inline_asm = [] -__check_build_system_using_ra = [] [package.metadata.rust-analyzer] rustc_private = true diff --git a/Readme.md b/Readme.md index 7e9945578a5..9469feea0cb 100644 --- a/Readme.md +++ b/Readme.md @@ -10,8 +10,8 @@ If not please open an issue. ```bash $ git clone https://github.com/bjorn3/rustc_codegen_cranelift $ cd rustc_codegen_cranelift -$ ./y.rs prepare -$ ./y.rs build +$ ./y.sh prepare +$ ./y.sh build ``` To run the test suite replace the last command with: @@ -20,7 +20,7 @@ To run the test suite replace the last command with: $ ./test.sh ``` -For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.rs`. +For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.sh`. ## Precompiled builds @@ -35,7 +35,7 @@ If you want to use `cargo clif build` instead of having to specify the full path rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`). In the directory with your project (where you can do the usual `cargo build`), run: diff --git a/build_system/Cargo.lock b/build_system/Cargo.lock new file mode 100644 index 00000000000..86268e19160 --- /dev/null +++ b/build_system/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "y" +version = "0.1.0" diff --git a/build_system/Cargo.toml b/build_system/Cargo.toml new file mode 100644 index 00000000000..f47b9bc5540 --- /dev/null +++ b/build_system/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "y" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "y" +path = "main.rs" + +[features] +unstable-features = [] # for rust-analyzer + +# Do not add any dependencies diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index dab9c77d1a4..2ed6272b2c5 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -225,7 +225,7 @@ fn build_clif_sysroot_for_triple( match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) { Err(e) => { eprintln!("Failed to get rustc version for patched sysroot source: {}", e); - eprintln!("Hint: Try `./y.rs prepare` to patch the sysroot source"); + eprintln!("Hint: Try `./y.sh prepare` to patch the sysroot source"); process::exit(1); } Ok(source_version) => { @@ -234,7 +234,7 @@ fn build_clif_sysroot_for_triple( eprintln!("The patched sysroot source is outdated"); eprintln!("Source version: {}", source_version.trim()); eprintln!("Rustc version: {}", rustc_version.trim()); - eprintln!("Hint: Try `./y.rs prepare` to update the patched sysroot source"); + eprintln!("Hint: Try `./y.sh prepare` to update the patched sysroot source"); process::exit(1); } } diff --git a/build_system/mod.rs b/build_system/main.rs similarity index 98% rename from build_system/mod.rs rename to build_system/main.rs index e1b46c3120e..06395eb141c 100644 --- a/build_system/mod.rs +++ b/build_system/main.rs @@ -1,3 +1,7 @@ +#![warn(rust_2018_idioms)] +#![warn(unused_lifetimes)] +#![warn(unreachable_pub)] + use std::env; use std::path::PathBuf; use std::process; @@ -37,19 +41,19 @@ enum Command { } #[derive(Copy, Clone, Debug)] -pub(crate) enum SysrootKind { +enum SysrootKind { None, Clif, Llvm, } #[derive(Clone, Debug)] -pub(crate) enum CodegenBackend { +enum CodegenBackend { Local(PathBuf), Builtin(String), } -pub(crate) fn main() { +fn main() { if env::var("RUST_BACKTRACE").is_err() { env::set_var("RUST_BACKTRACE", "1"); } diff --git a/build_system/usage.txt b/build_system/usage.txt index 98726ae5af1..9d20cdca6a7 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -1,11 +1,11 @@ The build system of cg_clif. USAGE: - ./y.rs prepare [--out-dir DIR] [--download-dir DIR] - ./y.rs build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.rs bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh prepare [--out-dir DIR] [--download-dir DIR] + ./y.sh build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] OPTIONS: --debug @@ -40,7 +40,7 @@ REQUIREMENTS: * Rustup: By default rustup is used to install the right nightly version. If you don't want to use rustup, you can manually install the nightly version indicated by rust-toolchain.toml and point the CARGO, RUSTC and RUSTDOC env vars to the right executables. - * Git: `./y.rs prepare` uses git for applying patches and on Windows for downloading test repos. - * Curl and tar (non-Windows only): Used by `./y.rs prepare` to download a single commit for + * Git: `./y.sh prepare` uses git for applying patches and on Windows for downloading test repos. + * Curl and tar (non-Windows only): Used by `./y.sh prepare` to download a single commit for repos. Git will be used to clone the whole repo when using Windows. - * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.rs bench`. + * [Hyperfine](https://github.com/sharkdp/hyperfine/): Used for benchmarking with `./y.sh bench`. diff --git a/clean_all.sh b/clean_all.sh index cdfc2e143e6..19405a53d1c 100755 --- a/clean_all.sh +++ b/clean_all.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -rm -rf target/ download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb +rm -rf target/ build_system/target download/ build/ dist/ y.bin y.bin.dSYM y.exe y.pdb # Kept for now in case someone updates their checkout of cg_clif before running clean_all.sh # FIXME remove at some point in the future diff --git a/docs/usage.md b/docs/usage.md index 4c2b0fa1704..c6210f958d6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -2,7 +2,7 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. -Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.rs prepare` and `y.rs build` or `test.sh`). +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`y.sh prepare` and `y.sh build` or `test.sh`). ## Cargo diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 3cbeb6375de..6b446aafb7b 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -32,7 +32,7 @@ case $1 in ./clean_all.sh - ./y.rs prepare + ./y.sh prepare (cd download/sysroot && cargo update && cargo fetch && cp Cargo.lock ../../build_sysroot/) ;; diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index abb09775d21..15b16b42be5 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -./y.rs build --no-unstable-features +./y.sh build --no-unstable-features echo "[SETUP] Rust fork" git clone https://github.com/rust-lang/rust.git || true diff --git a/test.sh b/test.sh index 13e7784539d..6357eebf026 100755 --- a/test.sh +++ b/test.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -exec ./y.rs test "$@" +exec ./y.sh test "$@" diff --git a/y.rs b/y.rs index a68a10500f5..e806a64d943 100755 --- a/y.rs +++ b/y.rs @@ -1,35 +1,6 @@ #!/usr/bin/env bash #![deny(unsafe_code)] /*This line is ignored by bash # This block is ignored by rustc -set -e -echo "[BUILD] y.rs" 1>&2 -rustc $0 -o ${0/.rs/.bin} -Cdebuginfo=1 --edition 2021 -exec ${0/.rs/.bin} $@ +echo "Warning: y.rs is a deprecated alias for y.sh" 1>&2 +exec ./y.sh "$@" */ - -#![warn(rust_2018_idioms)] -#![warn(unused_lifetimes)] -#![warn(unreachable_pub)] - -//! The build system for cg_clif -//! -//! # Manual compilation -//! -//! If your system doesn't support shell scripts you can manually compile and run this file using -//! for example: -//! -//! ```shell -//! $ rustc y.rs -o y.bin -//! $ ./y.bin -//! ``` -//! -//! # Naming -//! -//! The name `y.rs` was chosen to not conflict with rustc's `x.py`. - -#[path = "build_system/mod.rs"] -mod build_system; - -fn main() { - build_system::main(); -} diff --git a/y.sh b/y.sh new file mode 100755 index 00000000000..bc925a23e2a --- /dev/null +++ b/y.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e +echo "[BUILD] build system" 1>&2 +rustc build_system/main.rs -o y.bin -Cdebuginfo=1 --edition 2021 +exec ./y.bin "$@" From 0e4139922e08c306620cf5c43721670c55f3684f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:17:11 +0000 Subject: [PATCH 03/20] Put patched sources in build/ instead of download/ --- build_system/abi_cafe.rs | 2 +- build_system/build_sysroot.rs | 2 +- build_system/prepare.rs | 33 +++++++++++++++------------------ build_system/tests.rs | 8 ++++---- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index 9634430d116..ba15db9f966 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -7,7 +7,7 @@ use super::{CodegenBackend, SysrootKind}; static ABI_CAFE_REPO: GitRepo = GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); -static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe"); +static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); pub(crate) fn run( channel: &str, diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 2ed6272b2c5..7ceda34bfac 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -156,7 +156,7 @@ impl SysrootTarget { } pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); -pub(crate) static BUILD_SYSROOT: RelPath = RelPath::DOWNLOAD.join("sysroot"); +pub(crate) static BUILD_SYSROOT: RelPath = RelPath::BUILD.join("sysroot"); pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version"); pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src"); pub(crate) static STANDARD_LIBRARY: CargoProject = diff --git a/build_system/prepare.rs b/build_system/prepare.rs index ac2dc47dd7f..7bb9eca2945 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -39,9 +39,6 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); - eprintln!("[GIT] init"); - init_git_repo(&SYSROOT_SRC.to_path(dirs)); - apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs)); } @@ -51,15 +48,13 @@ fn prepare_coretests(dirs: &Dirs, rustc: &Path) { eprintln!("[COPY] coretests src"); - fs::create_dir_all(LIBCORE_TESTS_SRC.to_path(dirs)).unwrap(); + // FIXME ensure builds error out or update the copy if any of the files copied here change + LIBCORE_TESTS_SRC.ensure_fresh(dirs); copy_dir_recursively( &sysroot_src_orig.join("library/core/tests"), &LIBCORE_TESTS_SRC.to_path(dirs), ); - eprintln!("[GIT] init"); - init_git_repo(&LIBCORE_TESTS_SRC.to_path(dirs)); - apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs)); } @@ -85,23 +80,23 @@ impl GitRepo { pub(crate) const fn source_dir(&self) -> RelPath { match self.url { - GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo), + GitRepoUrl::Github { user: _, repo } => RelPath::BUILD.join(repo), } } pub(crate) fn fetch(&self, dirs: &Dirs) { + let download_dir = match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), + }; + let source_dir = self.source_dir(); match self.url { GitRepoUrl::Github { user, repo } => { - clone_repo_shallow_github( - dirs, - &self.source_dir().to_path(dirs), - user, - repo, - self.rev, - ); + clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev); } } - apply_patches(dirs, self.patch_name, &self.source_dir().to_path(dirs)); + source_dir.ensure_fresh(dirs); + copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); + apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); } } @@ -118,6 +113,8 @@ fn clone_repo(download_dir: &Path, repo: &str, rev: &str) { let mut checkout_cmd = git_command(download_dir, "checkout"); checkout_cmd.arg("-q").arg(rev); spawn_and_wait(checkout_cmd); + + std::fs::remove_dir_all(download_dir.join(".git")).unwrap(); } fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: &str, rev: &str) { @@ -165,8 +162,6 @@ fn clone_repo_shallow_github(dirs: &Dirs, download_dir: &Path, user: &str, repo: // Rename unpacked dir to the expected name std::fs::rename(archive_dir, &download_dir).unwrap(); - init_git_repo(&download_dir); - // Cleanup std::fs::remove_file(archive_file).unwrap(); } @@ -206,6 +201,8 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { } fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) { + init_git_repo(&target_dir); + if crate_name == "" { return; } diff --git a/build_system/tests.rs b/build_system/tests.rs index 7efb960697e..f975e43b13d 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -97,12 +97,12 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ pub(crate) static RAND_REPO: GitRepo = GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand"); -pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand"); +pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target"); pub(crate) static REGEX_REPO: GitRepo = GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex"); -pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex"); +pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex_target"); pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", @@ -112,9 +112,9 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( ); pub(crate) static PORTABLE_SIMD: CargoProject = - CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd"); + CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd_target"); -pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::DOWNLOAD.join("coretests_src"); +pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); From 75327f8587fad77a25cfe99e52376b6862d75b57 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:49:41 +0000 Subject: [PATCH 04/20] Reuse existing download in y.sh prepare if fresh --- build_system/abi_cafe.rs | 10 ++++- build_system/bench.rs | 6 +-- build_system/prepare.rs | 87 +++++++++++++++++++++++++++++++++++----- build_system/tests.rs | 19 +++++++-- 4 files changed, 103 insertions(+), 19 deletions(-) diff --git a/build_system/abi_cafe.rs b/build_system/abi_cafe.rs index ba15db9f966..29c127bf50e 100644 --- a/build_system/abi_cafe.rs +++ b/build_system/abi_cafe.rs @@ -4,8 +4,13 @@ use super::prepare::GitRepo; use super::utils::{spawn_and_wait, CargoProject, Compiler}; use super::{CodegenBackend, SysrootKind}; -static ABI_CAFE_REPO: GitRepo = - GitRepo::github("Gankra", "abi-cafe", "4c6dc8c9c687e2b3a760ff2176ce236872b37212", "abi-cafe"); +static ABI_CAFE_REPO: GitRepo = GitRepo::github( + "Gankra", + "abi-cafe", + "4c6dc8c9c687e2b3a760ff2176ce236872b37212", + "588df6d66abbe105", + "abi-cafe", +); static ABI_CAFE: CargoProject = CargoProject::new(&ABI_CAFE_REPO.source_dir(), "abi_cafe_target"); @@ -18,6 +23,7 @@ pub(crate) fn run( bootstrap_host_compiler: &Compiler, ) { ABI_CAFE_REPO.fetch(dirs); + ABI_CAFE_REPO.patch(dirs); eprintln!("Building sysroot for abi-cafe"); build_sysroot::build_sysroot( diff --git a/build_system/bench.rs b/build_system/bench.rs index d24803eb7c6..2bb11800034 100644 --- a/build_system/bench.rs +++ b/build_system/bench.rs @@ -10,6 +10,7 @@ static SIMPLE_RAYTRACER_REPO: GitRepo = GitRepo::github( "ebobby", "simple-raytracer", "804a7a21b9e673a482797aa289a18ed480e4d813", + "ad6f59a2331a3f56", "", ); @@ -24,9 +25,8 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) { std::process::exit(1); } - if !SIMPLE_RAYTRACER_REPO.source_dir().to_path(dirs).exists() { - SIMPLE_RAYTRACER_REPO.fetch(dirs); - } + SIMPLE_RAYTRACER_REPO.fetch(dirs); + SIMPLE_RAYTRACER_REPO.patch(dirs); let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap(); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 7bb9eca2945..b76164a1d00 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -10,14 +10,18 @@ use super::tests::LIBCORE_TESTS_SRC; use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait}; pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { - RelPath::DOWNLOAD.ensure_fresh(dirs); - - prepare_stdlib(dirs, rustc); - prepare_coretests(dirs, rustc); - + RelPath::DOWNLOAD.ensure_exists(dirs); super::tests::RAND_REPO.fetch(dirs); super::tests::REGEX_REPO.fetch(dirs); super::tests::PORTABLE_SIMD_REPO.fetch(dirs); + + // FIXME do this on the fly? + prepare_stdlib(dirs, rustc); + prepare_coretests(dirs, rustc); + + super::tests::RAND_REPO.patch(dirs); + super::tests::REGEX_REPO.patch(dirs); + super::tests::PORTABLE_SIMD_REPO.patch(dirs); } fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { @@ -61,6 +65,7 @@ fn prepare_coretests(dirs: &Dirs, rustc: &Path) { pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, + content_hash: &'static str, patch_name: &'static str, } @@ -68,14 +73,49 @@ enum GitRepoUrl { Github { user: &'static str, repo: &'static str }, } +// Note: This uses a hasher which is not cryptographically secure. This is fine as the hash is meant +// to protect against accidental modification and outdated downloads, not against manipulation. +fn hash_file(file: &std::path::Path) -> u64 { + let contents = std::fs::read(file).unwrap(); + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + std::hash::Hash::hash(&contents, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + +fn hash_dir(dir: &std::path::Path) -> u64 { + let mut sub_hashes = std::collections::BTreeMap::new(); + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_dir() { + sub_hashes + .insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path())); + } else { + sub_hashes + .insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path())); + } + } + #[allow(deprecated)] + let mut hasher = std::hash::SipHasher::new(); + std::hash::Hash::hash(&sub_hashes, &mut hasher); + std::hash::Hasher::finish(&hasher) +} + impl GitRepo { pub(crate) const fn github( user: &'static str, repo: &'static str, rev: &'static str, + content_hash: &'static str, patch_name: &'static str, ) -> GitRepo { - GitRepo { url: GitRepoUrl::Github { user, repo }, rev, patch_name } + GitRepo { url: GitRepoUrl::Github { user, repo }, rev, content_hash, patch_name } + } + + fn download_dir(&self, dirs: &Dirs) -> PathBuf { + match self.url { + GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), + } } pub(crate) const fn source_dir(&self) -> RelPath { @@ -85,15 +125,42 @@ impl GitRepo { } pub(crate) fn fetch(&self, dirs: &Dirs) { - let download_dir = match self.url { - GitRepoUrl::Github { user: _, repo } => RelPath::DOWNLOAD.join(repo).to_path(dirs), - }; - let source_dir = self.source_dir(); + let download_dir = self.download_dir(dirs); + + if download_dir.exists() { + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash == self.content_hash { + println!("[FRESH] {}", download_dir.display()); + return; + } else { + println!( + "Mismatched content hash for {download_dir}: {actual_hash} != {content_hash}. Downloading again.", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + } + } + match self.url { GitRepoUrl::Github { user, repo } => { clone_repo_shallow_github(dirs, &download_dir, user, repo, self.rev); } } + + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); + if actual_hash != self.content_hash { + println!( + "Download of {download_dir} failed with mismatched content hash: {actual_hash} != {content_hash}", + download_dir = download_dir.display(), + content_hash = self.content_hash, + ); + std::process::exit(1); + } + } + + pub(crate) fn patch(&self, dirs: &Dirs) { + let download_dir = self.download_dir(dirs); + let source_dir = self.source_dir(); source_dir.ensure_fresh(dirs); copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); diff --git a/build_system/tests.rs b/build_system/tests.rs index f975e43b13d..e60f0a4c612 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -94,13 +94,23 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[ // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is // fixed. -pub(crate) static RAND_REPO: GitRepo = - GitRepo::github("rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", "rand"); +pub(crate) static RAND_REPO: GitRepo = GitRepo::github( + "rust-random", + "rand", + "50b9a447410860af8d6db9a208c3576886955874", + "98b2276210b30e43", + "rand", +); pub(crate) static RAND: CargoProject = CargoProject::new(&RAND_REPO.source_dir(), "rand_target"); -pub(crate) static REGEX_REPO: GitRepo = - GitRepo::github("rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", "regex"); +pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( + "rust-lang", + "regex", + "32fed9429eafba0ae92a64b01796a0c5a75b88c8", + "d6af6507d565aa66", + "regex", +); pub(crate) static REGEX: CargoProject = CargoProject::new(®EX_REPO.source_dir(), "regex_target"); @@ -108,6 +118,7 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", + "1ba291009510070b", "portable-simd", ); From 2c38effe286e1b68a2f1125598ece6c8887d7d49 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 12:37:48 +0000 Subject: [PATCH 05/20] Don't patch in place in apply_patches This will make it easier to skip patching if unnecessary in the future --- build_system/prepare.rs | 42 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index b76164a1d00..0302946e654 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -7,7 +7,9 @@ use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERS use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; use super::tests::LIBCORE_TESTS_SRC; -use super::utils::{copy_dir_recursively, git_command, retry_spawn_and_wait, spawn_and_wait}; +use super::utils::{ + copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, +}; pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { RelPath::DOWNLOAD.ensure_exists(dirs); @@ -35,31 +37,24 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs)); fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap(); - copy_dir_recursively( - &sysroot_src_orig.join("library"), - &SYSROOT_SRC.to_path(dirs).join("library"), - ); + + apply_patches(dirs, "stdlib", &sysroot_src_orig, &SYSROOT_SRC.to_path(dirs)); let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); - - apply_patches(dirs, "stdlib", &SYSROOT_SRC.to_path(dirs)); } fn prepare_coretests(dirs: &Dirs, rustc: &Path) { let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); - eprintln!("[COPY] coretests src"); - // FIXME ensure builds error out or update the copy if any of the files copied here change - LIBCORE_TESTS_SRC.ensure_fresh(dirs); - copy_dir_recursively( + apply_patches( + dirs, + "coretests", &sysroot_src_orig.join("library/core/tests"), &LIBCORE_TESTS_SRC.to_path(dirs), ); - - apply_patches(dirs, "coretests", &LIBCORE_TESTS_SRC.to_path(dirs)); } pub(crate) struct GitRepo { @@ -159,11 +154,12 @@ impl GitRepo { } pub(crate) fn patch(&self, dirs: &Dirs) { - let download_dir = self.download_dir(dirs); - let source_dir = self.source_dir(); - source_dir.ensure_fresh(dirs); - copy_dir_recursively(&download_dir, &source_dir.to_path(dirs)); - apply_patches(dirs, self.patch_name, &source_dir.to_path(dirs)); + apply_patches( + dirs, + self.patch_name, + &self.download_dir(dirs), + &self.source_dir().to_path(dirs), + ); } } @@ -267,8 +263,14 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { patches } -fn apply_patches(dirs: &Dirs, crate_name: &str, target_dir: &Path) { - init_git_repo(&target_dir); +fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { + // FIXME avoid copy and patch if src, patches and target are unchanged + + remove_dir_if_exists(target_dir); + fs::create_dir_all(target_dir).unwrap(); + copy_dir_recursively(source_dir, target_dir); + + init_git_repo(target_dir); if crate_name == "" { return; From fc23a8a7e0fe32ef18214ced7ee79e111a40df69 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 12:45:05 +0000 Subject: [PATCH 06/20] Lazily patch coretests --- build_system/prepare.rs | 17 +---------------- build_system/tests.rs | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 0302946e654..8cf51db1770 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -6,7 +6,6 @@ use std::process::Command; use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; -use super::tests::LIBCORE_TESTS_SRC; use super::utils::{ copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, }; @@ -19,7 +18,6 @@ pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { // FIXME do this on the fly? prepare_stdlib(dirs, rustc); - prepare_coretests(dirs, rustc); super::tests::RAND_REPO.patch(dirs); super::tests::REGEX_REPO.patch(dirs); @@ -44,19 +42,6 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); } -fn prepare_coretests(dirs: &Dirs, rustc: &Path) { - let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); - assert!(sysroot_src_orig.exists()); - - // FIXME ensure builds error out or update the copy if any of the files copied here change - apply_patches( - dirs, - "coretests", - &sysroot_src_orig.join("library/core/tests"), - &LIBCORE_TESTS_SRC.to_path(dirs), - ); -} - pub(crate) struct GitRepo { url: GitRepoUrl, rev: &'static str, @@ -263,7 +248,7 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { patches } -fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { +pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { // FIXME avoid copy and patch if src, patches and target are unchanged remove_dir_if_exists(target_dir); diff --git a/build_system/tests.rs b/build_system/tests.rs index e60f0a4c612..ec97be2b2d9 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -1,12 +1,14 @@ use super::build_sysroot; use super::config; use super::path::{Dirs, RelPath}; -use super::prepare::GitRepo; +use super::prepare::{apply_patches, GitRepo}; +use super::rustc_info::get_default_sysroot; use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler}; use super::{CodegenBackend, SysrootKind}; use std::env; use std::ffi::OsStr; use std::fs; +use std::path::PathBuf; use std::process::Command; static BUILD_EXAMPLE_OUT_DIR: RelPath = RelPath::BUILD.join("example"); @@ -125,9 +127,9 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( pub(crate) static PORTABLE_SIMD: CargoProject = CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd_target"); -pub(crate) static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); +static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); -pub(crate) static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); +static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { @@ -145,6 +147,13 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.libcore", &|runner| { + apply_patches( + &runner.dirs, + "coretests", + &runner.stdlib_source.join("library/core/tests"), + &LIBCORE_TESTS_SRC.to_path(&runner.dirs), + ); + LIBCORE_TESTS.clean(&runner.dirs); if runner.is_native { @@ -231,6 +240,10 @@ pub(crate) fn run_tests( rustup_toolchain_name: Option<&str>, target_triple: String, ) { + let stdlib_source = + get_default_sysroot(&bootstrap_host_compiler.rustc).join("lib/rustlib/src/rust"); + assert!(stdlib_source.exists()); + if config::get_bool("testsuite.no_sysroot") { let target_compiler = build_sysroot::build_sysroot( dirs, @@ -247,6 +260,7 @@ pub(crate) fn run_tests( target_compiler, use_unstable_features, bootstrap_host_compiler.triple == target_triple, + stdlib_source.clone(), ); BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs); @@ -277,6 +291,7 @@ pub(crate) fn run_tests( target_compiler, use_unstable_features, bootstrap_host_compiler.triple == target_triple, + stdlib_source, ); if run_base_sysroot { @@ -299,6 +314,7 @@ struct TestRunner { use_unstable_features: bool, dirs: Dirs, target_compiler: Compiler, + stdlib_source: PathBuf, } impl TestRunner { @@ -307,6 +323,7 @@ impl TestRunner { mut target_compiler: Compiler, use_unstable_features: bool, is_native: bool, + stdlib_source: PathBuf, ) -> Self { if let Ok(rustflags) = env::var("RUSTFLAGS") { target_compiler.rustflags.push(' '); @@ -327,7 +344,14 @@ impl TestRunner { && target_compiler.triple.contains("x86_64") && !target_compiler.triple.contains("windows"); - Self { is_native, jit_supported, use_unstable_features, dirs, target_compiler } + Self { + is_native, + jit_supported, + use_unstable_features, + dirs, + target_compiler, + stdlib_source, + } } fn run_testsuite(&self, tests: &[TestCase]) { From b9129c0d6bb1e2adffb368f5062d4198bce6a7c7 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 13:00:28 +0000 Subject: [PATCH 07/20] Rename a couple of build dirs for consistency --- build_system/tests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_system/tests.rs b/build_system/tests.rs index ec97be2b2d9..5502e2a23c6 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -125,11 +125,11 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( ); pub(crate) static PORTABLE_SIMD: CargoProject = - CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable_simd_target"); + CargoProject::new(&PORTABLE_SIMD_REPO.source_dir(), "portable-simd_target"); -static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests_src"); +static LIBCORE_TESTS_SRC: RelPath = RelPath::BUILD.join("coretests"); -static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core_tests"); +static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "coretests_target"); const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { From 54eaa5382c8455be4390c7e1e1b9b9f86c5a159d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 28 May 2023 17:57:49 +0000 Subject: [PATCH 08/20] Enable compiler-builtins no-asm feature using --features flag --- build_sysroot/Cargo.lock | 1 - build_sysroot/Cargo.toml | 2 -- build_sysroot/src/lib.rs | 2 +- build_system/build_sysroot.rs | 1 + 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index ef99acb8211..934665a0537 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -270,7 +270,6 @@ name = "sysroot" version = "0.0.0" dependencies = [ "alloc", - "compiler_builtins", "core", "proc_macro", "std", diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index ea9d1c8df1c..194a5f27275 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -9,8 +9,6 @@ std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtra test = { path = "./sysroot_src/library/test" } proc_macro = { path = "./sysroot_src/library/proc_macro" } -compiler_builtins = { version = "0.1.87", default-features = false, features = ["no-asm"] } - [patch.crates-io] rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } diff --git a/build_sysroot/src/lib.rs b/build_sysroot/src/lib.rs index 0c9ac1ac8e4..8b137891791 100644 --- a/build_sysroot/src/lib.rs +++ b/build_sysroot/src/lib.rs @@ -1 +1 @@ -#![no_std] + diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 7ceda34bfac..5c1e30d4eef 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -277,6 +277,7 @@ fn build_clif_sysroot_for_triple( if channel == "release" { build_cmd.arg("--release"); } + build_cmd.arg("--features").arg("std/compiler-builtins-no-asm"); build_cmd.arg("--locked"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { From 3baee66f9b0cd440d9288ce757cc6caec93eeb67 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 09:58:59 +0000 Subject: [PATCH 09/20] Rework standard library building --- build_sysroot/Cargo.lock | 104 +++++++++++++++++++++++++++++++++- build_sysroot/Cargo.toml | 18 ++---- build_sysroot/src/lib.rs | 1 - build_system/build_sysroot.rs | 11 ++-- build_system/prepare.rs | 13 ++--- 5 files changed, 117 insertions(+), 30 deletions(-) delete mode 100644 build_sysroot/src/lib.rs diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 934665a0537..1dde9e54d7e 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -30,8 +30,26 @@ version = "0.0.0" dependencies = [ "compiler_builtins", "core", + "rand", + "rand_xorshift", ] +[[package]] +name = "auxv" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e50430f9beb8effb02399fa81c76eeaa26b05e4f03b09285cad8d079c1af5a3d" +dependencies = [ + "byteorder", + "gcc", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "cc" version = "1.0.79" @@ -54,12 +72,27 @@ version = "0.1.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76630810d973ecea3dbf611e1b7aecfb1012751ef1ff8de3998f89014a166781" dependencies = [ + "cc", "rustc-std-workspace-core", ] [[package]] name = "core" version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "cupid" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bad352a84b567cc38a5854e3aa8ee903cb8519a25d0b799b739bafffd1f91a1" +dependencies = [ + "gcc", + "rustc_version", +] [[package]] name = "dlmalloc" @@ -82,6 +115,12 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "gcc" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" + [[package]] name = "getopts" version = "0.2.21" @@ -200,6 +239,39 @@ dependencies = [ "std", ] +[[package]] +name = "profiler_builtins" +version = "0.0.0" +dependencies = [ + "cc", + "compiler_builtins", + "core", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -231,6 +303,30 @@ dependencies = [ "std", ] +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "std" version = "0.0.0" @@ -249,6 +345,9 @@ dependencies = [ "object", "panic_abort", "panic_unwind", + "profiler_builtins", + "rand", + "rand_xorshift", "rustc-demangle", "std_detect", "unwind", @@ -259,8 +358,11 @@ dependencies = [ name = "std_detect" version = "0.1.5" dependencies = [ + "auxv", "cfg-if", "compiler_builtins", + "cupid", + "libc", "rustc-std-workspace-alloc", "rustc-std-workspace-core", ] @@ -269,8 +371,6 @@ dependencies = [ name = "sysroot" version = "0.0.0" dependencies = [ - "alloc", - "core", "proc_macro", "std", "test", diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml index 194a5f27275..3e5d0c159f2 100644 --- a/build_sysroot/Cargo.toml +++ b/build_sysroot/Cargo.toml @@ -1,18 +1,10 @@ -[package] -name = "sysroot" -version = "0.0.0" - -[dependencies] -core = { path = "./sysroot_src/library/core" } -alloc = { path = "./sysroot_src/library/alloc" } -std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] } -test = { path = "./sysroot_src/library/test" } -proc_macro = { path = "./sysroot_src/library/proc_macro" } +[workspace] +members = ["./library/sysroot"] [patch.crates-io] -rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" } -rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" } -rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" } +rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } +rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } +rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } [profile.dev] lto = "off" diff --git a/build_sysroot/src/lib.rs b/build_sysroot/src/lib.rs deleted file mode 100644 index 8b137891791..00000000000 --- a/build_sysroot/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ - diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 5c1e30d4eef..09dc43b8136 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -156,11 +156,10 @@ impl SysrootTarget { } pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); -pub(crate) static BUILD_SYSROOT: RelPath = RelPath::BUILD.join("sysroot"); -pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = BUILD_SYSROOT.join("rustc_version"); -pub(crate) static SYSROOT_SRC: RelPath = BUILD_SYSROOT.join("sysroot_src"); +pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); +pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = STDLIB_SRC.join("rustc_version"); pub(crate) static STANDARD_LIBRARY: CargoProject = - CargoProject::new(&BUILD_SYSROOT, "build_sysroot"); + CargoProject::new(&STDLIB_SRC.join("library/sysroot"), "stdlib_target"); pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup"); #[must_use] @@ -277,7 +276,7 @@ fn build_clif_sysroot_for_triple( if channel == "release" { build_cmd.arg("--release"); } - build_cmd.arg("--features").arg("std/compiler-builtins-no-asm"); + build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); build_cmd.arg("--locked"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { @@ -307,7 +306,7 @@ fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option { RTSTARTUP_SYSROOT.ensure_fresh(dirs); - let rtstartup_src = SYSROOT_SRC.to_path(dirs).join("library").join("rtstartup"); + let rtstartup_src = STDLIB_SRC.to_path(dirs).join("library").join("rtstartup"); let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; for file in ["rsbegin", "rsend"] { diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 8cf51db1770..bc249f4cdab 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,7 +3,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{BUILD_SYSROOT, ORIG_BUILD_SYSROOT, SYSROOT_RUSTC_VERSION, SYSROOT_SRC}; +use super::build_sysroot::{ORIG_BUILD_SYSROOT, STDLIB_SRC, SYSROOT_RUSTC_VERSION}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; use super::utils::{ @@ -28,15 +28,10 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); - eprintln!("[COPY] stdlib src"); + apply_patches(dirs, "stdlib", &sysroot_src_orig, &STDLIB_SRC.to_path(dirs)); // FIXME ensure builds error out or update the copy if any of the files copied here change - BUILD_SYSROOT.ensure_fresh(dirs); - copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &BUILD_SYSROOT.to_path(dirs)); - - fs::create_dir_all(SYSROOT_SRC.to_path(dirs).join("library")).unwrap(); - - apply_patches(dirs, "stdlib", &sysroot_src_orig, &SYSROOT_SRC.to_path(dirs)); + copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &STDLIB_SRC.to_path(dirs)); let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); @@ -251,6 +246,8 @@ fn get_patches(dirs: &Dirs, crate_name: &str) -> Vec { pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, target_dir: &Path) { // FIXME avoid copy and patch if src, patches and target are unchanged + eprintln!("[COPY] {crate_name} source"); + remove_dir_if_exists(target_dir); fs::create_dir_all(target_dir).unwrap(); copy_dir_recursively(source_dir, target_dir); From 67f9fe6863cede6ca5f41b938e4f3188feb24c65 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 11:51:37 +0000 Subject: [PATCH 10/20] Lazily patch all test projects --- build_system/prepare.rs | 4 ---- build_system/tests.rs | 8 ++++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index bc249f4cdab..115575bff51 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -18,10 +18,6 @@ pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { // FIXME do this on the fly? prepare_stdlib(dirs, rustc); - - super::tests::RAND_REPO.patch(dirs); - super::tests::REGEX_REPO.patch(dirs); - super::tests::PORTABLE_SIMD_REPO.patch(dirs); } fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { diff --git a/build_system/tests.rs b/build_system/tests.rs index 5502e2a23c6..d104efd61e5 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -133,6 +133,8 @@ static LIBCORE_TESTS: CargoProject = CargoProject::new(&LIBCORE_TESTS_SRC, "core const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ TestCase::custom("test.rust-random/rand", &|runner| { + RAND_REPO.patch(&runner.dirs); + RAND.clean(&runner.dirs); if runner.is_native { @@ -168,6 +170,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.regex-shootout-regex-dna", &|runner| { + REGEX_REPO.patch(&runner.dirs); + REGEX.clean(&runner.dirs); let mut build_cmd = REGEX.build(&runner.target_compiler, &runner.dirs); @@ -200,6 +204,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.regex", &|runner| { + REGEX_REPO.patch(&runner.dirs); + REGEX.clean(&runner.dirs); if runner.is_native { @@ -216,6 +222,8 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ } }), TestCase::custom("test.portable-simd", &|runner| { + PORTABLE_SIMD_REPO.patch(&runner.dirs); + PORTABLE_SIMD.clean(&runner.dirs); let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs); From d3da972441f6efd16c1b1aa5c2672cc218949bdb Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 12:19:38 +0000 Subject: [PATCH 11/20] Write stdlib workspace Cargo.toml directly in prepare.rs --- build_sysroot/Cargo.toml | 26 -------------------------- build_system/prepare.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 26 deletions(-) delete mode 100644 build_sysroot/Cargo.toml diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml deleted file mode 100644 index 3e5d0c159f2..00000000000 --- a/build_sysroot/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[workspace] -members = ["./library/sysroot"] - -[patch.crates-io] -rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } -rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } -rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } - -[profile.dev] -lto = "off" - -[profile.release] -debug = true -incremental = true -lto = "off" - -# Mandatory for correctly compiling compiler-builtins -[profile.dev.package.compiler_builtins] -debug-assertions = false -overflow-checks = false -codegen-units = 10000 - -[profile.release.package.compiler_builtins] -debug-assertions = false -overflow-checks = false -codegen-units = 10000 diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 115575bff51..4f26cb5e85f 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -29,6 +29,39 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { // FIXME ensure builds error out or update the copy if any of the files copied here change copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &STDLIB_SRC.to_path(dirs)); + std::fs::write( + STDLIB_SRC.to_path(dirs).join("Cargo.toml"), + r#" +[workspace] +members = ["./library/sysroot"] + +[patch.crates-io] +rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } +rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } +rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } + +[profile.dev] +lto = "off" + +[profile.release] +debug = true +incremental = true +lto = "off" + +# Mandatory for correctly compiling compiler-builtins +[profile.dev.package.compiler_builtins] +debug-assertions = false +overflow-checks = false +codegen-units = 10000 + +[profile.release.package.compiler_builtins] +debug-assertions = false +overflow-checks = false +codegen-units = 10000 +"#, + ) + .unwrap(); + let rustc_version = get_rustc_version(rustc); fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); } From 8ad9e9f861c9cbdd85c99df73a4a2d03a0d15014 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 12:09:40 +0000 Subject: [PATCH 12/20] Ensure everything has a lockfile --- build_system/build_sysroot.rs | 2 - build_system/prepare.rs | 13 +- build_system/utils.rs | 3 +- patches/coretests-lock.toml | 35 ++ patches/portable-simd-lock.toml | 304 ++++++++++++ patches/rand-lock.toml | 346 ++++++++++++++ patches/regex-lock.toml | 439 ++++++++++++++++++ .../Cargo.lock => patches/stdlib-lock.toml | 0 scripts/rustup.sh | 4 +- 9 files changed, 1136 insertions(+), 10 deletions(-) create mode 100644 patches/coretests-lock.toml create mode 100644 patches/portable-simd-lock.toml create mode 100644 patches/rand-lock.toml create mode 100644 patches/regex-lock.toml rename build_sysroot/Cargo.lock => patches/stdlib-lock.toml (100%) diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 09dc43b8136..1045023c361 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -155,7 +155,6 @@ impl SysrootTarget { } } -pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot"); pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = STDLIB_SRC.join("rustc_version"); pub(crate) static STANDARD_LIBRARY: CargoProject = @@ -277,7 +276,6 @@ fn build_clif_sysroot_for_triple( build_cmd.arg("--release"); } build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); - build_cmd.arg("--locked"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed"); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 4f26cb5e85f..c6bc796f3a9 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,7 +3,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{ORIG_BUILD_SYSROOT, STDLIB_SRC, SYSROOT_RUSTC_VERSION}; +use super::build_sysroot::{STDLIB_SRC, SYSROOT_RUSTC_VERSION}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_default_sysroot, get_rustc_version}; use super::utils::{ @@ -26,9 +26,6 @@ fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { apply_patches(dirs, "stdlib", &sysroot_src_orig, &STDLIB_SRC.to_path(dirs)); - // FIXME ensure builds error out or update the copy if any of the files copied here change - copy_dir_recursively(&ORIG_BUILD_SYSROOT.to_path(dirs), &STDLIB_SRC.to_path(dirs)); - std::fs::write( STDLIB_SRC.to_path(dirs).join("Cargo.toml"), r#" @@ -297,4 +294,12 @@ pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, ta apply_patch_cmd.arg(patch).arg("-q"); spawn_and_wait(apply_patch_cmd); } + + let source_lockfile = RelPath::PATCHES.to_path(dirs).join(format!("{crate_name}-lock.toml")); + let target_lockfile = target_dir.join("Cargo.lock"); + if source_lockfile.exists() { + fs::copy(source_lockfile, target_lockfile).unwrap(); + } else { + assert!(target_lockfile.exists()); + } } diff --git a/build_system/utils.rs b/build_system/utils.rs index 3e12ed22ef6..e7fcb2644d7 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -81,7 +81,8 @@ impl CargoProject { .arg("--manifest-path") .arg(self.manifest_path(dirs)) .arg("--target-dir") - .arg(self.target_dir(dirs)); + .arg(self.target_dir(dirs)) + .arg("--locked"); if dirs.frozen { cmd.arg("--frozen"); diff --git a/patches/coretests-lock.toml b/patches/coretests-lock.toml new file mode 100644 index 00000000000..af8f28a193b --- /dev/null +++ b/patches/coretests-lock.toml @@ -0,0 +1,35 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "coretests" +version = "0.0.0" +dependencies = [ + "rand", + "rand_xorshift", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] diff --git a/patches/portable-simd-lock.toml b/patches/portable-simd-lock.toml new file mode 100644 index 00000000000..e7db1fd2c7f --- /dev/null +++ b/patches/portable-simd-lock.toml @@ -0,0 +1,304 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "core_simd" +version = "0.1.0" +dependencies = [ + "proptest", + "std_float", + "test_helpers", + "wasm-bindgen", + "wasm-bindgen-test", +] + +[[package]] +name = "js-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "log" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e6c80c1139113c28ee4670dc50cc42915228b51f56a9e407f0ec60f966646f" +dependencies = [ + "bitflags", + "byteorder", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77d416b86801d23dde1aa643023b775c3a462efc0ed96443add11546cdf1dca8" +dependencies = [ + "rand_core", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "std_float" +version = "0.1.0" +dependencies = [ + "core_simd", +] + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test_helpers" +version = "0.1.0" +dependencies = [ + "proptest", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "wasm-bindgen" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e636f3a428ff62b3742ebc3c70e254dfe12b8c2b469d688ea59cdd4abcf502" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18c1fad2f7c4958e7bcce014fa212f59a65d5e3721d0f77e6c0b27ede936ba3" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "web-sys" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] diff --git a/patches/rand-lock.toml b/patches/rand-lock.toml new file mode 100644 index 00000000000..66c515731c5 --- /dev/null +++ b/patches/rand-lock.toml @@ -0,0 +1,346 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "average" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843ec791d3f24503bbf72bbd5e49a3ab4dbb4bcd0a8ef6b0c908efa73caa27b1" +dependencies = [ + "easy-cast", + "float-ord", + "num-traits", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "easy-cast" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd102ee8c418348759919b83b81cdbdc933ffe29740b903df448b4bafaa348e" +dependencies = [ + "libm", +] + +[[package]] +name = "either" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" + +[[package]] +name = "float-ord" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce81f49ae8a0482e4c55ea62ebbd7e5a686af544c00b9d090bba3ff9be97b3d" + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "libm" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" + +[[package]] +name = "log" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" + +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.9.0" +dependencies = [ + "bincode", + "libc", + "log", + "rand_chacha", + "rand_core", + "rand_pcg", + "rayon", + "serde", +] + +[[package]] +name = "rand_chacha" +version = "0.4.0" +dependencies = [ + "ppv-lite86", + "rand_core", + "serde", + "serde_json", +] + +[[package]] +name = "rand_core" +version = "0.7.0" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "rand_distr" +version = "0.5.0" +dependencies = [ + "average", + "num-traits", + "rand", + "rand_pcg", + "serde", + "special", +] + +[[package]] +name = "rand_pcg" +version = "0.4.0" +dependencies = [ + "bincode", + "rand_core", + "serde", +] + +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "special" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a65e074159b75dcf173a4733ab2188baac24967b5c8ec9ed87ae15fcbc7636" +dependencies = [ + "libc", +] + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" diff --git a/patches/regex-lock.toml b/patches/regex-lock.toml new file mode 100644 index 00000000000..0e4a33b90ea --- /dev/null +++ b/patches/regex-lock.toml @@ -0,0 +1,439 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "docopt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" +dependencies = [ + "lazy_static", + "regex 1.8.3", + "serde", + "strsim", +] + +[[package]] +name = "filetime" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "windows-sys", +] + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.144" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" + +[[package]] +name = "libpcre-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff3dd28ba96d6fe6752882f2f1b25ba8e1646448e79042442347cf3a92a6666" +dependencies = [ + "bzip2", + "libc", + "pkg-config", + "tar", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "memmap" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2ffa2c986de11a9df78620c01eeaaf27d94d3ff02bf81bfcca953102dd0c6ff" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "onig" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5eeb268a4620c74ea5768c6d2ccd492d60a47a8754666b91a46bfc35cd4d1ba" +dependencies = [ + "bitflags", + "lazy_static", + "libc", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "68.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195ebddbb56740be48042ca117b8fb6e0d99fe392191a9362d82f5f69e510379" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "proc-macro2" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quickcheck" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" +dependencies = [ + "rand", +] + +[[package]] +name = "quote" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.7.2" +dependencies = [ + "aho-corasick", + "lazy_static", + "memchr", + "quickcheck", + "rand", + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +dependencies = [ + "regex-syntax 0.7.2", +] + +[[package]] +name = "regex-benchmark" +version = "0.1.0" +dependencies = [ + "cc", + "cfg-if 0.1.10", + "docopt", + "lazy_static", + "libc", + "libpcre-sys", + "memmap", + "onig", + "pkg-config", + "regex 1.7.2", + "regex-syntax 0.6.29", + "serde", +] + +[[package]] +name = "regex-debug" +version = "0.1.0" +dependencies = [ + "docopt", + "regex 1.7.2", + "regex-syntax 0.6.29", + "serde", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" + +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + +[[package]] +name = "rure" +version = "0.2.2" +dependencies = [ + "libc", + "regex 1.7.2", +] + +[[package]] +name = "serde" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.163" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] diff --git a/build_sysroot/Cargo.lock b/patches/stdlib-lock.toml similarity index 100% rename from build_sysroot/Cargo.lock rename to patches/stdlib-lock.toml diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 6b446aafb7b..e62788f2e50 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -33,11 +33,9 @@ case $1 in ./clean_all.sh ./y.sh prepare - - (cd download/sysroot && cargo update && cargo fetch && cp Cargo.lock ../../build_sysroot/) ;; "commit") - git add rust-toolchain build_sysroot/Cargo.lock + git add rust-toolchain git commit -m "Rustup to $(rustc -V)" ;; "push") From eb3e8bb7d729a2df19b1fe2a6a6d7b5e43673807 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 12:16:33 +0000 Subject: [PATCH 13/20] Move some profile settings out of the stdlib cargo workspace patch --- build_system/build_sysroot.rs | 10 +++++++++- build_system/prepare.rs | 8 -------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 1045023c361..5ea34f758b9 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -4,7 +4,9 @@ use std::process::{self, Command}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_file_name, get_rustc_version}; -use super::utils::{remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler}; +use super::utils::{ + is_ci, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, +}; use super::{CodegenBackend, SysrootKind}; static DIST_DIR: RelPath = RelPath::DIST; @@ -272,10 +274,16 @@ fn build_clif_sysroot_for_triple( } compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); + build_cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode + if is_ci() { + // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway + build_cmd.env("CARGO_BUILD_INCREMENTAL", "false"); + } if channel == "release" { build_cmd.arg("--release"); } build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind"); + build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true"); build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif"); if compiler.triple.contains("apple") { build_cmd.env("CARGO_PROFILE_RELEASE_SPLIT_DEBUGINFO", "packed"); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index c6bc796f3a9..350c3ad8fdc 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -37,14 +37,6 @@ rustc-std-workspace-core = { path = "./library/rustc-std-workspace-core" } rustc-std-workspace-alloc = { path = "./library/rustc-std-workspace-alloc" } rustc-std-workspace-std = { path = "./library/rustc-std-workspace-std" } -[profile.dev] -lto = "off" - -[profile.release] -debug = true -incremental = true -lto = "off" - # Mandatory for correctly compiling compiler-builtins [profile.dev.package.compiler_builtins] debug-assertions = false From 6b9af8cb36d2bc1ae57768e3c8703093870d919f Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 13:18:41 +0000 Subject: [PATCH 14/20] Disable all incremental compilation for CARGO_BUILD_INCREMENTAL=false --- build_system/build_backend.rs | 8 ++------ build_system/build_sysroot.rs | 8 ++------ build_system/utils.rs | 10 ++++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/build_system/build_backend.rs b/build_system/build_backend.rs index b88489a341c..6855c1a7fc5 100644 --- a/build_system/build_backend.rs +++ b/build_system/build_backend.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use super::path::{Dirs, RelPath}; use super::rustc_info::get_file_name; -use super::utils::{is_ci, is_ci_opt, CargoProject, Compiler}; +use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler}; pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif"); @@ -14,8 +14,7 @@ pub(crate) fn build_backend( use_unstable_features: bool, ) -> PathBuf { let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs); - - cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode + maybe_incremental(&mut cmd); let mut rustflags = env::var("RUSTFLAGS").unwrap_or_default(); @@ -23,9 +22,6 @@ pub(crate) fn build_backend( // Deny warnings on CI rustflags += " -Dwarnings"; - // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway - cmd.env("CARGO_BUILD_INCREMENTAL", "false"); - if !is_ci_opt() { cmd.env("CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS", "true"); cmd.env("CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS", "true"); diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 5ea34f758b9..04cd3894adc 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -5,7 +5,7 @@ use std::process::{self, Command}; use super::path::{Dirs, RelPath}; use super::rustc_info::{get_file_name, get_rustc_version}; use super::utils::{ - is_ci, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, + maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, }; use super::{CodegenBackend, SysrootKind}; @@ -274,11 +274,7 @@ fn build_clif_sysroot_for_triple( } compiler.rustflags += &rustflags; let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs); - build_cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode - if is_ci() { - // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway - build_cmd.env("CARGO_BUILD_INCREMENTAL", "false"); - } + maybe_incremental(&mut build_cmd); if channel == "release" { build_cmd.arg("--release"); } diff --git a/build_system/utils.rs b/build_system/utils.rs index e7fcb2644d7..41fc366e290 100644 --- a/build_system/utils.rs +++ b/build_system/utils.rs @@ -258,3 +258,13 @@ pub(crate) fn is_ci() -> bool { pub(crate) fn is_ci_opt() -> bool { env::var("CI_OPT").is_ok() } + +pub(crate) fn maybe_incremental(cmd: &mut Command) { + if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") { + // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway + cmd.env("CARGO_BUILD_INCREMENTAL", "false"); + } else { + // Force incr comp even in release mode unless in CI or incremental builds are explicitly disabled + cmd.env("CARGO_BUILD_INCREMENTAL", "true"); + } +} From d0ea8bbc5e8dcb2018a248ee9d88a02ce2319b3c Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 29 May 2023 13:41:55 +0000 Subject: [PATCH 15/20] Only copy library dir for stdlib When building as part of rust, the sysroot source dir is symlinked to the main source dir, which contains the build dir to which we are likely copying. --- build_system/prepare.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 350c3ad8fdc..748d92614b6 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -268,7 +268,12 @@ pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, ta remove_dir_if_exists(target_dir); fs::create_dir_all(target_dir).unwrap(); - copy_dir_recursively(source_dir, target_dir); + if crate_name == "stdlib" { + fs::create_dir(target_dir.join("library")).unwrap(); + copy_dir_recursively(&source_dir.join("library"), &target_dir.join("library")); + } else { + copy_dir_recursively(source_dir, target_dir); + } init_git_repo(target_dir); From bcac2220134a0b6ca858b674c00ad809c2ed446a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 4 Jun 2023 15:41:43 +0000 Subject: [PATCH 16/20] Lazily patch the standard library --- build_system/build_sysroot.rs | 27 ++++++--------------------- build_system/main.rs | 23 ++++++++++++++++------- build_system/prepare.rs | 14 ++++---------- build_system/rustc_info.rs | 6 ------ 4 files changed, 26 insertions(+), 44 deletions(-) diff --git a/build_system/build_sysroot.rs b/build_system/build_sysroot.rs index 04cd3894adc..74bba9ed5eb 100644 --- a/build_system/build_sysroot.rs +++ b/build_system/build_sysroot.rs @@ -1,9 +1,9 @@ use std::fs; use std::path::{Path, PathBuf}; -use std::process::{self, Command}; +use std::process::Command; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_file_name, get_rustc_version}; +use super::rustc_info::get_file_name; use super::utils::{ maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler, }; @@ -158,7 +158,6 @@ impl SysrootTarget { } pub(crate) static STDLIB_SRC: RelPath = RelPath::BUILD.join("stdlib"); -pub(crate) static SYSROOT_RUSTC_VERSION: RelPath = STDLIB_SRC.join("rustc_version"); pub(crate) static STANDARD_LIBRARY: CargoProject = CargoProject::new(&STDLIB_SRC.join("library/sysroot"), "stdlib_target"); pub(crate) static RTSTARTUP_SYSROOT: RelPath = RelPath::BUILD.join("rtstartup"); @@ -222,24 +221,6 @@ fn build_clif_sysroot_for_triple( mut compiler: Compiler, cg_clif_dylib_path: &CodegenBackend, ) -> SysrootTarget { - match fs::read_to_string(SYSROOT_RUSTC_VERSION.to_path(dirs)) { - Err(e) => { - eprintln!("Failed to get rustc version for patched sysroot source: {}", e); - eprintln!("Hint: Try `./y.sh prepare` to patch the sysroot source"); - process::exit(1); - } - Ok(source_version) => { - let rustc_version = get_rustc_version(&compiler.rustc); - if source_version != rustc_version { - eprintln!("The patched sysroot source is outdated"); - eprintln!("Source version: {}", source_version.trim()); - eprintln!("Rustc version: {}", rustc_version.trim()); - eprintln!("Hint: Try `./y.sh prepare` to update the patched sysroot source"); - process::exit(1); - } - } - } - let mut target_libs = SysrootTarget { triple: compiler.triple.clone(), libs: vec![] }; if let Some(rtstartup_target_libs) = build_rtstartup(dirs, &compiler) { @@ -302,6 +283,10 @@ fn build_clif_sysroot_for_triple( } fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option { + if !super::config::get_bool("keep_sysroot") { + super::prepare::prepare_stdlib(dirs, &compiler.rustc); + } + if !compiler.triple.ends_with("windows-gnu") { return None; } diff --git a/build_system/main.rs b/build_system/main.rs index 06395eb141c..d51e5027cf3 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -126,6 +126,22 @@ fn main() { } } + let current_dir = std::env::current_dir().unwrap(); + out_dir = current_dir.join(out_dir); + + if command == Command::Prepare { + prepare::prepare(&path::Dirs { + source_dir: current_dir.clone(), + download_dir: download_dir + .map(|dir| current_dir.join(dir)) + .unwrap_or_else(|| out_dir.join("download")), + build_dir: PathBuf::from("dummy_do_not_use"), + dist_dir: PathBuf::from("dummy_do_not_use"), + frozen, + }); + process::exit(0); + } + let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) { (Ok(_), Ok(_), Ok(_)) => None, (Err(_), Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()), @@ -158,8 +174,6 @@ fn main() { .unwrap_or_else(|| bootstrap_host_compiler.triple.clone()); // FIXME allow changing the location of these dirs using cli arguments - let current_dir = std::env::current_dir().unwrap(); - out_dir = current_dir.join(out_dir); let dirs = path::Dirs { source_dir: current_dir.clone(), download_dir: download_dir @@ -181,11 +195,6 @@ fn main() { std::fs::File::create(target).unwrap(); } - if command == Command::Prepare { - prepare::prepare(&dirs, &bootstrap_host_compiler.rustc); - process::exit(0); - } - env::set_var("RUSTC", "rustc_should_be_set_explicitly"); env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly"); diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 748d92614b6..77f7175b786 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -3,24 +3,21 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use super::build_sysroot::{STDLIB_SRC, SYSROOT_RUSTC_VERSION}; +use super::build_sysroot::STDLIB_SRC; use super::path::{Dirs, RelPath}; -use super::rustc_info::{get_default_sysroot, get_rustc_version}; +use super::rustc_info::get_default_sysroot; use super::utils::{ copy_dir_recursively, git_command, remove_dir_if_exists, retry_spawn_and_wait, spawn_and_wait, }; -pub(crate) fn prepare(dirs: &Dirs, rustc: &Path) { +pub(crate) fn prepare(dirs: &Dirs) { RelPath::DOWNLOAD.ensure_exists(dirs); super::tests::RAND_REPO.fetch(dirs); super::tests::REGEX_REPO.fetch(dirs); super::tests::PORTABLE_SIMD_REPO.fetch(dirs); - - // FIXME do this on the fly? - prepare_stdlib(dirs, rustc); } -fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { +pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) { let sysroot_src_orig = get_default_sysroot(rustc).join("lib/rustlib/src/rust"); assert!(sysroot_src_orig.exists()); @@ -50,9 +47,6 @@ codegen-units = 10000 "#, ) .unwrap(); - - let rustc_version = get_rustc_version(rustc); - fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap(); } pub(crate) struct GitRepo { diff --git a/build_system/rustc_info.rs b/build_system/rustc_info.rs index 42cec0c6935..5b71504e90a 100644 --- a/build_system/rustc_info.rs +++ b/build_system/rustc_info.rs @@ -1,12 +1,6 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -pub(crate) fn get_rustc_version(rustc: &Path) -> String { - let version_info = - Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout; - String::from_utf8(version_info).unwrap() -} - pub(crate) fn get_host_triple(rustc: &Path) -> String { let version_info = Command::new(rustc).stderr(Stdio::inherit()).args(&["-vV"]).output().unwrap().stdout; From 8c1c84d79e26e5250d3ad1ee894843c01785787a Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 5 Jun 2023 13:59:08 +0000 Subject: [PATCH 17/20] Copy Cargo.lock over in ./y.sh prepare This makes it easier for ./x.py to vendor all dependencies --- build_system/prepare.rs | 21 +++++++++++++-------- build_system/tests.rs | 10 +++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/build_system/prepare.rs b/build_system/prepare.rs index 77f7175b786..e31e39a483f 100644 --- a/build_system/prepare.rs +++ b/build_system/prepare.rs @@ -47,6 +47,10 @@ codegen-units = 10000 "#, ) .unwrap(); + + let source_lockfile = RelPath::PATCHES.to_path(dirs).join("stdlib-lock.toml"); + let target_lockfile = STDLIB_SRC.to_path(dirs).join("Cargo.lock"); + fs::copy(source_lockfile, target_lockfile).unwrap(); } pub(crate) struct GitRepo { @@ -134,6 +138,15 @@ impl GitRepo { } } + let source_lockfile = + RelPath::PATCHES.to_path(dirs).join(format!("{}-lock.toml", self.patch_name)); + let target_lockfile = download_dir.join("Cargo.lock"); + if source_lockfile.exists() { + fs::copy(source_lockfile, target_lockfile).unwrap(); + } else { + assert!(target_lockfile.exists()); + } + let actual_hash = format!("{:016x}", hash_dir(&download_dir)); if actual_hash != self.content_hash { println!( @@ -285,12 +298,4 @@ pub(crate) fn apply_patches(dirs: &Dirs, crate_name: &str, source_dir: &Path, ta apply_patch_cmd.arg(patch).arg("-q"); spawn_and_wait(apply_patch_cmd); } - - let source_lockfile = RelPath::PATCHES.to_path(dirs).join(format!("{crate_name}-lock.toml")); - let target_lockfile = target_dir.join("Cargo.lock"); - if source_lockfile.exists() { - fs::copy(source_lockfile, target_lockfile).unwrap(); - } else { - assert!(target_lockfile.exists()); - } } diff --git a/build_system/tests.rs b/build_system/tests.rs index d104efd61e5..733a572400b 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -100,7 +100,7 @@ pub(crate) static RAND_REPO: GitRepo = GitRepo::github( "rust-random", "rand", "50b9a447410860af8d6db9a208c3576886955874", - "98b2276210b30e43", + "446203b96054891e", "rand", ); @@ -110,7 +110,7 @@ pub(crate) static REGEX_REPO: GitRepo = GitRepo::github( "rust-lang", "regex", "32fed9429eafba0ae92a64b01796a0c5a75b88c8", - "d6af6507d565aa66", + "fcc4df7c5b902633", "regex", ); @@ -120,7 +120,7 @@ pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github( "rust-lang", "portable-simd", "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b", - "1ba291009510070b", + "800548f8000e31bd", "portable-simd", ); @@ -156,6 +156,10 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[ &LIBCORE_TESTS_SRC.to_path(&runner.dirs), ); + let source_lockfile = RelPath::PATCHES.to_path(&runner.dirs).join("coretests-lock.toml"); + let target_lockfile = LIBCORE_TESTS_SRC.to_path(&runner.dirs).join("Cargo.lock"); + fs::copy(source_lockfile, target_lockfile).unwrap(); + LIBCORE_TESTS.clean(&runner.dirs); if runner.is_native { From a691b14aeeea6de133c23050e33732b5fe620a7b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 5 Jun 2023 15:15:48 +0000 Subject: [PATCH 18/20] Allow skipping tests from the commandline --- build_system/main.rs | 8 ++++++++ build_system/tests.rs | 27 +++++++++++++++++++-------- build_system/usage.txt | 5 ++++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/build_system/main.rs b/build_system/main.rs index d51e5027cf3..41d0d6f2319 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -90,6 +90,7 @@ fn main() { let mut sysroot_kind = SysrootKind::Clif; let mut use_unstable_features = true; let mut frozen = false; + let mut skip_tests = vec![]; let mut use_backend = None; while let Some(arg) = args.next().as_deref() { match arg { @@ -115,6 +116,12 @@ fn main() { } "--no-unstable-features" => use_unstable_features = false, "--frozen" => frozen = true, + "--skip-test" => { + // FIXME check that all passed in tests actually exist + skip_tests.push(args.next().unwrap_or_else(|| { + arg_error!("--skip-test requires argument"); + })); + } "--use-backend" => { use_backend = Some(match args.next() { Some(name) => name, @@ -218,6 +225,7 @@ fn main() { channel, sysroot_kind, use_unstable_features, + &skip_tests.iter().map(|test| &**test).collect::>(), &cg_clif_dylib, &bootstrap_host_compiler, rustup_toolchain_name.as_deref(), diff --git a/build_system/tests.rs b/build_system/tests.rs index 733a572400b..08d8f708c7d 100644 --- a/build_system/tests.rs +++ b/build_system/tests.rs @@ -19,7 +19,7 @@ struct TestCase { } enum TestCaseCmd { - Custom { func: &'static dyn Fn(&TestRunner) }, + Custom { func: &'static dyn Fn(&TestRunner<'_>) }, BuildLib { source: &'static str, crate_types: &'static str }, BuildBinAndRun { source: &'static str, args: &'static [&'static str] }, JitBin { source: &'static str, args: &'static str }, @@ -27,7 +27,7 @@ enum TestCaseCmd { impl TestCase { // FIXME reduce usage of custom test case commands - const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner)) -> Self { + const fn custom(config: &'static str, func: &'static dyn Fn(&TestRunner<'_>)) -> Self { Self { config, cmd: TestCaseCmd::Custom { func } } } @@ -247,6 +247,7 @@ pub(crate) fn run_tests( channel: &str, sysroot_kind: SysrootKind, use_unstable_features: bool, + skip_tests: &[&str], cg_clif_dylib: &CodegenBackend, bootstrap_host_compiler: &Compiler, rustup_toolchain_name: Option<&str>, @@ -256,7 +257,7 @@ pub(crate) fn run_tests( get_default_sysroot(&bootstrap_host_compiler.rustc).join("lib/rustlib/src/rust"); assert!(stdlib_source.exists()); - if config::get_bool("testsuite.no_sysroot") { + if config::get_bool("testsuite.no_sysroot") && !skip_tests.contains(&"testsuite.no_sysroot") { let target_compiler = build_sysroot::build_sysroot( dirs, channel, @@ -271,6 +272,7 @@ pub(crate) fn run_tests( dirs.clone(), target_compiler, use_unstable_features, + skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source.clone(), ); @@ -281,8 +283,10 @@ pub(crate) fn run_tests( eprintln!("[SKIP] no_sysroot tests"); } - let run_base_sysroot = config::get_bool("testsuite.base_sysroot"); - let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot"); + let run_base_sysroot = config::get_bool("testsuite.base_sysroot") + && !skip_tests.contains(&"testsuite.base_sysroot"); + let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot") + && !skip_tests.contains(&"testsuite.extended_sysroot"); if run_base_sysroot || run_extended_sysroot { let mut target_compiler = build_sysroot::build_sysroot( @@ -302,6 +306,7 @@ pub(crate) fn run_tests( dirs.clone(), target_compiler, use_unstable_features, + skip_tests, bootstrap_host_compiler.triple == target_triple, stdlib_source, ); @@ -320,20 +325,22 @@ pub(crate) fn run_tests( } } -struct TestRunner { +struct TestRunner<'a> { is_native: bool, jit_supported: bool, use_unstable_features: bool, + skip_tests: &'a [&'a str], dirs: Dirs, target_compiler: Compiler, stdlib_source: PathBuf, } -impl TestRunner { +impl<'a> TestRunner<'a> { fn new( dirs: Dirs, mut target_compiler: Compiler, use_unstable_features: bool, + skip_tests: &'a [&'a str], is_native: bool, stdlib_source: PathBuf, ) -> Self { @@ -360,6 +367,7 @@ impl TestRunner { is_native, jit_supported, use_unstable_features, + skip_tests, dirs, target_compiler, stdlib_source, @@ -372,7 +380,10 @@ impl TestRunner { let tag = tag.to_uppercase(); let is_jit_test = tag == "JIT"; - if !config::get_bool(config) || (is_jit_test && !self.jit_supported) { + if !config::get_bool(config) + || (is_jit_test && !self.jit_supported) + || self.skip_tests.contains(&config) + { eprintln!("[{tag}] {testname} (skipped)"); continue; } else { diff --git a/build_system/usage.txt b/build_system/usage.txt index 9d20cdca6a7..6d3b3a13d6e 100644 --- a/build_system/usage.txt +++ b/build_system/usage.txt @@ -3,7 +3,7 @@ The build system of cg_clif. USAGE: ./y.sh prepare [--out-dir DIR] [--download-dir DIR] ./y.sh build [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] - ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] + ./y.sh test [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] [--skip-test TESTNAME] ./y.sh abi-cafe [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] ./y.sh bench [--debug] [--sysroot none|clif|llvm] [--out-dir DIR] [--download-dir DIR] [--no-unstable-features] [--frozen] @@ -32,6 +32,9 @@ OPTIONS: --frozen Require Cargo.lock and cache are up to date + --skip-test TESTNAME + Skip testing the TESTNAME test. The test name format is the same as config.txt. + --use-backend NAME Use the existing Cranelift (or other) backend of the rustc with which we built. Warning: This is meant for use in rust's CI only! From 485d7e129f5beb67e7769014959e07aff5989d22 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 5 Jun 2023 19:06:15 +0000 Subject: [PATCH 19/20] Remove outdated fixme --- build_system/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/build_system/main.rs b/build_system/main.rs index 41d0d6f2319..3bc78d5db94 100644 --- a/build_system/main.rs +++ b/build_system/main.rs @@ -180,7 +180,6 @@ fn main() { .or_else(|| config::get_value("target")) .unwrap_or_else(|| bootstrap_host_compiler.triple.clone()); - // FIXME allow changing the location of these dirs using cli arguments let dirs = path::Dirs { source_dir: current_dir.clone(), download_dir: download_dir From e9bd63af3cfc192f0f998debbf5ab3e597781cec Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 15 Jun 2023 17:14:49 +0000 Subject: [PATCH 20/20] Ignore -Clink-arg=-import-instr-limit --- src/config.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/config.rs b/src/config.rs index 263401e1c4b..9e92d656c76 100644 --- a/src/config.rs +++ b/src/config.rs @@ -82,6 +82,11 @@ impl BackendConfig { let mut config = BackendConfig::default(); for opt in opts { + if opt.starts_with("-import-instr-limit") { + // Silently ignore -import-instr-limit. It is set by rust's build system even when + // testing cg_clif. + continue; + } if let Some((name, value)) = opt.split_once('=') { match name { "mode" => config.codegen_mode = value.parse()?,