Merge commit '1eded3619d' into sync_cg_clif-2023-07-22

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -12,9 +12,11 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Install rustfmt
+    - name: Avoid installing rustc-dev
       run: |
-        rustup component add rustfmt
+        sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain
+        echo 'profile = "minimal"' >> rust-toolchain
+        rustfmt -v
 
     - name: Rustfmt
       run: |
@@ -127,7 +129,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Prepare dependencies
-      run: ./y.rs prepare
+      run: ./y.sh prepare
 
     - name: Disable JIT tests
       run: |
@@ -136,7 +138,7 @@ jobs:
     - name: Test
       env:
         TARGET_TRIPLE: x86_64-unknown-linux-gnu
-      run: ./y.rs test --use-backend llvm
+      run: ./y.sh test --use-backend llvm
 
   bench:
     runs-on: ubuntu-latest

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -51,18 +51,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b6160c0a96253993b79fb7e0983534a4515ecf666120ddf8f92068114997ebc"
+checksum = "ec27af72e56235eb326b5bf2de4e70ab7c5ac1fb683a1829595badaf821607fd"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b38da5f63562e42f3c929d7c76871098e5ad12c8ab44b0659ffc529f22a5b3a"
+checksum = "2231e12925e6c5f4bc9c95b62a798eea6ed669a95bc3e00f8b2adb3b7b9b7a80"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -81,39 +81,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "011371e213e163b55dd9e8404b3f2d9fa52cd14dc2f3dc5b83e61ffceff126db"
+checksum = "413b00b8dfb3aab85674a534677e7ca08854b503f164a70ec0634fce80996e2c"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bf97dde7f5ad571161cdd203a2c9c88682ef669830aea3c14ea5d164ef8bb43"
+checksum = "cd0feb9ecc8193ef5cb04f494c5bd835e5bfec4bde726e7ac0444fc9dd76229e"
 
 [[package]]
 name = "cranelift-control"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd9a9254aee733b0f2b68e5eaaf0337ad53cb23252a056c10a35370551be8d40"
+checksum = "72eedd2afcf5fee1e042eaaf18d3750e48ad0eca364a9f5971ecfdd5ef85bf71"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf39a33ee39479d1337cd9333f3c09786c5a0ca1ec509edcaf9d1346d5de0e5"
+checksum = "7af19157be42671073cf8c2a52d6a4ae1e7b11f1dcb4131fede356d9f91c29dd"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65e260b92a193a0a2dccc3938f133d9532e7dcfe8d03e36bf8b7d3518c1c1793"
+checksum = "c2dc7636c5fad156be7d9ae691cd1aaecd97326caf2ab534ba168056d56aa76c"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -123,15 +123,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9446c8e1aadfcdacee1a49592bc2c25d1d9bf5484782c163e7f5485c92cd3c1c"
+checksum = "c1111aea4fb6fade5779903f184249a3fc685a799fe4ec59126f9af59c7c2a74"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "689a6df165d0f860c1e1a3d53c28944e2743c3e9ee4c678cf190fe60ad7a6ef5"
+checksum = "dadf88076317f6286ec77ebbe65978734fb43b6befdc96f52ff4c4c511841644"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -149,9 +149,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b1402d6ff1695b429536b2eaa126db560fc94c375ed0e9cfb15051fc07427f7"
+checksum = "c6bae8a82dbf82241b1083e57e06870d2c2bdc9852727be99d58477513816953"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -160,9 +160,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eac916f3c5aff4b817e42fc2e682292b931495b3fe2603d5e3c3cf602d74e344"
+checksum = "1ecfc01a634448468a698beac433d98040033046678a0eed3ca39a3a9f63ae86"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -171,9 +171,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.96.1"
+version = "0.98.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23860f4cd064017f2108e6bc5d25660a77cd6eea77f1ac0756870a00abb12e93"
+checksum = "0ee14a7276999f0dcaae2de84043e2c2de50820fb89b3db56fab586a4ad26734"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -193,6 +193,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1"
+
 [[package]]
 name = "fallible-iterator"
 version = "0.2.0"
@@ -206,7 +212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
 dependencies = [
  "fallible-iterator",
- "indexmap",
+ "indexmap 1.9.3",
  "stable_deref_trait",
 ]
 
@@ -225,6 +231,12 @@ dependencies = [
  "ahash",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -235,6 +247,16 @@ dependencies = [
  "hashbrown 0.12.3",
 ]
 
+[[package]]
+name = "indexmap"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.138"
@@ -283,7 +305,7 @@ checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385"
 dependencies = [
  "crc32fast",
  "hashbrown 0.13.2",
- "indexmap",
+ "indexmap 1.9.3",
  "memchr",
 ]
 
@@ -295,9 +317,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
 
 [[package]]
 name = "regalloc2"
-version = "0.8.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4a52e724646c6c0800fc456ec43b4165d2f91fba88ceaca06d9e0b400023478"
+checksum = "5b4dcbd3a2ae7fb94b5813fa0e957c6ab51bf5d0a8ee1b69e0c2d0f1e6eb8485"
 dependencies = [
  "hashbrown 0.13.2",
  "log",
@@ -335,7 +357,7 @@ dependencies = [
  "cranelift-native",
  "cranelift-object",
  "gimli",
- "indexmap",
+ "indexmap 2.0.0",
  "libloading",
  "object",
  "smallvec",
@@ -374,9 +396,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "9.0.1"
+version = "11.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d90933b781e1cef7656baed671c7a90bdba0c1c694e04fdd4124419308f5cbb"
+checksum = "e34eb67f0829a5614ec54716c8e0c9fe68fab7b9df3686c85f719c9d247f7169"
 dependencies = [
  "cfg-if",
  "libc",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -8,17 +8,17 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = {  version = "0.96.1", features = ["unwind", "all-arch"] }
+cranelift-codegen = { version = "0.98", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.96.1" }
+cranelift-frontend = { version = "0.98" }
-cranelift-module = { version = "0.96.1" }
+cranelift-module = { version = "0.98" }
-cranelift-native = { version = "0.96.1" }
+cranelift-native = { version = "0.98" }
-cranelift-jit = { version = "0.96.1", optional = true }
+cranelift-jit = { version = "0.98", optional = true }
-cranelift-object = { version = "0.96.1" }
+cranelift-object = { version = "0.98" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.27.2", default-features = false, features = ["write"]}
 object = { version = "0.30.3", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
-indexmap = "1.9.3"
+indexmap = "2.0.0"
 libloading = { version = "0.7.3", optional = true }
 smallvec = "1.8.1"
 

compiler/rustc_codegen_cranelift/Readme.md

@@ -65,12 +65,12 @@ to `./build/host/stage2/bin/`. Note that you would need to do this every time yo
 5. Copy cargo from another toolchain: `cp $(rustup which cargo) .build/<your hostname triple>/stage2/bin/cargo`
    * Another option is to build it at step 3 and copy with other executables at step 4.
 6. Link your new `rustc` to toolchain: `rustup toolchain link stage2 ./build/host/stage2/`.
-7. (Windows only) compile y.rs: `rustc +stage2 -O y.rs`.
+7. (Windows only) compile the build system: `rustc +stage2 -O build_system/main.rs -o y.exe`.
-8. You need to prefix every `./y.rs` (or `y` if you built `y.rs`) command by `rustup run stage2` to make cg_clif use your local changes in rustc.
+8. You need to prefix every `./y.sh` (or `y` if you built `build_system/main.rs` as `y`) command by `rustup run stage2` to make cg_clif use your local changes in rustc.
 
-  * `rustup run stage2 ./y.rs prepare`
+  * `rustup run stage2 ./y.sh prepare`
-  * `rustup run stage2 ./y.rs build`
+  * `rustup run stage2 ./y.sh build`
-  * (Optional) run tests: `rustup run stage2 ./y.rs test`
+  * (Optional) run tests: `rustup run stage2 ./y.sh test`
 9. Now you can use your cg_clif build to compile other Rust programs, e.g. you can open any Rust crate and run commands like `$RustCheckoutDir/compiler/rustc_codegen_cranelift/dist/cargo-clif build --release`.
 
 ## Configuration

compiler/rustc_codegen_cranelift/build_system/bench.rs

@@ -1,4 +1,5 @@
 use std::env;
+use std::io::Write;
 use std::path::Path;
 
 use super::path::{Dirs, RelPath};
@@ -30,6 +31,12 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
 
     let bench_runs = env::var("BENCH_RUNS").unwrap_or_else(|_| "10".to_string()).parse().unwrap();
 
+    let mut gha_step_summary = if let Ok(file) = std::env::var("GITHUB_STEP_SUMMARY") {
+        Some(std::fs::OpenOptions::new().append(true).open(file).unwrap())
+    } else {
+        None
+    };
+
     eprintln!("[BENCH COMPILE] ebobby/simple-raytracer");
     let cargo_clif = RelPath::DIST
         .to_path(dirs)
@@ -60,36 +67,64 @@ fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
         target_dir = target_dir.display(),
     );
 
+    let bench_compile_markdown = RelPath::DIST.to_path(dirs).join("bench_compile.md");
+
     let bench_compile = hyperfine_command(
         1,
         bench_runs,
         Some(&clean_cmd),
-        &[&llvm_build_cmd, &clif_build_cmd, &clif_build_opt_cmd],
+        &[
+            ("cargo build", &llvm_build_cmd),
+            ("cargo-clif build", &clif_build_cmd),
+            ("cargo-clif build --release", &clif_build_opt_cmd),
+        ],
+        &bench_compile_markdown,
     );
 
     spawn_and_wait(bench_compile);
 
+    if let Some(gha_step_summary) = gha_step_summary.as_mut() {
+        gha_step_summary.write_all(b"## Compile ebobby/simple-raytracer\n\n").unwrap();
+        gha_step_summary.write_all(&std::fs::read(bench_compile_markdown).unwrap()).unwrap();
+        gha_step_summary.write_all(b"\n").unwrap();
+    }
+
     eprintln!("[BENCH RUN] ebobby/simple-raytracer");
 
+    let bench_run_markdown = RelPath::DIST.to_path(dirs).join("bench_run.md");
+
+    let raytracer_cg_llvm = Path::new(".").join(get_file_name(
+        &bootstrap_host_compiler.rustc,
+        "raytracer_cg_llvm",
+        "bin",
+    ));
+    let raytracer_cg_clif = Path::new(".").join(get_file_name(
+        &bootstrap_host_compiler.rustc,
+        "raytracer_cg_clif",
+        "bin",
+    ));
+    let raytracer_cg_clif_opt = Path::new(".").join(get_file_name(
+        &bootstrap_host_compiler.rustc,
+        "raytracer_cg_clif_opt",
+        "bin",
+    ));
     let mut bench_run = hyperfine_command(
         0,
         bench_runs,
         None,
         &[
-            Path::new(".")
+            ("", raytracer_cg_llvm.to_str().unwrap()),
-                .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_llvm", "bin"))
+            ("", raytracer_cg_clif.to_str().unwrap()),
-                .to_str()
+            ("", raytracer_cg_clif_opt.to_str().unwrap()),
-                .unwrap(),
-            Path::new(".")
-                .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif", "bin"))
-                .to_str()
-                .unwrap(),
-            Path::new(".")
-                .join(get_file_name(&bootstrap_host_compiler.rustc, "raytracer_cg_clif_opt", "bin"))
-                .to_str()
-                .unwrap(),
         ],
+        &bench_run_markdown,
     );
     bench_run.current_dir(RelPath::BUILD.to_path(dirs));
     spawn_and_wait(bench_run);
+
+    if let Some(gha_step_summary) = gha_step_summary.as_mut() {
+        gha_step_summary.write_all(b"## Run ebobby/simple-raytracer\n\n").unwrap();
+        gha_step_summary.write_all(&std::fs::read(bench_run_markdown).unwrap()).unwrap();
+        gha_step_summary.write_all(b"\n").unwrap();
+    }
 }

compiler/rustc_codegen_cranelift/build_system/build_backend.rs

@@ -3,7 +3,7 @@ use std::path::PathBuf;
 
 use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
-use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler};
+use super::utils::{is_ci, is_ci_opt, maybe_incremental, CargoProject, Compiler, LogGroup};
 
 pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "cg_clif");
 
@@ -13,6 +13,8 @@ pub(crate) fn build_backend(
     bootstrap_host_compiler: &Compiler,
     use_unstable_features: bool,
 ) -> PathBuf {
+    let _group = LogGroup::guard("Build backend");
+
     let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
     maybe_incremental(&mut cmd);
 

compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs

@@ -6,6 +6,7 @@ use super::path::{Dirs, RelPath};
 use super::rustc_info::get_file_name;
 use super::utils::{
     maybe_incremental, remove_dir_if_exists, spawn_and_wait, try_hard_link, CargoProject, Compiler,
+    LogGroup,
 };
 use super::{CodegenBackend, SysrootKind};
 
@@ -22,6 +23,8 @@ pub(crate) fn build_sysroot(
     rustup_toolchain_name: Option<&str>,
     target_triple: String,
 ) -> Compiler {
+    let _guard = LogGroup::guard("Build sysroot");
+
     eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
 
     DIST_DIR.ensure_fresh(dirs);
@@ -251,7 +254,10 @@ fn build_clif_sysroot_for_triple(
     rustflags
         .push_str(&format!(" --sysroot {}", RTSTARTUP_SYSROOT.to_path(dirs).to_str().unwrap()));
     if channel == "release" {
-        rustflags.push_str(" -Zmir-opt-level=3");
+        // Incremental compilation by default disables mir inlining. This leads to both a decent
+        // compile perf and a significant runtime perf regression. As such forcefully enable mir
+        // inlining.
+        rustflags.push_str(" -Zinline-mir");
     }
     compiler.rustflags += &rustflags;
     let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -27,6 +27,7 @@ pub(crate) fn prepare_stdlib(dirs: &Dirs, rustc: &Path) {
         STDLIB_SRC.to_path(dirs).join("Cargo.toml"),
         r#"
 [workspace]
+resolver = "1"
 members = ["./library/sysroot"]
 
 [patch.crates-io]

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -3,7 +3,7 @@ use super::config;
 use super::path::{Dirs, RelPath};
 use super::prepare::{apply_patches, GitRepo};
 use super::rustc_info::get_default_sysroot;
-use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
+use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler, LogGroup};
 use super::{CodegenBackend, SysrootKind};
 use std::env;
 use std::ffi::OsStr;
@@ -21,6 +21,7 @@ struct TestCase {
 enum TestCaseCmd {
     Custom { func: &'static dyn Fn(&TestRunner<'_>) },
     BuildLib { source: &'static str, crate_types: &'static str },
+    BuildBin { source: &'static str },
     BuildBinAndRun { source: &'static str, args: &'static [&'static str] },
     JitBin { source: &'static str, args: &'static str },
 }
@@ -39,6 +40,10 @@ impl TestCase {
         Self { config, cmd: TestCaseCmd::BuildLib { source, crate_types } }
     }
 
+    const fn build_bin(config: &'static str, source: &'static str) -> Self {
+        Self { config, cmd: TestCaseCmd::BuildBin { source } }
+    }
+
     const fn build_bin_and_run(
         config: &'static str,
         source: &'static str,
@@ -92,6 +97,7 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::build_bin_and_run("aot.float-minmax-pass", "example/float-minmax-pass.rs", &[]),
     TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]),
     TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]),
+    TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"),
 ];
 
 // FIXME(rust-random/rand#1293): Newer rand versions fail to test on Windows. Update once this is
@@ -119,8 +125,8 @@ pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "ad8afa8c81273b3b49acbea38cd3bcf17a34cf2b",
+    "7c7dbe0c505ccbc02ff30c1e37381ab1d47bf46f",
-    "800548f8000e31bd",
+    "5bcc9c544f6fa7bd",
     "portable-simd",
 );
 
@@ -380,15 +386,17 @@ impl<'a> TestRunner<'a> {
             let tag = tag.to_uppercase();
             let is_jit_test = tag == "JIT";
 
-            if !config::get_bool(config)
+            let _guard = if !config::get_bool(config)
                 || (is_jit_test && !self.jit_supported)
                 || self.skip_tests.contains(&config)
             {
                 eprintln!("[{tag}] {testname} (skipped)");
                 continue;
             } else {
+                let guard = LogGroup::guard(&format!("[{tag}] {testname}"));
                 eprintln!("[{tag}] {testname}");
-            }
+                guard
+            };
 
             match *cmd {
                 TestCaseCmd::Custom { func } => func(self),
@@ -405,6 +413,13 @@ impl<'a> TestRunner<'a> {
                         ]);
                     }
                 }
+                TestCaseCmd::BuildBin { source } => {
+                    if self.use_unstable_features {
+                        self.run_rustc([source]);
+                    } else {
+                        self.run_rustc([source, "--cfg", "no_unstable_features"]);
+                    }
+                }
                 TestCaseCmd::BuildBinAndRun { source, args } => {
                     if self.use_unstable_features {
                         self.run_rustc([source]);

compiler/rustc_codegen_cranelift/build_system/utils.rs

@@ -3,6 +3,7 @@ use std::fs;
 use std::io::{self, Write};
 use std::path::{Path, PathBuf};
 use std::process::{self, Command, Stdio};
+use std::sync::atomic::{AtomicBool, Ordering};
 
 use super::path::{Dirs, RelPath};
 
@@ -136,10 +137,13 @@ pub(crate) fn hyperfine_command(
     warmup: u64,
     runs: u64,
     prepare: Option<&str>,
-    cmds: &[&str],
+    cmds: &[(&str, &str)],
+    markdown_export: &Path,
 ) -> Command {
     let mut bench = Command::new("hyperfine");
 
+    bench.arg("--export-markdown").arg(markdown_export);
+
     if warmup != 0 {
         bench.arg("--warmup").arg(warmup.to_string());
     }
@@ -152,7 +156,12 @@ pub(crate) fn hyperfine_command(
         bench.arg("--prepare").arg(prepare);
     }
 
-    bench.args(cmds);
+    for &(name, cmd) in cmds {
+        if name != "" {
+            bench.arg("-n").arg(name);
+        }
+        bench.arg(cmd);
+    }
 
     bench
 }
@@ -167,6 +176,8 @@ pub(crate) fn git_command<'a>(repo_dir: impl Into<Option<&'a Path>>, cmd: &str)
         .arg("user.email=dummy@example.com")
         .arg("-c")
         .arg("core.autocrlf=false")
+        .arg("-c")
+        .arg("commit.gpgSign=false")
         .arg(cmd);
     if let Some(repo_dir) = repo_dir.into() {
         git_cmd.current_dir(repo_dir);
@@ -259,6 +270,33 @@ pub(crate) fn is_ci_opt() -> bool {
     env::var("CI_OPT").is_ok()
 }
 
+static IN_GROUP: AtomicBool = AtomicBool::new(false);
+pub(crate) struct LogGroup {
+    is_gha: bool,
+}
+
+impl LogGroup {
+    pub(crate) fn guard(name: &str) -> LogGroup {
+        let is_gha = env::var("GITHUB_ACTIONS").is_ok();
+
+        assert!(!IN_GROUP.swap(true, Ordering::SeqCst));
+        if is_gha {
+            eprintln!("::group::{name}");
+        }
+
+        LogGroup { is_gha }
+    }
+}
+
+impl Drop for LogGroup {
+    fn drop(&mut self) {
+        if self.is_gha {
+            eprintln!("::endgroup::");
+        }
+        IN_GROUP.store(false, Ordering::SeqCst);
+    }
+}
+
 pub(crate) fn maybe_incremental(cmd: &mut Command) {
     if is_ci() || std::env::var("CARGO_BUILD_INCREMENTAL").map_or(false, |val| val == "false") {
         // Disabling incr comp reduces cache size and incr comp doesn't save as much on CI anyway

compiler/rustc_codegen_cranelift/config.txt

@@ -41,6 +41,7 @@ aot.track-caller-attribute
 aot.float-minmax-pass
 aot.mod_bench
 aot.issue-72793
+aot.issue-59326
 
 testsuite.extended_sysroot
 test.rust-random/rand

compiler/rustc_codegen_cranelift/example/issue-59326.rs

@@ -0,0 +1,27 @@
+// Based on https://github.com/rust-lang/rust/blob/689511047a75a30825e367d4fd45c74604d0b15e/tests/ui/issues/issue-59326.rs#L1
+// check-pass
+trait Service {
+    type S;
+}
+
+trait Framing {
+    type F;
+}
+
+impl Framing for () {
+    type F = ();
+}
+
+trait HttpService<F: Framing>: Service<S = F::F> {}
+
+type BoxService = Box<dyn HttpService<(), S = ()>>;
+
+fn build_server<F: FnOnce() -> BoxService>(_: F) {}
+
+fn make_server<F: Framing>() -> Box<dyn HttpService<F, S = F::F>> {
+    unimplemented!()
+}
+
+fn main() {
+    build_server(|| make_server())
+}

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -547,7 +547,9 @@ impl<T> Box<T> {
 impl<T: ?Sized, A> Drop for Box<T, A> {
     fn drop(&mut self) {
         // inner value is dropped by compiler
-        libc::free(self.0.pointer.0 as *mut u8);
+        unsafe {
+            libc::free(self.0.pointer.0 as *mut u8);
+        }
     }
 }
 

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -1,4 +1,12 @@
-#![feature(core_intrinsics, generators, generator_trait, is_sorted, repr_simd)]
+#![feature(
+    core_intrinsics,
+    generators,
+    generator_trait,
+    is_sorted,
+    repr_simd,
+    tuple_trait,
+    unboxed_closures
+)]
 
 #[cfg(target_arch = "x86_64")]
 use std::arch::x86_64::*;
@@ -155,12 +163,34 @@ fn main() {
     }
 
     foo(I64X2(0, 0));
+
+    transmute_fat_pointer();
+
+    rust_call_abi();
 }
 
 fn panic(_: u128) {
     panic!();
 }
 
+use std::mem::transmute;
+
+#[cfg(target_pointer_width = "32")]
+type TwoPtrs = i64;
+#[cfg(target_pointer_width = "64")]
+type TwoPtrs = i128;
+
+fn transmute_fat_pointer() -> TwoPtrs {
+    unsafe { transmute::<_, TwoPtrs>("true !") }
+}
+
+extern "rust-call" fn rust_call_abi_callee<T: std::marker::Tuple>(_: T) {}
+
+fn rust_call_abi() {
+    rust_call_abi_callee(());
+    rust_call_abi_callee((1, 2));
+}
+
 #[repr(simd)]
 struct I64X2(i64, i64);
 

compiler/rustc_codegen_cranelift/patches/0027-coretests-128bit-atomic-operations.patch

@@ -10,6 +10,18 @@ Cranelift doesn't support them yet
  library/core/tests/atomic.rs          |  4 ---
  4 files changed, 4 insertions(+), 50 deletions(-)
 
+diff --git a/lib.rs b/lib.rs
+index 897a5e9..331f66f 100644
+--- a/lib.rs
++++ b/lib.rs
+@@ -93,7 +93,6 @@
+ #![feature(const_option)]
+ #![feature(const_option_ext)]
+ #![feature(const_result)]
+-#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))]
+ #![feature(int_roundings)]
+ #![feature(slice_group_by)]
+ #![feature(split_array)]
 diff --git a/atomic.rs b/atomic.rs
 index b735957..ea728b6 100644
 --- a/atomic.rs

compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch

@@ -38,9 +38,9 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs
 index d9de37e..8293fce 100644
 --- a/library/core/src/sync/atomic.rs
 +++ b/library/core/src/sync/atomic.rs
-@@ -2234,46 +2234,6 @@ atomic_int! {
+@@ -2996,42 +2996,6 @@ atomic_int! {
-     "AtomicU64::new(0)",
+     8,
-     u64 AtomicU64 ATOMIC_U64_INIT
+     u64 AtomicU64
  }
 -#[cfg(target_has_atomic_load_store = "128")]
 -atomic_int! {
@@ -53,14 +53,12 @@ index d9de37e..8293fce 100644
 -    unstable(feature = "integer_atomics", issue = "99069"),
 -    unstable(feature = "integer_atomics", issue = "99069"),
 -    rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
--    unstable(feature = "integer_atomics", issue = "99069"),
 -    cfg_attr(not(test), rustc_diagnostic_item = "AtomicI128"),
 -    "i128",
 -    "#![feature(integer_atomics)]\n\n",
 -    atomic_min, atomic_max,
 -    16,
--    "AtomicI128::new(0)",
+-    i128 AtomicI128
--    i128 AtomicI128 ATOMIC_I128_INIT
 -}
 -#[cfg(target_has_atomic_load_store = "128")]
 -atomic_int! {
@@ -73,16 +71,15 @@ index d9de37e..8293fce 100644
 -    unstable(feature = "integer_atomics", issue = "99069"),
 -    unstable(feature = "integer_atomics", issue = "99069"),
 -    rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
--    unstable(feature = "integer_atomics", issue = "99069"),
 -    cfg_attr(not(test), rustc_diagnostic_item = "AtomicU128"),
 -    "u128",
 -    "#![feature(integer_atomics)]\n\n",
 -    atomic_umin, atomic_umax,
 -    16,
--    "AtomicU128::new(0)",
+-    u128 AtomicU128
--    u128 AtomicU128 ATOMIC_U128_INIT
 -}
  
+ #[cfg(target_has_atomic_load_store = "ptr")]
  macro_rules! atomic_int_ptr_sized {
      ( $($target_pointer_width:literal $align:literal)* ) => { $(
 --

compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml

@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.19.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
+checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3"
 dependencies = [
  "compiler_builtins",
  "gimli",
@@ -34,6 +34,12 @@ dependencies = [
  "rand_xorshift",
 ]
 
+[[package]]
+name = "allocator-api2"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9"
+
 [[package]]
 name = "auxv"
 version = "0.3.3"
@@ -68,9 +74,9 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.93"
+version = "0.1.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76630810d973ecea3dbf611e1b7aecfb1012751ef1ff8de3998f89014a166781"
+checksum = "6866e0f3638013234db3c89ead7a14d278354338e7237257407500009012b23f"
 dependencies = [
  "cc",
  "rustc-std-workspace-core",
@@ -145,10 +151,11 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.13.2"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
+checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
 dependencies = [
+ "allocator-api2",
  "compiler_builtins",
  "rustc-std-workspace-alloc",
  "rustc-std-workspace-core",
@@ -156,9 +163,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
+checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -186,9 +193,9 @@ dependencies = [
 
 [[package]]
 name = "miniz_oxide"
-version = "0.6.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa"
+checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
 dependencies = [
  "adler",
  "compiler_builtins",
@@ -198,9 +205,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.30.4"
+version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385"
+checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1"
 dependencies = [
  "compiler_builtins",
  "memchr",

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-06-15"
+channel = "nightly-2023-07-22"
 components = ["rust-src", "rustc-dev", "llvm-tools"]

compiler/rustc_codegen_cranelift/scripts/cargo-clif.rs

@@ -40,14 +40,22 @@ fn main() {
         "cargo"
     };
 
-    let args: Vec<_> = match env::args().nth(1).as_deref() {
+    let mut args = env::args().skip(1).collect::<Vec<_>>();
+    if args.get(0).map(|arg| &**arg) == Some("clif") {
+        // Avoid infinite recursion when invoking `cargo-clif` as cargo subcommand using
+        // `cargo clif`.
+        args.remove(0);
+    }
+
+    let args: Vec<_> = match args.get(0).map(|arg| &**arg) {
         Some("jit") => {
             env::set_var(
                 "RUSTFLAGS",
                 env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic",
             );
+            args.remove(0);
             IntoIterator::into_iter(["rustc".to_string()])
-                .chain(env::args().skip(2))
+                .chain(args)
                 .chain([
                     "--".to_string(),
                     "-Zunstable-options".to_string(),
@@ -60,8 +68,9 @@ fn main() {
                 "RUSTFLAGS",
                 env::var("RUSTFLAGS").unwrap_or(String::new()) + " -Cprefer-dynamic",
             );
+            args.remove(0);
             IntoIterator::into_iter(["rustc".to_string()])
-                .chain(env::args().skip(2))
+                .chain(args)
                 .chain([
                     "--".to_string(),
                     "-Zunstable-options".to_string(),
@@ -69,7 +78,7 @@ fn main() {
                 ])
                 .collect()
         }
-        _ => env::args().skip(1).collect(),
+        _ => args,
     };
 
     #[cfg(unix)]

compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh

@@ -10,7 +10,8 @@ git fetch
 git checkout -- .
 git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
 
-git -c user.name=Dummy -c user.email=dummy@example.com am ../patches/*-stdlib-*.patch
+git -c user.name=Dummy -c user.email=dummy@example.com -c commit.gpgSign=false \
+    am ../patches/*-stdlib-*.patch
 
 git apply - <<EOF
 diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
@@ -51,7 +52,7 @@ popd
 # FIXME remove once inline asm is fully supported
 export RUSTFLAGS="$RUSTFLAGS --cfg=rustix_use_libc"
 
-export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd download/sysroot/sysroot_src; pwd)"
+export CFG_VIRTUAL_RUST_SOURCE_BASE_DIR="$(cd build/stdlib; pwd)"
 
 # Allow the testsuite to use llvm tools
 host_triple=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -32,6 +32,8 @@ rm tests/ui/parser/unclosed-delimiter-in-dep.rs # submodule contains //~ERROR
 # missing features
 # ================
 
+rm -r tests/run-make/comment-section # cg_clif doesn't yet write the .comment section
+
 # requires stack unwinding
 # FIXME add needs-unwind to this test
 rm -r tests/run-make/libtest-junit
@@ -98,8 +100,11 @@ rm -r tests/run-make/sepcomp-inlining # same
 rm -r tests/run-make/sepcomp-separate # same
 rm -r tests/run-make/sepcomp-cci-copies # same
 rm -r tests/run-make/volatile-intrinsics # same
+rm -r tests/run-make/llvm-ident # same
+rm -r tests/run-make/no-builtins-attribute # same
 rm tests/ui/abi/stack-protector.rs # requires stack protector support
 rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes
+rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
 
 # giving different but possibly correct results
 # =============================================
@@ -118,6 +123,7 @@ rm tests/ui/suggestions/derive-trait-for-method-call.rs # same
 rm tests/ui/typeck/issue-46112.rs # same
 rm tests/ui/consts/const_cmp_type_id.rs # same
 rm tests/ui/consts/issue-73976-monomorphic.rs # same
+rm tests/ui/rfcs/rfc-3348-c-string-literals/non-ascii.rs # same
 
 # rustdoc-clif passes extra args, suppressing the help message when no args are passed
 rm -r tests/run-make/issue-88756-default-output
@@ -143,6 +149,8 @@ rm -r tests/run-make/used # same
 rm -r tests/run-make/no-alloc-shim
 rm -r tests/run-make/emit-to-stdout
 
+rm -r tests/run-make/extern-fn-explicit-align # argument alignment not yet supported
+
 # bugs in the test suite
 # ======================
 rm tests/ui/backtrace.rs # TODO warning
@@ -150,6 +158,8 @@ rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
 
 rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
 
+rm tests/ui/panic-handler/weak-lang-item-2.rs # Will be fixed by #113568
+
 cp ../dist/bin/rustdoc-clif ../dist/bin/rustdoc # some tests expect bin/rustdoc to exist
 
 # prevent $(RUSTDOC) from picking up the sysroot built by x.py. It conflicts with the one used by

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -445,9 +445,14 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
     // Unpack arguments tuple for closures
     let mut args = if fn_sig.abi() == Abi::RustCall {
-        assert_eq!(args.len(), 2, "rust-call abi requires two arguments");
+        let (self_arg, pack_arg) = match args {
-        let self_arg = codegen_call_argument_operand(fx, &args[0]);
+            [pack_arg] => (None, codegen_call_argument_operand(fx, pack_arg)),
-        let pack_arg = codegen_call_argument_operand(fx, &args[1]);
+            [self_arg, pack_arg] => (
+                Some(codegen_call_argument_operand(fx, self_arg)),
+                codegen_call_argument_operand(fx, pack_arg),
+            ),
+            _ => panic!("rust-call abi requires one or two arguments"),
+        };
 
         let tupled_arguments = match pack_arg.value.layout().ty.kind() {
             ty::Tuple(ref tupled_arguments) => tupled_arguments,
@@ -455,7 +460,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         };
 
         let mut args = Vec::with_capacity(1 + tupled_arguments.len());
-        args.push(self_arg);
+        args.extend(self_arg);
         for i in 0..tupled_arguments.len() {
             args.push(CallArgument {
                 value: pack_arg.value.value_field(fx, FieldIdx::new(i)),

compiler/rustc_codegen_cranelift/src/base.rs

@@ -7,6 +7,8 @@ use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 
 use cranelift_codegen::ir::UserFuncName;
+use cranelift_codegen::CodegenError;
+use cranelift_module::ModuleError;
 
 use crate::constant::ConstantCx;
 use crate::debuginfo::FunctionDebugContext;
@@ -172,7 +174,21 @@ pub(crate) fn compile_fn(
     // Define function
     cx.profiler.generic_activity("define function").run(|| {
         context.want_disasm = cx.should_write_ir;
-        module.define_function(codegened_func.func_id, context).unwrap();
+        match module.define_function(codegened_func.func_id, context) {
+            Ok(()) => {}
+            Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => {
+                let handler = rustc_session::EarlyErrorHandler::new(
+                    rustc_session::config::ErrorOutputType::default(),
+                );
+                handler.early_error(format!(
+                    "backend implementation limit exceeded while compiling {name}",
+                    name = codegened_func.symbol_name
+                ));
+            }
+            Err(err) => {
+                panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name);
+            }
+        }
     });
 
     if cx.should_write_ir {
@@ -356,7 +372,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
 
                         codegen_panic_inner(
                             fx,
-                            rustc_hir::LangItem::PanicBoundsCheck,
+                            rustc_hir::LangItem::PanicMisalignedPointerDereference,
                             &[required, found, location],
                             source_info.span,
                         );

compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs

@@ -165,7 +165,7 @@ impl FunctionDebugContext {
         for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
             debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start);
             if !loc.is_default() {
-                let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap();
+                let source_loc = self.source_loc_set[loc.bits() as usize];
                 create_row_for_span(debug_context, source_loc);
             } else {
                 create_row_for_span(debug_context, self.function_source_loc);

compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs

@@ -38,7 +38,7 @@ pub(crate) struct DebugContext {
 pub(crate) struct FunctionDebugContext {
     entry_id: UnitEntryId,
     function_source_loc: (FileId, u64, u64),
-    source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>,
+    source_loc_set: IndexSet<(FileId, u64, u64)>,
 }
 
 impl DebugContext {

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -114,9 +114,9 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         .iter()
         .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
         .flatten()
-        .collect::<FxHashMap<_, (_, _)>>()
+        .collect::<FxHashMap<_, _>>()
         .into_iter()
-        .collect::<Vec<(_, (_, _))>>();
+        .collect::<Vec<(_, _)>>();
 
     tcx.sess.time("codegen mono items", || {
         super::predefine_mono_items(tcx, &mut jit_module, &mono_items);

compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs

@@ -18,6 +18,20 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             // Spin loop hint
         }
 
+        // Used by is_x86_feature_detected!();
+        "llvm.x86.xgetbv" => {
+            // FIXME use the actual xgetbv instruction
+            intrinsic_args!(fx, args => (v); intrinsic);
+
+            let v = v.load_scalar(fx);
+
+            // As of writing on XCR0 exists
+            fx.bcx.ins().trapnz(v, TrapCode::UnreachableCodeReached);
+
+            let res = fx.bcx.ins().iconst(types::I64, 1 /* bit 0 must be set */);
+            ret.write_cvalue(fx, CValue::by_val(res, fx.layout_of(fx.tcx.types.i64)));
+        }
+
         // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
         "llvm.x86.sse2.pmovmskb.128"
         | "llvm.x86.avx2.pmovmskb"
@@ -53,7 +67,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
             ret.write_cvalue(fx, res);
         }
-        "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
+        "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),
                 _ => bug!("wrong number of args for intrinsic {intrinsic}"),
@@ -66,18 +80,95 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let flt_cc = match kind
                 .try_to_bits(Size::from_bytes(1))
                 .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind))
+                .try_into()
+                .unwrap()
             {
-                0 => FloatCC::Equal,
+                _CMP_EQ_OQ | _CMP_EQ_OS => FloatCC::Equal,
-                1 => FloatCC::LessThan,
+                _CMP_LT_OS | _CMP_LT_OQ => FloatCC::LessThan,
-                2 => FloatCC::LessThanOrEqual,
+                _CMP_LE_OS | _CMP_LE_OQ => FloatCC::LessThanOrEqual,
-                7 => FloatCC::Ordered,
+                _CMP_UNORD_Q | _CMP_UNORD_S => FloatCC::Unordered,
-                3 => FloatCC::Unordered,
+                _CMP_NEQ_UQ | _CMP_NEQ_US => FloatCC::NotEqual,
-                4 => FloatCC::NotEqual,
+                _CMP_NLT_US | _CMP_NLT_UQ => FloatCC::UnorderedOrGreaterThanOrEqual,
-                5 => FloatCC::UnorderedOrGreaterThanOrEqual,
+                _CMP_NLE_US | _CMP_NLE_UQ => FloatCC::UnorderedOrGreaterThan,
-                6 => FloatCC::UnorderedOrGreaterThan,
+                _CMP_ORD_Q | _CMP_ORD_S => FloatCC::Ordered,
+                _CMP_EQ_UQ | _CMP_EQ_US => FloatCC::UnorderedOrEqual,
+                _CMP_NGE_US | _CMP_NGE_UQ => FloatCC::UnorderedOrLessThan,
+                _CMP_NGT_US | _CMP_NGT_UQ => FloatCC::UnorderedOrLessThanOrEqual,
+                _CMP_FALSE_OQ | _CMP_FALSE_OS => todo!(),
+                _CMP_NEQ_OQ | _CMP_NEQ_OS => FloatCC::OrderedNotEqual,
+                _CMP_GE_OS | _CMP_GE_OQ => FloatCC::GreaterThanOrEqual,
+                _CMP_GT_OS | _CMP_GT_OQ => FloatCC::GreaterThan,
+                _CMP_TRUE_UQ | _CMP_TRUE_US => todo!(),
+
                 kind => unreachable!("kind {:?}", kind),
             };
 
+            // Copied from stdarch
+            /// Equal (ordered, non-signaling)
+            const _CMP_EQ_OQ: i32 = 0x00;
+            /// Less-than (ordered, signaling)
+            const _CMP_LT_OS: i32 = 0x01;
+            /// Less-than-or-equal (ordered, signaling)
+            const _CMP_LE_OS: i32 = 0x02;
+            /// Unordered (non-signaling)
+            const _CMP_UNORD_Q: i32 = 0x03;
+            /// Not-equal (unordered, non-signaling)
+            const _CMP_NEQ_UQ: i32 = 0x04;
+            /// Not-less-than (unordered, signaling)
+            const _CMP_NLT_US: i32 = 0x05;
+            /// Not-less-than-or-equal (unordered, signaling)
+            const _CMP_NLE_US: i32 = 0x06;
+            /// Ordered (non-signaling)
+            const _CMP_ORD_Q: i32 = 0x07;
+            /// Equal (unordered, non-signaling)
+            const _CMP_EQ_UQ: i32 = 0x08;
+            /// Not-greater-than-or-equal (unordered, signaling)
+            const _CMP_NGE_US: i32 = 0x09;
+            /// Not-greater-than (unordered, signaling)
+            const _CMP_NGT_US: i32 = 0x0a;
+            /// False (ordered, non-signaling)
+            const _CMP_FALSE_OQ: i32 = 0x0b;
+            /// Not-equal (ordered, non-signaling)
+            const _CMP_NEQ_OQ: i32 = 0x0c;
+            /// Greater-than-or-equal (ordered, signaling)
+            const _CMP_GE_OS: i32 = 0x0d;
+            /// Greater-than (ordered, signaling)
+            const _CMP_GT_OS: i32 = 0x0e;
+            /// True (unordered, non-signaling)
+            const _CMP_TRUE_UQ: i32 = 0x0f;
+            /// Equal (ordered, signaling)
+            const _CMP_EQ_OS: i32 = 0x10;
+            /// Less-than (ordered, non-signaling)
+            const _CMP_LT_OQ: i32 = 0x11;
+            /// Less-than-or-equal (ordered, non-signaling)
+            const _CMP_LE_OQ: i32 = 0x12;
+            /// Unordered (signaling)
+            const _CMP_UNORD_S: i32 = 0x13;
+            /// Not-equal (unordered, signaling)
+            const _CMP_NEQ_US: i32 = 0x14;
+            /// Not-less-than (unordered, non-signaling)
+            const _CMP_NLT_UQ: i32 = 0x15;
+            /// Not-less-than-or-equal (unordered, non-signaling)
+            const _CMP_NLE_UQ: i32 = 0x16;
+            /// Ordered (signaling)
+            const _CMP_ORD_S: i32 = 0x17;
+            /// Equal (unordered, signaling)
+            const _CMP_EQ_US: i32 = 0x18;
+            /// Not-greater-than-or-equal (unordered, non-signaling)
+            const _CMP_NGE_UQ: i32 = 0x19;
+            /// Not-greater-than (unordered, non-signaling)
+            const _CMP_NGT_UQ: i32 = 0x1a;
+            /// False (ordered, signaling)
+            const _CMP_FALSE_OS: i32 = 0x1b;
+            /// Not-equal (ordered, signaling)
+            const _CMP_NEQ_OS: i32 = 0x1c;
+            /// Greater-than-or-equal (ordered, non-signaling)
+            const _CMP_GE_OQ: i32 = 0x1d;
+            /// Greater-than (ordered, non-signaling)
+            const _CMP_GT_OQ: i32 = 0x1e;
+            /// True (unordered, signaling)
+            const _CMP_TRUE_US: i32 = 0x1f;
+
             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
                 let res_lane = match lane_ty.kind() {
                     ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
@@ -103,6 +194,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.sse2.psrai.d" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.sse2.psrai.d imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.sse2.pslli.d" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -137,6 +245,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.sse2.psrai.w" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.sse2.psrai.d imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.sse2.pslli.w" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -171,6 +296,57 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.avx.psrai.d" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.psrai.d imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
+        "llvm.x86.sse2.psrli.q" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.psrli.q imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 64 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
+        "llvm.x86.sse2.pslli.q" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.pslli.q imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 64 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.avx.pslli.d" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -205,6 +381,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.avx2.psrai.w" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.psrai.w imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.avx2.pslli.w" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -313,7 +506,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             ret.place_lane(fx, 2).to_ptr().store(fx, res_2, MemFlags::trusted());
             ret.place_lane(fx, 3).to_ptr().store(fx, res_3, MemFlags::trusted());
         }
-        "llvm.x86.sse2.storeu.dq" => {
+        "llvm.x86.sse2.storeu.dq" | "llvm.x86.sse2.storeu.pd" => {
             intrinsic_args!(fx, args => (mem_addr, a); intrinsic);
             let mem_addr = mem_addr.load_scalar(fx);
 
@@ -321,17 +514,45 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
             dest.write_cvalue(fx, a);
         }
-        "llvm.x86.addcarry.64" => {
+        "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => {
+            let a = match args {
+                [a] => a,
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+                fx.bcx.ins().iabs(lane)
+            });
+        }
+        "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => {
             intrinsic_args!(fx, args => (c_in, a, b); intrinsic);
             let c_in = c_in.load_scalar(fx);
 
-            llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b);
+            let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b);
+
+            let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty]));
+            let val = CValue::by_val_pair(cb_out, c, layout);
+            ret.write_cvalue(fx, val);
         }
-        "llvm.x86.subborrow.64" => {
+        "llvm.x86.addcarryx.u32" | "llvm.x86.addcarryx.u64" => {
+            intrinsic_args!(fx, args => (c_in, a, b, out); intrinsic);
+            let c_in = c_in.load_scalar(fx);
+
+            let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b);
+
+            Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted());
+            ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8)));
+        }
+        "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => {
             intrinsic_args!(fx, args => (b_in, a, b); intrinsic);
             let b_in = b_in.load_scalar(fx);
 
-            llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b);
+            let (cb_out, c) = llvm_add_sub(fx, BinOp::Sub, b_in, a, b);
+
+            let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty]));
+            let val = CValue::by_val_pair(cb_out, c, layout);
+            ret.write_cvalue(fx, val);
         }
         _ => {
             fx.tcx
@@ -356,21 +577,11 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
 fn llvm_add_sub<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
-    ret: CPlace<'tcx>,
     cb_in: Value,
     a: CValue<'tcx>,
     b: CValue<'tcx>,
-) {
+) -> (Value, Value) {
-    assert_eq!(
+    assert_eq!(a.layout().ty, b.layout().ty);
-        a.layout().ty,
-        fx.tcx.types.u64,
-        "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
-    );
-    assert_eq!(
-        b.layout().ty,
-        fx.tcx.types.u64,
-        "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
-    );
 
     // c + carry -> c + first intermediate carry or borrow respectively
     let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
@@ -378,15 +589,14 @@ fn llvm_add_sub<'tcx>(
     let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx);
 
     // c + carry -> c + second intermediate carry or borrow respectively
-    let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
+    let clif_ty = fx.clif_type(a.layout().ty).unwrap();
-    let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
+    let cb_in_as_int = fx.bcx.ins().uextend(clif_ty, cb_in);
-    let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
+    let cb_in_as_int = CValue::by_val(cb_in_as_int, fx.layout_of(a.layout().ty));
+    let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_int);
     let (c, cb1) = int1.load_scalar_pair(fx);
 
     // carry0 | carry1 -> carry or borrow respectively
     let cb_out = fx.bcx.ins().bor(cb0, cb1);
 
-    let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, fx.tcx.types.u64]));
+    (cb_out, c)
-    let val = CValue::by_val_pair(cb_out, c, layout);
-    ret.write_cvalue(fx, val);
 }

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -647,12 +647,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
             ret.write_cvalue(fx, val);
         }
-        sym::volatile_store | sym::unaligned_volatile_store => {
+        sym::volatile_store | sym::unaligned_volatile_store | sym::nontemporal_store => {
             intrinsic_args!(fx, args => (ptr, val); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
             // Cranelift treats stores as volatile by default
             // FIXME correctly handle unaligned_volatile_store
+            // FIXME actually do nontemporal stores if requested
             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
             dest.write_cvalue(fx, val);
         }

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -268,8 +268,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar
     };
     flags_builder.set("tls_model", tls_model).unwrap();
 
-    flags_builder.set("enable_simd", "true").unwrap();
-
     flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
 
     use rustc_session::config::OptLevel;

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -2,6 +2,8 @@
 
 use crate::prelude::*;
 
+use rustc_middle::ty::FnSig;
+
 use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
@@ -160,6 +162,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value with layout.abi of scalar
+    #[track_caller]
     pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value {
         let layout = self.1;
         match self.0 {
@@ -182,6 +185,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value pair with layout.abi of scalar pair
+    #[track_caller]
     pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) {
         let layout = self.1;
         match self.0 {
@@ -583,17 +587,25 @@ impl<'tcx> CPlace<'tcx> {
         let dst_layout = self.layout();
         match self.inner {
             CPlaceInner::Var(_local, var) => {
-                let data = CValue(from.0, dst_layout).load_scalar(fx);
+                let data = match from.1.abi {
+                    Abi::Scalar(_) => CValue(from.0, dst_layout).load_scalar(fx),
+                    _ => {
+                        let (ptr, meta) = from.force_stack(fx);
+                        assert!(meta.is_none());
+                        CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar(fx)
+                    }
+                };
                 let dst_ty = fx.clif_type(self.layout().ty).unwrap();
                 transmute_scalar(fx, var, data, dst_ty);
             }
             CPlaceInner::VarPair(_local, var1, var2) => {
-                let (data1, data2) = if from.layout().ty == dst_layout.ty {
+                let (data1, data2) = match from.1.abi {
-                    CValue(from.0, dst_layout).load_scalar_pair(fx)
+                    Abi::ScalarPair(_, _) => CValue(from.0, dst_layout).load_scalar_pair(fx),
-                } else {
+                    _ => {
-                    let (ptr, meta) = from.force_stack(fx);
+                        let (ptr, meta) = from.force_stack(fx);
-                    assert!(meta.is_none());
+                        assert!(meta.is_none());
-                    CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
+                        CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
+                    }
                 };
                 let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap();
                 transmute_scalar(fx, var1, data1, dst_ty1);
@@ -607,30 +619,38 @@ impl<'tcx> CPlace<'tcx> {
 
                 let mut flags = MemFlags::new();
                 flags.set_notrap();
-                match from.layout().abi {
-                    Abi::Scalar(_) => {
-                        let val = from.load_scalar(fx);
-                        to_ptr.store(fx, val, flags);
-                        return;
-                    }
-                    Abi::ScalarPair(a_scalar, b_scalar) => {
-                        let (value, extra) = from.load_scalar_pair(fx);
-                        let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
-                        to_ptr.store(fx, value, flags);
-                        to_ptr.offset(fx, b_offset).store(fx, extra, flags);
-                        return;
-                    }
-                    _ => {}
-                }
 
                 match from.0 {
                     CValueInner::ByVal(val) => {
                         to_ptr.store(fx, val, flags);
                     }
-                    CValueInner::ByValPair(_, _) => {
+                    CValueInner::ByValPair(val1, val2) => match from.layout().abi {
-                        bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
+                        Abi::ScalarPair(a_scalar, b_scalar) => {
-                    }
+                            let b_offset =
+                                scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+                            to_ptr.store(fx, val1, flags);
+                            to_ptr.offset(fx, b_offset).store(fx, val2, flags);
+                        }
+                        _ => bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi),
+                    },
                     CValueInner::ByRef(from_ptr, None) => {
+                        match from.layout().abi {
+                            Abi::Scalar(_) => {
+                                let val = from.load_scalar(fx);
+                                to_ptr.store(fx, val, flags);
+                                return;
+                            }
+                            Abi::ScalarPair(a_scalar, b_scalar) => {
+                                let b_offset =
+                                    scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+                                let (val1, val2) = from.load_scalar_pair(fx);
+                                to_ptr.store(fx, val1, flags);
+                                to_ptr.offset(fx, b_offset).store(fx, val2, flags);
+                                return;
+                            }
+                            _ => {}
+                        }
+
                         let from_addr = from_ptr.get_addr(fx);
                         let to_addr = to_ptr.get_addr(fx);
                         let src_layout = from.1;
@@ -815,11 +835,42 @@ pub(crate) fn assert_assignable<'tcx>(
                 ParamEnv::reveal_all(),
                 from_ty.fn_sig(fx.tcx),
             );
+            let FnSig {
+                inputs_and_output: types_from,
+                c_variadic: c_variadic_from,
+                unsafety: unsafety_from,
+                abi: abi_from,
+            } = from_sig;
             let to_sig = fx
                 .tcx
                 .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx));
+            let FnSig {
+                inputs_and_output: types_to,
+                c_variadic: c_variadic_to,
+                unsafety: unsafety_to,
+                abi: abi_to,
+            } = to_sig;
+            let mut types_from = types_from.iter();
+            let mut types_to = types_to.iter();
+            loop {
+                match (types_from.next(), types_to.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+                    (None, None) => break,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
             assert_eq!(
-                from_sig, to_sig,
+                c_variadic_from, c_variadic_to,
+                "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
+                from_sig, to_sig, fx,
+            );
+            assert_eq!(
+                unsafety_from, unsafety_to,
+                "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
+                from_sig, to_sig, fx,
+            );
+            assert_eq!(
+                abi_from, abi_to,
                 "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
                 from_sig, to_sig, fx,
             );