Merge commit '9a0c32934e' into sync_cg_clif-2021-03-05

compiler/rustc_codegen_cranelift/.cirrus.yml

@@ -0,0 +1,25 @@
+task:
+  name: freebsd
+  freebsd_instance:
+    image: freebsd-12-1-release-amd64
+  setup_rust_script:
+    - pkg install -y curl git bash
+    - curl https://sh.rustup.rs -sSf --output rustup.sh
+    - sh rustup.sh --default-toolchain none -y --profile=minimal
+  cargo_bin_cache:
+    folder: ~/.cargo/bin
+  target_cache:
+    folder: target
+  prepare_script:
+    - . $HOME/.cargo/env
+    - git config --global user.email "user@example.com"
+    - git config --global user.name "User"
+    - ./prepare.sh
+  test_script:
+    - . $HOME/.cargo/env
+    - # Enable backtraces for easier debugging
+    - export RUST_BACKTRACE=1
+    - # Reduce amount of benchmark runs as they are slow
+    - export COMPILE_RUNS=2
+    - export RUN_RUNS=2
+    - ./test.sh

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -12,9 +12,6 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        env:
-          - BACKEND: ""
-          - BACKEND: --oldbe
 
     steps:
     - uses: actions/checkout@v2
@@ -54,7 +51,7 @@ jobs:
         export COMPILE_RUNS=2
         export RUN_RUNS=2
 
-        ./test.sh $BACKEND
+        ./test.sh
 
     - name: Package prebuilt cg_clif
       run: tar cvfJ cg_clif.tar.xz build

compiler/rustc_codegen_cranelift/.vscode/settings.json

@@ -1,6 +1,6 @@
 {
     // source for rustc_* is not included in the rust-src component; disable the errors about this
-    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"],
+    "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"],
     "rust-analyzer.assist.importMergeBehavior": "last",
     "rust-analyzer.cargo.loadOutDirsFromCheck": true,
     "rust-analyzer.linkedProjects": [

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "anyhow"
 version = "1.0.38"
@@ -29,18 +31,6 @@ version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b"
 
-[[package]]
-name = "cc"
-version = "1.0.66"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
-
-[[package]]
-name = "cfg-if"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -49,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -75,8 +65,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -84,18 +74,18 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 
 [[package]]
 name = "cranelift-entity"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -105,8 +95,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-jit"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -123,8 +113,8 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -135,18 +125,17 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "cranelift-codegen",
- "raw-cpuid",
  "target-lexicon",
 ]
 
 [[package]]
 name = "cranelift-object"
-version = "0.69.0"
+version = "0.70.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#986b5768f9e68f1564b43f32b8a4080a6582c8ca"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -162,7 +151,7 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
 ]
 
 [[package]]
@@ -219,9 +208,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.82"
+version = "0.2.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
+checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
 
 [[package]]
 name = "libloading"
@@ -229,17 +218,17 @@ version = "0.6.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "351a32417a12d5f7e82c368a66781e307834dae04c6ce0cd4456d52989229883"
 dependencies = [
- "cfg-if 1.0.0",
+ "cfg-if",
  "winapi",
 ]
 
 [[package]]
 name = "log"
-version = "0.4.13"
+version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if",
 ]
 
 [[package]]
@@ -253,9 +242,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.22.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
+checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4"
 dependencies = [
  "crc32fast",
  "indexmap",
@@ -272,24 +261,13 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.8"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
 dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "raw-cpuid"
-version = "8.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fdf7d9dbd43f3d81d94a49c1c3df73cc2b3827995147e6cf7f89d4ec5483e73"
-dependencies = [
- "bitflags",
- "cc",
- "rustc_version",
-]
-
 [[package]]
 name = "regalloc"
 version = "0.0.31"
@@ -337,30 +315,6 @@ dependencies = [
  "target-lexicon",
 ]
 
-[[package]]
-name = "rustc_version"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "semver"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
-dependencies = [
- "semver-parser",
-]
-
-[[package]]
-name = "semver-parser"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-
 [[package]]
 name = "smallvec"
 version = "1.6.1"
@@ -369,9 +323,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
 [[package]]
 name = "syn"
-version = "1.0.58"
+version = "1.0.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5"
+checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -380,24 +334,24 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.11.1"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9"
+checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95"
 
 [[package]]
 name = "thiserror"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146"
+checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.23"
+version = "1.0.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1"
+checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
 dependencies = [
  "proc-macro2",
  "quote",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -9,14 +9,14 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x86", "x64"] }
+cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] }
 cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
-object = { version = "0.22.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
+object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] }
 
 ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" }
 indexmap = "1.0.2"
@@ -38,7 +38,6 @@ smallvec = "1.6.1"
 default = ["jit", "inline_asm"]
 jit = ["cranelift-jit", "libloading"]
 inline_asm = []
-oldbe = []
 
 [profile.dev]
 # By compiling dependencies with optimizations, performing tests gets much faster.

compiler/rustc_codegen_cranelift/build.sh

@@ -1,11 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 # Settings
 export CHANNEL="release"
 build_sysroot="clif"
 target_dir='build'
-oldbe=''
 while [[ $# != 0 ]]; do
     case $1 in
         "--debug")
@@ -19,12 +18,9 @@ while [[ $# != 0 ]]; do
             target_dir=$2
             shift
             ;;
-        "--oldbe")
-            oldbe='--features oldbe'
-            ;;
         *)
             echo "Unknown flag '$1'"
-            echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR] [--oldbe]"
+            echo "Usage: ./build.sh [--debug] [--sysroot none|clif|llvm] [--target-dir DIR]"
             exit 1
             ;;
     esac
@@ -34,19 +30,19 @@ done
 # Build cg_clif
 unset CARGO_TARGET_DIR
 unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' ]]; then
+if [[ "$unamestr" == 'Linux' || "$unamestr" == "FreeBSD" ]]; then
    export RUSTFLAGS='-Clink-arg=-Wl,-rpath=$ORIGIN/../lib '$RUSTFLAGS
 elif [[ "$unamestr" == 'Darwin' ]]; then
    export RUSTFLAGS='-Csplit-debuginfo=unpacked -Clink-arg=-Wl,-rpath,@loader_path/../lib -Zosx-rpath-install-name '$RUSTFLAGS
    dylib_ext='dylib'
 else
-   echo "Unsupported os"
+   echo "Unsupported os $unamestr"
    exit 1
 fi
 if [[ "$CHANNEL" == "release" ]]; then
-    cargo build $oldbe --release
+    cargo build --release
 else
-    cargo build $oldbe
+    cargo build
 fi
 
 source scripts/ext_config.sh

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "addr2line"
 version = "0.14.1"
@@ -30,15 +32,6 @@ dependencies = [
  "core",
 ]
 
-[[package]]
-name = "alloc_system"
-version = "0.0.0"
-dependencies = [
- "compiler_builtins",
- "core",
- "libc",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.0.1"
@@ -47,9 +40,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "cc"
-version = "1.0.66"
+version = "1.0.67"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
+checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
 
 [[package]]
 name = "cfg-if"
@@ -139,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.84"
+version = "0.2.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
+checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -258,7 +251,6 @@ name = "sysroot"
 version = "0.0.0"
 dependencies = [
  "alloc",
- "alloc_system",
  "compiler_builtins",
  "core",
  "std",

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml

@@ -9,8 +9,6 @@ alloc = { path = "./sysroot_src/library/alloc" }
 std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
 test = { path = "./sysroot_src/library/test" }
 
-alloc_system = { path = "./alloc_system" }
-
 compiler_builtins = { version = "0.1.39", default-features = false, features = ["no-asm"] }
 
 [patch.crates-io]

compiler/rustc_codegen_cranelift/build_sysroot/alloc_system/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-authors = ["The Rust Project Developers", "bjorn3 (edited to be usable outside the rust source)"]
-name = "alloc_system"
-version = "0.0.0"
-[lib]
-name = "alloc_system"
-path = "lib.rs"
-test = false
-doc = false
-[dependencies]
-core = { path = "../sysroot_src/library/core" }
-libc = { version = "0.2.43", features = ['rustc-dep-of-std'], default-features = false }
-compiler_builtins = "0.1"

compiler/rustc_codegen_cranelift/build_sysroot/build_sysroot.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Requires the CHANNEL env var to be set to `debug` or `release.`
 

compiler/rustc_codegen_cranelift/build_sysroot/prepare_sysroot_src.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 cd "$(dirname "$0")"
 
@@ -33,7 +33,7 @@ git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/
 pushd compiler-builtins
 git checkout -- .
 git checkout 0.1.39
-git apply ../../crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch
+git apply ../../crate_patches/000*-compiler-builtins-*.patch
 popd
 
 echo "Successfully prepared sysroot source for building"

compiler/rustc_codegen_cranelift/clean_all.sh

@@ -1,4 +1,4 @@
-#!/bin/bash --verbose
+#!/usr/bin/env bash
 set -e
 
 rm -rf target/ build/ build_sysroot/{sysroot_src/,target/,compiler-builtins/} perf.data{,.old}

compiler/rustc_codegen_cranelift/crate_patches/0002-compiler-builtins-Disable-128bit-atomic-operations.patch

@@ -0,0 +1,48 @@
+From 1d574bf5e32d51641dcacaf8ef777e95b44f6f2a Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 18 Feb 2021 18:30:55 +0100
+Subject: [PATCH] Disable 128bit atomic operations
+
+Cranelift doesn't support them yet
+---
+ src/mem/mod.rs | 12 ------------
+ 1 file changed, 12 deletions(-)
+
+diff --git a/src/mem/mod.rs b/src/mem/mod.rs
+index 107762c..2d1ae10 100644
+--- a/src/mem/mod.rs
++++ b/src/mem/mod.rs
+@@ -137,10 +137,6 @@ intrinsics! {
+     pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+         memcpy_element_unordered_atomic(dest, src, bytes);
+     }
+-    #[cfg(target_has_atomic_load_store = "128")]
+-    pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+-        memcpy_element_unordered_atomic(dest, src, bytes);
+-    }
+ 
+     #[cfg(target_has_atomic_load_store = "8")]
+     pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () {
+@@ -158,10 +154,6 @@ intrinsics! {
+     pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () {
+         memmove_element_unordered_atomic(dest, src, bytes);
+     }
+-    #[cfg(target_has_atomic_load_store = "128")]
+-    pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () {
+-        memmove_element_unordered_atomic(dest, src, bytes);
+-    }
+ 
+     #[cfg(target_has_atomic_load_store = "8")]
+     pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () {
+@@ -179,8 +171,4 @@ intrinsics! {
+     pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () {
+         memset_element_unordered_atomic(s, c, bytes);
+     }
+-    #[cfg(target_has_atomic_load_store = "128")]
+-    pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () {
+-        memset_element_unordered_atomic(s, c, bytes);
+-    }
+ }
+-- 
+2.26.2.7.g19db9cfb68
+

compiler/rustc_codegen_cranelift/example/alloc_example.rs

@@ -1,4 +1,4 @@
-#![feature(start, box_syntax, alloc_system, core_intrinsics, alloc_prelude, alloc_error_handler)]
+#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler)]
 #![no_std]
 
 extern crate alloc;

compiler/rustc_codegen_cranelift/example/alloc_system.rs

@@ -8,66 +8,24 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 #![no_std]
-#![allow(unused_attributes)]
+#![feature(allocator_api, rustc_private)]
-#![unstable(feature = "alloc_system",
-            reason = "this library is unlikely to be stabilized in its current \
-                      form or name",
-            issue = "32838")]
-#![feature(allocator_api)]
-#![feature(core_intrinsics)]
-#![feature(nll)]
-#![feature(staged_api)]
-#![feature(rustc_attrs)]
-#![feature(alloc_layout_extra)]
-#![cfg_attr(
-    all(target_arch = "wasm32", not(target_os = "emscripten")),
-    feature(integer_atomics, stdsimd)
-)]
 #![cfg_attr(any(unix, target_os = "redox"), feature(libc))]
+
 // The minimum alignment guaranteed by the architecture. This value is used to
 // add fast paths for low alignment values.
 #[cfg(all(any(target_arch = "x86",
               target_arch = "arm",
               target_arch = "mips",
               target_arch = "powerpc",
-              target_arch = "powerpc64",
+              target_arch = "powerpc64")))]
-              target_arch = "asmjs",
-              target_arch = "wasm32")))]
-#[allow(dead_code)]
 const MIN_ALIGN: usize = 8;
 #[cfg(all(any(target_arch = "x86_64",
               target_arch = "aarch64",
               target_arch = "mips64",
               target_arch = "s390x",
               target_arch = "sparc64")))]
-#[allow(dead_code)]
 const MIN_ALIGN: usize = 16;
 
-/// The default memory allocator provided by the operating system.
-///
-/// This is based on `malloc` on Unix platforms and `HeapAlloc` on Windows,
-/// plus related functions.
-///
-/// This type can be used in a `static` item
-/// with the `#[global_allocator]` attribute
-/// to force the global allocator to be the system’s one.
-/// (The default is jemalloc for executables, on some platforms.)
-///
-/// ```rust
-/// use std::alloc::System;
-///
-/// #[global_allocator]
-/// static A: System = System;
-///
-/// fn main() {
-///     let a = Box::new(4); // Allocates from the system allocator.
-///     println!("{}", a);
-/// }
-/// ```
-///
-/// It can also be used directly to allocate memory
-/// independently of the standard library’s global allocator.
-#[stable(feature = "alloc_system_type", since = "1.28.0")]
 pub struct System;
 #[cfg(any(windows, unix, target_os = "redox"))]
 mod realloc_fallback {
@@ -96,7 +54,6 @@ mod platform {
     use MIN_ALIGN;
     use System;
     use core::alloc::{GlobalAlloc, Layout};
-    #[stable(feature = "alloc_system_type", since = "1.28.0")]
     unsafe impl GlobalAlloc for System {
         #[inline]
         unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
@@ -221,7 +178,6 @@ mod platform {
         };
         ptr as *mut u8
     }
-    #[stable(feature = "alloc_system_type", since = "1.28.0")]
     unsafe impl GlobalAlloc for System {
         #[inline]
         unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
@@ -254,89 +210,3 @@ mod platform {
         }
     }
 }
-// This is an implementation of a global allocator on the wasm32 platform when
-// emscripten is not in use. In that situation there's no actual runtime for us
-// to lean on for allocation, so instead we provide our own!
-//
-// The wasm32 instruction set has two instructions for getting the current
-// amount of memory and growing the amount of memory. These instructions are the
-// foundation on which we're able to build an allocator, so we do so! Note that
-// the instructions are also pretty "global" and this is the "global" allocator
-// after all!
-//
-// The current allocator here is the `dlmalloc` crate which we've got included
-// in the rust-lang/rust repository as a submodule. The crate is a port of
-// dlmalloc.c from C to Rust and is basically just so we can have "pure Rust"
-// for now which is currently technically required (can't link with C yet).
-//
-// The crate itself provides a global allocator which on wasm has no
-// synchronization as there are no threads!
-#[cfg(all(target_arch = "wasm32", not(target_os = "emscripten")))]
-mod platform {
-    extern crate dlmalloc;
-    use core::alloc::{GlobalAlloc, Layout};
-    use System;
-    static mut DLMALLOC: dlmalloc::Dlmalloc = dlmalloc::DLMALLOC_INIT;
-    #[stable(feature = "alloc_system_type", since = "1.28.0")]
-    unsafe impl GlobalAlloc for System {
-        #[inline]
-        unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
-            let _lock = lock::lock();
-            DLMALLOC.malloc(layout.size(), layout.align())
-        }
-        #[inline]
-        unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
-            let _lock = lock::lock();
-            DLMALLOC.calloc(layout.size(), layout.align())
-        }
-        #[inline]
-        unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
-            let _lock = lock::lock();
-            DLMALLOC.free(ptr, layout.size(), layout.align())
-        }
-        #[inline]
-        unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
-            let _lock = lock::lock();
-            DLMALLOC.realloc(ptr, layout.size(), layout.align(), new_size)
-        }
-    }
-    #[cfg(target_feature = "atomics")]
-    mod lock {
-        use core::arch::wasm32;
-        use core::sync::atomic::{AtomicI32, Ordering::SeqCst};
-        static LOCKED: AtomicI32 = AtomicI32::new(0);
-        pub struct DropLock;
-        pub fn lock() -> DropLock {
-            loop {
-                if LOCKED.swap(1, SeqCst) == 0 {
-                    return DropLock
-                }
-                unsafe {
-                    let r = wasm32::atomic::wait_i32(
-                        &LOCKED as *const AtomicI32 as *mut i32,
-                        1,  // expected value
-                        -1, // timeout
-                    );
-                    debug_assert!(r == 0 || r == 1);
-                }
-            }
-        }
-        impl Drop for DropLock {
-            fn drop(&mut self) {
-                let r = LOCKED.swap(0, SeqCst);
-                debug_assert_eq!(r, 1);
-                unsafe {
-                    wasm32::atomic::wake(
-                        &LOCKED as *const AtomicI32 as *mut i32,
-                        1, // only one thread
-                    );
-                }
-            }
-        }
-    }
-    #[cfg(not(target_feature = "atomics"))]
-    mod lock {
-        #[inline]
-        pub fn lock() {} // no atomics, no threads, that's easy!
-    }
-}

compiler/rustc_codegen_cranelift/example/arbitrary_self_types_pointers_and_wrappers.rs

@@ -1,22 +1,12 @@
 // Adapted from rustc run-pass test suite
 
-#![feature(no_core, arbitrary_self_types, box_syntax)]
+#![feature(arbitrary_self_types, unsize, coerce_unsized, dispatch_from_dyn)]
 #![feature(rustc_attrs)]
 
-#![feature(start, lang_items)]
+use std::{
-#![no_core]
+    ops::{Deref, CoerceUnsized, DispatchFromDyn},
-
+    marker::Unsize,
-extern crate mini_core;
+};
-
-use mini_core::*;
-
-macro_rules! assert_eq {
-    ($l:expr, $r: expr) => {
-        if $l != $r {
-            panic(stringify!($l != $r));
-        }
-    }
-}
 
 struct Ptr<T: ?Sized>(Box<T>);
 
@@ -67,16 +57,13 @@ impl Trait for i32 {
     }
 }
 
-#[start]
+fn main() {
-fn main(_: isize, _: *const *const u8) -> isize {
+    let pw = Ptr(Box::new(Wrapper(5))) as Ptr<Wrapper<dyn Trait>>;
-    let pw = Ptr(box Wrapper(5)) as Ptr<Wrapper<dyn Trait>>;
     assert_eq!(pw.ptr_wrapper(), 5);
 
-    let wp = Wrapper(Ptr(box 6)) as Wrapper<Ptr<dyn Trait>>;
+    let wp = Wrapper(Ptr(Box::new(6))) as Wrapper<Ptr<dyn Trait>>;
     assert_eq!(wp.wrapper_ptr(), 6);
 
-    let wpw = Wrapper(Ptr(box Wrapper(7))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
+    let wpw = Wrapper(Ptr(Box::new(Wrapper(7)))) as Wrapper<Ptr<Wrapper<dyn Trait>>>;
     assert_eq!(wpw.wrapper_ptr_wrapper(), 7);
-
-    0
 }

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -365,6 +365,22 @@ impl <T: PartialEq> PartialEq for Option<T> {
     }
 }
 
+#[lang = "shl"]
+pub trait Shl<RHS = Self> {
+    type Output;
+
+    #[must_use]
+    fn shl(self, rhs: RHS) -> Self::Output;
+}
+
+impl Shl for u128 {
+    type Output = u128;
+
+    fn shl(self, rhs: u128) -> u128 {
+        self << rhs
+    }
+}
+
 #[lang = "neg"]
 pub trait Neg {
     type Output;

compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs

@@ -264,6 +264,9 @@ fn main() {
     assert_eq!(f2 as i8, -128);
     assert_eq!(f2 as u8, 0);
 
+    let amount = 0;
+    assert_eq!(1u128 << amount, 1);
+
     static ANOTHER_STATIC: &u8 = &A_STATIC;
     assert_eq!(*ANOTHER_STATIC, 42);
 

compiler/rustc_codegen_cranelift/patches/0022-core-Disable-not-compiling-tests.patch

@@ -119,21 +119,5 @@ index 6609bc3..241b497 100644
  
  #[test]
  #[should_panic(expected = "index 0 greater than length of slice")]
-diff --git a/library/core/tests/num/ops.rs b/library/core/tests/num/ops.rs
-index 9979cc8..d5d1d83 100644
---- a/library/core/tests/num/ops.rs
-+++ b/library/core/tests/num/ops.rs
-@@ -238,7 +238,7 @@ macro_rules! test_shift_assign {
-         }
-     };
- }
--test_shift!(test_shl_defined, Shl::shl);
--test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
--test_shift!(test_shr_defined, Shr::shr);
--test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
-+//test_shift!(test_shl_defined, Shl::shl);
-+//test_shift_assign!(test_shl_assign_defined, ShlAssign::shl_assign);
-+//test_shift!(test_shr_defined, Shr::shr);
-+//test_shift_assign!(test_shr_assign_defined, ShrAssign::shr_assign);
 --
 2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/patches/0027-Disable-128bit-atomic-operations.patch

@@ -0,0 +1,103 @@
+From 894e07dfec2624ba539129b1c1d63e1d7d812bda Mon Sep 17 00:00:00 2001
+From: bjorn3 <bjorn3@users.noreply.github.com>
+Date: Thu, 18 Feb 2021 18:45:28 +0100
+Subject: [PATCH] Disable 128bit atomic operations
+
+Cranelift doesn't support them yet
+---
+ library/core/src/sync/atomic.rs | 38 ---------------------------------
+ library/core/tests/atomic.rs    |  4 ----
+ library/std/src/panic.rs        |  6 ------
+ 3 files changed, 48 deletions(-)
+
+diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs
+index 81c9e1d..65c9503 100644
+--- a/library/core/src/sync/atomic.rs
++++ b/library/core/src/sync/atomic.rs
+@@ -2228,44 +2228,6 @@ atomic_int! {
+     "AtomicU64::new(0)",
+     u64 AtomicU64 ATOMIC_U64_INIT
+ }
+-#[cfg(target_has_atomic_load_store = "128")]
+-atomic_int! {
+-    cfg(target_has_atomic = "128"),
+-    cfg(target_has_atomic_equal_alignment = "128"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    "i128",
+-    "#![feature(integer_atomics)]\n\n",
+-    atomic_min, atomic_max,
+-    16,
+-    "AtomicI128::new(0)",
+-    i128 AtomicI128 ATOMIC_I128_INIT
+-}
+-#[cfg(target_has_atomic_load_store = "128")]
+-atomic_int! {
+-    cfg(target_has_atomic = "128"),
+-    cfg(target_has_atomic_equal_alignment = "128"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    rustc_const_stable(feature = "const_integer_atomics", since = "1.34.0"),
+-    unstable(feature = "integer_atomics", issue = "32976"),
+-    "u128",
+-    "#![feature(integer_atomics)]\n\n",
+-    atomic_umin, atomic_umax,
+-    16,
+-    "AtomicU128::new(0)",
+-    u128 AtomicU128 ATOMIC_U128_INIT
+-}
+ 
+ macro_rules! atomic_int_ptr_sized {
+     ( $($target_pointer_width:literal $align:literal)* ) => { $(
+diff --git a/library/core/tests/atomic.rs b/library/core/tests/atomic.rs
+index 2d1e449..cb6da5d 100644
+--- a/library/core/tests/atomic.rs
++++ b/library/core/tests/atomic.rs
+@@ -145,10 +145,6 @@ fn atomic_alignment() {
+     assert_eq!(align_of::<AtomicU64>(), size_of::<AtomicU64>());
+     #[cfg(target_has_atomic = "64")]
+     assert_eq!(align_of::<AtomicI64>(), size_of::<AtomicI64>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicU128>(), size_of::<AtomicU128>());
+-    #[cfg(target_has_atomic = "128")]
+-    assert_eq!(align_of::<AtomicI128>(), size_of::<AtomicI128>());
+     #[cfg(target_has_atomic = "ptr")]
+     assert_eq!(align_of::<AtomicUsize>(), size_of::<AtomicUsize>());
+     #[cfg(target_has_atomic = "ptr")]
+diff --git a/library/std/src/panic.rs b/library/std/src/panic.rs
+index 89a822a..779fd88 100644
+--- a/library/std/src/panic.rs
++++ b/library/std/src/panic.rs
+@@ -279,9 +279,6 @@ impl RefUnwindSafe for atomic::AtomicI32 {}
+ #[cfg(target_has_atomic_load_store = "64")]
+ #[stable(feature = "integer_atomics_stable", since = "1.34.0")]
+ impl RefUnwindSafe for atomic::AtomicI64 {}
+-#[cfg(target_has_atomic_load_store = "128")]
+-#[unstable(feature = "integer_atomics", issue = "32976")]
+-impl RefUnwindSafe for atomic::AtomicI128 {}
+ 
+ #[cfg(target_has_atomic_load_store = "ptr")]
+ #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")]
+@@ -298,9 +295,6 @@ impl RefUnwindSafe for atomic::AtomicU32 {}
+ #[cfg(target_has_atomic_load_store = "64")]
+ #[stable(feature = "integer_atomics_stable", since = "1.34.0")]
+ impl RefUnwindSafe for atomic::AtomicU64 {}
+-#[cfg(target_has_atomic_load_store = "128")]
+-#[unstable(feature = "integer_atomics", issue = "32976")]
+-impl RefUnwindSafe for atomic::AtomicU128 {}
+ 
+ #[cfg(target_has_atomic_load_store = "8")]
+ #[stable(feature = "unwind_safe_atomic_refs", since = "1.14.0")]
+-- 
+2.26.2.7.g19db9cfb68
+

compiler/rustc_codegen_cranelift/prepare.sh

@@ -1,4 +1,4 @@
-#!/bin/bash --verbose
+#!/usr/bin/env bash
 set -e
 
 rustup component add rust-src rustc-dev llvm-tools-preview

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1 +1 @@
-nightly-2021-01-30
+nightly-2021-03-05

compiler/rustc_codegen_cranelift/rustfmt.toml

@@ -0,0 +1,4 @@
+# Matches rustfmt.toml of rustc
+version = "Two"
+use_small_heuristics = "Max"
+merge_derives = false

compiler/rustc_codegen_cranelift/scripts/cargo.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 dir=$(dirname "$0")
 source "$dir/config.sh"

compiler/rustc_codegen_cranelift/scripts/config.sh

@@ -3,7 +3,7 @@
 set -e
 
 unamestr=$(uname)
-if [[ "$unamestr" == 'Linux' ]]; then
+if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then
    dylib_ext='so'
 elif [[ "$unamestr" == 'Darwin' ]]; then
    dylib_ext='dylib'
@@ -26,7 +26,7 @@ export RUSTC=$dir"/bin/cg_clif"
 export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\
 '-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir
 
-# FIXME remove once the atomic shim is gone
+# FIXME fix `#[linkage = "extern_weak"]` without this
 if [[ "$unamestr" == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi

compiler/rustc_codegen_cranelift/scripts/rustup.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 

compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 cd "$(dirname "$0")/../"
@@ -14,21 +14,18 @@ git checkout -- .
 git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')"
 
 git apply - <<EOF
-diff --git a/.gitmodules b/.gitmodules
+diff --git a/Cargo.toml b/Cargo.toml
-index 984113151de..c1e9d960d56 100644
+index 5bd1147cad5..10d68a2ff14 100644
---- a/.gitmodules
+--- a/Cargo.toml
-+++ b/.gitmodules
++++ b/Cargo.toml
-@@ -34,10 +34,6 @@
+@@ -111,5 +111,7 @@ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- [submodule "src/doc/edition-guide"]
+ rustc-std-workspace-alloc = { path = 'library/rustc-std-workspace-alloc' }
- 	path = src/doc/edition-guide
+ rustc-std-workspace-std = { path = 'library/rustc-std-workspace-std' }
- 	url = https://github.com/rust-lang/edition-guide.git
+ 
--[submodule "src/llvm-project"]
++compiler_builtins = { path = "../build_sysroot/compiler-builtins" }
--	path = src/llvm-project
++
--	url = https://github.com/rust-lang/llvm-project.git
+ [patch."https://github.com/rust-lang/rust-clippy"]
--	branch = rustc/11.0-2020-10-12
+ clippy_lints = { path = "src/tools/clippy/clippy_lints" }
- [submodule "src/doc/embedded-book"]
- 	path = src/doc/embedded-book
- 	url = https://github.com/rust-embedded/book.git
 diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
 index 23e689fcae7..5f077b765b6 100644
 --- a/compiler/rustc_data_structures/Cargo.toml
@@ -41,6 +38,19 @@ index 23e689fcae7..5f077b765b6 100644
 
  [target.'cfg(windows)'.dependencies]
  winapi = { version = "0.3", features = ["fileapi", "psapi"] }
+diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
+index d95b5b7f17f..00b6f0e3635 100644
+--- a/library/alloc/Cargo.toml
++++ b/library/alloc/Cargo.toml
+@@ -8,7 +8,7 @@ edition = "2018"
+ 
+ [dependencies]
+ core = { path = "../core" }
+-compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] }
+ 
+ [dev-dependencies]
+ rand = "0.7"
 EOF
 
 cat > config.toml <<EOF

compiler/rustc_codegen_cranelift/scripts/tests.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
@@ -27,13 +27,16 @@ function no_sysroot_tests() {
     $MY_RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
     # (echo "break set -n main"; echo "run"; sleep 1; echo "si -c 10"; sleep 1; echo "frame variable") | lldb -- ./target/out/mini_core_hello_world abc bcd
-
-    echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
-    $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
-    $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
 }
 
 function base_sysroot_tests() {
+    echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
+    $MY_RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target "$TARGET_TRIPLE"
+    $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
+
+    echo "[AOT] alloc_system"
+    $MY_RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
+
     echo "[AOT] alloc_example"
     $MY_RUSTC example/alloc_example.rs --crate-type bin --target "$TARGET_TRIPLE"
     $RUN_WRAPPER ./target/out/alloc_example

compiler/rustc_codegen_cranelift/src/abi/comments.rs

@@ -10,14 +10,14 @@ use cranelift_codegen::entity::EntityRef;
 
 use crate::prelude::*;
 
-pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
+pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
     fx.add_global_comment(
         "kind  loc.idx   param    pass mode                            ty".to_string(),
     );
 }
 
 pub(super) fn add_arg_comment<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     kind: &str,
     local: Option<mir::Local>,
     local_field: Option<usize>,
@@ -42,11 +42,7 @@ pub(super) fn add_arg_comment<'tcx>(
         [param_a, param_b] => Cow::Owned(format!("= {:?},{:?}", param_a, param_b)),
         params => Cow::Owned(format!(
             "= {}",
-            params
+            params.iter().map(ToString::to_string).collect::<Vec<_>>().join(",")
-                .iter()
-                .map(ToString::to_string)
-                .collect::<Vec<_>>()
-                .join(",")
         )),
     };
 
@@ -62,7 +58,7 @@ pub(super) fn add_arg_comment<'tcx>(
     ));
 }
 
-pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>) {
+pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
     fx.add_global_comment(String::new());
     fx.add_global_comment(
         "kind  local ty                              size align (abi,pref)".to_string(),
@@ -70,19 +66,13 @@ pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, impl Module>
 }
 
 pub(super) fn add_local_place_comments<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     place: CPlace<'tcx>,
     local: Local,
 ) {
     let TyAndLayout { ty, layout } = place.layout();
-    let rustc_target::abi::Layout {
+    let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
-        size,
+        layout;
-        align,
-        abi: _,
-        variants: _,
-        fields: _,
-        largest_niche: _,
-    } = layout;
 
     let (kind, extra) = match *place.inner() {
         CPlaceInner::Var(place_local, var) => {
@@ -91,10 +81,7 @@ pub(super) fn add_local_place_comments<'tcx>(
         }
         CPlaceInner::VarPair(place_local, var1, var2) => {
             assert_eq!(local, place_local);
-            (
+            ("ssa", Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())))
-                "ssa",
-                Cow::Owned(format!(",var=({}, {})", var1.index(), var2.index())),
-            )
         }
         CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(),
         CPlaceInner::Addr(ptr, meta) => {
@@ -104,18 +91,15 @@ pub(super) fn add_local_place_comments<'tcx>(
                 Cow::Borrowed("")
             };
             match ptr.base_and_offset() {
-                (crate::pointer::PointerBase::Addr(addr), offset) => (
+                (crate::pointer::PointerBase::Addr(addr), offset) => {
-                    "reuse",
+                    ("reuse", format!("storage={}{}{}", addr, offset, meta).into())
-                    format!("storage={}{}{}", addr, offset, meta).into(),
+                }
-                ),
+                (crate::pointer::PointerBase::Stack(stack_slot), offset) => {
-                (crate::pointer::PointerBase::Stack(stack_slot), offset) => (
+                    ("stack", format!("storage={}{}{}", stack_slot, offset, meta).into())
-                    "stack",
+                }
-                    format!("storage={}{}{}", stack_slot, offset, meta).into(),
+                (crate::pointer::PointerBase::Dangling(align), offset) => {
-                ),
+                    ("zst", format!("align={},offset={}", align.bytes(), offset).into())
-                (crate::pointer::PointerBase::Dangling(align), offset) => (
+                }
-                    "zst",
-                    format!("align={},offset={}", align.bytes(), offset).into(),
-                ),
             }
         }
     };
@@ -128,11 +112,7 @@ pub(super) fn add_local_place_comments<'tcx>(
         size.bytes(),
         align.abi.bytes(),
         align.pref.bytes(),
-        if extra.is_empty() {
+        if extra.is_empty() { "" } else { "              " },
-            ""
-        } else {
-            "              "
-        },
         extra,
     ));
 }

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -38,25 +38,15 @@ fn clif_sig_from_fn_abi<'tcx>(
         | Conv::X86VectorCall
         | Conv::AmdGpuKernel
         | Conv::AvrInterrupt
-        | Conv::AvrNonBlockingInterrupt => {
+        | Conv::AvrNonBlockingInterrupt => todo!("{:?}", fn_abi.conv),
-            todo!("{:?}", fn_abi.conv)
-        }
     };
-    let inputs = fn_abi
+    let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
-        .args
-        .iter()
-        .map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter())
-        .flatten();
 
     let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx);
     // Sometimes the first param is an pointer to the place where the return value needs to be stored.
     let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect();
 
-    Signature {
+    Signature { params, returns, call_conv }
-        params,
-        returns,
-        call_conv,
-    }
 }
 
 pub(crate) fn get_function_sig<'tcx>(
@@ -65,34 +55,25 @@ pub(crate) fn get_function_sig<'tcx>(
     inst: Instance<'tcx>,
 ) -> Signature {
     assert!(!inst.substs.needs_infer());
-    clif_sig_from_fn_abi(
+    clif_sig_from_fn_abi(tcx, triple, &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]))
-        tcx,
-        triple,
-        &FnAbi::of_instance(&RevealAllLayoutCx(tcx), inst, &[]),
-    )
 }
 
 /// Instance must be monomorphized
 pub(crate) fn import_function<'tcx>(
     tcx: TyCtxt<'tcx>,
-    module: &mut impl Module,
+    module: &mut dyn Module,
     inst: Instance<'tcx>,
 ) -> FuncId {
     let name = tcx.symbol_name(inst).name.to_string();
     let sig = get_function_sig(tcx, module.isa().triple(), inst);
-    module
+    module.declare_function(&name, Linkage::Import, &sig).unwrap()
-        .declare_function(&name, Linkage::Import, &sig)
-        .unwrap()
 }
 
-impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     /// Instance must be monomorphized
     pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
-        let func_id = import_function(self.tcx, &mut self.cx.module, inst);
+        let func_id = import_function(self.tcx, self.cx.module, inst);
-        let func_ref = self
+        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
-            .cx
-            .module
-            .declare_func_in_func(func_id, &mut self.bcx.func);
 
         #[cfg(debug_assertions)]
         self.add_comment(func_ref, format!("{:?}", inst));
@@ -107,20 +88,9 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         returns: Vec<AbiParam>,
         args: &[Value],
     ) -> &[Value] {
-        let sig = Signature {
+        let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
-            params,
+        let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
-            returns,
+        let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
-            call_conv: CallConv::triple_default(self.triple()),
-        };
-        let func_id = self
-            .cx
-            .module
-            .declare_function(&name, Linkage::Import, &sig)
-            .unwrap();
-        let func_ref = self
-            .cx
-            .module
-            .declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
         #[cfg(debug_assertions)]
         {
@@ -140,17 +110,12 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         let (input_tys, args): (Vec<_>, Vec<_>) = args
             .iter()
             .map(|arg| {
-                (
+                (AbiParam::new(self.clif_type(arg.layout().ty).unwrap()), arg.load_scalar(self))
-                    AbiParam::new(self.clif_type(arg.layout().ty).unwrap()),
-                    arg.load_scalar(self),
-                )
             })
             .unzip();
         let return_layout = self.layout_of(return_ty);
         let return_tys = if let ty::Tuple(tup) = return_ty.kind() {
-            tup.types()
+            tup.types().map(|ty| AbiParam::new(self.clif_type(ty).unwrap())).collect()
-                .map(|ty| AbiParam::new(self.clif_type(ty).unwrap()))
-                .collect()
         } else {
             vec![AbiParam::new(self.clif_type(return_ty).unwrap())]
         };
@@ -169,7 +134,7 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
 
 /// Make a [`CPlace`] capable of holding value of the specified type.
 fn make_local_place<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     local: Local,
     layout: TyAndLayout<'tcx>,
     is_ssa: bool,
@@ -190,10 +155,7 @@ fn make_local_place<'tcx>(
     place
 }
 
-pub(crate) fn codegen_fn_prelude<'tcx>(
+pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) {
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    start_block: Block,
-) {
     fx.bcx.append_block_params_for_function_params(start_block);
 
     fx.bcx.switch_to_block(start_block);
@@ -204,13 +166,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
     #[cfg(debug_assertions)]
     self::comments::add_args_header_comment(fx);
 
-    let mut block_params_iter = fx
+    let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
-        .bcx
-        .func
-        .dfg
-        .block_params(start_block)
-        .to_vec()
-        .into_iter();
     let ret_place =
         self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter);
     assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE);
@@ -286,10 +242,10 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
             if let Some((addr, meta)) = val.try_to_ptr() {
                 let local_decl = &fx.mir.local_decls[local];
                 //                       v this ! is important
-                let internally_mutable = !val.layout().ty.is_freeze(
+                let internally_mutable = !val
-                    fx.tcx.at(local_decl.source_info.span),
+                    .layout()
-                    ParamEnv::reveal_all(),
+                    .ty
-                );
+                    .is_freeze(fx.tcx.at(local_decl.source_info.span), ParamEnv::reveal_all());
                 if local_decl.mutability == mir::Mutability::Not && !internally_mutable {
                     // We wont mutate this argument, so it is fine to borrow the backing storage
                     // of this argument, to prevent a copy.
@@ -321,9 +277,7 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
             ArgKind::Spread(params) => {
                 for (i, param) in params.into_iter().enumerate() {
                     if let Some(param) = param {
-                        place
+                        place.place_field(fx, mir::Field::new(i)).write_cvalue(fx, param);
-                            .place_field(fx, mir::Field::new(i))
-                            .write_cvalue(fx, param);
                     }
                 }
             }
@@ -340,13 +294,11 @@ pub(crate) fn codegen_fn_prelude<'tcx>(
         assert_eq!(fx.local_map.push(place), local);
     }
 
-    fx.bcx
+    fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
-        .ins()
-        .jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]);
 }
 
 pub(crate) fn codegen_terminator_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     span: Span,
     current_block: Block,
     func: &Operand<'tcx>,
@@ -354,9 +306,8 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     destination: Option<(Place<'tcx>, BasicBlock)>,
 ) {
     let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
-    let fn_sig = fx
+    let fn_sig =
-        .tcx
+        fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
-        .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
 
     let destination = destination.map(|(place, bb)| (codegen_place(fx, place), bb));
 
@@ -404,20 +355,11 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let fn_abi = if let Some(instance) = instance {
         FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
     } else {
-        FnAbi::of_fn_ptr(
+        FnAbi::of_fn_ptr(&RevealAllLayoutCx(fx.tcx), fn_ty.fn_sig(fx.tcx), &extra_args)
-            &RevealAllLayoutCx(fx.tcx),
-            fn_ty.fn_sig(fx.tcx),
-            &extra_args,
-        )
     };
 
     let is_cold = instance
-        .map(|inst| {
+        .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD))
-            fx.tcx
-                .codegen_fn_attrs(inst.def_id())
-                .flags
-                .contains(CodegenFnAttrFlags::COLD)
-        })
         .unwrap_or(false);
     if is_cold {
         fx.cold_blocks.insert(current_block);
@@ -441,9 +383,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         }
         args
     } else {
-        args.iter()
+        args.iter().map(|arg| codegen_operand(fx, arg)).collect::<Vec<_>>()
-            .map(|arg| codegen_operand(fx, arg))
-            .collect::<Vec<_>>()
     };
 
     //   | indirect call target
@@ -451,10 +391,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     //   v         v
     let (func_ref, first_arg) = match instance {
         // Trait object call
-        Some(Instance {
+        Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
-            def: InstanceDef::Virtual(_, idx),
-            ..
-        }) => {
             #[cfg(debug_assertions)]
             {
                 let nop_inst = fx.bcx.ins().nop();
@@ -511,10 +448,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 )
                 .collect::<Vec<_>>();
 
-            if instance
+            if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) {
-                .map(|inst| inst.def.requires_caller_location(fx.tcx))
-                .unwrap_or(false)
-            {
                 // Pass the caller location for `#[track_caller]`.
                 let caller_location = fx.get_caller_location(span);
                 call_args.extend(
@@ -543,10 +477,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
         if fn_sig.abi != Abi::C {
-            fx.tcx.sess.span_fatal(
+            fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
-                span,
-                &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
-            );
         }
         let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
         let abi_params = call_args
@@ -555,9 +486,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 let ty = fx.bcx.func.dfg.value_type(arg);
                 if !ty.is_int() {
                     // FIXME set %al to upperbound on float args once floats are supported
-                    fx.tcx
+                    fx.tcx.sess.span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
-                        .sess
-                        .span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
                 }
                 AbiParam::new(ty)
             })
@@ -574,7 +503,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 }
 
 pub(crate) fn codegen_drop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     span: Span,
     drop_place: CPlace<'tcx>,
 ) {
@@ -611,10 +540,7 @@ pub(crate) fn codegen_drop<'tcx>(
                     fx,
                     fx.layout_of(fx.tcx.mk_ref(
                         &ty::RegionKind::ReErased,
-                        TypeAndMut {
+                        TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut },
-                            ty,
-                            mutbl: crate::rustc_hir::Mutability::Mut,
-                        },
                     )),
                 );
                 let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0]);

compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

@@ -71,12 +71,7 @@ fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> {
         .prefix
         .iter()
         .flatten()
-        .map(|&kind| {
+        .map(|&kind| reg_to_abi_param(Reg { kind, size: cast.prefix_chunk_size }))
-            reg_to_abi_param(Reg {
-                kind,
-                size: cast.prefix_chunk_size,
-            })
-        })
         .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit)))
         .collect::<SmallVec<_>>();
 
@@ -98,12 +93,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match self.mode {
             PassMode::Ignore => smallvec![],
             PassMode::Direct(attrs) => match &self.layout.abi {
-                Abi::Scalar(scalar) => {
+                Abi::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param(
-                    smallvec![apply_arg_attrs_to_abi_param(
                     AbiParam::new(scalar_to_clif_type(tcx, scalar.clone())),
                     attrs
-                    )]
+                )],
-                }
                 Abi::Vector { .. } => {
                     let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
                     smallvec![AbiParam::new(vector_ty)]
@@ -122,11 +115,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 _ => unreachable!("{:?}", self.layout.abi),
             },
             PassMode::Cast(cast) => cast_target_to_abi_params(cast),
-            PassMode::Indirect {
+            PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
-                attrs,
-                extra_attrs: None,
-                on_stack,
-            } => {
                 if on_stack {
                     let size = u32::try_from(self.layout.size.bytes()).unwrap();
                     smallvec![apply_arg_attrs_to_abi_param(
@@ -134,17 +123,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                         attrs
                     )]
                 } else {
-                    smallvec![apply_arg_attrs_to_abi_param(
+                    smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)]
-                        AbiParam::new(pointer_ty(tcx)),
-                        attrs
-                    )]
                 }
             }
-            PassMode::Indirect {
+            PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => {
-                attrs,
-                extra_attrs: Some(extra_attrs),
-                on_stack,
-            } => {
                 assert!(!on_stack);
                 smallvec![
                     apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
@@ -158,10 +140,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match self.mode {
             PassMode::Ignore => (None, vec![]),
             PassMode::Direct(_) => match &self.layout.abi {
-                Abi::Scalar(scalar) => (
+                Abi::Scalar(scalar) => {
-                    None,
+                    (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))])
-                    vec![AbiParam::new(scalar_to_clif_type(tcx, scalar.clone()))],
+                }
-                ),
                 Abi::Vector { .. } => {
                     let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap();
                     (None, vec![AbiParam::new(vector_ty)])
@@ -177,31 +158,19 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 _ => unreachable!("{:?}", self.layout.abi),
             },
             PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()),
-            PassMode::Indirect {
+            PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => {
-                attrs: _,
-                extra_attrs: None,
-                on_stack,
-            } => {
                 assert!(!on_stack);
-                (
+                (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
-                    Some(AbiParam::special(
+            }
-                        pointer_ty(tcx),
+            PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
-                        ArgumentPurpose::StructReturn,
+                unreachable!("unsized return value")
-                    )),
-                    vec![],
-                )
             }
-            PassMode::Indirect {
-                attrs: _,
-                extra_attrs: Some(_),
-                on_stack: _,
-            } => unreachable!("unsized return value"),
         }
     }
 }
 
 pub(super) fn to_casted_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
     cast: CastTarget,
 ) -> SmallVec<[Value; 2]> {
@@ -211,9 +180,7 @@ pub(super) fn to_casted_value<'tcx>(
     cast_target_to_abi_params(cast)
         .into_iter()
         .map(|param| {
-            let val = ptr
+            let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new());
-                .offset_i64(fx, offset)
-                .load(fx, param.value_type, MemFlags::new());
             offset += i64::from(param.value_type.bytes());
             val
         })
@@ -221,16 +188,13 @@ pub(super) fn to_casted_value<'tcx>(
 }
 
 pub(super) fn from_casted_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     block_params: &[Value],
     layout: TyAndLayout<'tcx>,
     cast: CastTarget,
 ) -> CValue<'tcx> {
     let abi_params = cast_target_to_abi_params(cast);
-    let abi_param_size: u32 = abi_params
+    let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
-        .iter()
-        .map(|param| param.value_type.bytes())
-        .sum();
     let layout_size = u32::try_from(layout.size.bytes()).unwrap();
     let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
         kind: StackSlotKind::ExplicitSlot,
@@ -260,7 +224,7 @@ pub(super) fn from_casted_value<'tcx>(
 
 /// Get a set of values to be passed as function arguments.
 pub(super) fn adjust_arg_for_abi<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
 ) -> SmallVec<[Value; 2]> {
@@ -283,7 +247,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// Create a [`CValue`] containing the value of a function parameter adding clif function parameters
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
     #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
@@ -294,10 +258,7 @@ pub(super) fn cvalue_for_param<'tcx>(
         .into_iter()
         .map(|abi_param| {
             let block_param = block_params_iter.next().unwrap();
-            assert_eq!(
+            assert_eq!(fx.bcx.func.dfg.value_type(block_param), abi_param.value_type);
-                fx.bcx.func.dfg.value_type(block_param),
-                abi_param.value_type
-            );
             block_param
         })
         .collect::<SmallVec<[_; 2]>>();
@@ -321,29 +282,14 @@ pub(super) fn cvalue_for_param<'tcx>(
         }
         PassMode::Pair(_, _) => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
-            Some(CValue::by_val_pair(
+            Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout))
-                block_params[0],
-                block_params[1],
-                arg_abi.layout,
-            ))
         }
         PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)),
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {
             assert_eq!(block_params.len(), 1, "{:?}", block_params);
-            Some(CValue::by_ref(
+            Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout))
-                Pointer::new(block_params[0]),
-                arg_abi.layout,
-            ))
         }
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
             Some(CValue::by_ref_unsized(
                 Pointer::new(block_params[0]),

compiler/rustc_codegen_cranelift/src/abi/returning.rs

@@ -8,14 +8,13 @@ use smallvec::{smallvec, SmallVec};
 
 /// Can the given type be returned into an ssa var or does it need to be returned on the stack.
 pub(crate) fn can_return_to_ssa_var<'tcx>(
-    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    fx: &FunctionCx<'_, '_, 'tcx>,
     func: &mir::Operand<'tcx>,
     args: &[mir::Operand<'tcx>],
 ) -> bool {
     let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
-    let fn_sig = fx
+    let fn_sig =
-        .tcx
+        fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
-        .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), fn_ty.fn_sig(fx.tcx));
 
     // Handle special calls like instrinsics and empty drop glue.
     let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
@@ -42,11 +41,7 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
     let fn_abi = if let Some(instance) = instance {
         FnAbi::of_instance(&RevealAllLayoutCx(fx.tcx), instance, &extra_args)
     } else {
-        FnAbi::of_fn_ptr(
+        FnAbi::of_fn_ptr(&RevealAllLayoutCx(fx.tcx), fn_ty.fn_sig(fx.tcx), &extra_args)
-            &RevealAllLayoutCx(fx.tcx),
-            fn_ty.fn_sig(fx.tcx),
-            &extra_args,
-        )
     };
     match fn_abi.ret.mode {
         PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => true,
@@ -58,15 +53,12 @@ pub(crate) fn can_return_to_ssa_var<'tcx>(
 /// Return a place where the return value of the current function can be written to. If necessary
 /// this adds an extra parameter pointing to where the return value needs to be stored.
 pub(super) fn codegen_return_param<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     ssa_analyzed: &rustc_index::vec::IndexVec<Local, crate::analyze::SsaKind>,
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> CPlace<'tcx> {
     let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
-        PassMode::Ignore => (
+        PassMode::Ignore => (CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout), smallvec![]),
-            CPlace::no_place(fx.fn_abi.as_ref().unwrap().ret.layout),
-            smallvec![],
-        ),
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => {
             let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
             (
@@ -79,26 +71,17 @@ pub(super) fn codegen_return_param<'tcx>(
                 smallvec![],
             )
         }
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {
             let ret_param = block_params_iter.next().unwrap();
             assert_eq!(fx.bcx.func.dfg.value_type(ret_param), pointer_ty(fx.tcx));
             (
-                CPlace::for_ptr(
+                CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.as_ref().unwrap().ret.layout),
-                    Pointer::new(ret_param),
-                    fx.fn_abi.as_ref().unwrap().ret.layout,
-                ),
                 smallvec![ret_param],
             )
         }
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
-            attrs: _,
+            unreachable!("unsized return value")
-            extra_attrs: Some(_),
+        }
-            on_stack: _,
-        } => unreachable!("unsized return value"),
     };
 
     #[cfg(not(debug_assertions))]
@@ -120,27 +103,21 @@ pub(super) fn codegen_return_param<'tcx>(
 
 /// Invokes the closure with if necessary a value representing the return pointer. When the closure
 /// returns the call return value(s) if any are written to the correct place.
-pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
+pub(super) fn codegen_with_call_return_arg<'tcx, T>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     ret_place: Option<CPlace<'tcx>>,
-    f: impl FnOnce(&mut FunctionCx<'_, 'tcx, M>, Option<Value>) -> (Inst, T),
+    f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> (Inst, T),
 ) -> (Inst, T) {
     let return_ptr = match ret_arg_abi.mode {
         PassMode::Ignore => None,
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => match ret_place {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => match ret_place {
             Some(ret_place) => Some(ret_place.to_ptr().get_addr(fx)),
             None => Some(fx.bcx.ins().iconst(fx.pointer_type, 43)), // FIXME allocate temp stack slot
         },
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
-            attrs: _,
+            unreachable!("unsized return value")
-            extra_attrs: Some(_),
+        }
-            on_stack: _,
-        } => unreachable!("unsized return value"),
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => None,
     };
 
@@ -177,37 +154,24 @@ pub(super) fn codegen_with_call_return_arg<'tcx, M: Module, T>(
                 ret_place.write_cvalue(fx, result);
             }
         }
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {}
-            attrs: _,
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
-            extra_attrs: None,
+            unreachable!("unsized return value")
-            on_stack: _,
+        }
-        } => {}
-        PassMode::Indirect {
-            attrs: _,
-            extra_attrs: Some(_),
-            on_stack: _,
-        } => unreachable!("unsized return value"),
     }
 
     (call_inst, meta)
 }
 
 /// Codegen a return instruction with the right return value(s) if any.
-pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, impl Module>) {
+pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
     match fx.fn_abi.as_ref().unwrap().ret.mode {
-        PassMode::Ignore
+        PassMode::Ignore | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
-        | PassMode::Indirect {
-            attrs: _,
-            extra_attrs: None,
-            on_stack: _,
-        } => {
             fx.bcx.ins().return_(&[]);
         }
-        PassMode::Indirect {
+        PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
-            attrs: _,
+            unreachable!("unsized return value")
-            extra_attrs: Some(_),
+        }
-            on_stack: _,
-        } => unreachable!("unsized return value"),
         PassMode::Direct(_) => {
             let place = fx.get_local_place(RETURN_PLACE);
             let ret_val = place.to_cvalue(fx).load_scalar(fx);

compiler/rustc_codegen_cranelift/src/allocator.rs

@@ -66,13 +66,9 @@ fn codegen_inner(
         let callee_name = kind.fn_name(method.name);
         //eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns);
 
-        let func_id = module
+        let func_id = module.declare_function(&caller_name, Linkage::Export, &sig).unwrap();
-            .declare_function(&caller_name, Linkage::Export, &sig)
-            .unwrap();
 
-        let callee_func_id = module
+        let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
-            .declare_function(&callee_name, Linkage::Import, &sig)
-            .unwrap();
 
         let mut ctx = Context::new();
         ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone());
@@ -96,11 +92,7 @@ fn codegen_inner(
             bcx.finalize();
         }
         module
-            .define_function(
+            .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
-                func_id,
-                &mut ctx,
-                &mut cranelift_codegen::binemit::NullTrapSink {},
-            )
             .unwrap();
         unwind_context.add_function(func_id, &ctx, module.isa());
     }
@@ -114,13 +106,10 @@ fn codegen_inner(
     let callee_name = kind.fn_name(sym::oom);
     //eprintln!("Codegen allocator shim {} -> {} ({:?} -> {:?})", caller_name, callee_name, sig.params, sig.returns);
 
-    let func_id = module
+    let func_id =
-        .declare_function("__rust_alloc_error_handler", Linkage::Export, &sig)
+        module.declare_function("__rust_alloc_error_handler", Linkage::Export, &sig).unwrap();
-        .unwrap();
 
-    let callee_func_id = module
+    let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
-        .declare_function(&callee_name, Linkage::Import, &sig)
-        .unwrap();
 
     let mut ctx = Context::new();
     ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
@@ -143,11 +132,7 @@ fn codegen_inner(
         bcx.finalize();
     }
     module
-        .define_function(
+        .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
-            func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
         .unwrap();
     unwind_context.add_function(func_id, &ctx, module.isa());
 }

compiler/rustc_codegen_cranelift/src/analyze.rs

@@ -11,7 +11,7 @@ pub(crate) enum SsaKind {
     Ssa,
 }
 
-pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, SsaKind> {
+pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
     let mut flag_map = fx
         .mir
         .local_decls
@@ -40,12 +40,7 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, impl Module>) -> IndexVec<Local, S
         }
 
         match &bb.terminator().kind {
-            TerminatorKind::Call {
+            TerminatorKind::Call { destination, func, args, .. } => {
-                destination,
-                func,
-                args,
-                ..
-            } => {
                 if let Some((dest_place, _dest_bb)) = destination {
                     if !crate::abi::can_return_to_ssa_var(fx, func, args) {
                         not_ssa(&mut flag_map, dest_place.local)

compiler/rustc_codegen_cranelift/src/archive.rs

@@ -12,10 +12,7 @@ use object::{Object, ObjectSymbol, SymbolKind};
 
 #[derive(Debug)]
 enum ArchiveEntry {
-    FromArchive {
+    FromArchive { archive_index: usize, entry_index: usize },
-        archive_index: usize,
-        entry_index: usize,
-    },
     File(PathBuf),
 }
 
@@ -30,7 +27,6 @@ pub(crate) struct ArArchiveBuilder<'a> {
     // Don't use `HashMap` here, as the order is important. `rust.metadata.bin` must always be at
     // the end of an archive for linkers to not get confused.
     entries: Vec<(String, ArchiveEntry)>,
-    update_symbols: bool,
 }
 
 impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
@@ -46,10 +42,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                 let entry = entry.unwrap();
                 entries.push((
                     String::from_utf8(entry.header().identifier().to_vec()).unwrap(),
-                    ArchiveEntry::FromArchive {
+                    ArchiveEntry::FromArchive { archive_index: 0, entry_index: i },
-                        archive_index: 0,
-                        entry_index: i,
-                    },
                 ));
                 i += 1;
             }
@@ -69,7 +62,6 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
 
             src_archives,
             entries,
-            update_symbols: false,
         }
     }
 
@@ -95,13 +87,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
 
     fn add_native_library(&mut self, name: rustc_span::symbol::Symbol) {
         let location = find_library(name, &self.lib_search_paths, self.sess);
-        self.add_archive(location.clone(), |_| false)
+        self.add_archive(location.clone(), |_| false).unwrap_or_else(|e| {
-            .unwrap_or_else(|e| {
+            panic!("failed to add native library {}: {}", location.to_string_lossy(), e);
-                panic!(
-                    "failed to add native library {}: {}",
-                    location.to_string_lossy(),
-                    e
-                );
         });
     }
 
@@ -136,9 +123,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
         })
     }
 
-    fn update_symbols(&mut self) {
+    fn update_symbols(&mut self) {}
-        self.update_symbols = true;
-    }
 
     fn build(mut self) {
         enum BuilderKind {
@@ -156,10 +141,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
             // FIXME only read the symbol table of the object files to avoid having to keep all
             // object files in memory at once, or read them twice.
             let data = match entry {
-                ArchiveEntry::FromArchive {
+                ArchiveEntry::FromArchive { archive_index, entry_index } => {
-                    archive_index,
-                    entry_index,
-                } => {
                     // FIXME read symbols from symtab
                     use std::io::Read;
                     let (ref _src_archive_path, ref mut src_archive) =
@@ -225,10 +207,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                             err
                         ));
                     }),
-                    entries
+                    entries.iter().map(|(name, _)| name.as_bytes().to_vec()).collect(),
-                        .iter()
-                        .map(|(name, _)| name.as_bytes().to_vec())
-                        .collect(),
                     ar::GnuSymbolTableFormat::Size32,
                     symbol_table,
                 )
@@ -271,8 +250,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
                 .expect("Couldn't run ranlib");
 
             if !status.success() {
-                self.sess
+                self.sess.fatal(&format!("Ranlib exited with code {:?}", status.code()));
-                    .fatal(&format!("Ranlib exited with code {:?}", status.code()));
             }
         }
     }
@@ -292,13 +270,8 @@ impl<'a> ArArchiveBuilder<'a> {
             let file_name = String::from_utf8(entry.header().identifier().to_vec())
                 .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
             if !skip(&file_name) {
-                self.entries.push((
+                self.entries
-                    file_name,
+                    .push((file_name, ArchiveEntry::FromArchive { archive_index, entry_index: i }));
-                    ArchiveEntry::FromArchive {
-                        archive_index,
-                        entry_index: i,
-                    },
-                ));
             }
             i += 1;
         }

compiler/rustc_codegen_cranelift/src/atomic_shim.rs

@@ -1,185 +0,0 @@
-//! Atomic intrinsics are implemented using a global lock for now, as Cranelift doesn't support
-//! atomic operations yet.
-
-// FIXME implement atomic instructions in Cranelift.
-
-use crate::prelude::*;
-
-#[cfg(all(feature = "jit", unix))]
-#[no_mangle]
-static mut __cg_clif_global_atomic_mutex: libc::pthread_mutex_t = libc::PTHREAD_MUTEX_INITIALIZER;
-
-pub(crate) fn init_global_lock(
-    module: &mut impl Module,
-    bcx: &mut FunctionBuilder<'_>,
-    use_jit: bool,
-) {
-    if use_jit {
-        // When using JIT, dylibs won't find the __cg_clif_global_atomic_mutex data object defined here,
-        // so instead we define it in the cg_clif dylib.
-
-        return;
-    }
-
-    let mut data_ctx = DataContext::new();
-    data_ctx.define_zeroinit(1024); // 1024 bytes should be big enough on all platforms.
-    data_ctx.set_align(16);
-    let atomic_mutex = module
-        .declare_data(
-            "__cg_clif_global_atomic_mutex",
-            Linkage::Export,
-            true,
-            false,
-        )
-        .unwrap();
-    module.define_data(atomic_mutex, &data_ctx).unwrap();
-
-    let pthread_mutex_init = module
-        .declare_function(
-            "pthread_mutex_init",
-            Linkage::Import,
-            &cranelift_codegen::ir::Signature {
-                call_conv: module.target_config().default_call_conv,
-                params: vec![
-                    AbiParam::new(
-                        module.target_config().pointer_type(), /* *mut pthread_mutex_t */
-                    ),
-                    AbiParam::new(
-                        module.target_config().pointer_type(), /* *const pthread_mutex_attr_t */
-                    ),
-                ],
-                returns: vec![AbiParam::new(types::I32 /* c_int */)],
-            },
-        )
-        .unwrap();
-
-    let pthread_mutex_init = module.declare_func_in_func(pthread_mutex_init, bcx.func);
-
-    let atomic_mutex = module.declare_data_in_func(atomic_mutex, bcx.func);
-    let atomic_mutex = bcx
-        .ins()
-        .global_value(module.target_config().pointer_type(), atomic_mutex);
-
-    let nullptr = bcx.ins().iconst(module.target_config().pointer_type(), 0);
-
-    bcx.ins().call(pthread_mutex_init, &[atomic_mutex, nullptr]);
-}
-
-pub(crate) fn init_global_lock_constructor(
-    module: &mut impl Module,
-    constructor_name: &str,
-) -> FuncId {
-    let sig = Signature::new(CallConv::SystemV);
-    let init_func_id = module
-        .declare_function(constructor_name, Linkage::Export, &sig)
-        .unwrap();
-
-    let mut ctx = Context::new();
-    ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
-    {
-        let mut func_ctx = FunctionBuilderContext::new();
-        let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
-
-        let block = bcx.create_block();
-        bcx.switch_to_block(block);
-
-        crate::atomic_shim::init_global_lock(module, &mut bcx, false);
-
-        bcx.ins().return_(&[]);
-        bcx.seal_all_blocks();
-        bcx.finalize();
-    }
-    module
-        .define_function(
-            init_func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
-
-    init_func_id
-}
-
-pub(crate) fn lock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    let atomic_mutex = fx
-        .cx
-        .module
-        .declare_data(
-            "__cg_clif_global_atomic_mutex",
-            Linkage::Import,
-            true,
-            false,
-        )
-        .unwrap();
-
-    let pthread_mutex_lock = fx
-        .cx
-        .module
-        .declare_function(
-            "pthread_mutex_lock",
-            Linkage::Import,
-            &cranelift_codegen::ir::Signature {
-                call_conv: fx.cx.module.target_config().default_call_conv,
-                params: vec![AbiParam::new(
-                    fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */
-                )],
-                returns: vec![AbiParam::new(types::I32 /* c_int */)],
-            },
-        )
-        .unwrap();
-
-    let pthread_mutex_lock = fx
-        .cx
-        .module
-        .declare_func_in_func(pthread_mutex_lock, fx.bcx.func);
-
-    let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
-    let atomic_mutex = fx
-        .bcx
-        .ins()
-        .global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex);
-
-    fx.bcx.ins().call(pthread_mutex_lock, &[atomic_mutex]);
-}
-
-pub(crate) fn unlock_global_lock(fx: &mut FunctionCx<'_, '_, impl Module>) {
-    let atomic_mutex = fx
-        .cx
-        .module
-        .declare_data(
-            "__cg_clif_global_atomic_mutex",
-            Linkage::Import,
-            true,
-            false,
-        )
-        .unwrap();
-
-    let pthread_mutex_unlock = fx
-        .cx
-        .module
-        .declare_function(
-            "pthread_mutex_unlock",
-            Linkage::Import,
-            &cranelift_codegen::ir::Signature {
-                call_conv: fx.cx.module.target_config().default_call_conv,
-                params: vec![AbiParam::new(
-                    fx.cx.module.target_config().pointer_type(), /* *mut pthread_mutex_t */
-                )],
-                returns: vec![AbiParam::new(types::I32 /* c_int */)],
-            },
-        )
-        .unwrap();
-
-    let pthread_mutex_unlock = fx
-        .cx
-        .module
-        .declare_func_in_func(pthread_mutex_unlock, fx.bcx.func);
-
-    let atomic_mutex = fx.cx.module.declare_data_in_func(atomic_mutex, fx.bcx.func);
-    let atomic_mutex = fx
-        .bcx
-        .ins()
-        .global_value(fx.cx.module.target_config().pointer_type(), atomic_mutex);
-
-    fx.bcx.ins().call(pthread_mutex_unlock, &[atomic_mutex]);
-}

compiler/rustc_codegen_cranelift/src/backend.rs

@@ -8,7 +8,7 @@ use rustc_session::Session;
 use cranelift_module::FuncId;
 
 use object::write::*;
-use object::{RelocationEncoding, RelocationKind, SectionKind, SymbolFlags};
+use object::{RelocationEncoding, SectionKind, SymbolFlags};
 
 use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
 
@@ -22,9 +22,7 @@ pub(crate) trait WriteMetadata {
 
 impl WriteMetadata for object::write::Object {
     fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) {
-        let segment = self
+        let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
-            .segment_name(object::write::StandardSegment::Data)
-            .to_vec();
         let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
         let offset = self.append_section_data(section_id, &data, 1);
         // For MachO and probably PE this is necessary to prevent the linker from throwing away the
@@ -74,11 +72,7 @@ impl WriteDebugInfo for ObjectProduct {
         let section_id = self.object.add_section(
             segment,
             name,
-            if id == SectionId::EhFrame {
+            if id == SectionId::EhFrame { SectionKind::ReadOnlyData } else { SectionKind::Debug },
-                SectionKind::ReadOnlyData
-            } else {
-                SectionKind::Debug
-            },
         );
         self.object
             .section_mut(section_id)
@@ -118,49 +112,6 @@ impl WriteDebugInfo for ObjectProduct {
     }
 }
 
-// FIXME remove once atomic instructions are implemented in Cranelift.
-pub(crate) trait AddConstructor {
-    fn add_constructor(&mut self, func_id: FuncId);
-}
-
-impl AddConstructor for ObjectProduct {
-    fn add_constructor(&mut self, func_id: FuncId) {
-        let symbol = self.function_symbol(func_id);
-        let segment = self
-            .object
-            .segment_name(object::write::StandardSegment::Data);
-        let init_array_section =
-            self.object
-                .add_section(segment.to_vec(), b".init_array".to_vec(), SectionKind::Data);
-        let address_size = self
-            .object
-            .architecture()
-            .address_size()
-            .expect("address_size must be known")
-            .bytes();
-        self.object.append_section_data(
-            init_array_section,
-            &std::iter::repeat(0)
-                .take(address_size.into())
-                .collect::<Vec<u8>>(),
-            8,
-        );
-        self.object
-            .add_relocation(
-                init_array_section,
-                object::write::Relocation {
-                    offset: 0,
-                    size: address_size * 8,
-                    kind: RelocationKind::Absolute,
-                    encoding: RelocationEncoding::Generic,
-                    symbol,
-                    addend: 0,
-                },
-            )
-            .unwrap();
-    }
-}
-
 pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
     let triple = crate::build_isa(sess).triple().clone();
 
@@ -175,10 +126,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
         target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
         target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
         target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
-        architecture => sess.fatal(&format!(
+        architecture => {
-            "target architecture {:?} is unsupported",
+            sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
-            architecture,
+        }
-        )),
     };
     let endian = match triple.endianness().unwrap() {
         target_lexicon::Endianness::Little => object::Endianness::Little,

compiler/rustc_codegen_cranelift/src/base.rs

@@ -8,7 +8,7 @@ use rustc_target::abi::call::FnAbi;
 use crate::prelude::*;
 
 pub(crate) fn codegen_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'tcx, impl Module>,
+    cx: &mut crate::CodegenCx<'_, 'tcx>,
     instance: Instance<'tcx>,
     linkage: Linkage,
 ) {
@@ -38,9 +38,8 @@ pub(crate) fn codegen_fn<'tcx>(
 
     // Predefine blocks
     let start_block = bcx.create_block();
-    let block_map: IndexVec<BasicBlock, Block> = (0..mir.basic_blocks().len())
+    let block_map: IndexVec<BasicBlock, Block> =
-        .map(|_| bcx.create_block())
+        (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
-        .collect();
 
     // Make FunctionCx
     let pointer_type = cx.module.target_config().pointer_type();
@@ -68,22 +67,23 @@ pub(crate) fn codegen_fn<'tcx>(
         inline_asm_index: 0,
     };
 
-    let arg_uninhabited = fx.mir.args_iter().any(|arg| {
+    let arg_uninhabited = fx
-        fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty))
+        .mir
-            .abi
+        .args_iter()
-            .is_uninhabited()
+        .any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
-    });
 
-    if arg_uninhabited {
+    if !crate::constant::check_constants(&mut fx) {
-        fx.bcx
+        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
-            .append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+        fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+        crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
+    } else if arg_uninhabited {
+        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
         fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
         crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument");
     } else {
         tcx.sess.time("codegen clif ir", || {
-            tcx.sess.time("codegen prelude", || {
+            tcx.sess
-                crate::abi::codegen_fn_prelude(&mut fx, start_block)
+                .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
-            });
             codegen_fn_content(&mut fx);
         });
     }
@@ -131,11 +131,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let module = &mut cx.module;
     tcx.sess.time("define function", || {
         module
-            .define_function(
+            .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
-                func_id,
-                context,
-                &mut cranelift_codegen::binemit::NullTrapSink {},
-            )
             .unwrap()
     });
 
@@ -149,15 +145,13 @@ pub(crate) fn codegen_fn<'tcx>(
         &clif_comments,
     );
 
-    if let Some(mach_compile_result) = &context.mach_compile_result {
+    if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
-        if let Some(disasm) = &mach_compile_result.disasm {
         crate::pretty_clif::write_ir_file(
             tcx,
             &format!("{}.vcode", tcx.symbol_name(instance).name),
             |file| file.write_all(disasm.as_bytes()),
         )
     }
-    }
 
     // Define debuginfo for function
     let isa = cx.module.isa();
@@ -199,16 +193,13 @@ pub(crate) fn verify_func(
                     Some(Box::new(writer)),
                     err,
                 );
-                tcx.sess
+                tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error));
-                    .fatal(&format!("cranelift verify error:\n{}", pretty_error));
             }
         }
     });
 }
 
-fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
+fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
-    crate::constant::check_constants(fx);
-
     for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
         let block = fx.get_block(bb);
         fx.bcx.switch_to_block(block);
@@ -231,11 +222,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
         #[cfg(debug_assertions)]
         {
             let mut terminator_head = "\n".to_string();
-            bb_data
+            bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
-                .terminator()
-                .kind
-                .fmt_head(&mut terminator_head)
-                .unwrap();
             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
             fx.add_comment(inst, terminator_head);
         }
@@ -267,13 +254,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             TerminatorKind::Return => {
                 crate::abi::codegen_return(fx);
             }
-            TerminatorKind::Assert {
+            TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => {
-                cond,
-                expected,
-                msg,
-                target,
-                cleanup: _,
-            } => {
                 if !fx.tcx.sess.overflow_checks() {
                     if let mir::AssertKind::OverflowNeg(_) = *msg {
                         let target = fx.get_block(*target);
@@ -319,11 +300,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
                 }
             }
 
-            TerminatorKind::SwitchInt {
+            TerminatorKind::SwitchInt { discr, switch_ty, targets } => {
-                discr,
-                switch_ty,
-                targets,
-            } => {
                 let discr = codegen_operand(fx, discr).load_scalar(fx);
 
                 let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
@@ -433,11 +410,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             | TerminatorKind::GeneratorDrop => {
                 bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
             }
-            TerminatorKind::Drop {
+            TerminatorKind::Drop { place, target, unwind: _ } => {
-                place,
-                target,
-                unwind: _,
-            } => {
                 let drop_place = codegen_place(fx, *place);
                 crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place);
 
@@ -452,7 +425,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
 }
 
 fn codegen_stmt<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     #[allow(unused_variables)] cur_block: Block,
     stmt: &Statement<'tcx>,
 ) {
@@ -470,10 +443,7 @@ fn codegen_stmt<'tcx>(
     }
 
     match &stmt.kind {
-        StatementKind::SetDiscriminant {
+        StatementKind::SetDiscriminant { place, variant_index } => {
-            place,
-            variant_index,
-        } => {
             let place = codegen_place(fx, **place);
             crate::discriminant::codegen_set_discriminant(fx, place, *variant_index);
         }
@@ -594,19 +564,11 @@ fn codegen_stmt<'tcx>(
                     let from_ty = operand.layout().ty;
                     let to_ty = fx.monomorphize(to_ty);
 
-                    fn is_fat_ptr<'tcx>(
+                    fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool {
-                        fx: &FunctionCx<'_, 'tcx, impl Module>,
-                        ty: Ty<'tcx>,
-                    ) -> bool {
                         ty.builtin_deref(true)
-                            .map(
+                            .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| {
-                                |ty::TypeAndMut {
-                                     ty: pointee_ty,
-                                     mutbl: _,
-                                 }| {
                                 has_ptr_meta(fx.tcx, pointee_ty)
-                                },
+                            })
-                            )
                             .unwrap_or(false)
                     }
 
@@ -626,50 +588,22 @@ fn codegen_stmt<'tcx>(
                             ty::Uint(_) | ty::Int(_) => {}
                             _ => unreachable!("cast adt {} -> {}", from_ty, to_ty),
                         }
+                        let to_clif_ty = fx.clif_type(to_ty).unwrap();
 
-                        use rustc_target::abi::{Int, TagEncoding, Variants};
+                        let discriminant = crate::discriminant::codegen_get_discriminant(
+                            fx,
+                            operand,
+                            fx.layout_of(operand.layout().ty.discriminant_ty(fx.tcx)),
+                        )
+                        .load_scalar(fx);
 
-                        match operand.layout().variants {
+                        let res = crate::cast::clif_intcast(
-                            Variants::Single { index } => {
+                            fx,
-                                let discr = operand
+                            discriminant,
-                                    .layout()
+                            to_clif_ty,
-                                    .ty
+                            to_ty.is_signed(),
-                                    .discriminant_for_variant(fx.tcx, index)
+                        );
-                                    .unwrap();
+                        lval.write_cvalue(fx, CValue::by_val(res, dest_layout));
-                                let discr = if discr.ty.is_signed() {
-                                    fx.layout_of(discr.ty).size.sign_extend(discr.val)
-                                } else {
-                                    discr.val
-                                };
-                                let discr = discr.into();
-
-                                let discr = CValue::const_val(fx, fx.layout_of(to_ty), discr);
-                                lval.write_cvalue(fx, discr);
-                            }
-                            Variants::Multiple {
-                                ref tag,
-                                tag_field,
-                                tag_encoding: TagEncoding::Direct,
-                                variants: _,
-                            } => {
-                                let cast_to = fx.clif_type(dest_layout.ty).unwrap();
-
-                                // Read the tag/niche-encoded discriminant from memory.
-                                let encoded_discr =
-                                    operand.value_field(fx, mir::Field::new(tag_field));
-                                let encoded_discr = encoded_discr.load_scalar(fx);
-
-                                // Decode the discriminant (specifically if it's niche-encoded).
-                                let signed = match tag.value {
-                                    Int(_, signed) => signed,
-                                    _ => false,
-                                };
-                                let val = clif_intcast(fx, encoded_discr, cast_to, signed);
-                                let val = CValue::by_val(val, dest_layout);
-                                lval.write_cvalue(fx, val);
-                            }
-                            Variants::Multiple { .. } => unreachable!(),
-                        }
                     } else {
                         let to_clif_ty = fx.clif_type(to_ty).unwrap();
                         let from = operand.load_scalar(fx);
@@ -730,8 +664,7 @@ fn codegen_stmt<'tcx>(
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
                         let val = operand.load_scalar(fx);
-                        fx.bcx
+                        fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times);
-                            .call_memset(fx.cx.module.target_config(), addr, val, times);
                     } else {
                         let loop_block = fx.bcx.create_block();
                         let loop_block2 = fx.bcx.create_block();
@@ -766,18 +699,12 @@ fn codegen_stmt<'tcx>(
                     let content_ty = fx.monomorphize(content_ty);
                     let layout = fx.layout_of(content_ty);
                     let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64);
-                    let llalign = fx
+                    let llalign = fx.bcx.ins().iconst(usize_type, layout.align.abi.bytes() as i64);
-                        .bcx
-                        .ins()
-                        .iconst(usize_type, layout.align.abi.bytes() as i64);
                     let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty));
 
                     // Allocate space:
-                    let def_id = match fx
+                    let def_id =
-                        .tcx
+                        match fx.tcx.lang_items().require(rustc_hir::LangItem::ExchangeMalloc) {
-                        .lang_items()
-                        .require(rustc_hir::LangItem::ExchangeMalloc)
-                    {
                             Ok(id) => id,
                             Err(s) => {
                                 fx.tcx
@@ -792,10 +719,11 @@ fn codegen_stmt<'tcx>(
                     lval.write_cvalue(fx, CValue::by_val(ptr, box_layout));
                 }
                 Rvalue::NullaryOp(NullOp::SizeOf, ty) => {
-                    assert!(lval
+                    assert!(
-                        .layout()
+                        lval.layout()
                             .ty
-                        .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all()));
+                            .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all())
+                    );
                     let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes();
                     let val =
                         CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into());
@@ -823,11 +751,7 @@ fn codegen_stmt<'tcx>(
 
         StatementKind::LlvmInlineAsm(asm) => {
             use rustc_span::symbol::Symbol;
-            let LlvmInlineAsm {
+            let LlvmInlineAsm { asm, outputs, inputs } = &**asm;
-                asm,
-                outputs,
-                inputs,
-            } = &**asm;
             let rustc_hir::LlvmInlineAsmInner {
                 asm: asm_code,         // Name
                 outputs: output_names, // Vec<LlvmInlineAsmOutput>
@@ -843,15 +767,9 @@ fn codegen_stmt<'tcx>(
                     // Black box
                 }
                 "mov %rbx, %rsi\n                  cpuid\n                  xchg %rbx, %rsi" => {
-                    assert_eq!(
+                    assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]);
-                        input_names,
-                        &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]
-                    );
                     assert_eq!(output_names.len(), 4);
-                    for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"])
+                    for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() {
-                        .iter()
-                        .enumerate()
-                    {
                         assert_eq!(&output_names[i].constraint.as_str(), c);
                         assert!(!output_names[i].is_rw);
                         assert!(!output_names[i].is_indirect);
@@ -897,12 +815,7 @@ fn codegen_stmt<'tcx>(
                     crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported");
                 }
                 // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
-                _ if fx
+                _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => {
-                    .tcx
-                    .symbol_name(fx.instance)
-                    .name
-                    .starts_with("___chkstk") =>
-                {
                     crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
                 }
                 _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => {
@@ -922,27 +835,21 @@ fn codegen_stmt<'tcx>(
     }
 }
 
-fn codegen_array_len<'tcx>(
+fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value {
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    place: CPlace<'tcx>,
-) -> Value {
     match *place.layout().ty.kind() {
         ty::Array(_elem_ty, len) => {
-            let len = fx
+            let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
-                .monomorphize(len)
-                .eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64;
             fx.bcx.ins().iconst(fx.pointer_type, len)
         }
-        ty::Slice(_elem_ty) => place
+        ty::Slice(_elem_ty) => {
-            .to_ptr_maybe_unsized()
+            place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
-            .1
+        }
-            .expect("Length metadata for slice place"),
         _ => bug!("Rvalue::Len({:?})", place),
     }
 }
 
 pub(crate) fn codegen_place<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     place: Place<'tcx>,
 ) -> CPlace<'tcx> {
     let mut cplace = fx.get_local_place(place.local);
@@ -959,11 +866,7 @@ pub(crate) fn codegen_place<'tcx>(
                 let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx);
                 cplace = cplace.place_index(fx, index);
             }
-            PlaceElem::ConstantIndex {
+            PlaceElem::ConstantIndex { offset, min_length: _, from_end } => {
-                offset,
-                min_length: _,
-                from_end,
-            } => {
                 let offset: u64 = offset;
                 let index = if !from_end {
                     fx.bcx.ins().iconst(fx.pointer_type, offset as i64)
@@ -1014,7 +917,7 @@ pub(crate) fn codegen_place<'tcx>(
 }
 
 pub(crate) fn codegen_operand<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
 ) -> CValue<'tcx> {
     match operand {
@@ -1026,34 +929,24 @@ pub(crate) fn codegen_operand<'tcx>(
     }
 }
 
-pub(crate) fn codegen_panic<'tcx>(
+pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) {
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    msg_str: &str,
-    span: Span,
-) {
     let location = fx.get_caller_location(span).load_scalar(fx);
 
     let msg_ptr = fx.anonymous_str("assert", msg_str);
-    let msg_len = fx
+    let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
-        .bcx
-        .ins()
-        .iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len, location];
 
     codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span);
 }
 
 pub(crate) fn codegen_panic_inner<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     lang_item: rustc_hir::LangItem,
     args: &[Value],
     span: Span,
 ) {
-    let def_id = fx
+    let def_id =
-        .tcx
+        fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
-        .lang_items()
-        .require(lang_item)
-        .unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
 
     let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
     let symbol_name = fx.tcx.symbol_name(instance).name;

compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs

@@ -27,13 +27,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
         config.opts.cg.panic = Some(PanicStrategy::Abort);
         config.opts.debugging_opts.panic_abort_tests = true;
         config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| {
-            std::env::current_exe()
+            std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .to_owned()
         }));
     }
 }

compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs

@@ -46,15 +46,8 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
 
         config.opts.cg.panic = Some(PanicStrategy::Abort);
         config.opts.debugging_opts.panic_abort_tests = true;
-        config.opts.maybe_sysroot = Some(
+        config.opts.maybe_sysroot =
-            std::env::current_exe()
+            Some(std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned());
-                .unwrap()
-                .parent()
-                .unwrap()
-                .parent()
-                .unwrap()
-                .to_owned(),
-        );
     }
 }
 

compiler/rustc_codegen_cranelift/src/cast.rs

@@ -3,7 +3,7 @@
 use crate::prelude::*;
 
 pub(crate) fn clif_intcast(
-    fx: &mut FunctionCx<'_, '_, impl Module>,
+    fx: &mut FunctionCx<'_, '_, '_>,
     val: Value,
     to: Type,
     signed: bool,
@@ -40,18 +40,14 @@ pub(crate) fn clif_intcast(
         // reduce
         (types::I128, _) => {
             let (lsb, _msb) = fx.bcx.ins().isplit(val);
-            if to == types::I64 {
+            if to == types::I64 { lsb } else { fx.bcx.ins().ireduce(to, lsb) }
-                lsb
-            } else {
-                fx.bcx.ins().ireduce(to, lsb)
-            }
         }
         (_, _) => fx.bcx.ins().ireduce(to, val),
     }
 }
 
 pub(crate) fn clif_int_or_float_cast(
-    fx: &mut FunctionCx<'_, '_, impl Module>,
+    fx: &mut FunctionCx<'_, '_, '_>,
     from: Value,
     from_signed: bool,
     to_ty: Type,
@@ -87,11 +83,7 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            let from_rust_ty = if from_signed {
+            let from_rust_ty = if from_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-                fx.tcx.types.i128
-            } else {
-                fx.tcx.types.u128
-            };
 
             let to_rust_ty = match to_ty {
                 types::F32 => fx.tcx.types.f32,
@@ -100,11 +92,7 @@ pub(crate) fn clif_int_or_float_cast(
             };
 
             return fx
-                .easy_call(
+                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                    &name,
-                    &[CValue::by_val(from, fx.layout_of(from_rust_ty))],
-                    to_rust_ty,
-                )
                 .load_scalar(fx);
         }
 
@@ -138,18 +126,10 @@ pub(crate) fn clif_int_or_float_cast(
                 _ => unreachable!(),
             };
 
-            let to_rust_ty = if to_signed {
+            let to_rust_ty = if to_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-                fx.tcx.types.i128
-            } else {
-                fx.tcx.types.u128
-            };
 
             return fx
-                .easy_call(
+                .easy_call(&name, &[CValue::by_val(from, fx.layout_of(from_rust_ty))], to_rust_ty)
-                    &name,
-                    &[CValue::by_val(from, fx.layout_of(from_rust_ty))],
-                    to_rust_ty,
-                )
                 .load_scalar(fx);
         }
 

compiler/rustc_codegen_cranelift/src/codegen_i128.rs

@@ -5,13 +5,17 @@ use cranelift_codegen::ir::ArgumentPurpose;
 use crate::prelude::*;
 
 pub(crate) fn maybe_codegen<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     checked: bool,
     lhs: CValue<'tcx>,
     rhs: CValue<'tcx>,
 ) -> Option<CValue<'tcx>> {
-    if lhs.layout().ty != fx.tcx.types.u128 && lhs.layout().ty != fx.tcx.types.i128 {
+    if lhs.layout().ty != fx.tcx.types.u128
+        && lhs.layout().ty != fx.tcx.types.i128
+        && rhs.layout().ty != fx.tcx.types.u128
+        && rhs.layout().ty != fx.tcx.types.i128
+    {
         return None;
     }
 
@@ -27,11 +31,7 @@ pub(crate) fn maybe_codegen<'tcx>(
         }
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked => {
-            let val_ty = if is_signed {
+            let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-                fx.tcx.types.i128
-            } else {
-                fx.tcx.types.u128
-            };
             Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
         }
         BinOp::Add | BinOp::Sub | BinOp::Mul => {
@@ -43,11 +43,7 @@ pub(crate) fn maybe_codegen<'tcx>(
                 AbiParam::new(types::I128),
                 AbiParam::new(types::I128),
             ];
-            let args = [
+            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
-                out_place.to_ptr().get_addr(fx),
-                lhs.load_scalar(fx),
-                rhs.load_scalar(fx),
-            ];
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -97,70 +93,23 @@ pub(crate) fn maybe_codegen<'tcx>(
                 None
             };
 
-            // Optimize `val >> 64`, because compiler_builtins uses it to deconstruct an 128bit
-            // integer into its lsb and msb.
-            // https://github.com/rust-lang-nursery/compiler-builtins/blob/79a6a1603d5672cbb9187ff41ff4d9b5048ac1cb/src/int/mod.rs#L217
-            if resolve_value_imm(fx.bcx.func, rhs_val) == Some(64) {
-                let (lhs_lsb, lhs_msb) = fx.bcx.ins().isplit(lhs_val);
-                let all_zeros = fx.bcx.ins().iconst(types::I64, 0);
-                let val = match (bin_op, is_signed) {
-                    (BinOp::Shr, false) => {
-                        let val = fx.bcx.ins().iconcat(lhs_msb, all_zeros);
-                        Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.u128)))
-                    }
-                    (BinOp::Shr, true) => {
-                        let sign = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, lhs_msb, 0);
-                        let all_ones = fx.bcx.ins().iconst(types::I64, u64::MAX as i64);
-                        let all_sign_bits = fx.bcx.ins().select(sign, all_zeros, all_ones);
-
-                        let val = fx.bcx.ins().iconcat(lhs_msb, all_sign_bits);
-                        Some(CValue::by_val(val, fx.layout_of(fx.tcx.types.i128)))
-                    }
-                    (BinOp::Shl, _) => {
-                        let val_ty = if is_signed {
-                            fx.tcx.types.i128
-                        } else {
-                            fx.tcx.types.u128
-                        };
-                        let val = fx.bcx.ins().iconcat(all_zeros, lhs_lsb);
-                        Some(CValue::by_val(val, fx.layout_of(val_ty)))
-                    }
-                    _ => None,
-                };
-                if let Some(val) = val {
-                    if let Some(is_overflow) = is_overflow {
-                        let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
-                        let val = val.load_scalar(fx);
-                        return Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty)));
-                    } else {
-                        return Some(val);
-                    }
-                }
-            }
-
             let truncated_rhs = clif_intcast(fx, rhs_val, types::I32, false);
-            let truncated_rhs = CValue::by_val(truncated_rhs, fx.layout_of(fx.tcx.types.u32));
+            let val = match bin_op {
-            let val = match (bin_op, is_signed) {
+                BinOp::Shl => fx.bcx.ins().ishl(lhs_val, truncated_rhs),
-                (BinOp::Shl, false) => {
+                BinOp::Shr => {
-                    fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.u128)
+                    if is_signed {
+                        fx.bcx.ins().sshr(lhs_val, truncated_rhs)
+                    } else {
+                        fx.bcx.ins().ushr(lhs_val, truncated_rhs)
                     }
-                (BinOp::Shl, true) => {
-                    fx.easy_call("__ashlti3", &[lhs, truncated_rhs], fx.tcx.types.i128)
                 }
-                (BinOp::Shr, false) => {
+                _ => unreachable!(),
-                    fx.easy_call("__lshrti3", &[lhs, truncated_rhs], fx.tcx.types.u128)
-                }
-                (BinOp::Shr, true) => {
-                    fx.easy_call("__ashrti3", &[lhs, truncated_rhs], fx.tcx.types.i128)
-                }
-                (_, _) => unreachable!(),
             };
             if let Some(is_overflow) = is_overflow {
                 let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
-                let val = val.load_scalar(fx);
                 Some(CValue::by_val_pair(val, is_overflow, fx.layout_of(out_ty)))
             } else {
-                Some(val)
+                Some(CValue::by_val(val, lhs.layout()))
             }
         }
     }

compiler/rustc_codegen_cranelift/src/common.rs

@@ -3,8 +3,6 @@ use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{Integer, Primitive};
 use rustc_target::spec::{HasTargetSpec, Target};
 
-use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef};
-
 use crate::prelude::*;
 
 pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@@ -56,11 +54,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
             FloatTy::F64 => types::F64,
         },
         ty::FnPtr(_) => pointer_ty(tcx),
-        ty::RawPtr(TypeAndMut {
+        ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
-            ty: pointee_ty,
-            mutbl: _,
-        })
-        | ty::Ref(_, pointee_ty, _) => {
             if has_ptr_meta(tcx, pointee_ty) {
                 return None;
             } else {
@@ -99,11 +93,7 @@ fn clif_pair_type_from_ty<'tcx>(
             }
             (a, b)
         }
-        ty::RawPtr(TypeAndMut {
+        ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => {
-            ty: pointee_ty,
-            mutbl: _,
-        })
-        | ty::Ref(_, pointee_ty, _) => {
             if has_ptr_meta(tcx, pointee_ty) {
                 (pointer_ty(tcx), pointer_ty(tcx))
             } else {
@@ -116,15 +106,8 @@ fn clif_pair_type_from_ty<'tcx>(
 
 /// Is a pointer to this type a fat ptr?
 pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
-    let ptr_ty = tcx.mk_ptr(TypeAndMut {
+    let ptr_ty = tcx.mk_ptr(TypeAndMut { ty, mutbl: rustc_hir::Mutability::Not });
-        ty,
+    match &tcx.layout_of(ParamEnv::reveal_all().and(ptr_ty)).unwrap().abi {
-        mutbl: rustc_hir::Mutability::Not,
-    });
-    match &tcx
-        .layout_of(ParamEnv::reveal_all().and(ptr_ty))
-        .unwrap()
-        .abi
-    {
         Abi::Scalar(_) => false,
         Abi::ScalarPair(_, _) => true,
         abi => unreachable!("Abi of ptr to {:?} is {:?}???", ty, abi),
@@ -132,7 +115,7 @@ pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
 }
 
 pub(crate) fn codegen_icmp_imm(
-    fx: &mut FunctionCx<'_, '_, impl Module>,
+    fx: &mut FunctionCx<'_, '_, '_>,
     intcc: IntCC,
     lhs: Value,
     rhs: i128,
@@ -175,51 +158,6 @@ pub(crate) fn codegen_icmp_imm(
     }
 }
 
-fn resolve_normal_value_imm(func: &Function, val: Value) -> Option<i64> {
-    if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) {
-        if let InstructionData::UnaryImm {
-            opcode: Opcode::Iconst,
-            imm,
-        } = func.dfg[inst]
-        {
-            Some(imm.into())
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-fn resolve_128bit_value_imm(func: &Function, val: Value) -> Option<u128> {
-    let (lsb, msb) = if let ValueDef::Result(inst, 0 /*param*/) = func.dfg.value_def(val) {
-        if let InstructionData::Binary {
-            opcode: Opcode::Iconcat,
-            args: [lsb, msb],
-        } = func.dfg[inst]
-        {
-            (lsb, msb)
-        } else {
-            return None;
-        }
-    } else {
-        return None;
-    };
-
-    let lsb = u128::from(resolve_normal_value_imm(func, lsb)? as u64);
-    let msb = u128::from(resolve_normal_value_imm(func, msb)? as u64);
-
-    Some(msb << 64 | lsb)
-}
-
-pub(crate) fn resolve_value_imm(func: &Function, val: Value) -> Option<u128> {
-    if func.dfg.value_type(val) == types::I128 {
-        resolve_128bit_value_imm(func, val)
-    } else {
-        resolve_normal_value_imm(func, val).map(|imm| u128::from(imm as u64))
-    }
-}
-
 pub(crate) fn type_min_max_value(
     bcx: &mut FunctionBuilder<'_>,
     ty: Type,
@@ -288,8 +226,8 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
     }
 }
 
-pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
+pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
-    pub(crate) cx: &'clif mut crate::CodegenCx<'tcx, M>,
+    pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>,
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) pointer_type: Type, // Cached from module
 
@@ -316,7 +254,7 @@ pub(crate) struct FunctionCx<'clif, 'tcx, M: Module> {
     pub(crate) inline_asm_index: u32,
 }
 
-impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> LayoutOf for FunctionCx<'_, '_, 'tcx> {
     type Ty = Ty<'tcx>;
     type TyAndLayout = TyAndLayout<'tcx>;
 
@@ -325,31 +263,31 @@ impl<'tcx, M: Module> LayoutOf for FunctionCx<'_, 'tcx, M> {
     }
 }
 
-impl<'tcx, M: Module> layout::HasTyCtxt<'tcx> for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> layout::HasTyCtxt<'tcx> for FunctionCx<'_, '_, 'tcx> {
     fn tcx<'b>(&'b self) -> TyCtxt<'tcx> {
         self.tcx
     }
 }
 
-impl<'tcx, M: Module> rustc_target::abi::HasDataLayout for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> rustc_target::abi::HasDataLayout for FunctionCx<'_, '_, 'tcx> {
     fn data_layout(&self) -> &rustc_target::abi::TargetDataLayout {
         &self.tcx.data_layout
     }
 }
 
-impl<'tcx, M: Module> layout::HasParamEnv<'tcx> for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> layout::HasParamEnv<'tcx> for FunctionCx<'_, '_, 'tcx> {
     fn param_env(&self) -> ParamEnv<'tcx> {
         ParamEnv::reveal_all()
     }
 }
 
-impl<'tcx, M: Module> HasTargetSpec for FunctionCx<'_, 'tcx, M> {
+impl<'tcx> HasTargetSpec for FunctionCx<'_, '_, 'tcx> {
     fn target_spec(&self) -> &Target {
         &self.tcx.sess.target
     }
 }
 
-impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
+impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     pub(crate) fn monomorphize<T>(&self, value: T) -> T
     where
         T: TypeFoldable<'tcx> + Copy,
@@ -416,12 +354,7 @@ impl<'tcx, M: Module> FunctionCx<'_, 'tcx, M> {
         let msg_id = self
             .cx
             .module
-            .declare_data(
+            .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false)
-                &format!("__{}_{:08x}", prefix, msg_hash),
-                Linkage::Local,
-                false,
-                false,
-            )
             .unwrap();
 
         // Ignore DuplicateDefinition error, as the data will be the same
@@ -444,9 +377,7 @@ impl<'tcx> LayoutOf for RevealAllLayoutCx<'tcx> {
 
     fn layout_of(&self, ty: Ty<'tcx>) -> TyAndLayout<'tcx> {
         assert!(!ty.still_further_specializable());
-        self.0
+        self.0.layout_of(ParamEnv::reveal_all().and(&ty)).unwrap_or_else(|e| {
-            .layout_of(ParamEnv::reveal_all().and(&ty))
-            .unwrap_or_else(|e| {
             if let layout::LayoutError::SizeOverflow(_) = e {
                 self.0.sess.fatal(&e.to_string())
             } else {

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -28,7 +28,7 @@ enum TodoItem {
 }
 
 impl ConstantCx {
-    pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut impl Module) {
+    pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) {
         //println!("todo {:?}", self.todo);
         define_all_allocs(tcx, module, &mut self);
         //println!("done {:?}", self.done);
@@ -36,21 +36,20 @@ impl ConstantCx {
     }
 }
 
-pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) {
+pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
+    let mut all_constants_ok = true;
     for constant in &fx.mir.required_consts {
         let const_ = fx.monomorphize(constant.literal);
         match const_.val {
             ConstKind::Value(_) => {}
             ConstKind::Unevaluated(def, ref substs, promoted) => {
                 if let Err(err) =
-                    fx.tcx
+                    fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
-                        .const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
                 {
+                    all_constants_ok = false;
                     match err {
                         ErrorHandled::Reported(ErrorReported) | ErrorHandled::Linted => {
-                            fx.tcx
+                            fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
-                                .sess
-                                .span_err(constant.span, "erroneous constant encountered");
                         }
                         ErrorHandled::TooGeneric => {
                             span_bug!(
@@ -69,6 +68,7 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, impl Module>) {
             | ConstKind::Error(_) => unreachable!("{:?}", const_),
         }
     }
+    all_constants_ok
 }
 
 pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
@@ -76,11 +76,11 @@ pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
 }
 
 pub(crate) fn codegen_tls_ref<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CValue<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
+    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     #[cfg(debug_assertions)]
     fx.add_comment(local_data_id, format!("tls {:?}", def_id));
@@ -89,11 +89,11 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 }
 
 fn codegen_static_ref<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     def_id: DefId,
     layout: TyAndLayout<'tcx>,
 ) -> CPlace<'tcx> {
-    let data_id = data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
+    let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     #[cfg(debug_assertions)]
     fx.add_comment(local_data_id, format!("{:?}", def_id));
@@ -110,7 +110,7 @@ fn codegen_static_ref<'tcx>(
 }
 
 pub(crate) fn codegen_constant<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     constant: &Constant<'tcx>,
 ) -> CValue<'tcx> {
     let const_ = fx.monomorphize(constant.literal);
@@ -128,20 +128,10 @@ pub(crate) fn codegen_constant<'tcx>(
             .to_cvalue(fx);
         }
         ConstKind::Unevaluated(def, ref substs, promoted) => {
-            match fx
+            match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None) {
-                .tcx
-                .const_eval_resolve(ParamEnv::reveal_all(), def, substs, promoted, None)
-            {
                 Ok(const_val) => const_val,
                 Err(_) => {
-                    fx.tcx
+                    span_bug!(constant.span, "erroneous constant not captured by required_consts");
-                        .sess
-                        .span_err(constant.span, "erroneous constant encountered");
-                    return crate::trap::trap_unreachable_ret_value(
-                        fx,
-                        fx.layout_of(const_.ty),
-                        "erroneous constant encountered",
-                    );
                 }
             }
         }
@@ -156,7 +146,7 @@ pub(crate) fn codegen_constant<'tcx>(
 }
 
 pub(crate) fn codegen_const_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     const_val: ConstValue<'tcx>,
     ty: Ty<'tcx>,
 ) -> CValue<'tcx> {
@@ -172,9 +162,7 @@ pub(crate) fn codegen_const_value<'tcx>(
             if fx.clif_type(layout.ty).is_none() {
                 let (size, align) = (layout.size, layout.align.pref);
                 let mut alloc = Allocation::from_bytes(
-                    std::iter::repeat(0)
+                    std::iter::repeat(0).take(size.bytes_usize()).collect::<Vec<u8>>(),
-                        .take(size.bytes_usize())
-                        .collect::<Vec<u8>>(),
                     align,
                 );
                 let ptr = Pointer::new(AllocId(!0), Size::ZERO); // The alloc id is never used
@@ -190,11 +178,8 @@ pub(crate) fn codegen_const_value<'tcx>(
                     let base_addr = match alloc_kind {
                         Some(GlobalAlloc::Memory(alloc)) => {
                             fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
-                            let data_id = data_id_for_alloc_id(
+                            let data_id =
-                                &mut fx.cx.module,
+                                data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
-                                ptr.alloc_id,
-                                alloc.mutability,
-                            );
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
                             #[cfg(debug_assertions)]
@@ -203,15 +188,14 @@ pub(crate) fn codegen_const_value<'tcx>(
                         }
                         Some(GlobalAlloc::Function(instance)) => {
                             let func_id =
-                                crate::abi::import_function(fx.tcx, &mut fx.cx.module, instance);
+                                crate::abi::import_function(fx.tcx, fx.cx.module, instance);
                             let local_func_id =
                                 fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
                             fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
                         }
                         Some(GlobalAlloc::Static(def_id)) => {
                             assert!(fx.tcx.is_static(def_id));
-                            let data_id =
+                            let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
-                                data_id_for_static(fx.tcx, &mut fx.cx.module, def_id, false);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
                             #[cfg(debug_assertions)]
@@ -221,9 +205,7 @@ pub(crate) fn codegen_const_value<'tcx>(
                         None => bug!("missing allocation {:?}", ptr.alloc_id),
                     };
                     let val = if ptr.offset.bytes() != 0 {
-                        fx.bcx
+                        fx.bcx.ins().iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
-                            .ins()
-                            .iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
                     } else {
                         base_addr
                     };
@@ -240,22 +222,22 @@ pub(crate) fn codegen_const_value<'tcx>(
             let ptr = pointer_for_allocation(fx, data)
                 .offset_i64(fx, i64::try_from(start).unwrap())
                 .get_addr(fx);
-            let len = fx.bcx.ins().iconst(
+            let len = fx
-                fx.pointer_type,
+                .bcx
-                i64::try_from(end.checked_sub(start).unwrap()).unwrap(),
+                .ins()
-            );
+                .iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap());
             CValue::by_val_pair(ptr, len, layout)
         }
     }
 }
 
 fn pointer_for_allocation<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     alloc: &'tcx Allocation,
 ) -> crate::pointer::Pointer {
     let alloc_id = fx.tcx.create_memory_alloc(alloc);
     fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
-    let data_id = data_id_for_alloc_id(&mut fx.cx.module, alloc_id, alloc.mutability);
+    let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
 
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
     #[cfg(debug_assertions)]
@@ -265,7 +247,7 @@ fn pointer_for_allocation<'tcx>(
 }
 
 fn data_id_for_alloc_id(
-    module: &mut impl Module,
+    module: &mut dyn Module,
     alloc_id: AllocId,
     mutability: rustc_hir::Mutability,
 ) -> DataId {
@@ -281,7 +263,7 @@ fn data_id_for_alloc_id(
 
 fn data_id_for_static(
     tcx: TyCtxt<'_>,
-    module: &mut impl Module,
+    module: &mut dyn Module,
     def_id: DefId,
     definition: bool,
 ) -> DataId {
@@ -304,12 +286,7 @@ fn data_id_for_static(
     } else {
         !ty.is_freeze(tcx.at(DUMMY_SP), ParamEnv::reveal_all())
     };
-    let align = tcx
+    let align = tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().align.pref.bytes();
-        .layout_of(ParamEnv::reveal_all().and(ty))
-        .unwrap()
-        .align
-        .pref
-        .bytes();
 
     let attrs = tcx.codegen_fn_attrs(def_id);
 
@@ -332,17 +309,11 @@ fn data_id_for_static(
         // zero.
 
         let ref_name = format!("_rust_extern_with_linkage_{}", symbol_name);
-        let ref_data_id = module
+        let ref_data_id = module.declare_data(&ref_name, Linkage::Local, false, false).unwrap();
-            .declare_data(&ref_name, Linkage::Local, false, false)
-            .unwrap();
         let mut data_ctx = DataContext::new();
         data_ctx.set_align(align);
         let data = module.declare_data_in_data(data_id, &mut data_ctx);
-        data_ctx.define(
+        data_ctx.define(std::iter::repeat(0).take(pointer_ty(tcx).bytes() as usize).collect());
-            std::iter::repeat(0)
-                .take(pointer_ty(tcx).bytes() as usize)
-                .collect(),
-        );
         data_ctx.write_data_addr(0, data, 0);
         match module.define_data(ref_data_id, &data_ctx) {
             // Every time the static is referenced there will be another definition of this global,
@@ -356,7 +327,7 @@ fn data_id_for_static(
     }
 }
 
-fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut ConstantCx) {
+fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut ConstantCx) {
     while let Some(todo_item) = cx.todo.pop() {
         let (data_id, alloc, section_name) = match todo_item {
             TodoItem::Alloc(alloc_id) => {
@@ -371,10 +342,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
             TodoItem::Static(def_id) => {
                 //println!("static {:?}", def_id);
 
-                let section_name = tcx
+                let section_name = tcx.codegen_fn_attrs(def_id).link_section.map(|s| s.as_str());
-                    .codegen_fn_attrs(def_id)
-                    .link_section
-                    .map(|s| s.as_str());
 
                 let alloc = tcx.eval_static_initializer(def_id).unwrap();
 
@@ -396,9 +364,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
             data_ctx.set_segment_section("", &*section_name);
         }
 
-        let bytes = alloc
+        let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
-            .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len())
-            .to_vec();
         data_ctx.define(bytes.into_boxed_slice());
 
         for &(offset, (_tag, reloc)) in alloc.relocations().iter() {
@@ -426,10 +392,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
                     data_id_for_alloc_id(module, reloc, target_alloc.mutability)
                 }
                 GlobalAlloc::Static(def_id) => {
-                    if tcx
+                    if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)
-                        .codegen_fn_attrs(def_id)
-                        .flags
-                        .contains(CodegenFnAttrFlags::THREAD_LOCAL)
                     {
                         tcx.sess.fatal(&format!(
                             "Allocation {:?} contains reference to TLS value {:?}",
@@ -457,14 +420,13 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
 }
 
 pub(crate) fn mir_operand_get_const_val<'tcx>(
-    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    fx: &FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
 ) -> Option<&'tcx Const<'tcx>> {
     match operand {
         Operand::Copy(_) | Operand::Move(_) => None,
-        Operand::Constant(const_) => Some(
+        Operand::Constant(const_) => {
-            fx.monomorphize(const_.literal)
+            Some(fx.monomorphize(const_.literal).eval(fx.tcx, ParamEnv::reveal_all()))
-                .eval(fx.tcx, ParamEnv::reveal_all()),
+        }
-        ),
     }
 }

compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs

@@ -14,10 +14,7 @@ impl DebugContext<'_> {
         let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone());
         let root = self.dwarf.unit.root();
         let root = self.dwarf.unit.get_mut(root);
-        root.set(
+        root.set(gimli::DW_AT_ranges, AttributeValue::RangeListRef(unit_range_list_id));
-            gimli::DW_AT_ranges,
-            AttributeValue::RangeListRef(unit_range_list_id),
-        );
 
         let mut sections = Sections::new(WriterRelocate::new(self.endian));
         self.dwarf.write(&mut sections).unwrap();
@@ -66,10 +63,7 @@ pub(super) struct WriterRelocate {
 
 impl WriterRelocate {
     pub(super) fn new(endian: RunTimeEndian) -> Self {
-        WriterRelocate {
+        WriterRelocate { relocs: Vec::new(), writer: EndianVec::new(endian) }
-            relocs: Vec::new(),
-            writer: EndianVec::new(endian),
-        }
     }
 
     /// Perform the collected relocations to be usable for JIT usage.
@@ -85,9 +79,7 @@ impl WriterRelocate {
                         cranelift_module::FuncId::from_u32(sym.try_into().unwrap()),
                     );
                     let val = (addr as u64 as i64 + reloc.addend) as u64;
-                    self.writer
+                    self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap();
-                        .write_udata_at(reloc.offset as usize, val, reloc.size)
-                        .unwrap();
                 }
             }
         }

compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs

@@ -53,11 +53,7 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
     if hash.kind == SourceFileHashAlgorithm::Md5 {
         let mut buf = [0u8; MD5_LEN];
         buf.copy_from_slice(hash.hash_bytes());
-        Some(FileInfo {
+        Some(FileInfo { timestamp: 0, size: 0, md5: buf })
-            timestamp: 0,
-            size: 0,
-            md5: buf,
-        })
     } else {
         None
     }
@@ -112,24 +108,14 @@ impl<'tcx> DebugContext<'tcx> {
 
         let entry = self.dwarf.unit.get_mut(entry_id);
 
-        entry.set(
+        entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
-            gimli::DW_AT_decl_file,
+        entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
-            AttributeValue::FileIndex(Some(file_id)),
-        );
-        entry.set(
-            gimli::DW_AT_decl_line,
-            AttributeValue::Udata(loc.line as u64),
-        );
         // FIXME: probably omit this
-        entry.set(
+        entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
-            gimli::DW_AT_decl_column,
-            AttributeValue::Udata(loc.col.to_usize() as u64),
-        );
     }
 
     pub(super) fn create_debug_lines(
         &mut self,
-        isa: &dyn cranelift_codegen::isa::TargetIsa,
         symbol: usize,
         entry_id: UnitEntryId,
         context: &Context,
@@ -138,7 +124,6 @@ impl<'tcx> DebugContext<'tcx> {
     ) -> CodeOffset {
         let tcx = self.tcx;
         let line_program = &mut self.dwarf.unit.line_program;
-        let func = &context.func;
 
         let line_strings = &mut self.dwarf.line_strings;
         let mut last_span = None;
@@ -202,7 +187,7 @@ impl<'tcx> DebugContext<'tcx> {
 
         let mut func_end = 0;
 
-        if let Some(ref mcr) = &context.mach_compile_result {
+        let mcr = context.mach_compile_result.as_ref().unwrap();
         for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
             line_program.row().address_offset = u64::from(start);
             if !loc.is_default() {
@@ -216,28 +201,7 @@ impl<'tcx> DebugContext<'tcx> {
 
         line_program.end_sequence(u64::from(func_end));
 
-            func_end = mcr.buffer.total_size();
+        let func_end = mcr.buffer.total_size();
-        } else {
-            let encinfo = isa.encoding_info();
-            let mut blocks = func.layout.blocks().collect::<Vec<_>>();
-            blocks.sort_by_key(|block| func.offsets[*block]); // Ensure inst offsets always increase
-
-            for block in blocks {
-                for (offset, inst, size) in func.inst_offsets(block, &encinfo) {
-                    let srcloc = func.srclocs[inst];
-                    line_program.row().address_offset = u64::from(offset);
-                    if !srcloc.is_default() {
-                        let source_info =
-                            *source_info_set.get_index(srcloc.bits() as usize).unwrap();
-                        create_row_for_span(line_program, source_info.span);
-                    } else {
-                        create_row_for_span(line_program, function_span);
-                    }
-                    func_end = offset + size;
-                }
-            }
-            line_program.end_sequence(u64::from(func_end));
-        }
 
         assert_ne!(func_end, 0);
 
@@ -246,10 +210,7 @@ impl<'tcx> DebugContext<'tcx> {
             gimli::DW_AT_low_pc,
             AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
         );
-        entry.set(
+        entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
-            gimli::DW_AT_high_pc,
-            AttributeValue::Udata(u64::from(func_end)),
-        );
 
         self.emit_location(entry_id, function_span);
 

compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs

@@ -9,7 +9,7 @@ use crate::prelude::*;
 use rustc_index::vec::IndexVec;
 
 use cranelift_codegen::entity::EntityRef;
-use cranelift_codegen::ir::{StackSlots, ValueLabel, ValueLoc};
+use cranelift_codegen::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLoc};
 use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::ValueLocRange;
 
@@ -39,7 +39,6 @@ pub(crate) struct DebugContext<'tcx> {
     dwarf: DwarfUnit,
     unit_range_list: RangeList,
 
-    clif_types: FxHashMap<Type, UnitEntryId>,
     types: FxHashMap<Ty<'tcx>, UnitEntryId>,
 }
 
@@ -91,20 +90,11 @@ impl<'tcx> DebugContext<'tcx> {
 
             let root = dwarf.unit.root();
             let root = dwarf.unit.get_mut(root);
-            root.set(
+            root.set(gimli::DW_AT_producer, AttributeValue::StringRef(dwarf.strings.add(producer)));
-                gimli::DW_AT_producer,
+            root.set(gimli::DW_AT_language, AttributeValue::Language(gimli::DW_LANG_Rust));
-                AttributeValue::StringRef(dwarf.strings.add(producer)),
-            );
-            root.set(
-                gimli::DW_AT_language,
-                AttributeValue::Language(gimli::DW_LANG_Rust),
-            );
             root.set(gimli::DW_AT_name, AttributeValue::StringRef(name));
             root.set(gimli::DW_AT_comp_dir, AttributeValue::StringRef(comp_dir));
-            root.set(
+            root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
-                gimli::DW_AT_low_pc,
-                AttributeValue::Address(Address::Constant(0)),
-            );
         }
 
         DebugContext {
@@ -115,48 +105,10 @@ impl<'tcx> DebugContext<'tcx> {
             dwarf,
             unit_range_list: RangeList(Vec::new()),
 
-            clif_types: FxHashMap::default(),
             types: FxHashMap::default(),
         }
     }
 
-    fn dwarf_ty_for_clif_ty(&mut self, ty: Type) -> UnitEntryId {
-        if let Some(type_id) = self.clif_types.get(&ty) {
-            return *type_id;
-        }
-
-        let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag);
-
-        let primitive = |dwarf: &mut DwarfUnit, ate| {
-            let type_id = new_entry(dwarf, gimli::DW_TAG_base_type);
-            let type_entry = dwarf.unit.get_mut(type_id);
-            type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate));
-            type_id
-        };
-
-        let type_id = if ty.is_bool() {
-            primitive(&mut self.dwarf, gimli::DW_ATE_boolean)
-        } else if ty.is_int() {
-            primitive(&mut self.dwarf, gimli::DW_ATE_address)
-        } else if ty.is_float() {
-            primitive(&mut self.dwarf, gimli::DW_ATE_float)
-        } else {
-            new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type)
-        };
-
-        let type_entry = self.dwarf.unit.get_mut(type_id);
-        type_entry.set(
-            gimli::DW_AT_name,
-            AttributeValue::String(format!("{}", ty).replace('i', "u").into_bytes()),
-        );
-        type_entry.set(
-            gimli::DW_AT_byte_size,
-            AttributeValue::Udata(u64::from(ty.bytes())),
-        );
-
-        type_id
-    }
-
     fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
         if let Some(type_id) = self.types.get(ty) {
             return *type_id;
@@ -181,10 +133,7 @@ impl<'tcx> DebugContext<'tcx> {
             ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed),
             ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float),
             ty::Ref(_, pointee_ty, _mutbl)
-            | ty::RawPtr(ty::TypeAndMut {
+            | ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => {
-                ty: pointee_ty,
-                mutbl: _mutbl,
-            }) => {
                 let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type);
 
                 // Ensure that type is inserted before recursing to avoid duplicates
@@ -211,10 +160,7 @@ impl<'tcx> DebugContext<'tcx> {
                     let field_offset = layout.fields.offset(field_idx);
                     let field_layout = layout
                         .field(
-                            &layout::LayoutCx {
+                            &layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() },
-                                tcx: self.tcx,
-                                param_env: ParamEnv::reveal_all(),
-                            },
                             field_idx,
                         )
                         .unwrap();
@@ -243,10 +189,7 @@ impl<'tcx> DebugContext<'tcx> {
         let type_entry = self.dwarf.unit.get_mut(type_id);
 
         type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
-        type_entry.set(
+        type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
-            gimli::DW_AT_byte_size,
-            AttributeValue::Udata(layout.size.bytes()),
-        );
 
         self.types.insert(ty, type_id);
 
@@ -286,23 +229,15 @@ impl<'tcx> DebugContext<'tcx> {
         let name_id = self.dwarf.strings.add(name);
         // Gdb requires DW_AT_name. Otherwise the DW_TAG_subprogram is skipped.
         entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
-        entry.set(
+        entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
-            gimli::DW_AT_linkage_name,
-            AttributeValue::StringRef(name_id),
-        );
 
-        let end =
+        let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set);
-            self.create_debug_lines(isa, symbol, entry_id, context, mir.span, source_info_set);
 
         self.unit_range_list.0.push(Range::StartLength {
             begin: Address::Symbol { symbol, addend: 0 },
             length: u64::from(end),
         });
 
-        if isa.get_mach_backend().is_some() {
-            return; // Not yet implemented for the AArch64 backend.
-        }
-
         let func_entry = self.dwarf.unit.get_mut(entry_id);
         // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
         func_entry.set(
@@ -312,51 +247,6 @@ impl<'tcx> DebugContext<'tcx> {
         // Using Udata for DW_AT_high_pc requires at least DWARF4
         func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
 
-        // FIXME Remove once actual debuginfo for locals works.
-        for (i, (param, &val)) in context
-            .func
-            .signature
-            .params
-            .iter()
-            .zip(
-                context
-                    .func
-                    .dfg
-                    .block_params(context.func.layout.entry_block().unwrap()),
-            )
-            .enumerate()
-        {
-            use cranelift_codegen::ir::ArgumentPurpose;
-            let base_name = match param.purpose {
-                ArgumentPurpose::Normal => "arg",
-                ArgumentPurpose::StructArgument(_) => "struct_arg",
-                ArgumentPurpose::StructReturn => "sret",
-                ArgumentPurpose::Link
-                | ArgumentPurpose::FramePointer
-                | ArgumentPurpose::CalleeSaved => continue,
-                ArgumentPurpose::VMContext
-                | ArgumentPurpose::SignatureId
-                | ArgumentPurpose::CallerTLS
-                | ArgumentPurpose::CalleeTLS
-                | ArgumentPurpose::StackLimit => unreachable!(),
-            };
-            let name = format!("{}{}", base_name, i);
-
-            let dw_ty = self.dwarf_ty_for_clif_ty(param.value_type);
-            let loc =
-                translate_loc(isa, context.func.locations[val], &context.func.stack_slots).unwrap();
-
-            let arg_id = self
-                .dwarf
-                .unit
-                .add(entry_id, gimli::DW_TAG_formal_parameter);
-            let var_entry = self.dwarf.unit.get_mut(arg_id);
-
-            var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
-            var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
-            var_entry.set(gimli::DW_AT_location, AttributeValue::Exprloc(loc));
-        }
-
         // FIXME make it more reliable and implement scopes before re-enabling this.
         if false {
             let value_labels_ranges = context.build_value_labels_ranges(isa).unwrap();
@@ -376,10 +266,7 @@ impl<'tcx> DebugContext<'tcx> {
                     context,
                     &local_map,
                     &value_labels_ranges,
-                    Place {
+                    Place { local, projection: ty::List::empty() },
-                        local,
-                        projection: ty::List::empty(),
-                    },
                 );
 
                 let var_entry = self.dwarf.unit.get_mut(var_id);
@@ -417,10 +304,7 @@ fn place_location<'tcx>(
                                 symbol,
                                 addend: i64::from(value_loc_range.start),
                             },
-                            end: Address::Symbol {
+                            end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
-                                symbol,
-                                addend: i64::from(value_loc_range.end),
-                            },
                             data: translate_loc(
                                 isa,
                                 value_loc_range.loc,
@@ -463,17 +347,17 @@ fn place_location<'tcx>(
 // Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
 fn translate_loc(
     isa: &dyn TargetIsa,
-    loc: ValueLoc,
+    loc: LabelValueLoc,
     stack_slots: &StackSlots,
 ) -> Option<Expression> {
     match loc {
-        ValueLoc::Reg(reg) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
             let machine_reg = isa.map_dwarf_register(reg).unwrap();
             let mut expr = Expression::new();
             expr.op_reg(gimli::Register(machine_reg));
             Some(expr)
         }
-        ValueLoc::Stack(ss) => {
+        LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
             if let Some(ss_offset) = stack_slots[ss].offset {
                 let mut expr = Expression::new();
                 expr.op_breg(X86_64::RBP, i64::from(ss_offset) + 16);
@@ -482,6 +366,17 @@ fn translate_loc(
                 None
             }
         }
-        _ => None,
+        LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => unreachable!(),
+        LabelValueLoc::Reg(reg) => {
+            let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
+            let mut expr = Expression::new();
+            expr.op_reg(gimli::Register(machine_reg));
+            Some(expr)
+        }
+        LabelValueLoc::SPOffset(offset) => {
+            let mut expr = Expression::new();
+            expr.op_breg(X86_64::RSP, offset);
+            Some(expr)
+        }
     }
 }

compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs

@@ -28,11 +28,7 @@ impl<'tcx> UnwindContext<'tcx> {
             None
         };
 
-        UnwindContext {
+        UnwindContext { tcx, frame_table, cie_id }
-            tcx,
-            frame_table,
-            cie_id,
-        }
     }
 
     pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
@@ -46,10 +42,8 @@ impl<'tcx> UnwindContext<'tcx> {
             UnwindInfo::SystemV(unwind_info) => {
                 self.frame_table.add_fde(
                     self.cie_id.unwrap(),
-                    unwind_info.to_fde(Address::Symbol {
+                    unwind_info
-                        symbol: func_id.as_u32() as usize,
+                        .to_fde(Address::Symbol { symbol: func_id.as_u32() as usize, addend: 0 }),
-                        addend: 0,
-                    }),
                 );
             }
             UnwindInfo::WindowsX64(_) => {
@@ -60,9 +54,8 @@ impl<'tcx> UnwindContext<'tcx> {
     }
 
     pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) {
-        let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
+        let mut eh_frame =
-            self.tcx,
+            EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
-        )));
         self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
 
         if !eh_frame.0.writer.slice().is_empty() {
@@ -82,9 +75,8 @@ impl<'tcx> UnwindContext<'tcx> {
         self,
         jit_module: &cranelift_jit::JITModule,
     ) -> Option<UnwindRegistry> {
-        let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
+        let mut eh_frame =
-            self.tcx,
+            EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
-        )));
         self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
 
         if eh_frame.0.writer.slice().is_empty() {
@@ -130,10 +122,7 @@ impl<'tcx> UnwindContext<'tcx> {
             registrations.push(ptr as usize);
         }
 
-        Some(UnwindRegistry {
+        Some(UnwindRegistry { _frame_table: eh_frame, registrations })
-            _frame_table: eh_frame,
-            registrations,
-        })
     }
 }
 

compiler/rustc_codegen_cranelift/src/discriminant.rs

@@ -7,7 +7,7 @@ use rustc_target::abi::{Int, TagEncoding, Variants};
 use crate::prelude::*;
 
 pub(crate) fn codegen_set_discriminant<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     place: CPlace<'tcx>,
     variant_index: VariantIdx,
 ) {
@@ -26,11 +26,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
             variants: _,
         } => {
             let ptr = place.place_field(fx, mir::Field::new(tag_field));
-            let to = layout
+            let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
-                .ty
-                .discriminant_for_variant(fx.tcx, variant_index)
-                .unwrap()
-                .val;
             let to = if ptr.layout().abi.is_signed() {
                 ty::ScalarInt::try_from_int(
                     ptr.layout().size.sign_extend(to) as i128,
@@ -46,12 +42,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
         Variants::Multiple {
             tag: _,
             tag_field,
-            tag_encoding:
+            tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
-                TagEncoding::Niche {
-                    dataful_variant,
-                    ref niche_variants,
-                    niche_start,
-                },
             variants: _,
         } => {
             if variant_index != dataful_variant {
@@ -70,7 +61,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
 }
 
 pub(crate) fn codegen_get_discriminant<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     value: CValue<'tcx>,
     dest_layout: TyAndLayout<'tcx>,
 ) -> CValue<'tcx> {
@@ -101,12 +92,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             };
             return CValue::const_val(fx, dest_layout, discr_val);
         }
-        Variants::Multiple {
+        Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
-            tag,
+            (tag, *tag_field, tag_encoding)
-            tag_field,
+        }
-            tag_encoding,
-            variants: _,
-        } => (tag, *tag_field, tag_encoding),
     };
 
     let cast_to = fx.clif_type(dest_layout.ty).unwrap();
@@ -125,11 +113,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             let val = clif_intcast(fx, tag, cast_to, signed);
             CValue::by_val(val, dest_layout)
         }
-        TagEncoding::Niche {
+        TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
-            dataful_variant,
-            ref niche_variants,
-            niche_start,
-        } => {
             // Rebase from niche values to discriminants, and check
             // whether the result is in range for the niche variants.
 
@@ -146,9 +130,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 tag
             } else {
                 // FIXME handle niche_start > i64::MAX
-                fx.bcx
+                fx.bcx.ins().iadd_imm(tag, -i64::try_from(niche_start).unwrap())
-                    .ins()
-                    .iadd_imm(tag, -i64::try_from(niche_start).unwrap())
             };
             let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
             let is_niche = {
@@ -176,15 +158,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 } else {
                     clif_intcast(fx, relative_discr, cast_to, false)
                 };
-                fx.bcx
+                fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
-                    .ins()
-                    .iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
             };
 
-            let dataful_variant = fx
+            let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
-                .bcx
-                .ins()
-                .iconst(cast_to, i64::from(dataful_variant.as_u32()));
             let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
             CValue::by_val(discr, dest_layout)
         }

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -12,11 +12,9 @@ use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{DebugInfo, OutputType};
 
-use cranelift_object::{ObjectModule, ObjectProduct};
+use cranelift_object::ObjectModule;
 
-use crate::prelude::*;
+use crate::{prelude::*, BackendConfig};
-
-use crate::backend::AddConstructor;
 
 fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
     let module = crate::backend::make_module(tcx.sess, name);
@@ -39,7 +37,6 @@ fn emit_module(
     module: ObjectModule,
     debug: Option<DebugContext<'_>>,
     unwind_context: UnwindContext<'_>,
-    map_product: impl FnOnce(ObjectProduct) -> ObjectProduct,
 ) -> ModuleCodegenResult {
     let mut product = module.finish();
 
@@ -49,15 +46,10 @@ fn emit_module(
 
     unwind_context.emit(&mut product);
 
-    let product = map_product(product);
+    let tmp_file = tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(&name));
-
-    let tmp_file = tcx
-        .output_filenames(LOCAL_CRATE)
-        .temp_path(OutputType::Object, Some(&name));
     let obj = product.object.write().unwrap();
     if let Err(err) = std::fs::write(&tmp_file, obj) {
-        tcx.sess
+        tcx.sess.fatal(&format!("error writing object file: {}", err));
-            .fatal(&format!("error writing object file: {}", err));
     }
 
     let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() {
@@ -71,13 +63,7 @@ fn emit_module(
     };
 
     ModuleCodegenResult(
-        CompiledModule {
+        CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None },
-            name,
-            kind,
-            object: Some(tmp_file),
-            dwarf_object: None,
-            bytecode: None,
-        },
         work_product,
     )
 }
@@ -117,49 +103,27 @@ fn reuse_workproduct_for_cgu(
     }
 }
 
-fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodegenResult {
+fn module_codegen(
+    tcx: TyCtxt<'_>,
+    (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol),
+) -> ModuleCodegenResult {
     let cgu = tcx.codegen_unit(cgu_name);
     let mono_items = cgu.items_in_deterministic_order(tcx);
 
     let mut module = new_module(tcx, cgu_name.as_str().to_string());
 
-    // Initialize the global atomic mutex using a constructor for proc-macros.
-    // FIXME implement atomic instructions in Cranelift.
-    let mut init_atomics_mutex_from_constructor = None;
-    if tcx
-        .sess
-        .crate_types()
-        .contains(&rustc_session::config::CrateType::ProcMacro)
-    {
-        if mono_items.iter().any(|(mono_item, _)| match mono_item {
-            rustc_middle::mir::mono::MonoItem::Static(def_id) => tcx
-                .symbol_name(Instance::mono(tcx, *def_id))
-                .name
-                .contains("__rustc_proc_macro_decls_"),
-            _ => false,
-        }) {
-            init_atomics_mutex_from_constructor =
-                Some(crate::atomic_shim::init_global_lock_constructor(
-                    &mut module,
-                    &format!("{}_init_atomics_mutex", cgu_name.as_str()),
-                ));
-        }
-    }
-
     let mut cx = crate::CodegenCx::new(
         tcx,
-        module,
+        backend_config,
+        &mut module,
         tcx.sess.opts.debuginfo != DebugInfo::None,
-        true,
     );
     super::predefine_mono_items(&mut cx, &mono_items);
     for (mono_item, (linkage, visibility)) in mono_items {
         let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
         match mono_item {
             MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || {
+                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
-                    crate::base::codegen_fn(&mut cx, inst, linkage)
-                });
             }
             MonoItem::Static(def_id) => {
                 crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@@ -175,9 +139,9 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
             }
         }
     }
-    let (mut module, global_asm, debug, mut unwind_context) =
+    let (global_asm, debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context, false);
+    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
 
     let codegen_result = emit_module(
         tcx,
@@ -186,13 +150,6 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
         module,
         debug,
         unwind_context,
-        |mut product| {
-            if let Some(func_id) = init_atomics_mutex_from_constructor {
-                product.add_constructor(func_id);
-            }
-
-            product
-        },
     );
 
     codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
@@ -202,6 +159,7 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
 
 pub(super) fn run_aot(
     tcx: TyCtxt<'_>,
+    backend_config: BackendConfig,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
@@ -225,9 +183,7 @@ pub(super) fn run_aot(
         cgus.iter()
             .map(|cgu| {
                 let cgu_reuse = determine_cgu_reuse(tcx, cgu);
-                tcx.sess
+                tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
-                    .cgu_reuse_tracker
-                    .set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
 
                 match cgu_reuse {
                     _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {}
@@ -242,7 +198,7 @@ pub(super) fn run_aot(
                 let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
                     dep_node,
                     tcx,
-                    cgu.name(),
+                    (backend_config, cgu.name()),
                     module_codegen,
                     rustc_middle::dep_graph::hash_result,
                 );
@@ -271,7 +227,6 @@ pub(super) fn run_aot(
             allocator_module,
             None,
             allocator_unwind_context,
-            |product| product,
         );
         if let Some((id, product)) = work_product {
             work_products.insert(id, product);
@@ -301,8 +256,7 @@ pub(super) fn run_aot(
             });
 
             if let Err(err) = std::fs::write(&tmp_file, obj) {
-                tcx.sess
+                tcx.sess.fatal(&format!("error writing metadata object file: {}", err));
-                    .fatal(&format!("error writing metadata object file: {}", err));
             }
 
             (metadata_cgu_name, tmp_file)
@@ -356,8 +310,7 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
                 "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
             );
         } else {
-            tcx.sess
+            tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
-                .fatal("asm! and global_asm! are not yet supported on macOS and Windows");
         }
     }
 
@@ -367,19 +320,12 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
     // Remove all LLVM style comments
     let global_asm = global_asm
         .lines()
-        .map(|line| {
+        .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
-            if let Some(index) = line.find("//") {
-                &line[0..index]
-            } else {
-                line
-            }
-        })
         .collect::<Vec<_>>()
         .join("\n");
 
-    let output_object_file = tcx
+    let output_object_file =
-        .output_filenames(LOCAL_CRATE)
+        tcx.output_filenames(LOCAL_CRATE).temp_path(OutputType::Object, Some(cgu_name));
-        .temp_path(OutputType::Object, Some(cgu_name));
 
     // Assemble `global_asm`
     let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
@@ -389,16 +335,10 @@ fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
         .stdin(Stdio::piped())
         .spawn()
         .expect("Failed to spawn `as`.");
-    child
+    child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
-        .stdin
-        .take()
-        .unwrap()
-        .write_all(global_asm.as_bytes())
-        .unwrap();
     let status = child.wait().expect("Failed to wait for `as`.");
     if !status.success() {
-        tcx.sess
+        tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm));
-            .fatal(&format!("Failed to assemble `{}`", global_asm));
     }
 
     // Link the global asm and main object file together
@@ -442,11 +382,7 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
     }
 
     let work_product_id = &cgu.work_product_id();
-    if tcx
+    if tcx.dep_graph.previous_work_product(work_product_id).is_none() {
-        .dep_graph
-        .previous_work_product(work_product_id)
-        .is_none()
-    {
         // We don't have anything cached for this CGU. This can happen
         // if the CGU did not exist in the previous session.
         return CguReuse::No;

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -10,43 +10,24 @@ use rustc_middle::mir::mono::MonoItem;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
-use crate::prelude::*;
+use crate::{prelude::*, BackendConfig};
 use crate::{CodegenCx, CodegenMode};
 
 thread_local! {
+    pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
     pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
 }
 
-pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
+pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     if !tcx.sess.opts.output_types.should_codegen() {
         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
     }
 
-    #[cfg(unix)]
-    unsafe {
-        // When not using our custom driver rustc will open us without the RTLD_GLOBAL flag, so
-        // __cg_clif_global_atomic_mutex will not be exported. We fix this by opening ourself again
-        // as global.
-        // FIXME remove once atomic_shim is gone
-
-        let mut dl_info: libc::Dl_info = std::mem::zeroed();
-        assert_ne!(
-            libc::dladdr(run_jit as *const libc::c_void, &mut dl_info),
-            0
-        );
-        assert_ne!(
-            libc::dlopen(dl_info.dli_fname, libc::RTLD_NOW | libc::RTLD_GLOBAL),
-            std::ptr::null_mut(),
-        );
-    }
-
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
-    let mut jit_builder = JITBuilder::with_isa(
+    let mut jit_builder =
-        crate::build_isa(tcx.sess),
+        JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
-        cranelift_module::default_libcall_names(),
+    jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
-    );
-    jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
@@ -56,14 +37,10 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
             AbiParam::new(jit_module.target_config().pointer_type()),
             AbiParam::new(jit_module.target_config().pointer_type()),
         ],
-        returns: vec![AbiParam::new(
+        returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
-            jit_module.target_config().pointer_type(), /*isize*/
-        )],
         call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
     };
-    let main_func_id = jit_module
+    let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
-        .declare_function("main", Linkage::Import, &sig)
-        .unwrap();
 
     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
     let mono_items = cgus
@@ -74,19 +51,19 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+    let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
 
     super::time(tcx, "codegen mono items", || {
         super::predefine_mono_items(&mut cx, &mono_items);
         for (mono_item, (linkage, visibility)) in mono_items {
             let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
             match mono_item {
-                MonoItem::Fn(inst) => match codegen_mode {
+                MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx.sess.time("codegen fn", || {
+                        cx.tcx
-                            crate::base::codegen_fn(&mut cx, inst, linkage)
+                            .sess
-                        });
+                            .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
                     }
                     CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
                 },
@@ -101,7 +78,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
         }
     });
 
-    let (mut jit_module, global_asm, _debug, mut unwind_context) =
+    let (global_asm, _debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
     jit_module.finalize_definitions();
 
@@ -109,7 +86,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
+    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
     tcx.sess.abort_if_errors();
@@ -120,7 +97,9 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
 
     let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
 
-    println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
+    println!(
+        "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
+    );
 
     let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
         unsafe { ::std::mem::transmute(finalized_main) };
@@ -136,6 +115,9 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
     // useful as some dynamic linkers use it as a marker to jump over.
     argv.push(std::ptr::null());
 
+    BACKEND_CONFIG.with(|tls_backend_config| {
+        assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
+    });
     CURRENT_MODULE
         .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
 
@@ -153,21 +135,19 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
         CURRENT_MODULE.with(|jit_module| {
             let mut jit_module = jit_module.borrow_mut();
             let jit_module = jit_module.as_mut().unwrap();
-            let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+            let backend_config =
+                BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
 
             let name = tcx.symbol_name(instance).name.to_string();
-            let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), instance);
+            let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
-            let func_id = cx
+            let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
-                .module
+            jit_module.prepare_for_function_redefine(func_id).unwrap();
-                .declare_function(&name, Linkage::Export, &sig)
-                .unwrap();
-            cx.module.prepare_for_function_redefine(func_id).unwrap();
 
-            tcx.sess.time("codegen fn", || {
+            let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
-                crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
+            tcx.sess
-            });
+                .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
 
-            let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
+            let (global_asm, _debug_context, unwind_context) = cx.finalize();
             assert!(global_asm.is_empty());
             jit_module.finalize_definitions();
             std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
@@ -194,9 +174,8 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
             Linkage::Static => {
                 let name = tcx.crate_name(cnum);
-                let mut err = tcx
+                let mut err =
-                    .sess
+                    tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
-                    .struct_err(&format!("Can't load static lib {}", name.as_str()));
                 err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
                 err.emit();
             }
@@ -217,6 +196,11 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
             if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
                 return None;
             }
+            if name.starts_with("rust_metadata_") {
+                // The metadata is part of a section that is not loaded by the dynamic linker in
+                // case of cg_llvm.
+                return None;
+            }
             let dlsym_name = if cfg!(target_os = "macos") {
                 // On macOS `dlsym` expects the name without leading `_`.
                 assert!(name.starts_with('_'), "{:?}", name);
@@ -236,17 +220,14 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     imported_symbols
 }
 
-pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
+pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let pointer_type = cx.module.target_config().pointer_type();
 
     let name = tcx.symbol_name(inst).name.to_string();
     let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
-    let func_id = cx
+    let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
-        .module
-        .declare_function(&name, Linkage::Export, &sig)
-        .unwrap();
 
     let instance_ptr = Box::into_raw(Box::new(inst));
 
@@ -267,28 +248,18 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: In
     let mut builder_ctx = FunctionBuilderContext::new();
     let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
 
-    let jit_fn = cx
+    let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
-        .module
-        .declare_func_in_func(jit_fn, trampoline_builder.func);
     let sig_ref = trampoline_builder.func.import_signature(sig);
 
     let entry_block = trampoline_builder.create_block();
     trampoline_builder.append_block_params_for_function_params(entry_block);
-    let fn_args = trampoline_builder
+    let fn_args = trampoline_builder.func.dfg.block_params(entry_block).to_vec();
-        .func
-        .dfg
-        .block_params(entry_block)
-        .to_vec();
 
     trampoline_builder.switch_to_block(entry_block);
-    let instance_ptr = trampoline_builder
+    let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
-        .ins()
-        .iconst(pointer_type, instance_ptr as u64 as i64);
     let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
     let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
-    let call_inst = trampoline_builder
+    let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
-        .ins()
-        .call_indirect(sig_ref, jitted_fn, &fn_args);
     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
     trampoline_builder.ins().return_(&ret_vals);
 

compiler/rustc_codegen_cranelift/src/driver/mod.rs

@@ -17,33 +17,30 @@ pub(crate) fn codegen_crate(
     tcx: TyCtxt<'_>,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
-    config: crate::BackendConfig,
+    backend_config: crate::BackendConfig,
 ) -> Box<dyn Any> {
     tcx.sess.abort_if_errors();
 
-    match config.codegen_mode {
+    match backend_config.codegen_mode {
-        CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module),
+        CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module),
         CodegenMode::Jit | CodegenMode::JitLazy => {
-            let is_executable = tcx
+            let is_executable =
-                .sess
+                tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable);
-                .crate_types()
-                .contains(&rustc_session::config::CrateType::Executable);
             if !is_executable {
                 tcx.sess.fatal("can't jit non-executable crate");
             }
 
             #[cfg(feature = "jit")]
-            let _: ! = jit::run_jit(tcx, config.codegen_mode);
+            let _: ! = jit::run_jit(tcx, backend_config);
 
             #[cfg(not(feature = "jit"))]
-            tcx.sess
+            tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
-                .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         }
     }
 }
 
 fn predefine_mono_items<'tcx>(
-    cx: &mut crate::CodegenCx<'tcx, impl Module>,
+    cx: &mut crate::CodegenCx<'_, 'tcx>,
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
     cx.tcx.sess.time("predefine functions", || {
@@ -63,21 +60,12 @@ fn predefine_mono_items<'tcx>(
 }
 
 fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
-    if std::env::var("CG_CLIF_DISPLAY_CG_TIME")
+    if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") {
-        .as_ref()
-        .map(|val| &**val)
-        == Ok("1")
-    {
         println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
         let before = std::time::Instant::now();
         let res = tcx.sess.time(name, f);
         let after = std::time::Instant::now();
-        println!(
+        println!("[{:<30}: {}] end time: {:?}", tcx.crate_name(LOCAL_CRATE), name, after - before);
-            "[{:<30}: {}] end time: {:?}",
-            tcx.crate_name(LOCAL_CRATE),
-            name,
-            after - before
-        );
         res
     } else {
         tcx.sess.time(name, f)

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -9,7 +9,7 @@ use rustc_middle::mir::InlineAsmOperand;
 use rustc_target::asm::*;
 
 pub(crate) fn codegen_inline_asm<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     _span: Span,
     template: &[InlineAsmTemplatePiece],
     operands: &[InlineAsmOperand<'tcx>],
@@ -53,11 +53,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                     crate::base::codegen_operand(fx, value).load_scalar(fx),
                 ));
             }
-            InlineAsmOperand::Out {
+            InlineAsmOperand::Out { reg, late: _, place } => {
-                reg,
-                late: _,
-                place,
-            } => {
                 let reg = expect_reg(reg);
                 clobbered_regs.push((reg, new_slot(reg.reg_class())));
                 if let Some(place) = place {
@@ -68,12 +64,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                     ));
                 }
             }
-            InlineAsmOperand::InOut {
+            InlineAsmOperand::InOut { reg, late: _, ref in_value, out_place } => {
-                reg,
-                late: _,
-                ref in_value,
-                out_place,
-            } => {
                 let reg = expect_reg(reg);
                 clobbered_regs.push((reg, new_slot(reg.reg_class())));
                 inputs.push((
@@ -97,11 +88,8 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 
     let inline_asm_index = fx.inline_asm_index;
     fx.inline_asm_index += 1;
-    let asm_name = format!(
+    let asm_name =
-        "{}__inline_asm_{}",
+        format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index);
-        fx.tcx.symbol_name(fx.instance).name,
-        inline_asm_index
-    );
 
     let generated_asm = generate_asm_wrapper(
         &asm_name,
@@ -129,12 +117,7 @@ fn generate_asm_wrapper(
     let mut generated_asm = String::new();
     writeln!(generated_asm, ".globl {}", asm_name).unwrap();
     writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
-    writeln!(
+    writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
-        generated_asm,
-        ".section .text.{},\"ax\",@progbits",
-        asm_name
-    )
-    .unwrap();
     writeln!(generated_asm, "{}:", asm_name).unwrap();
 
     generated_asm.push_str(".intel_syntax noprefix\n");
@@ -164,11 +147,7 @@ fn generate_asm_wrapper(
             InlineAsmTemplatePiece::String(s) => {
                 generated_asm.push_str(s);
             }
-            InlineAsmTemplatePiece::Placeholder {
+            InlineAsmTemplatePiece::Placeholder { operand_idx: _, modifier: _, span: _ } => todo!(),
-                operand_idx: _,
-                modifier: _,
-                span: _,
-            } => todo!(),
         }
     }
     generated_asm.push('\n');
@@ -203,7 +182,7 @@ fn generate_asm_wrapper(
 }
 
 fn call_inline_asm<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     asm_name: &str,
     slot_size: Size,
     inputs: Vec<(InlineAsmReg, Size, Value)>,
@@ -230,17 +209,12 @@ fn call_inline_asm<'tcx>(
             },
         )
         .unwrap();
-    let inline_asm_func = fx
+    let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
-        .cx
-        .module
-        .declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
     #[cfg(debug_assertions)]
     fx.add_comment(inline_asm_func, asm_name);
 
     for (_reg, offset, value) in inputs {
-        fx.bcx
+        fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
-            .ins()
-            .stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
     }
 
     let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
@@ -248,10 +222,7 @@ fn call_inline_asm<'tcx>(
 
     for (_reg, offset, place) in outputs {
         let ty = fx.clif_type(place.layout().ty).unwrap();
-        let value = fx
+        let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
-            .bcx
-            .ins()
-            .stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
@@ -267,8 +238,7 @@ fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsm
     match arch {
         InlineAsmArch::X86_64 => {
             write!(generated_asm, "    mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
-            reg.emit(generated_asm, InlineAsmArch::X86_64, None)
+            reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
-                .unwrap();
             generated_asm.push('\n');
         }
         _ => unimplemented!("save_register for {:?}", arch),
@@ -284,8 +254,7 @@ fn restore_register(
     match arch {
         InlineAsmArch::X86_64 => {
             generated_asm.push_str("    mov ");
-            reg.emit(generated_asm, InlineAsmArch::X86_64, None)
+            reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
-                .unwrap();
             writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
         }
         _ => unimplemented!("restore_register for {:?}", arch),

compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs

@@ -6,7 +6,7 @@ use crate::prelude::*;
 ///
 /// This emulates an intel cpu with sse and sse2 support, but which doesn't support anything else.
 pub(crate) fn codegen_cpuid_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     leaf: Value,
     _subleaf: Value,
 ) -> (Value, Value, Value, Value) {
@@ -31,54 +31,28 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
 
     fx.bcx.switch_to_block(leaf_0);
     let max_basic_leaf = fx.bcx.ins().iconst(types::I32, 1);
-    let vend0 = fx
+    let vend0 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
-        .bcx
+    let vend2 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
-        .ins()
+    let vend1 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
-        .iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
+    fx.bcx.ins().jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
-    let vend2 = fx
-        .bcx
-        .ins()
-        .iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
-    let vend1 = fx
-        .bcx
-        .ins()
-        .iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
-    fx.bcx
-        .ins()
-        .jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
 
     fx.bcx.switch_to_block(leaf_1);
     let cpu_signature = fx.bcx.ins().iconst(types::I32, 0);
     let additional_information = fx.bcx.ins().iconst(types::I32, 0);
     let ecx_features = fx.bcx.ins().iconst(types::I32, 0);
-    let edx_features = fx
+    let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
-        .bcx
+    fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]);
-        .ins()
-        .iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
-    fx.bcx.ins().jump(
-        dest,
-        &[
-            cpu_signature,
-            additional_information,
-            ecx_features,
-            edx_features,
-        ],
-    );
 
     fx.bcx.switch_to_block(leaf_8000_0000);
     let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0);
     let zero = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx
+    fx.bcx.ins().jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
-        .ins()
-        .jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
 
     fx.bcx.switch_to_block(leaf_8000_0001);
     let zero = fx.bcx.ins().iconst(types::I32, 0);
     let proc_info_ecx = fx.bcx.ins().iconst(types::I32, 0);
     let proc_info_edx = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx
+    fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
-        .ins()
-        .jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
 
     fx.bcx.switch_to_block(unsupported_leaf);
     crate::trap::trap_unreachable(

compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

@@ -6,7 +6,7 @@ use crate::prelude::*;
 use rustc_middle::ty::subst::SubstsRef;
 
 pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: &str,
     substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -9,6 +9,7 @@ pub(crate) use cpuid::codegen_cpuid_call;
 pub(crate) use llvm::codegen_llvm_intrinsic_call;
 
 use crate::prelude::*;
+use cranelift_codegen::ir::AtomicRmwOp;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 
 macro intrinsic_pat {
@@ -112,38 +113,6 @@ macro call_intrinsic_match {
     }
 }
 
-macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
-    crate::atomic_shim::lock_global_lock($fx);
-
-    let clif_ty = $fx.clif_type($T).unwrap();
-    let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
-    let new = $fx.bcx.ins().$op(old, $src);
-    $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
-    $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
-
-    crate::atomic_shim::unlock_global_lock($fx);
-}
-
-macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
-    crate::atomic_shim::lock_global_lock($fx);
-
-    // Read old
-    let clif_ty = $fx.clif_type($T).unwrap();
-    let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
-
-    // Compare
-    let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
-    let new = $fx.bcx.ins().select(is_eq, old, $src);
-
-    // Write new
-    $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
-
-    let ret_val = CValue::by_val(old, $ret.layout());
-    $ret.write_cvalue($fx, ret_val);
-
-    crate::atomic_shim::unlock_global_lock($fx);
-}
-
 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     match $ty.kind() {
         ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@@ -184,12 +153,12 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
     }
 }
 
-fn simd_for_each_lane<'tcx, M: Module>(
+fn simd_for_each_lane<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
     f: impl Fn(
-        &mut FunctionCx<'_, 'tcx, M>,
+        &mut FunctionCx<'_, '_, 'tcx>,
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
         Value,
@@ -213,13 +182,13 @@ fn simd_for_each_lane<'tcx, M: Module>(
     }
 }
 
-fn simd_pair_for_each_lane<'tcx, M: Module>(
+fn simd_pair_for_each_lane<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     x: CValue<'tcx>,
     y: CValue<'tcx>,
     ret: CPlace<'tcx>,
     f: impl Fn(
-        &mut FunctionCx<'_, 'tcx, M>,
+        &mut FunctionCx<'_, '_, 'tcx>,
         TyAndLayout<'tcx>,
         TyAndLayout<'tcx>,
         Value,
@@ -246,11 +215,11 @@ fn simd_pair_for_each_lane<'tcx, M: Module>(
     }
 }
 
-fn simd_reduce<'tcx, M: Module>(
+fn simd_reduce<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value,
+    f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     let lane_layout = fx.layout_of(lane_ty);
@@ -258,20 +227,19 @@ fn simd_reduce<'tcx, M: Module>(
 
     let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     for lane_idx in 1..lane_count {
-        let lane = val
+        let lane =
-            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
+            val.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())).load_scalar(fx);
-            .load_scalar(fx);
         res_val = f(fx, lane_layout, res_val, lane);
     }
     let res = CValue::by_val(res_val, lane_layout);
     ret.write_cvalue(fx, res);
 }
 
-fn simd_reduce_bool<'tcx, M: Module>(
+fn simd_reduce_bool<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, M>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     val: CValue<'tcx>,
     ret: CPlace<'tcx>,
-    f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value,
+    f: impl Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
 ) {
     let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     assert!(ret.layout().ty.is_bool());
@@ -279,9 +247,8 @@ fn simd_reduce_bool<'tcx, M: Module>(
     let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
     for lane_idx in 1..lane_count {
-        let lane = val
+        let lane =
-            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
+            val.value_field(fx, mir::Field::new(lane_idx.try_into().unwrap())).load_scalar(fx);
-            .load_scalar(fx);
         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
         res_val = f(fx, res_val, lane);
     }
@@ -290,7 +257,7 @@ fn simd_reduce_bool<'tcx, M: Module>(
 }
 
 fn bool_to_zero_or_max_uint<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     val: Value,
 ) -> CValue<'tcx> {
@@ -424,7 +391,7 @@ macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 }
 
 pub(crate) fn codegen_intrinsic_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     instance: Instance<'tcx>,
     args: &[mir::Operand<'tcx>],
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
@@ -912,136 +879,175 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
         };
 
         _ if intrinsic.starts_with("atomic_fence"), () {
-            crate::atomic_shim::lock_global_lock(fx);
+            fx.bcx.ins().fence();
-            crate::atomic_shim::unlock_global_lock(fx);
         };
         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
-            crate::atomic_shim::lock_global_lock(fx);
+            // FIXME use a compiler fence once Cranelift supports it
-            crate::atomic_shim::unlock_global_lock(fx);
+            fx.bcx.ins().fence();
         };
-        _ if intrinsic.starts_with("atomic_load"), (c ptr) {
+        _ if intrinsic.starts_with("atomic_load"), <T> (v ptr) {
-            crate::atomic_shim::lock_global_lock(fx);
+            validate_atomic_type!(fx, intrinsic, span, T);
+            let ty = fx.clif_type(T).unwrap();
 
-            let inner_layout =
+            let val = fx.bcx.ins().atomic_load(ty, MemFlags::trusted(), ptr);
-                fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
+
-            validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
+            let val = CValue::by_val(val, fx.layout_of(T));
-            let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
             ret.write_cvalue(fx, val);
-
-            crate::atomic_shim::unlock_global_lock(fx);
         };
         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 
-            crate::atomic_shim::lock_global_lock(fx);
+            let val = val.load_scalar(fx);
 
-            let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
+            fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
-            dest.write_cvalue(fx, val);
-
-            crate::atomic_shim::unlock_global_lock(fx);
         };
-        _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
+        _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) {
-            validate_atomic_type!(fx, intrinsic, span, T);
+            let layout = new.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
 
-            crate::atomic_shim::lock_global_lock(fx);
+            let new = new.load_scalar(fx);
 
-            // Read old
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
-            let clif_ty = fx.clif_type(T).unwrap();
-            let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
-            ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 
-            // Write new
+            let old = CValue::by_val(old, layout);
-            let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
+            ret.write_cvalue(fx, old);
-            dest.write_cvalue(fx, src);
-
-            crate::atomic_shim::unlock_global_lock(fx);
         };
-        _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
+        _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
-            validate_atomic_type!(fx, intrinsic, span, T);
+            let layout = new.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
 
             let test_old = test_old.load_scalar(fx);
             let new = new.load_scalar(fx);
 
-            crate::atomic_shim::lock_global_lock(fx);
+            let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
-
-            // Read old
-            let clif_ty = fx.clif_type(T).unwrap();
-            let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
-
-            // Compare
             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
-            let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
-
-            // Write new
-            fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 
             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
-            ret.write_cvalue(fx, ret_val);
+            ret.write_cvalue(fx, ret_val)
-
-            crate::atomic_shim::unlock_global_lock(fx);
         };
 
-        _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
+        _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = amount.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let amount = amount.load_scalar(fx);
-            atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
+        _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = amount.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let amount = amount.load_scalar(fx);
-            atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
+        _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = src.layout();
-            let src = src.load_scalar(fx);
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
-            atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
+            let ty = fx.clif_type(layout.ty).unwrap();
-        };
-        _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, T);
 
             let src = src.load_scalar(fx);
 
-            crate::atomic_shim::lock_global_lock(fx);
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
 
-            let clif_ty = fx.clif_type(T).unwrap();
+            let old = CValue::by_val(old, layout);
-            let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
+            ret.write_cvalue(fx, old);
-            let and = fx.bcx.ins().band(old, src);
+        };
-            let new = fx.bcx.ins().bnot(and);
+        _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) {
-            fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
+            let layout = src.layout();
-            ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
 
-            crate::atomic_shim::unlock_global_lock(fx);
-        };
-        _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
             let src = src.load_scalar(fx);
-            atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
+        _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
 
-        _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
+        // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+        _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
+        _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
+        _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
-        _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
+        _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) {
-            validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
             let src = src.load_scalar(fx);
-            atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
+        };
+        _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) {
+            let layout = src.layout();
+            validate_atomic_type!(fx, intrinsic, span, layout.ty);
+            let ty = fx.clif_type(layout.ty).unwrap();
+
+            let src = src.load_scalar(fx);
+
+            let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
+
+            let old = CValue::by_val(old, layout);
+            ret.write_cvalue(fx, old);
         };
 
         minnumf32, (v a, v b) {

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -4,7 +4,7 @@ use super::*;
 use crate::prelude::*;
 
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     instance: Instance<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -11,8 +11,6 @@
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
 
-#[cfg(feature = "jit")]
-extern crate libc;
 extern crate snap;
 #[macro_use]
 extern crate rustc_middle;
@@ -53,7 +51,6 @@ mod abi;
 mod allocator;
 mod analyze;
 mod archive;
-mod atomic_shim;
 mod backend;
 mod base;
 mod cast;
@@ -129,9 +126,9 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
     }
 }
 
-struct CodegenCx<'tcx, M: Module> {
+struct CodegenCx<'m, 'tcx: 'm> {
     tcx: TyCtxt<'tcx>,
-    module: M,
+    module: &'m mut dyn Module,
     global_asm: String,
     constants_cx: ConstantCx,
     cached_context: Context,
@@ -140,14 +137,20 @@ struct CodegenCx<'tcx, M: Module> {
     unwind_context: UnwindContext<'tcx>,
 }
 
-impl<'tcx, M: Module> CodegenCx<'tcx, M> {
+impl<'m, 'tcx> CodegenCx<'m, 'tcx> {
-    fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self {
+    fn new(
-        let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame);
+        tcx: TyCtxt<'tcx>,
-        let debug_context = if debug_info {
+        backend_config: BackendConfig,
-            Some(DebugContext::new(tcx, module.isa()))
+        module: &'m mut dyn Module,
-        } else {
+        debug_info: bool,
-            None
+    ) -> Self {
-        };
+        let unwind_context = UnwindContext::new(
+            tcx,
+            module.isa(),
+            matches!(backend_config.codegen_mode, CodegenMode::Aot),
+        );
+        let debug_context =
+            if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None };
         CodegenCx {
             tcx,
             module,
@@ -160,14 +163,9 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> {
         }
     }
 
-    fn finalize(mut self) -> (M, String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
+    fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
-        self.constants_cx.finalize(self.tcx, &mut self.module);
+        self.constants_cx.finalize(self.tcx, self.module);
-        (
+        (self.global_asm, self.debug_context, self.unwind_context)
-            self.module,
-            self.global_asm,
-            self.debug_context,
-            self.unwind_context,
-        )
     }
 }
 
@@ -302,14 +300,7 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     flags_builder.enable("is_pic").unwrap();
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
     flags_builder
-        .set(
+        .set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
-            "enable_verifier",
-            if cfg!(debug_assertions) {
-                "true"
-            } else {
-                "false"
-            },
-        )
         .unwrap();
 
     let tls_model = match target_triple.binary_format {
@@ -338,11 +329,7 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
 
     let flags = settings::Flags::new(flags_builder);
 
-    let variant = if cfg!(feature = "oldbe") {
+    let variant = cranelift_codegen::isa::BackendVariant::MachInst;
-        cranelift_codegen::isa::BackendVariant::Legacy
-    } else {
-        cranelift_codegen::isa::BackendVariant::MachInst
-    };
     let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
     // Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
     // is interpreted as `bsr`.

compiler/rustc_codegen_cranelift/src/main_shim.rs

@@ -9,7 +9,6 @@ pub(crate) fn maybe_create_entry_wrapper(
     tcx: TyCtxt<'_>,
     module: &mut impl Module,
     unwind_context: &mut UnwindContext<'_>,
-    use_jit: bool,
 ) {
     let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) {
         Some((def_id, entry_ty)) => (
@@ -27,14 +26,7 @@ pub(crate) fn maybe_create_entry_wrapper(
         return;
     }
 
-    create_entry_fn(
+    create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item);
-        tcx,
-        module,
-        unwind_context,
-        main_def_id,
-        use_start_lang_item,
-        use_jit,
-    );
 
     fn create_entry_fn(
         tcx: TyCtxt<'_>,
@@ -42,7 +34,6 @@ pub(crate) fn maybe_create_entry_wrapper(
         unwind_context: &mut UnwindContext<'_>,
         rust_main_def_id: DefId,
         use_start_lang_item: bool,
-        use_jit: bool,
     ) {
         let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
         // Given that `main()` has no arguments,
@@ -57,23 +48,17 @@ pub(crate) fn maybe_create_entry_wrapper(
                 AbiParam::new(m.target_config().pointer_type()),
                 AbiParam::new(m.target_config().pointer_type()),
             ],
-            returns: vec![AbiParam::new(
+            returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)],
-                m.target_config().pointer_type(), /*isize*/
-            )],
             call_conv: CallConv::triple_default(m.isa().triple()),
         };
 
-        let cmain_func_id = m
+        let cmain_func_id = m.declare_function("main", Linkage::Export, &cmain_sig).unwrap();
-            .declare_function("main", Linkage::Export, &cmain_sig)
-            .unwrap();
 
         let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
 
         let main_name = tcx.symbol_name(instance).name.to_string();
         let main_sig = get_function_sig(tcx, m.isa().triple(), instance);
-        let main_func_id = m
+        let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap();
-            .declare_function(&main_name, Linkage::Import, &main_sig)
-            .unwrap();
 
         let mut ctx = Context::new();
         ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig);
@@ -86,8 +71,6 @@ pub(crate) fn maybe_create_entry_wrapper(
             let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type());
             let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type());
 
-            crate::atomic_shim::init_global_lock(m, &mut bcx, use_jit);
-
             let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
 
             let call_inst = if use_start_lang_item {
@@ -103,9 +86,7 @@ pub(crate) fn maybe_create_entry_wrapper(
                 .polymorphize(tcx);
                 let start_func_id = import_function(tcx, m, start_instance);
 
-                let main_val = bcx
+                let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
-                    .ins()
-                    .func_addr(m.target_config().pointer_type(), main_func_ref);
 
                 let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
                 bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv])

compiler/rustc_codegen_cranelift/src/metadata.rs

@@ -94,9 +94,7 @@ pub(crate) fn write_metadata<P: WriteMetadata>(
 
     assert!(kind == MetadataKind::Compressed);
     let mut compressed = tcx.metadata_encoding_version();
-    FrameEncoder::new(&mut compressed)
+    FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
-        .write_all(&metadata.raw_data)
-        .unwrap();
 
     product.add_rustc_section(
         rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx),

compiler/rustc_codegen_cranelift/src/num.rs

@@ -41,7 +41,7 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
 }
 
 fn codegen_compare_bin_op<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     signed: bool,
     lhs: Value,
@@ -54,7 +54,7 @@ fn codegen_compare_bin_op<'tcx>(
 }
 
 pub(crate) fn codegen_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -93,17 +93,12 @@ pub(crate) fn codegen_binop<'tcx>(
         ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
         ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
         ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
-        _ => unreachable!(
+        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
-            "{:?}({:?}, {:?})",
-            bin_op,
-            in_lhs.layout().ty,
-            in_rhs.layout().ty
-        ),
     }
 }
 
 pub(crate) fn codegen_bool_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -124,7 +119,7 @@ pub(crate) fn codegen_bool_binop<'tcx>(
 }
 
 pub(crate) fn codegen_int_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -185,19 +180,14 @@ pub(crate) fn codegen_int_binop<'tcx>(
             }
         }
         // Compare binops handles by `codegen_binop`.
-        _ => unreachable!(
+        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
-            "{:?}({:?}, {:?})",
-            bin_op,
-            in_lhs.layout().ty,
-            in_rhs.layout().ty
-        ),
     };
 
     CValue::by_val(val, in_lhs.layout())
 }
 
 pub(crate) fn codegen_checked_int_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -268,9 +258,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
                     let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
                     let val = fx.bcx.ins().imul(lhs, rhs);
                     let has_underflow =
-                        fx.bcx
+                        fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
-                            .ins()
-                            .icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
                     let has_overflow = fx.bcx.ins().icmp_imm(
                         IntCC::SignedGreaterThan,
                         val,
@@ -309,10 +297,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
             let val = fx.bcx.ins().ishl(lhs, actual_shift);
             let ty = fx.bcx.func.dfg.value_type(val);
             let max_shift = i64::from(ty.bits()) - 1;
-            let has_overflow = fx
+            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
-                .bcx
-                .ins()
-                .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
             (val, has_overflow)
         }
         BinOp::Shr => {
@@ -326,38 +311,20 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
             };
             let ty = fx.bcx.func.dfg.value_type(val);
             let max_shift = i64::from(ty.bits()) - 1;
-            let has_overflow = fx
+            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
-                .bcx
-                .ins()
-                .icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
             (val, has_overflow)
         }
-        _ => bug!(
+        _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
-            "binop {:?} on checked int/uint lhs: {:?} rhs: {:?}",
-            bin_op,
-            in_lhs,
-            in_rhs
-        ),
     };
 
     let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow);
 
-    // FIXME directly write to result place instead
+    let out_layout = fx.layout_of(fx.tcx.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()));
-    let out_place = CPlace::new_stack_slot(
+    CValue::by_val_pair(res, has_overflow, out_layout)
-        fx,
-        fx.layout_of(
-            fx.tcx
-                .mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()),
-        ),
-    );
-    let out_layout = out_place.layout();
-    out_place.write_cvalue(fx, CValue::by_val_pair(res, has_overflow, out_layout));
-
-    out_place.to_cvalue(fx)
 }
 
 pub(crate) fn codegen_float_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -402,7 +369,7 @@ pub(crate) fn codegen_float_binop<'tcx>(
 }
 
 pub(crate) fn codegen_ptr_binop<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
     in_rhs: CValue<'tcx>,
@@ -452,9 +419,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
 
                 let ptr_cmp =
-                    fx.bcx
+                    fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
-                        .ins()
-                        .icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
                 let extra_cmp = fx.bcx.ins().icmp(
                     bin_op_to_intcc(bin_op, false).unwrap(),
                     lhs_extra,
@@ -466,9 +431,6 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
             _ => panic!("bin_op {:?} on ptr", bin_op),
         };
 
-        CValue::by_val(
+        CValue::by_val(fx.bcx.ins().bint(types::I8, res), fx.layout_of(fx.tcx.types.bool))
-            fx.bcx.ins().bint(types::I8, res),
-            fx.layout_of(fx.tcx.types.bool),
-        )
     }
 }

compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs

@@ -15,10 +15,7 @@ pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block
     // bytecodealliance/cranelift#1339 is implemented.
 
     let mut block_insts = FxHashMap::default();
-    for block in cold_blocks
+    for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
-        .keys()
-        .filter(|&block| cold_blocks.contains(block))
-    {
         let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>();
         for &inst in &insts {
             ctx.func.layout.remove_inst(inst);
@@ -28,10 +25,7 @@ pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block
     }
 
     // And then append them at the back again.
-    for block in cold_blocks
+    for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
-        .keys()
-        .filter(|&block| cold_blocks.contains(block))
-    {
         ctx.func.layout.append_block(block);
         for inst in block_insts.remove(&block).unwrap() {
             ctx.func.layout.append_inst(inst, block);

compiler/rustc_codegen_cranelift/src/optimize/mod.rs

@@ -19,7 +19,12 @@ pub(crate) fn optimize_function<'tcx>(
     if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No {
         return; // FIXME classify optimizations over opt levels
     }
+
+    // FIXME(#1142) stack2reg miscompiles lewton
+    if false {
         self::stack2reg::optimize_function(ctx, clif_comments);
+    }
+
     crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments);
     crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
 }

compiler/rustc_codegen_cranelift/src/optimize/peephole.rs

@@ -10,10 +10,7 @@ use cranelift_frontend::FunctionBuilder;
 pub(crate) fn maybe_unwrap_bint(bcx: &mut FunctionBuilder<'_>, arg: Value) -> Value {
     if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
         match bcx.func.dfg[arg_inst] {
-            InstructionData::Unary {
+            InstructionData::Unary { opcode: Opcode::Bint, arg } => arg,
-                opcode: Opcode::Bint,
-                arg,
-            } => arg,
             _ => arg,
         }
     } else {
@@ -54,12 +51,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
 
         match bcx.func.dfg[arg_inst] {
             // This is the lowering of Rvalue::Not
-            InstructionData::Load {
+            InstructionData::Load { opcode: Opcode::Load, arg: ptr, flags, offset } => {
-                opcode: Opcode::Load,
-                arg: ptr,
-                flags,
-                offset,
-            } => {
                 // Using `load.i8 + uextend.i32` would legalize to `uload8 + ireduce.i8 +
                 // uextend.i32`. Just `uload8` is much faster.
                 match bcx.func.dfg.ctrl_typevar(arg_inst) {
@@ -95,20 +87,14 @@ pub(crate) fn maybe_known_branch_taken(
     };
 
     match bcx.func.dfg[arg_inst] {
-        InstructionData::UnaryBool {
+        InstructionData::UnaryBool { opcode: Opcode::Bconst, imm } => {
-            opcode: Opcode::Bconst,
-            imm,
-        } => {
             if test_zero {
                 Some(!imm)
             } else {
                 Some(imm)
             }
         }
-        InstructionData::UnaryImm {
+        InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => {
-            opcode: Opcode::Iconst,
-            imm,
-        } => {
             if test_zero {
                 Some(imm.bits() == 0)
             } else {

compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs

@@ -175,16 +175,14 @@ impl<'a> OptimizeContext<'a> {
             }
         }
 
-        OptimizeContext {
+        OptimizeContext { ctx, stack_slot_usage_map }
-            ctx,
-            stack_slot_usage_map,
-        }
     }
 }
 
 pub(super) fn optimize_function(
     ctx: &mut Context,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] clif_comments: &mut crate::pretty_clif::CommentWriter,
+    #[cfg_attr(not(debug_assertions), allow(unused_variables))]
+    clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
     combine_stack_addr_with_load_store(&mut ctx.func);
 
@@ -296,12 +294,7 @@ fn combine_stack_addr_with_load_store(func: &mut Function) {
     while let Some(_block) = cursor.next_block() {
         while let Some(inst) = cursor.next_inst() {
             match cursor.func.dfg[inst] {
-                InstructionData::Load {
+                InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => {
-                    opcode: Opcode::Load,
-                    arg: addr,
-                    flags: _,
-                    offset,
-                } => {
                     if cursor.func.dfg.ctrl_typevar(inst) == types::I128
                         || cursor.func.dfg.ctrl_typevar(inst).is_vector()
                     {
@@ -391,20 +384,14 @@ fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) {
         stack_slot_users
             .stack_addr
             .drain_filter(|inst| {
-                stack_addr_load_insts_users
+                stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
-                    .get(inst)
-                    .map(|users| users.is_empty())
-                    .unwrap_or(true)
             })
             .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst));
 
         stack_slot_users
             .stack_load
             .drain_filter(|inst| {
-                stack_addr_load_insts_users
+                stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
-                    .get(inst)
-                    .map(|users| users.is_empty())
-                    .unwrap_or(true)
             })
             .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst));
     }
@@ -415,11 +402,8 @@ fn try_get_stack_slot_and_offset_for_addr(
     addr: Value,
 ) -> Option<(StackSlot, Offset32)> {
     if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) {
-        if let InstructionData::StackLoad {
+        if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } =
-            opcode: Opcode::StackAddr,
+            func.dfg[addr_inst]
-            stack_slot,
-            offset,
-        } = func.dfg[addr_inst]
         {
             return Some((stack_slot, offset));
         }
@@ -437,16 +421,8 @@ enum SpatialOverlap {
 fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
     fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) {
         match func.dfg[inst] {
-            InstructionData::StackLoad {
+            InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset }
-                opcode: Opcode::StackAddr,
+            | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset }
-                stack_slot,
-                offset,
-            }
-            | InstructionData::StackLoad {
-                opcode: Opcode::StackLoad,
-                stack_slot,
-                offset,
-            }
             | InstructionData::StackStore {
                 opcode: Opcode::StackStore,
                 stack_slot,
@@ -471,10 +447,7 @@ fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
     }
 
     let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into();
-    let dest_end: i64 = dest_offset
+    let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into();
-        .try_add_i64(i64::from(dest_size))
-        .unwrap()
-        .into();
     if src_end <= dest_offset.into() || dest_end <= src_offset.into() {
         return SpatialOverlap::No;
     }

compiler/rustc_codegen_cranelift/src/pointer.rs

@@ -23,35 +23,20 @@ pub(crate) enum PointerBase {
 
 impl Pointer {
     pub(crate) fn new(addr: Value) -> Self {
-        Pointer {
+        Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
-            base: PointerBase::Addr(addr),
-            offset: Offset32::new(0),
-        }
     }
 
     pub(crate) fn stack_slot(stack_slot: StackSlot) -> Self {
-        Pointer {
+        Pointer { base: PointerBase::Stack(stack_slot), offset: Offset32::new(0) }
-            base: PointerBase::Stack(stack_slot),
-            offset: Offset32::new(0),
-        }
     }
 
-    pub(crate) fn const_addr<'a, 'tcx>(
+    pub(crate) fn const_addr(fx: &mut FunctionCx<'_, '_, '_>, addr: i64) -> Self {
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        addr: i64,
-    ) -> Self {
         let addr = fx.bcx.ins().iconst(fx.pointer_type, addr);
-        Pointer {
+        Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
-            base: PointerBase::Addr(addr),
-            offset: Offset32::new(0),
-        }
     }
 
     pub(crate) fn dangling(align: Align) -> Self {
-        Pointer {
+        Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
-            base: PointerBase::Dangling(align),
-            offset: Offset32::new(0),
-        }
     }
 
     #[cfg(debug_assertions)]
@@ -59,46 +44,28 @@ impl Pointer {
         (self.base, self.offset)
     }
 
-    pub(crate) fn get_addr<'a, 'tcx>(self, fx: &mut FunctionCx<'a, 'tcx, impl Module>) -> Value {
+    pub(crate) fn get_addr(self, fx: &mut FunctionCx<'_, '_, '_>) -> Value {
         match self.base {
             PointerBase::Addr(base_addr) => {
                 let offset: i64 = self.offset.into();
-                if offset == 0 {
+                if offset == 0 { base_addr } else { fx.bcx.ins().iadd_imm(base_addr, offset) }
-                    base_addr
-                } else {
-                    fx.bcx.ins().iadd_imm(base_addr, offset)
-                }
             }
             PointerBase::Stack(stack_slot) => {
-                fx.bcx
+                fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset)
-                    .ins()
+            }
-                    .stack_addr(fx.pointer_type, stack_slot, self.offset)
+            PointerBase::Dangling(align) => {
+                fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap())
             }
-            PointerBase::Dangling(align) => fx
-                .bcx
-                .ins()
-                .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()),
         }
     }
 
-    pub(crate) fn offset<'a, 'tcx>(
+    pub(crate) fn offset(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Offset32) -> Self {
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        extra_offset: Offset32,
-    ) -> Self {
         self.offset_i64(fx, extra_offset.into())
     }
 
-    pub(crate) fn offset_i64<'a, 'tcx>(
+    pub(crate) fn offset_i64(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: i64) -> Self {
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        extra_offset: i64,
-    ) -> Self {
         if let Some(new_offset) = self.offset.try_add_i64(extra_offset) {
-            Pointer {
+            Pointer { base: self.base, offset: new_offset }
-                base: self.base,
-                offset: new_offset,
-            }
         } else {
             let base_offset: i64 = self.offset.into();
             if let Some(new_offset) = base_offset.checked_add(extra_offset) {
@@ -107,16 +74,12 @@ impl Pointer {
                     PointerBase::Stack(stack_slot) => {
                         fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0)
                     }
-                    PointerBase::Dangling(align) => fx
+                    PointerBase::Dangling(align) => {
-                        .bcx
+                        fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap())
-                        .ins()
+                    }
-                        .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap()),
                 };
                 let addr = fx.bcx.ins().iadd_imm(base_addr, new_offset);
-                Pointer {
+                Pointer { base: PointerBase::Addr(addr), offset: Offset32::new(0) }
-                    base: PointerBase::Addr(addr),
-                    offset: Offset32::new(0),
-                }
             } else {
                 panic!(
                     "self.offset ({}) + extra_offset ({}) not representable in i64",
@@ -126,31 +89,22 @@ impl Pointer {
         }
     }
 
-    pub(crate) fn offset_value<'a, 'tcx>(
+    pub(crate) fn offset_value(self, fx: &mut FunctionCx<'_, '_, '_>, extra_offset: Value) -> Self {
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        extra_offset: Value,
-    ) -> Self {
         match self.base {
             PointerBase::Addr(addr) => Pointer {
                 base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)),
                 offset: self.offset,
             },
             PointerBase::Stack(stack_slot) => {
-                let base_addr = fx
+                let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, self.offset);
-                    .bcx
-                    .ins()
-                    .stack_addr(fx.pointer_type, stack_slot, self.offset);
                 Pointer {
                     base: PointerBase::Addr(fx.bcx.ins().iadd(base_addr, extra_offset)),
                     offset: Offset32::new(0),
                 }
             }
             PointerBase::Dangling(align) => {
-                let addr = fx
+                let addr =
-                    .bcx
+                    fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap());
-                    .ins()
-                    .iconst(fx.pointer_type, i64::try_from(align.bytes()).unwrap());
                 Pointer {
                     base: PointerBase::Addr(fx.bcx.ins().iadd(addr, extra_offset)),
                     offset: self.offset,
@@ -159,47 +113,22 @@ impl Pointer {
         }
     }
 
-    pub(crate) fn load<'a, 'tcx>(
+    pub(crate) fn load(self, fx: &mut FunctionCx<'_, '_, '_>, ty: Type, flags: MemFlags) -> Value {
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        ty: Type,
-        flags: MemFlags,
-    ) -> Value {
         match self.base {
             PointerBase::Addr(base_addr) => fx.bcx.ins().load(ty, flags, base_addr, self.offset),
-            PointerBase::Stack(stack_slot) => {
+            PointerBase::Stack(stack_slot) => fx.bcx.ins().stack_load(ty, stack_slot, self.offset),
-                if ty == types::I128 || ty.is_vector() {
-                    // WORKAROUND for stack_load.i128 and stack_load.iXxY not being implemented
-                    let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
-                    fx.bcx.ins().load(ty, flags, base_addr, self.offset)
-                } else {
-                    fx.bcx.ins().stack_load(ty, stack_slot, self.offset)
-                }
-            }
             PointerBase::Dangling(_align) => unreachable!(),
         }
     }
 
-    pub(crate) fn store<'a, 'tcx>(
+    pub(crate) fn store(self, fx: &mut FunctionCx<'_, '_, '_>, value: Value, flags: MemFlags) {
-        self,
-        fx: &mut FunctionCx<'a, 'tcx, impl Module>,
-        value: Value,
-        flags: MemFlags,
-    ) {
         match self.base {
             PointerBase::Addr(base_addr) => {
                 fx.bcx.ins().store(flags, value, base_addr, self.offset);
             }
             PointerBase::Stack(stack_slot) => {
-                let val_ty = fx.bcx.func.dfg.value_type(value);
-                if val_ty == types::I128 || val_ty.is_vector() {
-                    // WORKAROUND for stack_store.i128 and stack_store.iXxY not being implemented
-                    let base_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
-                    fx.bcx.ins().store(flags, value, base_addr, self.offset);
-                } else {
                 fx.bcx.ins().stack_store(value, stack_slot, self.offset);
             }
-            }
             PointerBase::Dangling(_align) => unreachable!(),
         }
     }

compiler/rustc_codegen_cranelift/src/pretty_clif.rs

@@ -79,20 +79,14 @@ impl CommentWriter {
             vec![
                 format!("symbol {}", tcx.symbol_name(instance).name),
                 format!("instance {:?}", instance),
-                format!(
+                format!("abi {:?}", FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])),
-                    "abi {:?}",
-                    FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])
-                ),
                 String::new(),
             ]
         } else {
             vec![]
         };
 
-        CommentWriter {
+        CommentWriter { global_comments, entity_comments: FxHashMap::default() }
-            global_comments,
-            entity_comments: FxHashMap::default(),
-        }
     }
 }
 
@@ -186,7 +180,7 @@ impl FuncWriter for &'_ CommentWriter {
 }
 
 #[cfg(debug_assertions)]
-impl<M: Module> FunctionCx<'_, '_, M> {
+impl FunctionCx<'_, '_, '_> {
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
         self.clif_comments.add_global_comment(comment);
     }
@@ -201,12 +195,7 @@ impl<M: Module> FunctionCx<'_, '_, M> {
 }
 
 pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
-    cfg!(debug_assertions)
+    tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
-        || tcx
-            .sess
-            .opts
-            .output_types
-            .contains_key(&OutputType::LlvmAssembly)
 }
 
 pub(crate) fn write_ir_file<'tcx>(
@@ -245,15 +234,9 @@ pub(crate) fn write_clif_file<'tcx>(
     context: &cranelift_codegen::Context,
     mut clif_comments: &CommentWriter,
 ) {
-    write_ir_file(
+    write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| {
-        tcx,
+        let value_ranges =
-        &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
+            isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
-        |file| {
-            let value_ranges = isa.map(|isa| {
-                context
-                    .build_value_labels_ranges(isa)
-                    .expect("value location ranges")
-            });
 
         let mut clif = String::new();
         cranelift_codegen::write::decorate_function(
@@ -274,11 +257,10 @@ pub(crate) fn write_clif_file<'tcx>(
         writeln!(file)?;
         file.write_all(clif.as_bytes())?;
         Ok(())
-        },
+    });
-    );
 }
 
-impl<M: Module> fmt::Debug for FunctionCx<'_, '_, M> {
+impl fmt::Debug for FunctionCx<'_, '_, '_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         writeln!(f, "{:?}", self.instance.substs)?;
         writeln!(f, "{:?}", self.local_map)?;

compiler/rustc_codegen_cranelift/src/toolchain.rs

@@ -71,10 +71,7 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
                 flavor,
             )),
             (Some(linker), None) => {
-                let stem = linker
+                let stem = linker.file_stem().and_then(|stem| stem.to_str()).unwrap_or_else(|| {
-                    .file_stem()
-                    .and_then(|stem| stem.to_str())
-                    .unwrap_or_else(|| {
                     sess.fatal("couldn't extract file stem from specified linker")
                 });
 
@@ -105,11 +102,7 @@ fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
 
     // linker and linker flavor specified via command line have precedence over what the target
     // specification specifies
-    if let Some(ret) = infer_from(
+    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
-        sess,
-        sess.opts.cg.linker.clone(),
-        sess.opts.cg.linker_flavor,
-    ) {
         return ret;
     }
 

compiler/rustc_codegen_cranelift/src/trap.rs

@@ -2,7 +2,7 @@
 
 use crate::prelude::*;
 
-fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
+fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
     let puts = fx
         .cx
         .module
@@ -29,7 +29,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, impl Module>, msg: &str) {
 }
 
 /// Trap code: user1
-pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
+pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
     fx.bcx.ins().trap(TrapCode::User(1));
 }
@@ -38,7 +38,7 @@ pub(crate) fn trap_abort(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsR
 /// so you can **not** add instructions to it afterwards.
 ///
 /// Trap code: user65535
-pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
+pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
     fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
 }
@@ -47,7 +47,7 @@ pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, impl Module>, msg: im
 ///
 /// Trap code: user65535
 pub(crate) fn trap_unreachable_ret_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     dest_layout: TyAndLayout<'tcx>,
     msg: impl AsRef<str>,
 ) -> CValue<'tcx> {
@@ -62,7 +62,7 @@ pub(crate) fn trap_unreachable_ret_value<'tcx>(
 /// to it afterwards.
 ///
 /// Trap code: user65535
-pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg: impl AsRef<str>) {
+pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
     let true_ = fx.bcx.ins().iconst(types::I32, 1);
     fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
@@ -72,7 +72,7 @@ pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, impl Module>, msg:
 ///
 /// Trap code: user65535
 pub(crate) fn trap_unimplemented_ret_value<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     dest_layout: TyAndLayout<'tcx>,
     msg: impl AsRef<str>,
 ) -> CValue<'tcx> {

compiler/rustc_codegen_cranelift/src/unsize.rs

@@ -13,19 +13,18 @@ use crate::prelude::*;
 /// in an upcast, where the new vtable for an object will be derived
 /// from the old one.
 pub(crate) fn unsized_info<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     source: Ty<'tcx>,
     target: Ty<'tcx>,
     old_info: Option<Value>,
 ) -> Value {
     let (source, target) =
-        fx.tcx
+        fx.tcx.struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all());
-            .struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all());
     match (&source.kind(), &target.kind()) {
-        (&ty::Array(_, len), &ty::Slice(_)) => fx.bcx.ins().iconst(
+        (&ty::Array(_, len), &ty::Slice(_)) => fx
-            fx.pointer_type,
+            .bcx
-            len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64,
+            .ins()
-        ),
+            .iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64),
         (&ty::Dynamic(..), &ty::Dynamic(..)) => {
             // For now, upcasts are limited to changes in marker
             // traits, and hence never actually require an actual
@@ -35,17 +34,13 @@ pub(crate) fn unsized_info<'tcx>(
         (_, &ty::Dynamic(ref data, ..)) => {
             crate::vtable::get_vtable(fx, fx.layout_of(source), data.principal())
         }
-        _ => bug!(
+        _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
-            "unsized_info: invalid unsizing {:?} -> {:?}",
-            source,
-            target
-        ),
     }
 }
 
 /// Coerce `src` to `dst_ty`. `src_ty` must be a thin pointer.
 fn unsize_thin_ptr<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     src: Value,
     src_layout: TyAndLayout<'tcx>,
     dst_layout: TyAndLayout<'tcx>,
@@ -89,17 +84,15 @@ fn unsize_thin_ptr<'tcx>(
 /// Coerce `src`, which is a reference to a value of type `src_ty`,
 /// to a value of type `dst_ty` and store the result in `dst`
 pub(crate) fn coerce_unsized_into<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     src: CValue<'tcx>,
     dst: CPlace<'tcx>,
 ) {
     let src_ty = src.layout().ty;
     let dst_ty = dst.layout().ty;
     let mut coerce_ptr = || {
-        let (base, info) = if fx
+        let (base, info) =
-            .layout_of(src.layout().ty.builtin_deref(true).unwrap().ty)
+            if fx.layout_of(src.layout().ty.builtin_deref(true).unwrap().ty).is_unsized() {
-            .is_unsized()
-        {
                 // fat-ptr to fat-ptr unsize preserves the vtable
                 // i.e., &'a fmt::Debug+Send => &'a fmt::Debug
                 src.load_scalar_pair(fx)
@@ -131,39 +124,26 @@ pub(crate) fn coerce_unsized_into<'tcx>(
                 }
             }
         }
-        _ => bug!(
+        _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty),
-            "coerce_unsized_into: invalid coercion {:?} -> {:?}",
-            src_ty,
-            dst_ty
-        ),
     }
 }
 
 // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs
 
 pub(crate) fn size_and_align_of_dst<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     info: Value,
 ) -> (Value, Value) {
     if !layout.is_unsized() {
-        let size = fx
+        let size = fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64);
-            .bcx
+        let align = fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64);
-            .ins()
-            .iconst(fx.pointer_type, layout.size.bytes() as i64);
-        let align = fx
-            .bcx
-            .ins()
-            .iconst(fx.pointer_type, layout.align.abi.bytes() as i64);
         return (size, align);
     }
     match layout.ty.kind() {
         ty::Dynamic(..) => {
             // load size/align from vtable
-            (
+            (crate::vtable::size_of_obj(fx, info), crate::vtable::min_align_of_obj(fx, info))
-                crate::vtable::size_of_obj(fx, info),
-                crate::vtable::min_align_of_obj(fx, info),
-            )
         }
         ty::Slice(_) | ty::Str => {
             let unit = layout.field(fx, 0);
@@ -171,9 +151,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
             // times the unit size.
             (
                 fx.bcx.ins().imul_imm(info, unit.size.bytes() as i64),
-                fx.bcx
+                fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
-                    .ins()
-                    .iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
             )
         }
         _ => {
@@ -211,10 +189,7 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
 
             // Choose max of two known alignments (combined value must
             // be aligned according to more restrictive of the two).
-            let cmp = fx
+            let cmp = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
-                .bcx
-                .ins()
-                .icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
             let align = fx.bcx.ins().select(cmp, sized_align, unsized_align);
 
             // Issue #27023: must add any necessary padding to `size`

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -6,7 +6,7 @@ use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
 fn codegen_field<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     base: Pointer,
     extra: Option<Value>,
     layout: TyAndLayout<'tcx>,
@@ -15,11 +15,8 @@ fn codegen_field<'tcx>(
     let field_offset = layout.fields.offset(field.index());
     let field_layout = layout.field(&*fx, field.index());
 
-    let simple = |fx: &mut FunctionCx<'_, '_, _>| {
+    let simple = |fx: &mut FunctionCx<'_, '_, '_>| {
-        (
+        (base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout)
-            base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()),
-            field_layout,
-        )
     };
 
     if let Some(extra) = extra {
@@ -58,10 +55,7 @@ fn scalar_pair_calculate_b_offset(
     a_scalar: &Scalar,
     b_scalar: &Scalar,
 ) -> Offset32 {
-    let b_offset = a_scalar
+    let b_offset = a_scalar.value.size(&tcx).align_to(b_scalar.value.align(&tcx).abi);
-        .value
-        .size(&tcx)
-        .align_to(b_scalar.value.align(&tcx).abi);
     Offset32::new(b_offset.bytes().try_into().unwrap())
 }
 
@@ -106,10 +100,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     // FIXME remove
-    pub(crate) fn force_stack(
+    pub(crate) fn force_stack(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Pointer, Option<Value>) {
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    ) -> (Pointer, Option<Value>) {
         let layout = self.1;
         match self.0 {
             CValueInner::ByRef(ptr, meta) => (ptr, meta),
@@ -129,7 +120,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value with layout.abi of scalar
-    pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> Value {
+    pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value {
         let layout = self.1;
         match self.0 {
             CValueInner::ByRef(ptr, None) => {
@@ -153,10 +144,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value pair with layout.abi of scalar pair
-    pub(crate) fn load_scalar_pair(
+    pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) {
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-    ) -> (Value, Value) {
         let layout = self.1;
         match self.0 {
             CValueInner::ByRef(ptr, None) => {
@@ -183,7 +171,7 @@ impl<'tcx> CValue<'tcx> {
 
     pub(crate) fn value_field(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         field: mir::Field,
     ) -> CValue<'tcx> {
         let layout = self.1;
@@ -219,21 +207,17 @@ impl<'tcx> CValue<'tcx> {
         }
     }
 
-    pub(crate) fn unsize_value(
+    pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-        dest: CPlace<'tcx>,
-    ) {
         crate::unsize::coerce_unsized_into(fx, self, dest);
     }
 
     /// If `ty` is signed, `const_val` must already be sign extended.
     pub(crate) fn const_val(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
         const_val: ty::ScalarInt,
     ) -> CValue<'tcx> {
-        assert_eq!(const_val.size(), layout.size);
+        assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout);
         use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
 
         let clif_ty = fx.clif_type(layout.ty).unwrap();
@@ -250,18 +234,11 @@ impl<'tcx> CValue<'tcx> {
             ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
                 let const_val = const_val.to_bits(layout.size).unwrap();
                 let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64);
-                let msb = fx
+                let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64);
-                    .bcx
-                    .ins()
-                    .iconst(types::I64, (const_val >> 64) as u64 as i64);
                 fx.bcx.ins().iconcat(lsb, msb)
             }
-            ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..)
+            ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => {
-            | ty::RawPtr(..) => {
+                fx.bcx.ins().iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
-                fx
-                    .bcx
-                    .ins()
-                    .iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
             }
             ty::Float(FloatTy::F32) => {
                 fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))
@@ -279,14 +256,8 @@ impl<'tcx> CValue<'tcx> {
     }
 
     pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self {
-        assert!(matches!(
+        assert!(matches!(self.layout().ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
-            self.layout().ty.kind(),
+        assert!(matches!(layout.ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)));
-            ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)
-        ));
-        assert!(matches!(
-            layout.ty.kind(),
-            ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..)
-        ));
         assert_eq!(self.layout().abi, layout.abi);
         CValue(self.0, layout)
     }
@@ -317,14 +288,11 @@ impl<'tcx> CPlace<'tcx> {
     }
 
     pub(crate) fn no_place(layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
-        CPlace {
+        CPlace { inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None), layout }
-            inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None),
-            layout,
-        }
     }
 
     pub(crate) fn new_stack_slot(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
         assert!(!layout.is_unsized());
@@ -339,28 +307,22 @@ impl<'tcx> CPlace<'tcx> {
             size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
             offset: None,
         });
-        CPlace {
+        CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
-            inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None),
-            layout,
-        }
     }
 
     pub(crate) fn new_var(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         local: Local,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
         let var = Variable::with_u32(fx.next_ssa_var);
         fx.next_ssa_var += 1;
         fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap());
-        CPlace {
+        CPlace { inner: CPlaceInner::Var(local, var), layout }
-            inner: CPlaceInner::Var(local, var),
-            layout,
-        }
     }
 
     pub(crate) fn new_var_pair(
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         local: Local,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
@@ -372,17 +334,11 @@ impl<'tcx> CPlace<'tcx> {
         let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap();
         fx.bcx.declare_var(var1, ty1);
         fx.bcx.declare_var(var2, ty2);
-        CPlace {
+        CPlace { inner: CPlaceInner::VarPair(local, var1, var2), layout }
-            inner: CPlaceInner::VarPair(local, var1, var2),
-            layout,
-        }
     }
 
     pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> {
-        CPlace {
+        CPlace { inner: CPlaceInner::Addr(ptr, None), layout }
-            inner: CPlaceInner::Addr(ptr, None),
-            layout,
-        }
     }
 
     pub(crate) fn for_ptr_with_extra(
@@ -390,34 +346,27 @@ impl<'tcx> CPlace<'tcx> {
         extra: Value,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
-        CPlace {
+        CPlace { inner: CPlaceInner::Addr(ptr, Some(extra)), layout }
-            inner: CPlaceInner::Addr(ptr, Some(extra)),
-            layout,
-        }
     }
 
-    pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CValue<'tcx> {
+    pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CValue<'tcx> {
         let layout = self.layout();
         match self.inner {
             CPlaceInner::Var(_local, var) => {
                 let val = fx.bcx.use_var(var);
-                fx.bcx
+                //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                    .set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
                 CValue::by_val(val, layout)
             }
             CPlaceInner::VarPair(_local, var1, var2) => {
                 let val1 = fx.bcx.use_var(var1);
-                fx.bcx
+                //fx.bcx.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
-                    .set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index()));
                 let val2 = fx.bcx.use_var(var2);
-                fx.bcx
+                //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
-                    .set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index()));
                 CValue::by_val_pair(val1, val2, layout)
             }
             CPlaceInner::VarLane(_local, var, lane) => {
                 let val = fx.bcx.use_var(var);
-                fx.bcx
+                //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                    .set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index()));
                 let val = fx.bcx.ins().extractlane(val, lane);
                 CValue::by_val(val, layout)
             }
@@ -447,11 +396,7 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
-    pub(crate) fn write_cvalue(
+    pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) {
-        self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
-        from: CValue<'tcx>,
-    ) {
         assert_assignable(fx, from.layout().ty, self.layout().ty);
 
         self.write_cvalue_maybe_transmute(fx, from, "write_cvalue");
@@ -459,7 +404,7 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn write_cvalue_transmute(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
     ) {
         self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute");
@@ -467,12 +412,12 @@ impl<'tcx> CPlace<'tcx> {
 
     fn write_cvalue_maybe_transmute(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
         #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
     ) {
         fn transmute_value<'tcx>(
-            fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+            fx: &mut FunctionCx<'_, '_, 'tcx>,
             var: Variable,
             data: Value,
             dst_ty: Type,
@@ -511,8 +456,7 @@ impl<'tcx> CPlace<'tcx> {
                 }
                 _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty),
             };
-            fx.bcx
+            //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                .set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
             fx.bcx.def_var(var, data);
         }
 
@@ -558,15 +502,13 @@ impl<'tcx> CPlace<'tcx> {
 
                 // First get the old vector
                 let vector = fx.bcx.use_var(var);
-                fx.bcx
+                //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                    .set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
 
                 // Next insert the written lane into the vector
                 let vector = fx.bcx.ins().insertlane(vector, data, lane);
 
                 // Finally write the new vector
-                fx.bcx
+                //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
-                    .set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index()));
                 fx.bcx.def_var(var, vector);
 
                 return;
@@ -604,10 +546,7 @@ impl<'tcx> CPlace<'tcx> {
                 to_ptr.store(fx, val, flags);
             }
             CValueInner::ByValPair(_, _) => {
-                bug!(
+                bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
-                    "Non ScalarPair abi {:?} for ByValPair CValue",
-                    dst_layout.abi
-                );
             }
             CValueInner::ByRef(from_ptr, None) => {
                 let from_addr = from_ptr.get_addr(fx);
@@ -632,7 +571,7 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn place_field(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         field: mir::Field,
     ) -> CPlace<'tcx> {
         let layout = self.layout();
@@ -650,18 +589,8 @@ impl<'tcx> CPlace<'tcx> {
                 let layout = layout.field(&*fx, field.index());
 
                 match field.as_u32() {
-                    0 => {
+                    0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout },
-                        return CPlace {
+                    1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout },
-                            inner: CPlaceInner::Var(local, var1),
-                            layout,
-                        }
-                    }
-                    1 => {
-                        return CPlace {
-                            inner: CPlaceInner::Var(local, var2),
-                            layout,
-                        }
-                    }
                     _ => unreachable!("field should be 0 or 1"),
                 }
             }
@@ -680,7 +609,7 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn place_index(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         index: Value,
     ) -> CPlace<'tcx> {
         let (elem_layout, ptr) = match self.layout().ty.kind() {
@@ -689,30 +618,24 @@ impl<'tcx> CPlace<'tcx> {
             _ => bug!("place_index({:?})", self.layout().ty),
         };
 
-        let offset = fx
+        let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64);
-            .bcx
-            .ins()
-            .imul_imm(index, elem_layout.size.bytes() as i64);
 
         CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout)
     }
 
-    pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, 'tcx, impl Module>) -> CPlace<'tcx> {
+    pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> {
         let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty);
         if has_ptr_meta(fx.tcx, inner_layout.ty) {
             let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx);
             CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout)
         } else {
-            CPlace::for_ptr(
+            CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout)
-                Pointer::new(self.to_cvalue(fx).load_scalar(fx)),
-                inner_layout,
-            )
         }
     }
 
     pub(crate) fn place_ref(
         self,
-        fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
     ) -> CValue<'tcx> {
         if has_ptr_meta(fx.tcx, self.layout().ty) {
@@ -729,21 +652,18 @@ impl<'tcx> CPlace<'tcx> {
 
     pub(crate) fn downcast_variant(
         self,
-        fx: &FunctionCx<'_, 'tcx, impl Module>,
+        fx: &FunctionCx<'_, '_, 'tcx>,
         variant: VariantIdx,
     ) -> Self {
         assert!(!self.layout().is_unsized());
         let layout = self.layout().for_variant(fx, variant);
-        CPlace {
+        CPlace { inner: self.inner, layout }
-            inner: self.inner,
-            layout,
-        }
     }
 }
 
 #[track_caller]
 pub(crate) fn assert_assignable<'tcx>(
-    fx: &FunctionCx<'_, 'tcx, impl Module>,
+    fx: &FunctionCx<'_, '_, 'tcx>,
     from_ty: Ty<'tcx>,
     to_ty: Ty<'tcx>,
 ) {
@@ -776,12 +696,9 @@ pub(crate) fn assert_assignable<'tcx>(
         }
         (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
             for (from, to) in from_traits.iter().zip(to_traits) {
-                let from = fx
+                let from =
-                    .tcx
+                    fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
-                    .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);
+                let to = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
-                let to = fx
-                    .tcx
-                    .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to);
                 assert_eq!(
                     from, to,
                     "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}",

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -15,7 +15,7 @@ fn vtable_memflags() -> MemFlags {
     flags
 }
 
-pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
+pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
     let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
     fx.bcx.ins().load(
         pointer_ty(fx.tcx),
@@ -25,7 +25,7 @@ pub(crate) fn drop_fn_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: V
     )
 }
 
-pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
+pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
     let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
     fx.bcx.ins().load(
         pointer_ty(fx.tcx),
@@ -35,7 +35,7 @@ pub(crate) fn size_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Valu
     )
 }
 
-pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable: Value) -> Value {
+pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -> Value {
     let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes() as usize;
     fx.bcx.ins().load(
         pointer_ty(fx.tcx),
@@ -46,7 +46,7 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, impl Module>, vtable:
 }
 
 pub(crate) fn get_ptr_and_method_ref<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
     idx: usize,
 ) -> (Value, Value) {
@@ -68,7 +68,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
 }
 
 pub(crate) fn get_vtable<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
 ) -> Value {
@@ -85,7 +85,7 @@ pub(crate) fn get_vtable<'tcx>(
 }
 
 fn build_vtable<'tcx>(
-    fx: &mut FunctionCx<'_, 'tcx, impl Module>,
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
     trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
 ) -> DataId {
@@ -94,7 +94,7 @@ fn build_vtable<'tcx>(
 
     let drop_in_place_fn = import_function(
         tcx,
-        &mut fx.cx.module,
+        fx.cx.module,
         Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx),
     );
 
@@ -111,7 +111,7 @@ fn build_vtable<'tcx>(
         opt_mth.map(|(def_id, substs)| {
             import_function(
                 tcx,
-                &mut fx.cx.module,
+                fx.cx.module,
                 Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs)
                     .unwrap()
                     .polymorphize(fx.tcx),
@@ -165,11 +165,8 @@ fn build_vtable<'tcx>(
 }
 
 fn write_usize(tcx: TyCtxt<'_>, buf: &mut [u8], idx: usize, num: u64) {
-    let pointer_size = tcx
+    let pointer_size =
-        .layout_of(ParamEnv::reveal_all().and(tcx.types.usize))
+        tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.usize)).unwrap().size.bytes() as usize;
-        .unwrap()
-        .size
-        .bytes() as usize;
     let target = &mut buf[idx * pointer_size..(idx + 1) * pointer_size];
 
     match tcx.data_layout.endian {

compiler/rustc_codegen_cranelift/test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 ./build.sh --sysroot none "$@"