Merge commit 'dbee13661e' into sync_cg_clif-2020-12-27

compiler/rustc_codegen_cranelift/.vscode/settings.json

@@ -1,6 +1,7 @@
 {
     // source for rustc_* is not included in the rust-src component; disable the errors about this
     "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate"],
+    "rust-analyzer.assist.importMergeBehaviour": "last",
     "rust-analyzer.cargo.loadOutDirsFromCheck": true,
     "rust-analyzer.linkedProjects": [
         "./Cargo.toml",

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -50,7 +50,7 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 [[package]]
 name = "cranelift-bforest"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "cranelift-entity",
 ]
@@ -58,7 +58,7 @@ dependencies = [
 [[package]]
 name = "cranelift-codegen"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "byteorder",
  "cranelift-bforest",
@@ -76,7 +76,7 @@ dependencies = [
 [[package]]
 name = "cranelift-codegen-meta"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "cranelift-codegen-shared",
  "cranelift-entity",
@@ -85,17 +85,17 @@ dependencies = [
 [[package]]
 name = "cranelift-codegen-shared"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 
 [[package]]
 name = "cranelift-entity"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 
 [[package]]
 name = "cranelift-frontend"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -103,10 +103,28 @@ dependencies = [
  "target-lexicon",
 ]
 
+[[package]]
+name = "cranelift-jit"
+version = "0.68.0"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
+dependencies = [
+ "anyhow",
+ "cranelift-codegen",
+ "cranelift-entity",
+ "cranelift-module",
+ "cranelift-native",
+ "errno",
+ "libc",
+ "log",
+ "region",
+ "target-lexicon",
+ "winapi",
+]
+
 [[package]]
 name = "cranelift-module"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -118,7 +136,7 @@ dependencies = [
 [[package]]
 name = "cranelift-native"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "cranelift-codegen",
  "raw-cpuid",
@@ -128,7 +146,7 @@ dependencies = [
 [[package]]
 name = "cranelift-object"
 version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
+source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8f7f8ee0b4c5007ace6de29b45505c360450b1bb"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -138,23 +156,6 @@ dependencies = [
  "target-lexicon",
 ]
 
-[[package]]
-name = "cranelift-simplejit"
-version = "0.68.0"
-source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#19640367dbf0da7093e61add3306c8d092644fb3"
-dependencies = [
- "cranelift-codegen",
- "cranelift-entity",
- "cranelift-module",
- "cranelift-native",
- "errno",
- "libc",
- "log",
- "region",
- "target-lexicon",
- "winapi",
-]
-
 [[package]]
 name = "crc32fast"
 version = "1.2.1"
@@ -325,9 +326,9 @@ dependencies = [
  "ar",
  "cranelift-codegen",
  "cranelift-frontend",
+ "cranelift-jit",
  "cranelift-module",
  "cranelift-object",
- "cranelift-simplejit",
  "gimli",
  "indexmap",
  "libloading",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -12,7 +12,7 @@ crate-type = ["dylib"]
 cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] }
 cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
-cranelift-simplejit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
+cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true }
 cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" }
 target-lexicon = "0.11.0"
 gimli = { version = "0.23.0", default-features = false, features = ["write"]}
@@ -27,7 +27,7 @@ libloading = { version = "0.6.0", optional = true }
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
 #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" }
 #cranelift-module = { path = "../wasmtime/cranelift/module" }
-#cranelift-simplejit = { path = "../wasmtime/cranelift/simplejit" }
+#cranelift-jit = { path = "../wasmtime/cranelift/jit" }
 #cranelift-object = { path = "../wasmtime/cranelift/object" }
 
 #[patch.crates-io]
@@ -35,7 +35,7 @@ libloading = { version = "0.6.0", optional = true }
 
 [features]
 default = ["jit", "inline_asm"]
-jit = ["cranelift-simplejit", "libloading"]
+jit = ["cranelift-jit", "libloading"]
 inline_asm = []
 
 [profile.dev]

compiler/rustc_codegen_cranelift/Readme.md

@@ -2,7 +2,7 @@
 
 > ⚠⚠⚠ Certain kinds of FFI don't work yet. ⚠⚠⚠
 
-The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/master/cranelift).
+The goal of this project is to create an alternative codegen backend for the rust compiler based on [Cranelift](https://github.com/bytecodealliance/wasmtime/blob/main/cranelift).
 This has the potential to improve compilation times in debug mode.
 If your project doesn't use any of the things listed under "Not yet supported", it should work fine.
 If not please open an issue.
@@ -68,7 +68,15 @@ $ $cg_clif_dir/build/cargo.sh jit
 or
 
 ```bash
-$ $cg_clif_dir/build/bin/cg_clif --jit my_crate.rs
+$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs
+```
+
+There is also an experimental lazy jit mode. In this mode functions are only compiled once they are
+first called. It currently does not work with multi-threaded programs. When a not yet compiled
+function is called from another thread than the main thread, you will get an ICE.
+
+```bash
+$ $cg_clif_dir/build/cargo.sh lazy-jit
 ```
 
 ### Shell
@@ -77,7 +85,7 @@ These are a few functions that allow you to easily run rust code from the shell
 
 ```bash
 function jit_naked() {
-    echo "$@" | $cg_clif_dir/build/bin/cg_clif - --jit
+    echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic
 }
 
 function jit() {

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.lock

@@ -47,9 +47,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
 [[package]]
 name = "cc"
-version = "1.0.65"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15"
+checksum = "4c0496836a84f8d0495758516b8621a622beb77c0fed418570e50764093ced48"
 
 [[package]]
 name = "cfg-if"
@@ -141,9 +141,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"
+checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb"
 dependencies = [
  "rustc-std-workspace-core",
 ]

compiler/rustc_codegen_cranelift/build_sysroot/Cargo.toml

@@ -5,13 +5,14 @@ version = "0.0.0"
 
 [dependencies]
 core = { path = "./sysroot_src/library/core" }
-compiler_builtins = "0.1"
 alloc = { path = "./sysroot_src/library/alloc" }
 std = { path = "./sysroot_src/library/std", features = ["panic_unwind", "backtrace"] }
 test = { path = "./sysroot_src/library/test" }
 
 alloc_system = { path = "./alloc_system" }
 
+compiler_builtins = { version = "=0.1.36", default-features = false }
+
 [patch.crates-io]
 rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-core" }
 rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" }

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -15,6 +15,8 @@ fn main() {
     let stderr = ::std::io::stderr();
     let mut stderr = stderr.lock();
 
+    // FIXME support lazy jit when multi threading
+    #[cfg(not(lazy_jit))]
     std::thread::spawn(move || {
         println!("Hello from another thread!");
     });

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1 +1 @@
-nightly-2020-11-27
+nightly-2020-12-23

compiler/rustc_codegen_cranelift/scripts/cargo.sh

@@ -10,7 +10,9 @@ cmd=$1
 shift || true
 
 if [[ "$cmd" = "jit" ]]; then
-cargo "+${TOOLCHAIN}" rustc "$@" -- --jit
+cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit -Cprefer-dynamic
+elif [[ "$cmd" = "lazy-jit" ]]; then
+cargo "+${TOOLCHAIN}" rustc "$@" -- -Cllvm-args=mode=jit-lazy -Cprefer-dynamic
 else
 cargo "+${TOOLCHAIN}" "$cmd" "$@"
 fi

compiler/rustc_codegen_cranelift/scripts/filter_profile.rs

@@ -4,7 +4,7 @@
 pushd $(dirname "$0")/../
 source build/config.sh
 popd
-PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS --jit $0
+PROFILE=$1 OUTPUT=$2 exec $RUSTC $RUSTFLAGS -Cllvm-args=mode=jit -Cprefer-dynamic $0
 #*/
 
 //! This program filters away uninteresting samples and trims uninteresting frames for stackcollapse

compiler/rustc_codegen_cranelift/scripts/tests.sh

@@ -15,7 +15,10 @@ function no_sysroot_tests() {
 
     if [[ "$JIT_SUPPORTED" = "1" ]]; then
         echo "[JIT] mini_core_hello_world"
-        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC --jit example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
+        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
+
+        echo "[JIT-lazy] mini_core_hello_world"
+        CG_CLIF_JIT_ARGS="abc bcd" $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/mini_core_hello_world.rs --cfg jit --target "$HOST_TRIPLE"
     else
         echo "[JIT] mini_core_hello_world (skipped)"
     fi
@@ -37,7 +40,10 @@ function base_sysroot_tests() {
 
     if [[ "$JIT_SUPPORTED" = "1" ]]; then
         echo "[JIT] std_example"
-        $MY_RUSTC --jit example/std_example.rs --target "$HOST_TRIPLE"
+        $MY_RUSTC -Cllvm-args=mode=jit -Cprefer-dynamic example/std_example.rs --target "$HOST_TRIPLE"
+
+        echo "[JIT-lazy] std_example"
+        $MY_RUSTC -Cllvm-args=mode=jit-lazy -Cprefer-dynamic example/std_example.rs --cfg lazy_jit --target "$HOST_TRIPLE"
     else
         echo "[JIT] std_example (skipped)"
     fi

compiler/rustc_codegen_cranelift/src/backend.rs

@@ -162,7 +162,7 @@ impl AddConstructor for ObjectProduct {
 }
 
 pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
-    let triple = crate::build_isa(sess, true).triple().clone();
+    let triple = crate::build_isa(sess).triple().clone();
 
     let binary_format = match triple.binary_format {
         target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
@@ -193,7 +193,7 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
 
 pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule {
     let mut builder = ObjectBuilder::new(
-        crate::build_isa(sess, true),
+        crate::build_isa(sess),
         name + ".o",
         cranelift_module::default_libcall_names(),
     )

compiler/rustc_codegen_cranelift/src/base.rs

@@ -118,6 +118,8 @@ pub(crate) fn codegen_fn<'tcx>(
     context.eliminate_unreachable_code(cx.module.isa()).unwrap();
     context.dce(cx.module.isa()).unwrap();
 
+    context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
+
     // Define function
     let module = &mut cx.module;
     tcx.sess.time("define function", || {
@@ -140,6 +142,16 @@ pub(crate) fn codegen_fn<'tcx>(
         &clif_comments,
     );
 
+    if let Some(mach_compile_result) = &context.mach_compile_result {
+        if let Some(disasm) = &mach_compile_result.disasm {
+            crate::pretty_clif::write_ir_file(
+                tcx,
+                &format!("{}.vcode", tcx.symbol_name(instance).name),
+                |file| file.write_all(disasm.as_bytes()),
+            )
+        }
+    }
+
     // Define debuginfo for function
     let isa = cx.module.isa();
     let debug_context = &mut cx.debug_context;
@@ -307,7 +319,9 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
             } => {
                 let discr = codegen_operand(fx, discr).load_scalar(fx);
 
-                if switch_ty.kind() == fx.tcx.types.bool.kind() {
+                let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind()
+                    || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0);
+                if use_bool_opt {
                     assert_eq!(targets.iter().count(), 1);
                     let (then_value, then_block) = targets.iter().next().unwrap();
                     let then_block = fx.get_block(then_block);
@@ -325,6 +339,15 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
                     let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr);
                     let discr =
                         crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr);
+                    if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken(
+                        &fx.bcx, discr, test_zero,
+                    ) {
+                        if taken {
+                            fx.bcx.ins().jump(then_block, &[]);
+                        } else {
+                            fx.bcx.ins().jump(else_block, &[]);
+                        }
+                    } else {
                         if test_zero {
                             fx.bcx.ins().brz(discr, then_block, &[]);
                             fx.bcx.ins().jump(else_block, &[]);
@@ -332,6 +355,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, impl Module>) {
                             fx.bcx.ins().brnz(discr, then_block, &[]);
                             fx.bcx.ins().jump(else_block, &[]);
                         }
+                    }
                 } else {
                     let mut switch = ::cranelift_frontend::Switch::new();
                     for (value, block) in targets.iter() {

compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs

@@ -44,9 +44,7 @@ fn main() {
     let mut callbacks = CraneliftPassesCallbacks::default();
     rustc_driver::install_ice_hook();
     let exit_code = rustc_driver::catch_with_exit_code(|| {
-        let mut use_jit = false;
+        let args = std::env::args_os()
-
-        let mut args = std::env::args_os()
             .enumerate()
             .map(|(i, arg)| {
                 arg.into_string().unwrap_or_else(|arg| {
@@ -56,23 +54,10 @@ fn main() {
                     )
                 })
             })
-            .filter(|arg| {
-                if arg == "--jit" {
-                    use_jit = true;
-                    false
-                } else {
-                    true
-                }
-            })
             .collect::<Vec<_>>();
-        if use_jit {
-            args.push("-Cprefer-dynamic".to_string());
-        }
         let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
         run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
-            Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend {
+            Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
-                config: rustc_codegen_cranelift::BackendConfig { use_jit },
-            })
         })));
         run_compiler.run()
     });

compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs

@@ -92,9 +92,7 @@ fn main() {
         let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
         if use_clif {
             run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
-                Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend {
+                Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
-                    config: rustc_codegen_cranelift::BackendConfig { use_jit: false },
-                })
             })));
         }
         run_compiler.run()

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -100,7 +100,10 @@ fn codegen_static_ref<'tcx>(
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     assert!(!layout.is_unsized(), "unsized statics aren't supported");
     assert!(
-        matches!(fx.bcx.func.global_values[local_data_id], GlobalValueData::Symbol { tls: false, ..}),
+        matches!(
+            fx.bcx.func.global_values[local_data_id],
+            GlobalValueData::Symbol { tls: false, .. }
+        ),
         "tls static referenced without Rvalue::ThreadLocalRef"
     );
     CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout)
@@ -447,7 +450,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut impl Module, cx: &mut Constan
             data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64);
         }
 
-        module.define_data(data_id, &data_ctx).unwrap();
+        // FIXME don't duplicate definitions in lazy jit mode
+        let _ = module.define_data(data_id, &data_ctx);
         cx.done.insert(data_id);
     }
 

compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs

@@ -74,10 +74,7 @@ impl WriterRelocate {
 
     /// Perform the collected relocations to be usable for JIT usage.
     #[cfg(feature = "jit")]
-    pub(super) fn relocate_for_jit(
+    pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
-        mut self,
-        jit_module: &cranelift_simplejit::SimpleJITModule,
-    ) -> Vec<u8> {
         use std::convert::TryInto;
 
         for reloc in self.relocs.drain(..) {

compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs

@@ -15,11 +15,11 @@ pub(crate) struct UnwindContext<'tcx> {
 }
 
 impl<'tcx> UnwindContext<'tcx> {
-    pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self {
+    pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
         let mut frame_table = FrameTable::default();
 
         let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
-            if isa.flags().is_pic() {
+            if pic_eh_frame {
                 cie.fde_address_encoding =
                     gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0);
             }
@@ -80,7 +80,7 @@ impl<'tcx> UnwindContext<'tcx> {
     #[cfg(feature = "jit")]
     pub(crate) unsafe fn register_jit(
         self,
-        jit_module: &cranelift_simplejit::SimpleJITModule,
+        jit_module: &cranelift_jit::JITModule,
     ) -> Option<UnwindRegistry> {
         let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(
             self.tcx,

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::middle::cstore::EncodedMetadata;
-use rustc_middle::mir::mono::CodegenUnit;
+use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{DebugInfo, OutputType};
 
@@ -146,11 +146,34 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
         }
     }
 
-    let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None);
+    let mut cx = crate::CodegenCx::new(
+        tcx,
+        module,
+        tcx.sess.opts.debuginfo != DebugInfo::None,
+        true,
+    );
     super::predefine_mono_items(&mut cx, &mono_items);
     for (mono_item, (linkage, visibility)) in mono_items {
         let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
-        super::codegen_mono_item(&mut cx, mono_item, linkage);
+        match mono_item {
+            MonoItem::Fn(inst) => {
+                cx.tcx.sess.time("codegen fn", || {
+                    crate::base::codegen_fn(&mut cx, inst, linkage)
+                });
+            }
+            MonoItem::Static(def_id) => {
+                crate::constant::codegen_static(&mut cx.constants_cx, def_id)
+            }
+            MonoItem::GlobalAsm(hir_id) => {
+                let item = cx.tcx.hir().expect_item(hir_id);
+                if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
+                    cx.global_asm.push_str(&*asm.as_str());
+                    cx.global_asm.push_str("\n\n");
+                } else {
+                    bug!("Expected GlobalAsm found {:?}", item);
+                }
+            }
+        }
     }
     let (mut module, global_asm, debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
@@ -236,7 +259,7 @@ pub(super) fn run_aot(
     tcx.sess.abort_if_errors();
 
     let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
-    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa());
+    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
     let created_alloc_shim =
         crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
 

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -1,16 +1,23 @@
 //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
 //! files.
 
+use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
 use rustc_codegen_ssa::CrateInfo;
+use rustc_middle::mir::mono::MonoItem;
 
-use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule};
+use cranelift_jit::{JITBuilder, JITModule};
 
 use crate::prelude::*;
+use crate::{CodegenCx, CodegenMode};
 
-pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
+thread_local! {
+    pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
+}
+
+pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
     if !tcx.sess.opts.output_types.should_codegen() {
         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
     }
@@ -35,12 +42,13 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
 
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
-    let mut jit_builder = SimpleJITBuilder::with_isa(
+    let mut jit_builder = JITBuilder::with_isa(
-        crate::build_isa(tcx.sess, false),
+        crate::build_isa(tcx.sess),
         cranelift_module::default_libcall_names(),
     );
+    jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
     jit_builder.symbols(imported_symbols);
-    let mut jit_module = SimpleJITModule::new(jit_builder);
+    let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
     let sig = Signature {
@@ -66,20 +74,42 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    let mut cx = crate::CodegenCx::new(tcx, jit_module, false);
+    let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
 
-    let (mut jit_module, global_asm, _debug, mut unwind_context) =
     super::time(tcx, "codegen mono items", || {
         super::predefine_mono_items(&mut cx, &mono_items);
         for (mono_item, (linkage, visibility)) in mono_items {
             let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
-                super::codegen_mono_item(&mut cx, mono_item, linkage);
+            match mono_item {
-            }
+                MonoItem::Fn(inst) => match codegen_mode {
-            tcx.sess.time("finalize CodegenCx", || cx.finalize())
+                    CodegenMode::Aot => unreachable!(),
+                    CodegenMode::Jit => {
+                        cx.tcx.sess.time("codegen fn", || {
+                            crate::base::codegen_fn(&mut cx, inst, linkage)
                         });
-    if !global_asm.is_empty() {
-        tcx.sess.fatal("Global asm is not supported in JIT mode");
                     }
+                    CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
+                },
+                MonoItem::Static(def_id) => {
+                    crate::constant::codegen_static(&mut cx.constants_cx, def_id);
+                }
+                MonoItem::GlobalAsm(hir_id) => {
+                    let item = cx.tcx.hir().expect_item(hir_id);
+                    tcx.sess
+                        .span_fatal(item.span, "Global asm is not supported in JIT mode");
+                }
+            }
+        }
+    });
+
+    let (mut jit_module, global_asm, _debug, mut unwind_context) =
+        tcx.sess.time("finalize CodegenCx", || cx.finalize());
+    jit_module.finalize_definitions();
+
+    if !global_asm.is_empty() {
+        tcx.sess.fatal("Inline asm is not supported in JIT mode");
+    }
+
     crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
@@ -91,7 +121,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
 
     let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
 
-    println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed");
+    println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
 
     let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
         unsafe { ::std::mem::transmute(finalized_main) };
@@ -107,11 +137,50 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
     // useful as some dynamic linkers use it as a marker to jump over.
     argv.push(std::ptr::null());
 
+    CURRENT_MODULE
+        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
+
     let ret = f(args.len() as c_int, argv.as_ptr());
 
     std::process::exit(ret);
 }
 
+#[no_mangle]
+extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
+    rustc_middle::ty::tls::with(|tcx| {
+        // lift is used to ensure the correct lifetime for instance.
+        let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
+
+        CURRENT_MODULE.with(|jit_module| {
+            let mut jit_module = jit_module.borrow_mut();
+            let jit_module = jit_module.as_mut().unwrap();
+            let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+
+            let (name, sig) = crate::abi::get_function_name_and_sig(
+                tcx,
+                cx.module.isa().triple(),
+                instance,
+                true,
+            );
+            let func_id = cx
+                .module
+                .declare_function(&name, Linkage::Export, &sig)
+                .unwrap();
+            cx.module.prepare_for_function_redefine(func_id).unwrap();
+
+            tcx.sess.time("codegen fn", || {
+                crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
+            });
+
+            let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
+            assert!(global_asm.is_empty());
+            jit_module.finalize_definitions();
+            std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
+            jit_module.get_finalized_function(func_id)
+        })
+    })
+}
+
 fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     use rustc_middle::middle::dependency_format::Linkage;
 
@@ -171,3 +240,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
 
     imported_symbols
 }
+
+pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
+    let tcx = cx.tcx;
+
+    let pointer_type = cx.module.target_config().pointer_type();
+
+    let (name, sig) =
+        crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true);
+    let func_id = cx
+        .module
+        .declare_function(&name, Linkage::Export, &sig)
+        .unwrap();
+
+    let instance_ptr = Box::into_raw(Box::new(inst));
+
+    let jit_fn = cx
+        .module
+        .declare_function(
+            "__clif_jit_fn",
+            Linkage::Import,
+            &Signature {
+                call_conv: cx.module.target_config().default_call_conv,
+                params: vec![AbiParam::new(pointer_type)],
+                returns: vec![AbiParam::new(pointer_type)],
+            },
+        )
+        .unwrap();
+
+    let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
+    let mut builder_ctx = FunctionBuilderContext::new();
+    let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
+
+    let jit_fn = cx
+        .module
+        .declare_func_in_func(jit_fn, trampoline_builder.func);
+    let sig_ref = trampoline_builder.func.import_signature(sig);
+
+    let entry_block = trampoline_builder.create_block();
+    trampoline_builder.append_block_params_for_function_params(entry_block);
+    let fn_args = trampoline_builder
+        .func
+        .dfg
+        .block_params(entry_block)
+        .to_vec();
+
+    trampoline_builder.switch_to_block(entry_block);
+    let instance_ptr = trampoline_builder
+        .ins()
+        .iconst(pointer_type, instance_ptr as u64 as i64);
+    let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
+    let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
+    let call_inst = trampoline_builder
+        .ins()
+        .call_indirect(sig_ref, jitted_fn, &fn_args);
+    let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
+    trampoline_builder.ins().return_(&ret_vals);
+
+    cx.module
+        .define_function(
+            func_id,
+            &mut Context::for_function(trampoline),
+            &mut cranelift_codegen::binemit::NullTrapSink {},
+        )
+        .unwrap();
+}

compiler/rustc_codegen_cranelift/src/driver/mod.rs

@@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility};
 
 use crate::prelude::*;
+use crate::CodegenMode;
 
 mod aot;
 #[cfg(feature = "jit")]
@@ -20,7 +21,9 @@ pub(crate) fn codegen_crate(
 ) -> Box<dyn Any> {
     tcx.sess.abort_if_errors();
 
-    if config.use_jit {
+    match config.codegen_mode {
+        CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module),
+        CodegenMode::Jit | CodegenMode::JitLazy => {
             let is_executable = tcx
                 .sess
                 .crate_types()
@@ -30,14 +33,13 @@ pub(crate) fn codegen_crate(
             }
 
             #[cfg(feature = "jit")]
-        let _: ! = jit::run_jit(tcx);
+            let _: ! = jit::run_jit(tcx, config.codegen_mode);
 
             #[cfg(not(feature = "jit"))]
             tcx.sess
                 .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         }
-
+    }
-    aot::run_aot(tcx, metadata, need_metadata_module)
 }
 
 fn predefine_mono_items<'tcx>(
@@ -63,30 +65,6 @@ fn predefine_mono_items<'tcx>(
     });
 }
 
-fn codegen_mono_item<'tcx, M: Module>(
-    cx: &mut crate::CodegenCx<'tcx, M>,
-    mono_item: MonoItem<'tcx>,
-    linkage: Linkage,
-) {
-    match mono_item {
-        MonoItem::Fn(inst) => {
-            cx.tcx
-                .sess
-                .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage));
-        }
-        MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id),
-        MonoItem::GlobalAsm(hir_id) => {
-            let item = cx.tcx.hir().expect_item(hir_id);
-            if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
-                cx.global_asm.push_str(&*asm.as_str());
-                cx.global_asm.push_str("\n\n");
-            } else {
-                bug!("Expected GlobalAsm found {:?}", item);
-            }
-        }
-    }
-}
-
 fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
     if std::env::var("CG_CLIF_DISPLAY_CG_TIME")
         .as_ref()

compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

@@ -23,8 +23,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
 
         // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
         llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) {
-            let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout());
+            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
-            let lane_ty = fx.clif_type(lane_layout.ty).unwrap();
+            let lane_ty = fx.clif_type(lane_ty).unwrap();
             assert!(lane_count <= 32);
 
             let mut res = fx.bcx.ins().iconst(types::I32, 0);

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -171,27 +171,6 @@ macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
     }
 }
 
-fn lane_type_and_count<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    layout: TyAndLayout<'tcx>,
-) -> (TyAndLayout<'tcx>, u16) {
-    assert!(layout.ty.is_simd());
-    let lane_count = match layout.fields {
-        rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(),
-        _ => unreachable!("lane_type_and_count({:?})", layout),
-    };
-    let lane_layout = layout
-        .field(
-            &ty::layout::LayoutCx {
-                tcx,
-                param_env: ParamEnv::reveal_all(),
-            },
-            0,
-        )
-        .unwrap();
-    (lane_layout, lane_count)
-}
-
 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
     let (element, count) = match &layout.abi {
         Abi::Vector { element, count } => (element.clone(), *count),
@@ -218,8 +197,10 @@ fn simd_for_each_lane<'tcx, M: Module>(
 ) {
     let layout = val.layout();
 
-    let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
+    let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-    let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+    let lane_layout = fx.layout_of(lane_ty);
+    let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+    let ret_lane_layout = fx.layout_of(ret_lane_ty);
     assert_eq!(lane_count, ret_lane_count);
 
     for lane_idx in 0..lane_count {
@@ -248,8 +229,10 @@ fn simd_pair_for_each_lane<'tcx, M: Module>(
     assert_eq!(x.layout(), y.layout());
     let layout = x.layout();
 
-    let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
+    let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-    let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+    let lane_layout = fx.layout_of(lane_ty);
+    let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+    let ret_lane_layout = fx.layout_of(ret_lane_ty);
     assert_eq!(lane_count, ret_lane_count);
 
     for lane in 0..lane_count {
@@ -269,13 +252,14 @@ fn simd_reduce<'tcx, M: Module>(
     ret: CPlace<'tcx>,
     f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, TyAndLayout<'tcx>, Value, Value) -> Value,
 ) {
-    let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout());
+    let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+    let lane_layout = fx.layout_of(lane_ty);
     assert_eq!(lane_layout, ret.layout());
 
     let mut res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     for lane_idx in 1..lane_count {
         let lane = val
-            .value_field(fx, mir::Field::new(lane_idx.into()))
+            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
             .load_scalar(fx);
         res_val = f(fx, lane_layout, res_val, lane);
     }
@@ -289,14 +273,14 @@ fn simd_reduce_bool<'tcx, M: Module>(
     ret: CPlace<'tcx>,
     f: impl Fn(&mut FunctionCx<'_, 'tcx, M>, Value, Value) -> Value,
 ) {
-    let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, val.layout());
+    let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
     assert!(ret.layout().ty.is_bool());
 
     let res_val = val.value_field(fx, mir::Field::new(0)).load_scalar(fx);
     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
     for lane_idx in 1..lane_count {
         let lane = val
-            .value_field(fx, mir::Field::new(lane_idx.into()))
+            .value_field(fx, mir::Field::new(lane_idx.try_into().unwrap()))
             .load_scalar(fx);
         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
         res_val = f(fx, res_val, lane);
@@ -460,9 +444,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 "abort" => {
                     trap_abort(fx, "Called intrinsic::abort.");
                 }
-                "unreachable" => {
-                    trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
-                }
                 "transmute" => {
                     crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
                 }
@@ -575,12 +556,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
             }
         };
-        discriminant_value, (c ptr) {
-            let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
-            let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
-            let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
-            ret.write_cvalue(fx, discr);
-        };
         size_of_val, <T> (c ptr) {
             let layout = fx.layout_of(T);
             let size = if layout.is_unsized() {
@@ -641,22 +616,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             );
             ret.write_cvalue(fx, res);
         };
-        _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
-            assert_eq!(x.layout().ty, y.layout().ty);
-            let bin_op = match intrinsic {
-                "wrapping_add" => BinOp::Add,
-                "wrapping_sub" => BinOp::Sub,
-                "wrapping_mul" => BinOp::Mul,
-                _ => unreachable!("intrinsic {}", intrinsic),
-            };
-            let res = crate::num::codegen_int_binop(
-                fx,
-                bin_op,
-                x,
-                y,
-            );
-            ret.write_cvalue(fx, res);
-        };
         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
             assert_eq!(lhs.layout().ty, rhs.layout().ty);
             let bin_op = match intrinsic {
@@ -916,7 +875,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             dest.write_cvalue(fx, val);
         };
 
-        size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
+        pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
             let const_val =
                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
             let val = crate::constant::codegen_const_value(

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -73,11 +73,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             assert_eq!(x.layout(), y.layout());
             let layout = x.layout();
 
-            let (lane_type, lane_count) = lane_type_and_count(fx.tcx, layout);
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-            let (ret_lane_type, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 
-            assert_eq!(lane_type, ret_lane_type);
+            assert_eq!(lane_ty, ret_lane_ty);
-            assert_eq!(n, ret_lane_count);
+            assert_eq!(u64::from(n), ret_lane_count);
 
             let total_len = lane_count * 2;
 
@@ -105,14 +105,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             for &idx in &indexes {
-                assert!(idx < total_len, "idx {} out of range 0..{}", idx, total_len);
+                assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
             }
 
             for (out_idx, in_idx) in indexes.into_iter().enumerate() {
-                let in_lane = if in_idx < lane_count {
+                let in_lane = if u64::from(in_idx) < lane_count {
                     x.value_field(fx, mir::Field::new(in_idx.into()))
                 } else {
-                    y.value_field(fx, mir::Field::new((in_idx - lane_count).into()))
+                    y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap()))
                 };
                 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
                 out_lane.write_cvalue(fx, in_lane);
@@ -131,7 +131,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
-            let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, base.layout());
+            let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
             }
@@ -160,7 +160,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             let idx = idx_const.val.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
-            let (_lane_type, lane_count) = lane_type_and_count(fx.tcx, v.layout());
+            let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
             if idx >= lane_count.into() {
                 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
             }
@@ -212,12 +212,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             assert_eq!(a.layout(), c.layout());
             let layout = a.layout();
 
-            let (_lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
+            let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
-            let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
             assert_eq!(lane_count, ret_lane_count);
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
 
             for lane in 0..lane_count {
-                let lane = mir::Field::new(lane.into());
+                let lane = mir::Field::new(lane.try_into().unwrap());
                 let a_lane = a.value_field(fx, lane).load_scalar(fx);
                 let b_lane = b.value_field(fx, lane).load_scalar(fx);
                 let c_lane = c.value_field(fx, lane).load_scalar(fx);

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -5,7 +5,8 @@
     associated_type_bounds,
     never_type,
     try_blocks,
-    hash_drain_filter
+    hash_drain_filter,
+    str_split_once
 )]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
@@ -34,6 +35,7 @@ extern crate rustc_target;
 extern crate rustc_driver;
 
 use std::any::Any;
+use std::str::FromStr;
 
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
@@ -141,8 +143,8 @@ struct CodegenCx<'tcx, M: Module> {
 }
 
 impl<'tcx, M: Module> CodegenCx<'tcx, M> {
-    fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool) -> Self {
+    fn new(tcx: TyCtxt<'tcx>, module: M, debug_info: bool, pic_eh_frame: bool) -> Self {
-        let unwind_context = UnwindContext::new(tcx, module.isa());
+        let unwind_context = UnwindContext::new(tcx, module.isa(), pic_eh_frame);
         let debug_context = if debug_info {
             Some(DebugContext::new(tcx, module.isa()))
         } else {
@@ -172,12 +174,55 @@ impl<'tcx, M: Module> CodegenCx<'tcx, M> {
 }
 
 #[derive(Copy, Clone, Debug)]
+pub enum CodegenMode {
+    Aot,
+    Jit,
+    JitLazy,
+}
+
+impl Default for CodegenMode {
+    fn default() -> Self {
+        CodegenMode::Aot
+    }
+}
+
+impl FromStr for CodegenMode {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "aot" => Ok(CodegenMode::Aot),
+            "jit" => Ok(CodegenMode::Jit),
+            "jit-lazy" => Ok(CodegenMode::JitLazy),
+            _ => Err(format!("Unknown codegen mode `{}`", s)),
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, Default)]
 pub struct BackendConfig {
-    pub use_jit: bool,
+    pub codegen_mode: CodegenMode,
+}
+
+impl BackendConfig {
+    fn from_opts(opts: &[String]) -> Result<Self, String> {
+        let mut config = BackendConfig::default();
+        for opt in opts {
+            if let Some((name, value)) = opt.split_once('=') {
+                match name {
+                    "mode" => config.codegen_mode = value.parse()?,
+                    _ => return Err(format!("Unknown option `{}`", name)),
+                }
+            } else {
+                return Err(format!("Invalid option `{}`", opt));
+            }
+        }
+        Ok(config)
+    }
 }
 
 pub struct CraneliftCodegenBackend {
-    pub config: BackendConfig,
+    pub config: Option<BackendConfig>,
 }
 
 impl CodegenBackend for CraneliftCodegenBackend {
@@ -204,7 +249,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, self.config);
+        let config = if let Some(config) = self.config {
+            config
+        } else {
+            BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
+                .unwrap_or_else(|err| tcx.sess.fatal(&err))
+        };
+        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
 
         rustc_symbol_mangling::test::report_symbol_names(tcx);
 
@@ -250,17 +301,13 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
     sess.target.llvm_target.parse().unwrap()
 }
 
-fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'static> {
+fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     use target_lexicon::BinaryFormat;
 
     let target_triple = crate::target_triple(sess);
 
     let mut flags_builder = settings::builder();
-    if enable_pic {
     flags_builder.enable("is_pic").unwrap();
-    } else {
-        flags_builder.set("is_pic", "false").unwrap();
-    }
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
     flags_builder
         .set(
@@ -283,8 +330,6 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'stat
 
     flags_builder.set("enable_simd", "true").unwrap();
 
-    // FIXME(CraneStation/cranelift#732) fix LICM in presence of jump tables
-    /*
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
@@ -297,7 +342,7 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'stat
         OptLevel::Size | OptLevel::SizeMin => {
             sess.warn("Optimizing for size is not supported. Just ignoring the request");
         }
-    }*/
+    }
 
     let flags = settings::Flags::new(flags_builder);
 
@@ -311,7 +356,5 @@ fn build_isa(sess: &Session, enable_pic: bool) -> Box<dyn isa::TargetIsa + 'stat
 /// This is the entrypoint for a hot plugged rustc_codegen_cranelift
 #[no_mangle]
 pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
-    Box::new(CraneliftCodegenBackend {
+    Box::new(CraneliftCodegenBackend { config: None })
-        config: BackendConfig { use_jit: false },
-    })
 }

compiler/rustc_codegen_cranelift/src/optimize/peephole.rs

@@ -73,7 +73,7 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
     })()
     .unwrap_or_else(|| {
         match bcx.func.dfg.value_type(arg) {
-            types::I8 | types::I32 => {
+            types::I8 | types::I16 => {
                 // WORKAROUND for brz.i8 and brnz.i8 not yet being implemented
                 bcx.ins().uextend(types::I32, arg)
             }
@@ -81,3 +81,40 @@ pub(crate) fn make_branchable_value(bcx: &mut FunctionBuilder<'_>, arg: Value) -
         }
     })
 }
+
+/// Returns whether the branch is statically known to be taken or `None` if it isn't statically known.
+pub(crate) fn maybe_known_branch_taken(
+    bcx: &FunctionBuilder<'_>,
+    arg: Value,
+    test_zero: bool,
+) -> Option<bool> {
+    let arg_inst = if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
+        arg_inst
+    } else {
+        return None;
+    };
+
+    match bcx.func.dfg[arg_inst] {
+        InstructionData::UnaryBool {
+            opcode: Opcode::Bconst,
+            imm,
+        } => {
+            if test_zero {
+                Some(!imm)
+            } else {
+                Some(imm)
+            }
+        }
+        InstructionData::UnaryImm {
+            opcode: Opcode::Iconst,
+            imm,
+        } => {
+            if test_zero {
+                Some(imm.bits() == 0)
+            } else {
+                Some(imm.bits() != 0)
+            }
+        }
+        _ => None,
+    }
+}

compiler/rustc_codegen_cranelift/src/pretty_clif.rs

@@ -53,6 +53,7 @@
 //! ```
 
 use std::fmt;
+use std::io::Write;
 
 use cranelift_codegen::{
     entity::SecondaryMap,
@@ -200,31 +201,23 @@ impl<M: Module> FunctionCx<'_, '_, M> {
     }
 }
 
-pub(crate) fn write_clif_file<'tcx>(
+pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
-    tcx: TyCtxt<'tcx>,
+    cfg!(debug_assertions)
-    postfix: &str,
+        || tcx
-    isa: Option<&dyn cranelift_codegen::isa::TargetIsa>,
-    instance: Instance<'tcx>,
-    context: &cranelift_codegen::Context,
-    mut clif_comments: &CommentWriter,
-) {
-    use std::io::Write;
-
-    if !cfg!(debug_assertions)
-        && !tcx
             .sess
             .opts
             .output_types
             .contains_key(&OutputType::LlvmAssembly)
-    {
-        return;
 }
 
-    let value_ranges = isa.map(|isa| {
+pub(crate) fn write_ir_file<'tcx>(
-        context
+    tcx: TyCtxt<'tcx>,
-            .build_value_labels_ranges(isa)
+    name: &str,
-            .expect("value location ranges")
+    write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
-    });
+) {
+    if !should_write_ir(tcx) {
+        return;
+    }
 
     let clif_output_dir = tcx.output_filenames(LOCAL_CRATE).with_extension("clif");
 
@@ -234,11 +227,34 @@ pub(crate) fn write_clif_file<'tcx>(
         res @ Err(_) => res.unwrap(),
     }
 
-    let clif_file_name = clif_output_dir.join(format!(
+    let clif_file_name = clif_output_dir.join(name);
-        "{}.{}.clif",
+
-        tcx.symbol_name(instance).name,
+    let res: std::io::Result<()> = try {
-        postfix
+        let mut file = std::fs::File::create(clif_file_name)?;
-    ));
+        write(&mut file)?;
+    };
+    if let Err(err) = res {
+        tcx.sess.warn(&format!("error writing ir file: {}", err));
+    }
+}
+
+pub(crate) fn write_clif_file<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    postfix: &str,
+    isa: Option<&dyn cranelift_codegen::isa::TargetIsa>,
+    instance: Instance<'tcx>,
+    context: &cranelift_codegen::Context,
+    mut clif_comments: &CommentWriter,
+) {
+    write_ir_file(
+        tcx,
+        &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
+        |file| {
+            let value_ranges = isa.map(|isa| {
+                context
+                    .build_value_labels_ranges(isa)
+                    .expect("value location ranges")
+            });
 
             let mut clif = String::new();
             cranelift_codegen::write::decorate_function(
@@ -246,27 +262,21 @@ pub(crate) fn write_clif_file<'tcx>(
                 &mut clif,
                 &context.func,
                 &DisplayFunctionAnnotations {
-            isa: Some(&*crate::build_isa(
+                    isa: Some(&*crate::build_isa(tcx.sess)),
-                tcx.sess, true, /* PIC doesn't matter here */
-            )),
                     value_ranges: value_ranges.as_ref(),
                 },
             )
             .unwrap();
 
-    let res: std::io::Result<()> = try {
-        let mut file = std::fs::File::create(clif_file_name)?;
-        let target_triple = crate::target_triple(tcx.sess);
             writeln!(file, "test compile")?;
             writeln!(file, "set is_pic")?;
             writeln!(file, "set enable_simd")?;
-        writeln!(file, "target {} haswell", target_triple)?;
+            writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
             writeln!(file)?;
             file.write_all(clif.as_bytes())?;
-    };
+            Ok(())
-    if let Err(err) = res {
+        },
-        tcx.sess.warn(&format!("err writing clif file: {}", err));
+    );
-    }
 }
 
 impl<M: Module> fmt::Debug for FunctionCx<'_, '_, M> {

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -158,7 +158,8 @@ fn build_vtable<'tcx>(
         )
         .unwrap();
 
-    fx.cx.module.define_data(data_id, &data_ctx).unwrap();
+    // FIXME don't duplicate definitions in lazy jit mode
+    let _ = fx.cx.module.define_data(data_id, &data_ctx);
 
     data_id
 }