Merge commit 'dbee13661e' into sync_cg_clif-2020-12-27

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -8,7 +8,7 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::middle::cstore::EncodedMetadata;
-use rustc_middle::mir::mono::CodegenUnit;
+use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{DebugInfo, OutputType};
 
@@ -146,11 +146,34 @@ fn module_codegen(tcx: TyCtxt<'_>, cgu_name: rustc_span::Symbol) -> ModuleCodege
         }
     }
 
-    let mut cx = crate::CodegenCx::new(tcx, module, tcx.sess.opts.debuginfo != DebugInfo::None);
+    let mut cx = crate::CodegenCx::new(
+        tcx,
+        module,
+        tcx.sess.opts.debuginfo != DebugInfo::None,
+        true,
+    );
     super::predefine_mono_items(&mut cx, &mono_items);
     for (mono_item, (linkage, visibility)) in mono_items {
         let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
-        super::codegen_mono_item(&mut cx, mono_item, linkage);
+        match mono_item {
+            MonoItem::Fn(inst) => {
+                cx.tcx.sess.time("codegen fn", || {
+                    crate::base::codegen_fn(&mut cx, inst, linkage)
+                });
+            }
+            MonoItem::Static(def_id) => {
+                crate::constant::codegen_static(&mut cx.constants_cx, def_id)
+            }
+            MonoItem::GlobalAsm(hir_id) => {
+                let item = cx.tcx.hir().expect_item(hir_id);
+                if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
+                    cx.global_asm.push_str(&*asm.as_str());
+                    cx.global_asm.push_str("\n\n");
+                } else {
+                    bug!("Expected GlobalAsm found {:?}", item);
+                }
+            }
+        }
     }
     let (mut module, global_asm, debug, mut unwind_context) =
         tcx.sess.time("finalize CodegenCx", || cx.finalize());
@@ -236,7 +259,7 @@ pub(super) fn run_aot(
     tcx.sess.abort_if_errors();
 
     let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
-    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa());
+    let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
     let created_alloc_shim =
         crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
 

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -1,16 +1,23 @@
 //! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
 //! files.
 
+use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
 use rustc_codegen_ssa::CrateInfo;
+use rustc_middle::mir::mono::MonoItem;
 
-use cranelift_simplejit::{SimpleJITBuilder, SimpleJITModule};
+use cranelift_jit::{JITBuilder, JITModule};
 
 use crate::prelude::*;
+use crate::{CodegenCx, CodegenMode};
 
-pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
+thread_local! {
+    pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
+}
+
+pub(super) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode) -> ! {
     if !tcx.sess.opts.output_types.should_codegen() {
         tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
     }
@@ -35,12 +42,13 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
 
     let imported_symbols = load_imported_symbols_for_jit(tcx);
 
-    let mut jit_builder = SimpleJITBuilder::with_isa(
-        crate::build_isa(tcx.sess, false),
+    let mut jit_builder = JITBuilder::with_isa(
+        crate::build_isa(tcx.sess),
         cranelift_module::default_libcall_names(),
     );
+    jit_builder.hotswap(matches!(codegen_mode, CodegenMode::JitLazy));
     jit_builder.symbols(imported_symbols);
-    let mut jit_module = SimpleJITModule::new(jit_builder);
+    let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
     let sig = Signature {
@@ -66,20 +74,42 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
         .into_iter()
         .collect::<Vec<(_, (_, _))>>();
 
-    let mut cx = crate::CodegenCx::new(tcx, jit_module, false);
+    let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+
+    super::time(tcx, "codegen mono items", || {
+        super::predefine_mono_items(&mut cx, &mono_items);
+        for (mono_item, (linkage, visibility)) in mono_items {
+            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+            match mono_item {
+                MonoItem::Fn(inst) => match codegen_mode {
+                    CodegenMode::Aot => unreachable!(),
+                    CodegenMode::Jit => {
+                        cx.tcx.sess.time("codegen fn", || {
+                            crate::base::codegen_fn(&mut cx, inst, linkage)
+                        });
+                    }
+                    CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
+                },
+                MonoItem::Static(def_id) => {
+                    crate::constant::codegen_static(&mut cx.constants_cx, def_id);
+                }
+                MonoItem::GlobalAsm(hir_id) => {
+                    let item = cx.tcx.hir().expect_item(hir_id);
+                    tcx.sess
+                        .span_fatal(item.span, "Global asm is not supported in JIT mode");
+                }
+            }
+        }
+    });
 
     let (mut jit_module, global_asm, _debug, mut unwind_context) =
-        super::time(tcx, "codegen mono items", || {
-            super::predefine_mono_items(&mut cx, &mono_items);
-            for (mono_item, (linkage, visibility)) in mono_items {
-                let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
-                super::codegen_mono_item(&mut cx, mono_item, linkage);
-            }
-            tcx.sess.time("finalize CodegenCx", || cx.finalize())
-        });
+        tcx.sess.time("finalize CodegenCx", || cx.finalize());
+    jit_module.finalize_definitions();
+
     if !global_asm.is_empty() {
-        tcx.sess.fatal("Global asm is not supported in JIT mode");
+        tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
+
     crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context, true);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
@@ -91,7 +121,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
 
     let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
 
-    println!("Rustc codegen cranelift will JIT run the executable, because --jit was passed");
+    println!("Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed");
 
     let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
         unsafe { ::std::mem::transmute(finalized_main) };
@@ -107,11 +137,50 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>) -> ! {
     // useful as some dynamic linkers use it as a marker to jump over.
     argv.push(std::ptr::null());
 
+    CURRENT_MODULE
+        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
+
     let ret = f(args.len() as c_int, argv.as_ptr());
 
     std::process::exit(ret);
 }
 
+#[no_mangle]
+extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
+    rustc_middle::ty::tls::with(|tcx| {
+        // lift is used to ensure the correct lifetime for instance.
+        let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
+
+        CURRENT_MODULE.with(|jit_module| {
+            let mut jit_module = jit_module.borrow_mut();
+            let jit_module = jit_module.as_mut().unwrap();
+            let mut cx = crate::CodegenCx::new(tcx, jit_module, false, false);
+
+            let (name, sig) = crate::abi::get_function_name_and_sig(
+                tcx,
+                cx.module.isa().triple(),
+                instance,
+                true,
+            );
+            let func_id = cx
+                .module
+                .declare_function(&name, Linkage::Export, &sig)
+                .unwrap();
+            cx.module.prepare_for_function_redefine(func_id).unwrap();
+
+            tcx.sess.time("codegen fn", || {
+                crate::base::codegen_fn(&mut cx, instance, Linkage::Export)
+            });
+
+            let (jit_module, global_asm, _debug_context, unwind_context) = cx.finalize();
+            assert!(global_asm.is_empty());
+            jit_module.finalize_definitions();
+            std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
+            jit_module.get_finalized_function(func_id)
+        })
+    })
+}
+
 fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     use rustc_middle::middle::dependency_format::Linkage;
 
@@ -171,3 +240,68 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
 
     imported_symbols
 }
+
+pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx, impl Module>, inst: Instance<'tcx>) {
+    let tcx = cx.tcx;
+
+    let pointer_type = cx.module.target_config().pointer_type();
+
+    let (name, sig) =
+        crate::abi::get_function_name_and_sig(tcx, cx.module.isa().triple(), inst, true);
+    let func_id = cx
+        .module
+        .declare_function(&name, Linkage::Export, &sig)
+        .unwrap();
+
+    let instance_ptr = Box::into_raw(Box::new(inst));
+
+    let jit_fn = cx
+        .module
+        .declare_function(
+            "__clif_jit_fn",
+            Linkage::Import,
+            &Signature {
+                call_conv: cx.module.target_config().default_call_conv,
+                params: vec![AbiParam::new(pointer_type)],
+                returns: vec![AbiParam::new(pointer_type)],
+            },
+        )
+        .unwrap();
+
+    let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
+    let mut builder_ctx = FunctionBuilderContext::new();
+    let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
+
+    let jit_fn = cx
+        .module
+        .declare_func_in_func(jit_fn, trampoline_builder.func);
+    let sig_ref = trampoline_builder.func.import_signature(sig);
+
+    let entry_block = trampoline_builder.create_block();
+    trampoline_builder.append_block_params_for_function_params(entry_block);
+    let fn_args = trampoline_builder
+        .func
+        .dfg
+        .block_params(entry_block)
+        .to_vec();
+
+    trampoline_builder.switch_to_block(entry_block);
+    let instance_ptr = trampoline_builder
+        .ins()
+        .iconst(pointer_type, instance_ptr as u64 as i64);
+    let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
+    let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
+    let call_inst = trampoline_builder
+        .ins()
+        .call_indirect(sig_ref, jitted_fn, &fn_args);
+    let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
+    trampoline_builder.ins().return_(&ret_vals);
+
+    cx.module
+        .define_function(
+            func_id,
+            &mut Context::for_function(trampoline),
+            &mut cranelift_codegen::binemit::NullTrapSink {},
+        )
+        .unwrap();
+}

compiler/rustc_codegen_cranelift/src/driver/mod.rs

@@ -7,6 +7,7 @@ use rustc_middle::middle::cstore::EncodedMetadata;
 use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility};
 
 use crate::prelude::*;
+use crate::CodegenMode;
 
 mod aot;
 #[cfg(feature = "jit")]
@@ -20,24 +21,25 @@ pub(crate) fn codegen_crate(
 ) -> Box<dyn Any> {
     tcx.sess.abort_if_errors();
 
-    if config.use_jit {
-        let is_executable = tcx
-            .sess
-            .crate_types()
-            .contains(&rustc_session::config::CrateType::Executable);
-        if !is_executable {
-            tcx.sess.fatal("can't jit non-executable crate");
+    match config.codegen_mode {
+        CodegenMode::Aot => aot::run_aot(tcx, metadata, need_metadata_module),
+        CodegenMode::Jit | CodegenMode::JitLazy => {
+            let is_executable = tcx
+                .sess
+                .crate_types()
+                .contains(&rustc_session::config::CrateType::Executable);
+            if !is_executable {
+                tcx.sess.fatal("can't jit non-executable crate");
+            }
+
+            #[cfg(feature = "jit")]
+            let _: ! = jit::run_jit(tcx, config.codegen_mode);
+
+            #[cfg(not(feature = "jit"))]
+            tcx.sess
+                .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         }
-
-        #[cfg(feature = "jit")]
-        let _: ! = jit::run_jit(tcx);
-
-        #[cfg(not(feature = "jit"))]
-        tcx.sess
-            .fatal("jit support was disabled when compiling rustc_codegen_cranelift");
     }
-
-    aot::run_aot(tcx, metadata, need_metadata_module)
 }
 
 fn predefine_mono_items<'tcx>(
@@ -63,30 +65,6 @@ fn predefine_mono_items<'tcx>(
     });
 }
 
-fn codegen_mono_item<'tcx, M: Module>(
-    cx: &mut crate::CodegenCx<'tcx, M>,
-    mono_item: MonoItem<'tcx>,
-    linkage: Linkage,
-) {
-    match mono_item {
-        MonoItem::Fn(inst) => {
-            cx.tcx
-                .sess
-                .time("codegen fn", || crate::base::codegen_fn(cx, inst, linkage));
-        }
-        MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx.constants_cx, def_id),
-        MonoItem::GlobalAsm(hir_id) => {
-            let item = cx.tcx.hir().expect_item(hir_id);
-            if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
-                cx.global_asm.push_str(&*asm.as_str());
-                cx.global_asm.push_str("\n\n");
-            } else {
-                bug!("Expected GlobalAsm found {:?}", item);
-            }
-        }
-    }
-}
-
 fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
     if std::env::var("CG_CLIF_DISPLAY_CG_TIME")
         .as_ref()