bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Sync rustc_codegen_cranelift 'ddd4ce2553'

Browse source
This commit is contained in:
Erin Power 2021-04-30 14:49:58 +02:00
commit 505aa48f24
43 changed files with 784 additions and 1243 deletions
Download patch file Download diff file Expand all files Collapse all files

69
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -16,12 +16,6 @@ use cranelift_object::ObjectModule;
use crate::{prelude::*, BackendConfig};
fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
let module = crate::backend::make_module(tcx.sess, name);
assert_eq!(pointer_ty(tcx), module.target_config().pointer_type());
module
}
struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
impl<HCX> HashStable<HCX> for ModuleCodegenResult {
@ -32,11 +26,12 @@ impl<HCX> HashStable<HCX> for ModuleCodegenResult {
fn emit_module(
tcx: TyCtxt<'_>,
backend_config: &BackendConfig,
name: String,
kind: ModuleKind,
module: ObjectModule,
debug: Option<DebugContext<'_>>,
unwind_context: UnwindContext<'_>,
unwind_context: UnwindContext,
) -> ModuleCodegenResult {
let mut product = module.finish();
@ -52,7 +47,7 @@ fn emit_module(
tcx.sess.fatal(&format!("error writing object file: {}", err));
}
let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() {
let work_product = if backend_config.disable_incr_cache {
None
} else {
rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
@ -110,23 +105,24 @@ fn module_codegen(
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let mut module = new_module(tcx, cgu_name.as_str().to_string());
let isa = crate::build_isa(tcx.sess, &backend_config);
let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string());
let mut cx = crate::CodegenCx::new(
tcx,
backend_config,
&mut module,
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
);
super::predefine_mono_items(&mut cx, &mono_items);
super::predefine_mono_items(tcx, &mut module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(&mut cx.constants_cx, def_id)
cx.tcx
.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst));
}
MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id),
MonoItem::GlobalAsm(item_id) => {
let item = cx.tcx.hir().item(item_id);
if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
@ -138,25 +134,28 @@ fn module_codegen(
}
}
}
let (global_asm, debug, mut unwind_context) =
tcx.sess.time("finalize CodegenCx", || cx.finalize());
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false);
let codegen_result = emit_module(
tcx,
cgu.name().as_str().to_string(),
ModuleKind::Regular,
module,
debug,
unwind_context,
);
let debug_context = cx.debug_context;
let unwind_context = cx.unwind_context;
let codegen_result = tcx.sess.time("write object file", || {
emit_module(
tcx,
&backend_config,
cgu.name().as_str().to_string(),
ModuleKind::Regular,
module,
debug_context,
unwind_context,
)
});
codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm);
codegen_result
}
pub(super) fn run_aot(
pub(crate) fn run_aot(
tcx: TyCtxt<'_>,
backend_config: BackendConfig,
metadata: EncodedMetadata,
@ -193,14 +192,14 @@ pub(super) fn run_aot(
}
}
let modules = super::time(tcx, "codegen mono items", || {
let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
cgus.iter()
.map(|cgu| {
let cgu_reuse = determine_cgu_reuse(tcx, cgu);
tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
match cgu_reuse {
_ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {}
_ if backend_config.disable_incr_cache => {}
CguReuse::No => {}
CguReuse::PreLto => {
return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products);
@ -212,7 +211,7 @@ pub(super) fn run_aot(
let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
dep_node,
tcx,
(backend_config, cgu.name()),
(backend_config.clone(), cgu.name()),
module_codegen,
rustc_middle::dep_graph::hash_result,
);
@ -228,7 +227,10 @@ pub(super) fn run_aot(
tcx.sess.abort_if_errors();
let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
let isa = crate::build_isa(tcx.sess, &backend_config);
let mut allocator_module =
crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string());
assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
let created_alloc_shim =
crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
@ -236,6 +238,7 @@ pub(super) fn run_aot(
let allocator_module = if created_alloc_shim {
let ModuleCodegenResult(module, work_product) = emit_module(
tcx,
&backend_config,
"allocator_shim".to_string(),
ModuleKind::Allocator,
allocator_module,

155
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -1,4 +1,4 @@
//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object
//! files.
use std::cell::RefCell;
@ -8,32 +8,62 @@ use std::os::raw::{c_char, c_int};
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::config::EntryFnType;
use cranelift_jit::{JITBuilder, JITModule};
use crate::{prelude::*, BackendConfig};
use crate::{CodegenCx, CodegenMode};
thread_local! {
pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
struct JitState {
backend_config: BackendConfig,
jit_module: JITModule,
}
pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
if !tcx.sess.opts.output_types.should_codegen() {
tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
}
thread_local! {
static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
}
fn create_jit_module<'tcx>(
tcx: TyCtxt<'tcx>,
backend_config: &BackendConfig,
hotswap: bool,
) -> (JITModule, CodegenCx<'tcx>) {
let imported_symbols = load_imported_symbols_for_jit(tcx);
let mut jit_builder =
JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
let isa = crate::build_isa(tcx.sess, backend_config);
let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
jit_builder.hotswap(hotswap);
crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
jit_builder.symbols(imported_symbols);
let mut jit_module = JITModule::new(jit_builder);
assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false);
crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut jit_module,
&mut cx.unwind_context,
true,
);
(jit_module, cx)
}
pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
if !tcx.sess.opts.output_types.should_codegen() {
tcx.sess.fatal("JIT mode doesn't work with `cargo check`");
}
if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) {
tcx.sess.fatal("can't jit non-executable crate");
}
let (mut jit_module, mut cx) = create_jit_module(
tcx,
&backend_config,
matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
);
let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
let mono_items = cgus
@ -44,52 +74,45 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
.into_iter()
.collect::<Vec<(_, (_, _))>>();
let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
super::time(tcx, "codegen mono items", || {
super::predefine_mono_items(&mut cx, &mono_items);
super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
cx.tcx.sess.time("codegen fn", || {
crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
});
}
CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
},
MonoItem::Static(def_id) => {
crate::constant::codegen_static(&mut cx.constants_cx, def_id);
crate::constant::codegen_static(tcx, &mut jit_module, def_id);
}
MonoItem::GlobalAsm(item_id) => {
let item = cx.tcx.hir().item(item_id);
let item = tcx.hir().item(item_id);
tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode");
}
}
}
});
let (global_asm, _debug, mut unwind_context) =
tcx.sess.time("finalize CodegenCx", || cx.finalize());
jit_module.finalize_definitions();
if !global_asm.is_empty() {
if !cx.global_asm.is_empty() {
tcx.sess.fatal("Inline asm is not supported in JIT mode");
}
crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
tcx.sess.abort_if_errors();
jit_module.finalize_definitions();
let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
unsafe { cx.unwind_context.register_jit(&jit_module) };
println!(
"Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
);
let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
.chain(args.split(' '))
.chain(backend_config.jit_args.iter().map(|arg| &**arg))
.map(|arg| CString::new(arg).unwrap())
.collect::<Vec<_>>();
let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
@ -98,8 +121,21 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
// useful as some dynamic linkers use it as a marker to jump over.
argv.push(std::ptr::null());
BACKEND_CONFIG.with(|tls_backend_config| {
assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
let start_sig = Signature {
params: vec![
AbiParam::new(jit_module.target_config().pointer_type()),
AbiParam::new(jit_module.target_config().pointer_type()),
],
returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
};
let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
LAZY_JIT_STATE.with(|lazy_jit_state| {
let mut lazy_jit_state = lazy_jit_state.borrow_mut();
assert!(lazy_jit_state.is_none());
*lazy_jit_state = Some(JitState { backend_config, jit_module });
});
let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
@ -161,24 +197,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
// lift is used to ensure the correct lifetime for instance.
let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
CURRENT_MODULE.with(|jit_module| {
let mut jit_module = jit_module.borrow_mut();
let jit_module = jit_module.as_mut().unwrap();
let backend_config =
BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
LAZY_JIT_STATE.with(|lazy_jit_state| {
let mut lazy_jit_state = lazy_jit_state.borrow_mut();
let lazy_jit_state = lazy_jit_state.as_mut().unwrap();
let jit_module = &mut lazy_jit_state.jit_module;
let backend_config = lazy_jit_state.backend_config.clone();
let name = tcx.symbol_name(instance).name.to_string();
let name = tcx.symbol_name(instance).name;
let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap();
jit_module.prepare_for_function_redefine(func_id).unwrap();
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
let (global_asm, _debug_context, unwind_context) = cx.finalize();
assert!(global_asm.is_empty());
assert!(cx.global_asm.is_empty());
jit_module.finalize_definitions();
std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
unsafe { cx.unwind_context.register_jit(&jit_module) };
jit_module.get_finalized_function(func_id)
})
})
@ -248,35 +283,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
imported_symbols
}
fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
let tcx = cx.tcx;
let pointer_type = cx.module.target_config().pointer_type();
let pointer_type = module.target_config().pointer_type();
let name = tcx.symbol_name(inst).name.to_string();
let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
let name = tcx.symbol_name(inst).name;
let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst);
let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap();
let instance_ptr = Box::into_raw(Box::new(inst));
let jit_fn = cx
.module
let jit_fn = module
.declare_function(
"__clif_jit_fn",
Linkage::Import,
&Signature {
call_conv: cx.module.target_config().default_call_conv,
call_conv: module.target_config().default_call_conv,
params: vec![AbiParam::new(pointer_type)],
returns: vec![AbiParam::new(pointer_type)],
},
)
.unwrap();
let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
let mut builder_ctx = FunctionBuilderContext::new();
let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
cx.cached_context.clear();
let trampoline = &mut cx.cached_context.func;
trampoline.signature = sig.clone();
let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
let mut builder_ctx = FunctionBuilderContext::new();
let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx);
let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func);
let sig_ref = trampoline_builder.func.import_signature(sig);
let entry_block = trampoline_builder.create_block();
@ -291,10 +328,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
trampoline_builder.ins().return_(&ret_vals);
cx.module
module
.define_function(
func_id,
&mut Context::for_function(trampoline),
&mut cx.cached_context,
&mut NullTrapSink {},
&mut NullStackMapSink {},
)

58
compiler/rustc_codegen_cranelift/src/driver/mod.rs
View file

@ -1,63 +1,37 @@
//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions
//! like JIT executing or writing object files.
//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and
//! performing any further actions like JIT executing or writing object files.
//!
//! [`codegen_fn`]: crate::base::codegen_fn
//! [`codegen_static`]: crate::constant::codegen_static
use std::any::Any;
use rustc_middle::middle::cstore::EncodedMetadata;
use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility};
use crate::prelude::*;
use crate::CodegenMode;
mod aot;
pub(crate) mod aot;
#[cfg(feature = "jit")]
mod jit;
pub(crate) fn codegen_crate(
tcx: TyCtxt<'_>,
metadata: EncodedMetadata,
need_metadata_module: bool,
backend_config: crate::BackendConfig,
) -> Box<dyn Any> {
tcx.sess.abort_if_errors();
match backend_config.codegen_mode {
CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
let is_executable =
tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable);
if !is_executable {
tcx.sess.fatal("can't jit non-executable crate");
}
#[cfg(feature = "jit")]
let _: ! = jit::run_jit(tcx, backend_config);
#[cfg(not(feature = "jit"))]
tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
}
}
}
pub(crate) mod jit;
fn predefine_mono_items<'tcx>(
cx: &mut crate::CodegenCx<'_, 'tcx>,
tcx: TyCtxt<'tcx>,
module: &mut dyn Module,
mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
) {
cx.tcx.sess.time("predefine functions", || {
let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
tcx.sess.time("predefine functions", || {
let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE);
for &(mono_item, (linkage, visibility)) in mono_items {
match mono_item {
MonoItem::Fn(instance) => {
let name = cx.tcx.symbol_name(instance).name.to_string();
let name = tcx.symbol_name(instance).name;
let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
let sig = get_function_sig(tcx, module.isa().triple(), instance);
let linkage = crate::linkage::get_clif_linkage(
mono_item,
linkage,
visibility,
is_compiler_builtins,
);
cx.module.declare_function(&name, linkage, &sig).unwrap();
module.declare_function(name, linkage, &sig).unwrap();
}
MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
}
@ -65,8 +39,8 @@ fn predefine_mono_items<'tcx>(
});
}
fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") {
fn time<R>(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R {
if display {
println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
let before = std::time::Instant::now();
let res = tcx.sess.time(name, f);