bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '0969bc6dde' into sync_cg_clif-2021-03-29

Browse source
This commit is contained in:
bjorn3 2021-03-29 10:45:09 +02:00
commit 5444b46234
49 changed files with 880 additions and 495 deletions
Download patch file Download diff file Expand all files Collapse all files

27
compiler/rustc_codegen_cranelift/src/abi/comments.rs
View file

@ -11,9 +11,11 @@ use cranelift_codegen::entity::EntityRef;
use crate::prelude::*;
pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
fx.add_global_comment(
"kind loc.idx param pass mode ty".to_string(),
);
if fx.clif_comments.enabled() {
fx.add_global_comment(
"kind loc.idx param pass mode ty".to_string(),
);
}
}
pub(super) fn add_arg_comment<'tcx>(
@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>(
arg_abi_mode: PassMode,
arg_layout: TyAndLayout<'tcx>,
) {
if !fx.clif_comments.enabled() {
return;
}
let local = if let Some(local) = local {
Cow::Owned(format!("{:?}", local))
} else {
@ -59,10 +65,12 @@ pub(super) fn add_arg_comment<'tcx>(
}
pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
fx.add_global_comment(String::new());
fx.add_global_comment(
"kind local ty size align (abi,pref)".to_string(),
);
if fx.clif_comments.enabled() {
fx.add_global_comment(String::new());
fx.add_global_comment(
"kind local ty size align (abi,pref)".to_string(),
);
}
}
pub(super) fn add_local_place_comments<'tcx>(
@ -70,6 +78,9 @@ pub(super) fn add_local_place_comments<'tcx>(
place: CPlace<'tcx>,
local: Local,
) {
if !fx.clif_comments.enabled() {
return;
}
let TyAndLayout { ty, layout } = place.layout();
let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
layout;
@ -90,7 +101,7 @@ pub(super) fn add_local_place_comments<'tcx>(
} else {
Cow::Borrowed("")
};
match ptr.base_and_offset() {
match ptr.debug_base_and_offset() {
(crate::pointer::PointerBase::Addr(addr), offset) => {
("reuse", format!("storage={}{}{}", addr, offset, meta).into())
}

24
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -1,6 +1,5 @@
//! Handling of everything related to the calling convention. Also fills `fx.local_map`.
#[cfg(debug_assertions)]
mod comments;
mod pass_mode;
mod returning;
@ -75,8 +74,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
let func_id = import_function(self.tcx, self.cx.module, inst);
let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
#[cfg(debug_assertions)]
self.add_comment(func_ref, format!("{:?}", inst));
if self.clif_comments.enabled() {
self.add_comment(func_ref, format!("{:?}", inst));
}
func_ref
}
@ -92,8 +92,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
let call_inst = self.bcx.ins().call(func_ref, args);
#[cfg(debug_assertions)]
{
if self.clif_comments.enabled() {
self.add_comment(call_inst, format!("easy_call {}", name));
}
let results = self.bcx.inst_results(call_inst);
@ -149,7 +148,6 @@ fn make_local_place<'tcx>(
CPlace::new_stack_slot(fx, layout)
};
#[cfg(debug_assertions)]
self::comments::add_local_place_comments(fx, place, local);
place
@ -163,7 +161,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
let ssa_analyzed = crate::analyze::analyze(fx);
#[cfg(debug_assertions)]
self::comments::add_args_header_comment(fx);
let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
@ -228,7 +225,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
fx.fn_abi = Some(fn_abi);
assert!(block_params_iter.next().is_none(), "arg_value left behind");
#[cfg(debug_assertions)]
self::comments::add_locals_header_comment(fx);
for (local, arg_kind, ty) in func_params {
@ -256,7 +252,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
CPlace::for_ptr(addr, val.layout())
};
#[cfg(debug_assertions)]
self::comments::add_local_place_comments(fx, place, local);
assert_eq!(fx.local_map.push(place), local);
@ -392,8 +387,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
let (func_ref, first_arg) = match instance {
// Trait object call
Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
#[cfg(debug_assertions)]
{
if fx.clif_comments.enabled() {
let nop_inst = fx.bcx.ins().nop();
fx.add_comment(
nop_inst,
@ -414,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
// Indirect call
None => {
#[cfg(debug_assertions)]
{
if fx.clif_comments.enabled() {
let nop_inst = fx.bcx.ins().nop();
fx.add_comment(nop_inst, "indirect call");
}
@ -477,10 +470,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
// FIXME find a cleaner way to support varargs
if fn_sig.c_variadic {
if !matches!(fn_sig.abi, Abi::C { .. }) {
fx.tcx.sess.span_fatal(
span,
&format!("Variadic call for non-C abi {:?}", fn_sig.abi),
);
fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
}
let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
let abi_params = call_args

7
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -208,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
});
let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
let mut offset = 0;
let mut block_params_iter = block_params.into_iter().copied();
let mut block_params_iter = block_params.iter().copied();
for param in abi_params {
let val = ptr.offset_i64(fx, offset).store(
fx,
@ -248,8 +248,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
/// as necessary.
pub(super) fn cvalue_for_param<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
local: Option<mir::Local>,
local_field: Option<usize>,
arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
block_params_iter: &mut impl Iterator<Item = Value>,
) -> Option<CValue<'tcx>> {
@ -263,7 +263,6 @@ pub(super) fn cvalue_for_param<'tcx>(
})
.collect::<SmallVec<[_; 2]>>();
#[cfg(debug_assertions)]
crate::abi::comments::add_arg_comment(
fx,
"arg",

6
compiler/rustc_codegen_cranelift/src/abi/returning.rs
View file

@ -84,10 +84,6 @@ pub(super) fn codegen_return_param<'tcx>(
}
};
#[cfg(not(debug_assertions))]
let _ = ret_param;
#[cfg(debug_assertions)]
crate::abi::comments::add_arg_comment(
fx,
"ret",
@ -146,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, T>(
let results = fx
.bcx
.inst_results(call_inst)
.into_iter()
.iter()
.copied()
.collect::<SmallVec<[Value; 2]>>();
let result =

5
compiler/rustc_codegen_cranelift/src/allocator.rs
View file

@ -3,6 +3,7 @@
use crate::prelude::*;
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
use rustc_span::symbol::sym;
@ -92,7 +93,7 @@ fn codegen_inner(
bcx.finalize();
}
module
.define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
.define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
.unwrap();
unwind_context.add_function(func_id, &ctx, module.isa());
}
@ -132,7 +133,7 @@ fn codegen_inner(
bcx.finalize();
}
module
.define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
.define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
.unwrap();
unwind_context.add_function(func_id, &ctx, module.isa());
}

63
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -1,5 +1,6 @@
//! Codegen of a single function
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_index::vec::IndexVec;
use rustc_middle::ty::adjustment::PointerCast;
use rustc_middle::ty::layout::FnAbiExt;
@ -7,11 +8,7 @@ use rustc_target::abi::call::FnAbi;
use crate::prelude::*;
pub(crate) fn codegen_fn<'tcx>(
cx: &mut crate::CodegenCx<'_, 'tcx>,
instance: Instance<'tcx>,
linkage: Linkage,
) {
pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
let tcx = cx.tcx;
let _inst_guard =
@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>(
// Declare function
let name = tcx.symbol_name(instance).name.to_string();
let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
cx.cached_context.clear();
@ -131,7 +128,7 @@ pub(crate) fn codegen_fn<'tcx>(
let module = &mut cx.module;
tcx.sess.time("define function", || {
module
.define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
.define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
.unwrap()
});
@ -219,8 +216,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
codegen_stmt(fx, block, stmt);
}
#[cfg(debug_assertions)]
{
if fx.clif_comments.enabled() {
let mut terminator_head = "\n".to_string();
bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
let inst = fx.bcx.func.layout.last_inst(block).unwrap();
@ -433,12 +429,14 @@ fn codegen_stmt<'tcx>(
fx.set_debug_loc(stmt.source_info);
#[cfg(false_debug_assertions)]
#[cfg(disabled)]
match &stmt.kind {
StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
_ => {
let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
fx.add_comment(inst, format!("{:?}", stmt));
if fx.clif_comments.enabled() {
let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
fx.add_comment(inst, format!("{:?}", stmt));
}
}
}
@ -464,16 +462,16 @@ fn codegen_stmt<'tcx>(
let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
lval.write_cvalue(fx, val);
}
Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => {
let lhs = codegen_operand(fx, lhs);
let rhs = codegen_operand(fx, rhs);
Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
let lhs = codegen_operand(fx, &lhs_rhs.0);
let rhs = codegen_operand(fx, &lhs_rhs.1);
let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
lval.write_cvalue(fx, res);
}
Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => {
let lhs = codegen_operand(fx, lhs);
let rhs = codegen_operand(fx, rhs);
Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
let lhs = codegen_operand(fx, &lhs_rhs.0);
let rhs = codegen_operand(fx, &lhs_rhs.1);
let res = if !fx.tcx.sess.overflow_checks() {
let val =
@ -659,7 +657,9 @@ fn codegen_stmt<'tcx>(
.val
.try_to_bits(fx.tcx.data_layout.pointer_size)
.unwrap();
if fx.clif_type(operand.layout().ty) == Some(types::I8) {
if operand.layout().size.bytes() == 0 {
// Do nothing for ZST's
} else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
// FIXME use emit_small_memset where possible
let addr = lval.to_ptr().get_addr(fx);
@ -832,25 +832,18 @@ fn codegen_stmt<'tcx>(
}
}
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
src,
dst,
count,
}) => {
let dst = codegen_operand(fx, dst);
StatementKind::CopyNonOverlapping(inner) => {
let dst = codegen_operand(fx, &inner.dst);
let pointee = dst
.layout()
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
.expect("Expected pointer");
.layout()
.pointee_info_at(fx, rustc_target::abi::Size::ZERO)
.expect("Expected pointer");
let dst = dst.load_scalar(fx);
let src = codegen_operand(fx, src).load_scalar(fx);
let count = codegen_operand(fx, count).load_scalar(fx);
let src = codegen_operand(fx, &inner.src).load_scalar(fx);
let count = codegen_operand(fx, &inner.count).load_scalar(fx);
let elem_size: u64 = pointee.size.bytes();
let bytes = if elem_size != 1 {
fx.bcx.ins().imul_imm(count, elem_size as i64)
} else {
count
};
let bytes =
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
}
}

89
compiler/rustc_codegen_cranelift/src/codegen_i128.rs
View file

@ -32,18 +32,56 @@ pub(crate) fn maybe_codegen<'tcx>(
BinOp::Add | BinOp::Sub if !checked => None,
BinOp::Mul if !checked => {
let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
if fx.tcx.sess.target.is_like_windows {
let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
assert!(lhs_extra.is_none());
assert!(rhs_extra.is_none());
let args =
[ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
fx.lib_call(
"__multi3",
vec![
AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
AbiParam::new(pointer_ty(fx.tcx)),
AbiParam::new(pointer_ty(fx.tcx)),
],
vec![],
&args,
);
Some(ret_place.to_cvalue(fx))
} else {
Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
}
}
BinOp::Add | BinOp::Sub | BinOp::Mul => {
assert!(checked);
let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
let param_types = vec![
AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
AbiParam::new(types::I128),
AbiParam::new(types::I128),
];
let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
assert!(lhs_extra.is_none());
assert!(rhs_extra.is_none());
(
vec![
AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
AbiParam::new(pointer_ty(fx.tcx)),
AbiParam::new(pointer_ty(fx.tcx)),
],
[out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
)
} else {
(
vec![
AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
AbiParam::new(types::I128),
AbiParam::new(types::I128),
],
[out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
)
};
let name = match (bin_op, is_signed) {
(BinOp::Add, false) => "__rust_u128_addo",
(BinOp::Add, true) => "__rust_i128_addo",
@ -57,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>(
Some(out_place.to_cvalue(fx))
}
BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
BinOp::Div => {
BinOp::Div | BinOp::Rem => {
assert!(!checked);
if is_signed {
Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128))
let name = match (bin_op, is_signed) {
(BinOp::Div, false) => "__udivti3",
(BinOp::Div, true) => "__divti3",
(BinOp::Rem, false) => "__umodti3",
(BinOp::Rem, true) => "__modti3",
_ => unreachable!(),
};
if fx.tcx.sess.target.is_like_windows {
let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
assert!(lhs_extra.is_none());
assert!(rhs_extra.is_none());
let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
let ret = fx.lib_call(
name,
vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))],
vec![AbiParam::new(types::I64X2)],
&args,
)[0];
// FIXME use bitcast instead of store to get from i64x2 to i128
let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
Some(ret_place.to_cvalue(fx))
} else {
Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128))
}
}
BinOp::Rem => {
assert!(!checked);
if is_signed {
Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128))
} else {
Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128))
Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
}
}
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {

3
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -361,8 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
let _ = self.cx.module.define_data(msg_id, &data_ctx);
let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
#[cfg(debug_assertions)]
{
if self.clif_comments.enabled() {
self.add_comment(local_msg_id, msg);
}
self.bcx.ins().global_value(self.pointer_type, local_msg_id)

41
compiler/rustc_codegen_cranelift/src/compiler_builtins.rs Normal file
View file

@ -0,0 +1,41 @@
macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
#[cfg(feature = "jit")]
#[allow(improper_ctypes)]
extern "C" {
$(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
}
#[cfg(feature = "jit")]
pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] {
builder.symbol(name, val);
}
}
}
builtin_functions! {
register_functions_for_jit;
// integers
fn __multi3(a: i128, b: i128) -> i128;
fn __udivti3(n: u128, d: u128) -> u128;
fn __divti3(n: i128, d: i128) -> i128;
fn __umodti3(n: u128, d: u128) -> u128;
fn __modti3(n: i128, d: i128) -> i128;
fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
// floats
fn __floattisf(i: i128) -> f32;
fn __floattidf(i: i128) -> f64;
fn __floatuntisf(i: u128) -> f32;
fn __floatuntidf(i: u128) -> f64;
fn __fixsfti(f: f32) -> i128;
fn __fixdfti(f: f64) -> i128;
fn __fixunssfti(f: f32) -> u128;
fn __fixunsdfti(f: f64) -> u128;
}

29
compiler/rustc_codegen_cranelift/src/constant.rs
View file

@ -85,8 +85,9 @@ pub(crate) fn codegen_tls_ref<'tcx>(
) -> CValue<'tcx> {
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("tls {:?}", def_id));
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("tls {:?}", def_id));
}
let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
CValue::by_val(tls_ptr, layout)
}
@ -98,8 +99,9 @@ fn codegen_static_ref<'tcx>(
) -> CPlace<'tcx> {
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("{:?}", def_id));
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", def_id));
}
let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
assert!(!layout.is_unsized(), "unsized statics aren't supported");
assert!(
@ -122,7 +124,9 @@ pub(crate) fn codegen_constant<'tcx>(
};
let const_val = match const_.val {
ConstKind::Value(const_val) => const_val,
ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) if fx.tcx.is_static(def.did) => {
ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
if fx.tcx.is_static(def.did) =>
{
assert!(substs.is_empty());
assert!(promoted.is_none());
@ -183,8 +187,9 @@ pub(crate) fn codegen_const_value<'tcx>(
data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
let local_data_id =
fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
}
fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
}
Some(GlobalAlloc::Function(instance)) => {
@ -199,8 +204,9 @@ pub(crate) fn codegen_const_value<'tcx>(
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id =
fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("{:?}", def_id));
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", def_id));
}
fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
}
None => bug!("missing allocation {:?}", ptr.alloc_id),
@ -241,8 +247,9 @@ fn pointer_for_allocation<'tcx>(
let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(local_data_id, format!("{:?}", alloc_id));
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", alloc_id));
}
let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
crate::pointer::Pointer::new(global_ptr)
}

4
compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
View file

@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
return path.as_bytes();
path.as_bytes()
}
#[cfg(not(unix))]
{
return path.to_str().unwrap().as_bytes();
path.to_str().unwrap().as_bytes()
}
}

22
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -119,11 +119,10 @@ fn module_codegen(
tcx.sess.opts.debuginfo != DebugInfo::None,
);
super::predefine_mono_items(&mut cx, &mono_items);
for (mono_item, (linkage, visibility)) in mono_items {
let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@ -163,6 +162,21 @@ pub(super) fn run_aot(
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
use rustc_span::symbol::sym;
let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
let windows_subsystem = subsystem.map(|subsystem| {
if subsystem != sym::windows && subsystem != sym::console {
tcx.sess.fatal(&format!(
"invalid windows subsystem `{}`, only \
`windows` and `console` are allowed",
subsystem
));
}
subsystem.to_string()
});
let mut work_products = FxHashMap::default();
let cgus = if tcx.sess.opts.output_types.should_codegen() {
@ -280,7 +294,7 @@ pub(super) fn run_aot(
allocator_module,
metadata_module,
metadata,
windows_subsystem: None, // Windows is not yet supported
windows_subsystem,
linker_info: LinkerInfo::new(tcx),
crate_info: CrateInfo::new(tcx),
},

89
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -5,8 +5,10 @@ use std::cell::RefCell;
use std::ffi::CString;
use std::os::raw::{c_char, c_int};
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::config::EntryFnType;
use cranelift_jit::{JITBuilder, JITModule};
@ -28,20 +30,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
let mut jit_builder =
JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
jit_builder.symbols(imported_symbols);
let mut jit_module = JITModule::new(jit_builder);
assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
let sig = Signature {
params: vec![
AbiParam::new(jit_module.target_config().pointer_type()),
AbiParam::new(jit_module.target_config().pointer_type()),
],
returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
};
let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
let mono_items = cgus
.iter()
@ -55,15 +48,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
super::time(tcx, "codegen mono items", || {
super::predefine_mono_items(&mut cx, &mono_items);
for (mono_item, (linkage, visibility)) in mono_items {
let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
cx.tcx
.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
}
CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
},
@ -86,24 +76,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
tcx.sess.fatal("Inline asm is not supported in JIT mode");
}
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
tcx.sess.abort_if_errors();
jit_module.finalize_definitions();
let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
println!(
"Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
);
let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
unsafe { ::std::mem::transmute(finalized_main) };
let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
.chain(args.split(' '))
@ -118,12 +101,58 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
BACKEND_CONFIG.with(|tls_backend_config| {
assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
});
CURRENT_MODULE
.with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
let ret = f(args.len() as c_int, argv.as_ptr());
let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx);
std::process::exit(ret);
match entry_ty {
EntryFnType::Main => {
// FIXME set program arguments somehow
let main_sig = Signature {
params: vec![],
returns: vec![],
call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
};
let main_func_id = jit_module
.declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
.unwrap();
let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
CURRENT_MODULE.with(|current_module| {
assert!(current_module.borrow_mut().replace(jit_module).is_none())
});
let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
f();
std::process::exit(0);
}
EntryFnType::Start => {
let start_sig = Signature {
params: vec![
AbiParam::new(jit_module.target_config().pointer_type()),
AbiParam::new(jit_module.target_config().pointer_type()),
],
returns: vec![AbiParam::new(
jit_module.target_config().pointer_type(), /*isize*/
)],
call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
};
let start_func_id = jit_module
.declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
.unwrap();
let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
CURRENT_MODULE.with(|current_module| {
assert!(current_module.borrow_mut().replace(jit_module).is_none())
});
let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
unsafe { ::std::mem::transmute(finalized_start) };
let ret = f(args.len() as c_int, argv.as_ptr());
std::process::exit(ret);
}
}
}
#[no_mangle]
@ -144,8 +173,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
jit_module.prepare_for_function_redefine(func_id).unwrap();
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
tcx.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
let (global_asm, _debug_context, unwind_context) = cx.finalize();
assert!(global_asm.is_empty());
@ -220,7 +248,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
imported_symbols
}
pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
let tcx = cx.tcx;
let pointer_type = cx.module.target_config().pointer_type();
@ -267,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'t
.define_function(
func_id,
&mut Context::for_function(trampoline),
&mut cranelift_codegen::binemit::NullTrapSink {},
&mut NullTrapSink {},
&mut NullStackMapSink {},
)
.unwrap();
}

8
compiler/rustc_codegen_cranelift/src/driver/mod.rs
View file

@ -44,13 +44,19 @@ fn predefine_mono_items<'tcx>(
mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
) {
cx.tcx.sess.time("predefine functions", || {
let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
for &(mono_item, (linkage, visibility)) in mono_items {
match mono_item {
MonoItem::Fn(instance) => {
let name = cx.tcx.symbol_name(instance).name.to_string();
let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
let linkage = crate::linkage::get_clif_linkage(
mono_item,
linkage,
visibility,
is_compiler_builtins,
);
cx.module.declare_function(&name, linkage, &sig).unwrap();
}
MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}

14
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
if template.is_empty() {
// Black box
return;
} else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::User(1));
return;
}
let mut slot_size = Size::from_bytes(0);
@ -193,8 +197,9 @@ fn call_inline_asm<'tcx>(
offset: None,
size: u32::try_from(slot_size.bytes()).unwrap(),
});
#[cfg(debug_assertions)]
fx.add_comment(stack_slot, "inline asm scratch slot");
if fx.clif_comments.enabled() {
fx.add_comment(stack_slot, "inline asm scratch slot");
}
let inline_asm_func = fx
.cx
@ -210,8 +215,9 @@ fn call_inline_asm<'tcx>(
)
.unwrap();
let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
#[cfg(debug_assertions)]
fx.add_comment(inline_asm_func, asm_name);
if fx.clif_comments.enabled() {
fx.add_comment(inline_asm_func, asm_name);
}
for (_reg, offset, value) in inputs {
fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());

2
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -88,7 +88,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let idx_bytes = match idx_const {
ConstValue::ByRef { alloc, offset } => {
let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
alloc.get_bytes(fx, ptr, size).unwrap()
}
_ => unreachable!("{:?}", idx_const),

39
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -1,13 +1,4 @@
#![feature(
rustc_private,
decl_macro,
type_alias_impl_trait,
associated_type_bounds,
never_type,
try_blocks,
box_patterns,
hash_drain_filter
)]
#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
#![warn(rust_2018_idioms)]
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
@ -57,6 +48,7 @@ mod base;
mod cast;
mod codegen_i128;
mod common;
mod compiler_builtins;
mod constant;
mod debuginfo;
mod discriminant;
@ -224,8 +216,10 @@ pub struct CraneliftCodegenBackend {
impl CodegenBackend for CraneliftCodegenBackend {
fn init(&self, sess: &Session) {
if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode {
sess.warn("LTO is not supported. You may get a linker error.");
use rustc_session::config::Lto;
match sess.lto() {
Lto::No | Lto::ThinLocal => {}
Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
}
}
@ -240,9 +234,9 @@ impl CodegenBackend for CraneliftCodegenBackend {
vec![]
}
fn codegen_crate<'tcx>(
fn codegen_crate(
&self,
tcx: TyCtxt<'tcx>,
tcx: TyCtxt<'_>,
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<dyn Any> {
@ -252,9 +246,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
.unwrap_or_else(|err| tcx.sess.fatal(&err))
};
let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
res
driver::codegen_crate(tcx, metadata, need_metadata_module, config)
}
fn join_codegen(
@ -300,9 +292,9 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
let mut flags_builder = settings::builder();
flags_builder.enable("is_pic").unwrap();
flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
flags_builder
.set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
.unwrap();
let enable_verifier =
cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
let tls_model = match target_triple.binary_format {
BinaryFormat::Elf => "elf_gd",
@ -314,18 +306,17 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
flags_builder.set("enable_simd", "true").unwrap();
flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
use rustc_session::config::OptLevel;
match sess.opts.optimize {
OptLevel::No => {
flags_builder.set("opt_level", "none").unwrap();
}
OptLevel::Less | OptLevel::Default => {}
OptLevel::Aggressive => {
OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
flags_builder.set("opt_level", "speed_and_size").unwrap();
}
OptLevel::Size | OptLevel::SizeMin => {
sess.warn("Optimizing for size is not supported. Just ignoring the request");
}
}
let flags = settings::Flags::new(flags_builder);

2
compiler/rustc_codegen_cranelift/src/linkage.rs
View file

@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage(
mono_item: MonoItem<'_>,
linkage: RLinkage,
visibility: Visibility,
is_compiler_builtins: bool,
) -> Linkage {
match (linkage, visibility) {
(RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden,
(RLinkage::External, Visibility::Default) => Linkage::Export,
(RLinkage::Internal, Visibility::Default) => Linkage::Local,
(RLinkage::External, Visibility::Hidden) => Linkage::Hidden,

9
compiler/rustc_codegen_cranelift/src/main_shim.rs
View file

@ -1,3 +1,4 @@
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_hir::LangItem;
use rustc_session::config::EntryFnType;
@ -100,12 +101,8 @@ pub(crate) fn maybe_create_entry_wrapper(
bcx.seal_all_blocks();
bcx.finalize();
}
m.define_function(
cmain_func_id,
&mut ctx,
&mut cranelift_codegen::binemit::NullTrapSink {},
)
.unwrap();
m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
.unwrap();
unwind_context.add_function(cmain_func_id, &ctx, m.isa());
}
}

72
compiler/rustc_codegen_cranelift/src/metadata.rs
View file

@ -1,11 +1,11 @@
//! Reading and writing of the rustc metadata for rlibs and dylibs
use std::convert::TryFrom;
use std::fs::File;
use std::ops::Deref;
use std::path::Path;
use rustc_codegen_ssa::METADATA_FILENAME;
use rustc_data_structures::owning_ref::OwningRef;
use rustc_data_structures::owning_ref::{OwningRef, StableAddress};
use rustc_data_structures::rustc_erase_owner;
use rustc_data_structures::sync::MetadataRef;
use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
@ -17,38 +17,56 @@ use crate::backend::WriteMetadata;
pub(crate) struct CraneliftMetadataLoader;
struct StableMmap(memmap2::Mmap);
impl Deref for StableMmap {
type Target = [u8];
fn deref(&self) -> &[u8] {
&*self.0
}
}
unsafe impl StableAddress for StableMmap {}
fn load_metadata_with(
path: &Path,
f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
) -> Result<MetadataRef, String> {
let file = File::open(path).map_err(|e| format!("{:?}", e))?;
let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) }
.map_err(|e| format!("{:?}", e))?;
let metadata = OwningRef::new(StableMmap(data)).try_map(f)?;
return Ok(rustc_erase_owner!(metadata.map_owner_box()));
}
impl MetadataLoader for CraneliftMetadataLoader {
fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
// Iterate over all entries in the archive:
while let Some(entry_result) = archive.next_entry() {
let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
let mut buf = Vec::with_capacity(
usize::try_from(entry.header().size())
.expect("Rlib metadata file too big to load into memory."),
);
::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
return Ok(rustc_erase_owner!(buf.map_owner_box()));
}
}
load_metadata_with(path, |data| {
let archive = object::read::archive::ArchiveFile::parse(&*data)
.map_err(|e| format!("{:?}", e))?;
Err("couldn't find metadata entry".to_string())
for entry_result in archive.members() {
let entry = entry_result.map_err(|e| format!("{:?}", e))?;
if entry.name() == METADATA_FILENAME.as_bytes() {
return Ok(entry.data());
}
}
Err("couldn't find metadata entry".to_string())
})
}
fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
use object::{Object, ObjectSection};
let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
let buf = file
.section_by_name(".rustc")
.ok_or("no .rustc section")?
.data()
.map_err(|e| format!("failed to read .rustc section: {:?}", e))?
.to_owned();
let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
Ok(rustc_erase_owner!(buf.map_owner_box()))
load_metadata_with(path, |data| {
let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
file.section_by_name(".rustc")
.ok_or("no .rustc section")?
.data()
.map_err(|e| format!("failed to read .rustc section: {:?}", e))
})
}
}

8
compiler/rustc_codegen_cranelift/src/num.rs
View file

@ -166,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
BinOp::Shl => {
let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
fx.bcx.ins().ishl(lhs, actual_shift)
}
BinOp::Shr => {
let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
if signed {
fx.bcx.ins().sshr(lhs, actual_shift)
} else {
@ -387,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
let lhs = in_lhs.load_scalar(fx);
let rhs = in_rhs.load_scalar(fx);
return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
}
BinOp::Offset => {
let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
@ -396,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
let base_val = base.load_scalar(fx);
let res = fx.bcx.ins().iadd(base_val, ptr_diff);
return CValue::by_val(res, base.layout());
CValue::by_val(res, base.layout())
}
_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
};
}
} else {
let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);

89
compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
View file

@ -181,7 +181,6 @@ impl<'a> OptimizeContext<'a> {
pub(super) fn optimize_function(
ctx: &mut Context,
#[cfg_attr(not(debug_assertions), allow(unused_variables))]
clif_comments: &mut crate::pretty_clif::CommentWriter,
) {
combine_stack_addr_with_load_store(&mut ctx.func);
@ -192,8 +191,7 @@ pub(super) fn optimize_function(
remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
#[cfg(debug_assertions)]
{
if clif_comments.enabled() {
for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
}
@ -209,25 +207,27 @@ pub(super) fn optimize_function(
for load in users.stack_load.clone().into_iter() {
let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
#[cfg(debug_assertions)]
for &store in &potential_stores {
clif_comments.add_comment(
load,
format!(
"Potential store -> load forwarding {} -> {} ({:?}, {:?})",
opt_ctx.ctx.func.dfg.display_inst(store, None),
opt_ctx.ctx.func.dfg.display_inst(load, None),
spatial_overlap(&opt_ctx.ctx.func, store, load),
temporal_order(&opt_ctx.ctx, store, load),
),
);
if clif_comments.enabled() {
for &store in &potential_stores {
clif_comments.add_comment(
load,
format!(
"Potential store -> load forwarding {} -> {} ({:?}, {:?})",
opt_ctx.ctx.func.dfg.display_inst(store, None),
opt_ctx.ctx.func.dfg.display_inst(load, None),
spatial_overlap(&opt_ctx.ctx.func, store, load),
temporal_order(&opt_ctx.ctx, store, load),
),
);
}
}
match *potential_stores {
[] => {
#[cfg(debug_assertions)]
clif_comments
.add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
if clif_comments.enabled() {
clif_comments
.add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
}
}
[store]
if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
@ -237,9 +237,12 @@ pub(super) fn optimize_function(
// Only one store could have been the origin of the value.
let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
#[cfg(debug_assertions)]
clif_comments
.add_comment(load, format!("Store to load forward {} -> {}", store, load));
if clif_comments.enabled() {
clif_comments.add_comment(
load,
format!("Store to load forward {} -> {}", store, load),
);
}
users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
}
@ -250,33 +253,35 @@ pub(super) fn optimize_function(
for store in users.stack_store.clone().into_iter() {
let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
#[cfg(debug_assertions)]
for &load in &potential_loads {
clif_comments.add_comment(
store,
format!(
"Potential load from store {} <- {} ({:?}, {:?})",
opt_ctx.ctx.func.dfg.display_inst(load, None),
opt_ctx.ctx.func.dfg.display_inst(store, None),
spatial_overlap(&opt_ctx.ctx.func, store, load),
temporal_order(&opt_ctx.ctx, store, load),
),
);
if clif_comments.enabled() {
for &load in &potential_loads {
clif_comments.add_comment(
store,
format!(
"Potential load from store {} <- {} ({:?}, {:?})",
opt_ctx.ctx.func.dfg.display_inst(load, None),
opt_ctx.ctx.func.dfg.display_inst(store, None),
spatial_overlap(&opt_ctx.ctx.func, store, load),
temporal_order(&opt_ctx.ctx, store, load),
),
);
}
}
if potential_loads.is_empty() {
// Never loaded; can safely remove all stores and the stack slot.
// FIXME also remove stores when there is always a next store before a load.
#[cfg(debug_assertions)]
clif_comments.add_comment(
store,
format!(
"Remove dead stack store {} of {}",
opt_ctx.ctx.func.dfg.display_inst(store, None),
stack_slot.0
),
);
if clif_comments.enabled() {
clif_comments.add_comment(
store,
format!(
"Remove dead stack store {} of {}",
opt_ctx.ctx.func.dfg.display_inst(store, None),
stack_slot.0
),
);
}
users.remove_dead_store(&mut opt_ctx.ctx.func, store);
}

3
compiler/rustc_codegen_cranelift/src/pointer.rs
View file

@ -39,8 +39,7 @@ impl Pointer {
Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
}
#[cfg(debug_assertions)]
pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) {
pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) {
(self.base, self.offset)
}

24
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -69,13 +69,15 @@ use crate::prelude::*;
#[derive(Debug)]
pub(crate) struct CommentWriter {
enabled: bool,
global_comments: Vec<String>,
entity_comments: FxHashMap<AnyEntity, String>,
}
impl CommentWriter {
pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
let global_comments = if cfg!(debug_assertions) {
let enabled = should_write_ir(tcx);
let global_comments = if enabled {
vec![
format!("symbol {}", tcx.symbol_name(instance).name),
format!("instance {:?}", instance),
@ -86,13 +88,17 @@ impl CommentWriter {
vec![]
};
CommentWriter { global_comments, entity_comments: FxHashMap::default() }
CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
}
}
#[cfg(debug_assertions)]
impl CommentWriter {
pub(crate) fn enabled(&self) -> bool {
self.enabled
}
pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
debug_assert!(self.enabled);
self.global_comments.push(comment.into());
}
@ -101,6 +107,8 @@ impl CommentWriter {
entity: E,
comment: S,
) {
debug_assert!(self.enabled);
use std::collections::hash_map::Entry;
match self.entity_comments.entry(entity.into()) {
Entry::Occupied(mut occ) => {
@ -179,7 +187,6 @@ impl FuncWriter for &'_ CommentWriter {
}
}
#[cfg(debug_assertions)]
impl FunctionCx<'_, '_, '_> {
pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
self.clif_comments.add_global_comment(comment);
@ -198,8 +205,8 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
}
pub(crate) fn write_ir_file<'tcx>(
tcx: TyCtxt<'tcx>,
pub(crate) fn write_ir_file(
tcx: TyCtxt<'_>,
name: &str,
write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
) {
@ -217,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>(
let clif_file_name = clif_output_dir.join(name);
let res: std::io::Result<()> = try {
let mut file = std::fs::File::create(clif_file_name)?;
write(&mut file)?;
};
let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
if let Err(err) = res {
tcx.sess.warn(&format!("error writing ir file: {}", err));
}

3
compiler/rustc_codegen_cranelift/src/trap.rs
View file

@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
)
.unwrap();
let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
#[cfg(debug_assertions)]
{
if fx.clif_comments.enabled() {
fx.add_comment(puts, "puts");
}

19
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -2,7 +2,6 @@
use crate::prelude::*;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::immediates::Offset32;
fn codegen_field<'tcx>(
@ -414,7 +413,7 @@ impl<'tcx> CPlace<'tcx> {
self,
fx: &mut FunctionCx<'_, '_, 'tcx>,
from: CValue<'tcx>,
#[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
method: &'static str,
) {
fn transmute_value<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
@ -462,8 +461,7 @@ impl<'tcx> CPlace<'tcx> {
assert_eq!(self.layout().size, from.layout().size);
#[cfg(debug_assertions)]
{
if fx.clif_comments.enabled() {
use cranelift_codegen::cursor::{Cursor, CursorPosition};
let cur_block = match fx.bcx.cursor().position() {
CursorPosition::After(block) => block,
@ -707,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>(
}
// dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
}
(&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
if adt_def_a.did == adt_def_b.did =>
{
let mut types_a = substs_a.types();
let mut types_b = substs_b.types();
loop {
match (types_a.next(), types_b.next()) {
(Some(a), Some(b)) => assert_assignable(fx, a, b),
(None, None) => return,
(Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
}
}
}
_ => {
assert_eq!(
from_ty, to_ty,