bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Merge commit '266e96785a' into sync_cg_clif-2022-10-23

Browse source
This commit is contained in:
bjorn3 2022-10-23 16:22:55 +02:00
commit e54a13f18b
45 changed files with 1041 additions and 634 deletions
Download patch file Download diff file Expand all files Collapse all files

53
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -465,7 +465,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
(CallTarget::Indirect(sig, method), Some(ptr))
(CallTarget::Indirect(sig, method), Some(ptr.get_addr(fx)))
}
// Normal call
@ -560,7 +560,19 @@ pub(crate) fn codegen_drop<'tcx>(
// we don't actually need to drop anything
} else {
match ty.kind() {
ty::Dynamic(..) => {
ty::Dynamic(_, _, ty::Dyn) => {
// IN THIS ARM, WE HAVE:
// ty = *mut (dyn Trait)
// which is: exists<T> ( *mut T, Vtable<T: Trait> )
// args[0] args[1]
//
// args = ( Data, Vtable )
// |
// v
// /-------\
// | ... |
// \-------/
//
let (ptr, vtable) = drop_place.to_ptr_maybe_unsized();
let ptr = ptr.get_addr(fx);
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap());
@ -578,6 +590,43 @@ pub(crate) fn codegen_drop<'tcx>(
let sig = fx.bcx.import_signature(sig);
fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
}
ty::Dynamic(_, _, ty::DynStar) => {
// IN THIS ARM, WE HAVE:
// ty = *mut (dyn* Trait)
// which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
//
// args = [ * ]
// |
// v
// ( Data, Vtable )
// |
// v
// /-------\
// | ... |
// \-------/
//
//
// WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
//
// data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
// vtable = (*args[0]).1 // loads the vtable out
// (data, vtable) // an equivalent Rust `*mut dyn Trait`
//
// SO THEN WE CAN USE THE ABOVE CODE.
let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx);
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
let virtual_drop = Instance {
def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
substs: drop_instance.substs,
};
let fn_abi =
RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty());
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
fx.bcx.ins().call_indirect(sig, drop_fn, &[data]);
}
_ => {
assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _)));

4
compiler/rustc_codegen_cranelift/src/allocator.rs
View file

@ -78,7 +78,7 @@ fn codegen_inner(
let callee_func_id = module.declare_function(&callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig.clone());
ctx.func.signature = sig.clone();
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
@ -116,7 +116,7 @@ fn codegen_inner(
let callee_func_id = module.declare_function(callee_name, Linkage::Import, &sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), sig);
ctx.func.signature = sig;
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);

2
compiler/rustc_codegen_cranelift/src/archive.rs
View file

@ -159,6 +159,8 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
let err = err.to_string();
if err == "Unknown file magic" {
// Not an object file; skip it.
} else if object::read::archive::ArchiveFile::parse(&*data).is_ok() {
// Nested archive file; skip it.
} else {
sess.fatal(&format!(
"error parsing `{}` during archive creation: {}",

12
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -6,6 +6,8 @@ use rustc_middle::ty::adjustment::PointerCast;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
use cranelift_codegen::ir::UserFuncName;
use crate::constant::ConstantCx;
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
@ -64,7 +66,7 @@ pub(crate) fn codegen_fn<'tcx>(
let mut func_ctx = FunctionBuilderContext::new();
let mut func = cached_func;
func.clear();
func.name = ExternalName::user(0, func_id.as_u32());
func.name = UserFuncName::user(0, func_id.as_u32());
func.signature = sig;
func.collect_debug_info();
@ -706,9 +708,9 @@ fn codegen_stmt<'tcx>(
let operand = codegen_operand(fx, operand);
operand.unsize_value(fx, lval);
}
Rvalue::Cast(CastKind::DynStar, _, _) => {
// FIXME(dyn-star)
unimplemented!()
Rvalue::Cast(CastKind::DynStar, ref operand, _) => {
let operand = codegen_operand(fx, operand);
operand.coerce_dyn_star(fx, lval);
}
Rvalue::Discriminant(place) => {
let place = codegen_place(fx, place);
@ -922,7 +924,7 @@ pub(crate) fn codegen_operand<'tcx>(
let cplace = codegen_place(fx, *place);
cplace.to_cvalue(fx)
}
Operand::Constant(const_) => crate::constant::codegen_constant(fx, const_),
Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_),
}
}

17
compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
View file

@ -10,6 +10,7 @@ pub(super) struct ConcurrencyLimiter {
helper_thread: Option<HelperThread>,
state: Arc<Mutex<state::ConcurrencyLimiterState>>,
available_token_condvar: Arc<Condvar>,
finished: bool,
}
impl ConcurrencyLimiter {
@ -32,6 +33,7 @@ impl ConcurrencyLimiter {
helper_thread: Some(helper_thread),
state,
available_token_condvar: Arc::new(Condvar::new()),
finished: false,
}
}
@ -56,16 +58,23 @@ impl ConcurrencyLimiter {
let mut state = self.state.lock().unwrap();
state.job_already_done();
}
}
impl Drop for ConcurrencyLimiter {
fn drop(&mut self) {
//
pub(crate) fn finished(mut self) {
self.helper_thread.take();
// Assert that all jobs have finished
let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap();
state.assert_done();
self.finished = true;
}
}
impl Drop for ConcurrencyLimiter {
fn drop(&mut self) {
if !self.finished && !std::thread::panicking() {
panic!("Forgot to call finished() on ConcurrencyLimiter");
}
}
}

87
compiler/rustc_codegen_cranelift/src/constant.rs
View file

@ -7,7 +7,6 @@ use rustc_middle::mir::interpret::{
};
use rustc_span::DUMMY_SP;
use cranelift_codegen::ir::GlobalValueData;
use cranelift_module::*;
use crate::prelude::*;
@ -81,52 +80,45 @@ pub(crate) fn codegen_tls_ref<'tcx>(
CValue::by_val(tls_ptr, layout)
}
fn codegen_static_ref<'tcx>(
pub(crate) fn eval_mir_constant<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
def_id: DefId,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", def_id));
}
let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
assert!(!layout.is_unsized(), "unsized statics aren't supported");
assert!(
matches!(
fx.bcx.func.global_values[local_data_id],
GlobalValueData::Symbol { tls: false, .. }
),
"tls static referenced without Rvalue::ThreadLocalRef"
);
CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout)
constant: &Constant<'tcx>,
) -> (ConstValue<'tcx>, Ty<'tcx>) {
let constant_kind = fx.monomorphize(constant.literal);
let uv = match constant_kind {
ConstantKind::Ty(const_) => match const_.kind() {
ty::ConstKind::Unevaluated(uv) => uv.expand(),
ty::ConstKind::Value(val) => {
return (fx.tcx.valtree_to_const_val((const_.ty(), val)), const_.ty());
}
err => span_bug!(
constant.span,
"encountered bad ConstKind after monomorphizing: {:?}",
err
),
},
ConstantKind::Unevaluated(mir::UnevaluatedConst { def, .. }, _)
if fx.tcx.is_static(def.did) =>
{
span_bug!(constant.span, "MIR constant refers to static");
}
ConstantKind::Unevaluated(uv, _) => uv,
ConstantKind::Val(val, _) => return (val, constant_kind.ty()),
};
(
fx.tcx.const_eval_resolve(ty::ParamEnv::reveal_all(), uv, None).unwrap_or_else(|_err| {
span_bug!(constant.span, "erroneous constant not captured by required_consts");
}),
constant_kind.ty(),
)
}
pub(crate) fn codegen_constant<'tcx>(
pub(crate) fn codegen_constant_operand<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
constant: &Constant<'tcx>,
) -> CValue<'tcx> {
let (const_val, ty) = match fx.monomorphize(constant.literal) {
ConstantKind::Ty(const_) => unreachable!("{:?}", const_),
ConstantKind::Unevaluated(mir::UnevaluatedConst { def, substs, promoted }, ty)
if fx.tcx.is_static(def.did) =>
{
assert!(substs.is_empty());
assert!(promoted.is_none());
return codegen_static_ref(fx, def.did, fx.layout_of(ty)).to_cvalue(fx);
}
ConstantKind::Unevaluated(unevaluated, ty) => {
match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
Ok(const_val) => (const_val, ty),
Err(_) => {
span_bug!(constant.span, "erroneous constant not captured by required_consts");
}
}
}
ConstantKind::Val(val, ty) => (val, ty),
};
let (const_val, ty) = eval_mir_constant(fx, constant);
codegen_const_value(fx, const_val, ty)
}
@ -244,7 +236,7 @@ pub(crate) fn codegen_const_value<'tcx>(
}
}
pub(crate) fn pointer_for_allocation<'tcx>(
fn pointer_for_allocation<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
alloc: ConstAllocation<'tcx>,
) -> crate::pointer::Pointer {
@ -467,14 +459,13 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
operand: &Operand<'tcx>,
) -> Option<ConstValue<'tcx>> {
match operand {
Operand::Constant(const_) => match const_.literal {
ConstantKind::Ty(const_) => fx
.monomorphize(const_)
.eval_for_mir(fx.tcx, ParamEnv::reveal_all())
.try_to_value(fx.tcx),
Operand::Constant(const_) => match fx.monomorphize(const_.literal) {
ConstantKind::Ty(const_) => Some(
const_.eval_for_mir(fx.tcx, ParamEnv::reveal_all()).try_to_value(fx.tcx).unwrap(),
),
ConstantKind::Val(val, _) => Some(val),
ConstantKind::Unevaluated(uv, _) => {
fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).ok()
Some(fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).unwrap())
}
},
// FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored

2
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -106,7 +106,7 @@ impl OngoingCodegen {
}
}
drop(self.concurrency_limiter);
self.concurrency_limiter.finished();
(
CodegenResults {

51
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -67,13 +67,12 @@ fn create_jit_module(
hotswap: bool,
) -> (JITModule, CodegenCx) {
let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info);
let isa = crate::build_isa(tcx.sess, backend_config);
let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
jit_builder.hotswap(hotswap);
crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
jit_builder.symbols(imported_symbols);
jit_builder.symbol_lookup_fn(dep_symbol_lookup_fn(tcx.sess, crate_info));
jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8);
let mut jit_module = JITModule::new(jit_builder);
@ -286,10 +285,10 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
})
}
fn load_imported_symbols_for_jit(
fn dep_symbol_lookup_fn(
sess: &Session,
crate_info: CrateInfo,
) -> Vec<(String, *const u8)> {
) -> Box<dyn Fn(&str) -> Option<*const u8>> {
use rustc_middle::middle::dependency_format::Linkage;
let mut dylib_paths = Vec::new();
@ -316,39 +315,23 @@ fn load_imported_symbols_for_jit(
}
}
let mut imported_symbols = Vec::new();
for path in dylib_paths {
use object::{Object, ObjectSymbol};
let lib = libloading::Library::new(&path).unwrap();
let obj = std::fs::read(path).unwrap();
let obj = object::File::parse(&*obj).unwrap();
imported_symbols.extend(obj.dynamic_symbols().filter_map(|symbol| {
let name = symbol.name().unwrap().to_string();
if name.is_empty() || !symbol.is_global() || symbol.is_undefined() {
return None;
}
if name.starts_with("rust_metadata_") {
// The metadata is part of a section that is not loaded by the dynamic linker in
// case of cg_llvm.
return None;
}
let dlsym_name = if cfg!(target_os = "macos") {
// On macOS `dlsym` expects the name without leading `_`.
assert!(name.starts_with('_'), "{:?}", name);
&name[1..]
} else {
&name
};
let symbol: libloading::Symbol<'_, *const u8> =
unsafe { lib.get(dlsym_name.as_bytes()) }.unwrap();
Some((name, *symbol))
}));
std::mem::forget(lib)
}
let imported_dylibs = Box::leak(
dylib_paths
.into_iter()
.map(|path| unsafe { libloading::Library::new(&path).unwrap() })
.collect::<Box<[_]>>(),
);
sess.abort_if_errors();
imported_symbols
Box::new(move |sym_name| {
for dylib in &*imported_dylibs {
if let Ok(sym) = unsafe { dylib.get::<*const u8>(sym_name.as_bytes()) } {
return Some(*sym);
}
}
None
})
}
fn codegen_shim<'tcx>(

22
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -27,7 +27,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
}
// Used by stdarch
if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
if template[0] == InlineAsmTemplatePiece::String("mov ".to_string())
&& matches!(
template[1],
InlineAsmTemplatePiece::Placeholder {
@ -36,24 +36,26 @@ pub(crate) fn codegen_inline_asm<'tcx>(
span: _
}
)
&& template[2] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[3] == InlineAsmTemplatePiece::String("cpuid".to_string())
&& template[4] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[5] == InlineAsmTemplatePiece::String("xchgq %rbx, ".to_string())
&& template[2] == InlineAsmTemplatePiece::String(", rbx".to_string())
&& template[3] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[4] == InlineAsmTemplatePiece::String("cpuid".to_string())
&& template[5] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[6] == InlineAsmTemplatePiece::String("xchg ".to_string())
&& matches!(
template[6],
template[7],
InlineAsmTemplatePiece::Placeholder {
operand_idx: 0,
modifier: Some('r'),
span: _
}
)
&& template[8] == InlineAsmTemplatePiece::String(", rbx".to_string())
{
assert_eq!(operands.len(), 4);
let (leaf, eax_place) = match operands[1] {
InlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
late: true,
late: _,
ref in_value,
out_place: Some(out_place),
} => (
@ -68,7 +70,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
X86InlineAsmRegClass::reg,
)),
late: true,
late: _,
place: Some(place),
} => crate::base::codegen_place(fx, place),
_ => unreachable!(),
@ -76,7 +78,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
let (sub_leaf, ecx_place) = match operands[2] {
InlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
late: true,
late: _,
ref in_value,
out_place: Some(out_place),
} => (
@ -88,7 +90,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
let edx_place = match operands[3] {
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
late: true,
late: _,
place: Some(place),
} => crate::base::codegen_place(fx, place),
_ => unreachable!(),

7
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View file

@ -14,6 +14,10 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
target: Option<BasicBlock>,
) {
match intrinsic {
"llvm.x86.sse2.pause" | "llvm.aarch64.isb" => {
// Spin loop hint
}
// Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
"llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd" => {
intrinsic_args!(fx, args => (a); intrinsic);
@ -25,8 +29,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
let mut res = fx.bcx.ins().iconst(types::I32, 0);
for lane in (0..lane_count).rev() {
let a_lane =
a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
let a_lane = a.value_lane(fx, lane).load_scalar(fx);
// cast float to int
let a_lane = match lane_ty {

62
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -84,6 +84,30 @@ fn simd_for_each_lane<'tcx>(
}
}
fn simd_pair_for_each_lane_typed<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
x: CValue<'tcx>,
y: CValue<'tcx>,
ret: CPlace<'tcx>,
f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, CValue<'tcx>, CValue<'tcx>) -> CValue<'tcx>,
) {
assert_eq!(x.layout(), y.layout());
let layout = x.layout();
let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
let (ret_lane_count, _ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
assert_eq!(lane_count, ret_lane_count);
for lane_idx in 0..lane_count {
let x_lane = x.value_lane(fx, lane_idx);
let y_lane = y.value_lane(fx, lane_idx);
let res_lane = f(fx, x_lane, y_lane);
ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
}
}
fn simd_pair_for_each_lane<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
x: CValue<'tcx>,
@ -504,37 +528,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
_ => unreachable!(),
};
let signed = type_sign(lhs.layout().ty);
let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs);
let (val, has_overflow) = checked_res.load_scalar_pair(fx);
let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
let val = match (intrinsic, signed) {
(sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val),
(sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
(sym::saturating_add, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero =
fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
(sym::saturating_sub, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero =
fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
_ => unreachable!(),
};
let res = CValue::by_val(val, lhs.layout());
let res = crate::num::codegen_saturating_int_binop(fx, bin_op, lhs, rhs);
ret.write_cvalue(fx, res);
}
sym::rotate_left => {
@ -819,8 +813,8 @@ fn codegen_regular_intrinsic_call<'tcx>(
sym::ptr_guaranteed_cmp => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
ret.write_cvalue(fx, val);
let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b).load_scalar(fx);
ret.write_cvalue(fx, CValue::by_val(val, fx.layout_of(fx.tcx.types.u8)));
}
sym::caller_location => {
@ -1206,7 +1200,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
// FIXME once unwinding is supported, change this to actually catch panics
let f_sig = fx.bcx.func.import_signature(Signature {
call_conv: fx.target_config.default_call_conv,
params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
params: vec![AbiParam::new(pointer_ty(fx.tcx))],
returns: vec![],
});

128
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -2,6 +2,7 @@
use rustc_middle::ty::subst::SubstsRef;
use rustc_span::Symbol;
use rustc_target::abi::Endian;
use super::*;
use crate::prelude::*;
@ -26,7 +27,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
span: Span,
) {
match intrinsic {
sym::simd_cast => {
sym::simd_as | sym::simd_cast => {
intrinsic_args!(fx, args => (a); intrinsic);
if !a.layout().ty.is_simd() {
@ -162,6 +163,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
}
}
} else {
// FIXME remove this case
intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
};
@ -650,8 +652,128 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
}
}
// simd_saturating_*
// simd_bitmask
sym::simd_select_bitmask => {
intrinsic_args!(fx, args => (m, a, b); intrinsic);
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
assert_eq!(a.layout(), b.layout());
let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
let lane_layout = fx.layout_of(lane_ty);
let m = m.load_scalar(fx);
for lane in 0..lane_count {
let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64);
let m_lane = fx.bcx.ins().band_imm(m_lane, 1);
let a_lane = a.value_lane(fx, lane).load_scalar(fx);
let b_lane = b.value_lane(fx, lane).load_scalar(fx);
let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
let res_lane =
CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
}
}
sym::simd_bitmask => {
intrinsic_args!(fx, args => (a); intrinsic);
let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
let lane_clif_ty = fx.clif_type(lane_ty).unwrap();
// The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
// vector mask and returns the most significant bit (MSB) of each lane in the form
// of either:
// * an unsigned integer
// * an array of `u8`
// If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
//
// The bit order of the result depends on the byte endianness, LSB-first for little
// endian and MSB-first for big endian.
let expected_int_bits = lane_count.max(8);
let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64);
match lane_ty.kind() {
ty::Int(_) | ty::Uint(_) => {}
_ => {
fx.tcx.sess.span_fatal(
span,
&format!(
"invalid monomorphization of `simd_bitmask` intrinsic: \
vector argument `{}`'s element type `{}`, expected integer element \
type",
a.layout().ty,
lane_ty
),
);
}
}
let res_type =
Type::int_with_byte_size(u16::try_from(expected_bytes).unwrap()).unwrap();
let mut res = fx.bcx.ins().iconst(res_type, 0);
let lanes = match fx.tcx.sess.target.endian {
Endian::Big => Box::new(0..lane_count) as Box<dyn Iterator<Item = u64>>,
Endian::Little => Box::new((0..lane_count).rev()) as Box<dyn Iterator<Item = u64>>,
};
for lane in lanes {
let a_lane = a.value_lane(fx, lane).load_scalar(fx);
// extract sign bit of an int
let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1));
// shift sign bit into result
let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false);
res = fx.bcx.ins().ishl_imm(res, 1);
res = fx.bcx.ins().bor(res, a_lane_sign);
}
match ret.layout().ty.kind() {
ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {}
ty::Array(elem, len)
if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
&& len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
== Some(expected_bytes) => {}
_ => {
fx.tcx.sess.span_fatal(
span,
&format!(
"invalid monomorphization of `simd_bitmask` intrinsic: \
cannot return `{}`, expected `u{}` or `[u8; {}]`",
ret.layout().ty,
expected_int_bits,
expected_bytes
),
);
}
}
let res = CValue::by_val(res, ret.layout());
ret.write_cvalue(fx, res);
}
sym::simd_saturating_add | sym::simd_saturating_sub => {
intrinsic_args!(fx, args => (x, y); intrinsic);
let bin_op = match intrinsic {
sym::simd_saturating_add => BinOp::Add,
sym::simd_saturating_sub => BinOp::Sub,
_ => unreachable!(),
};
// FIXME use vector instructions when possible
simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| {
crate::num::codegen_saturating_int_binop(fx, bin_op, x_lane, y_lane)
});
}
// simd_arith_offset
// simd_scatter
// simd_gather
_ => {

14
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -96,8 +96,8 @@ mod prelude {
pub(crate) use cranelift_codegen::ir::function::Function;
pub(crate) use cranelift_codegen::ir::types;
pub(crate) use cranelift_codegen::ir::{
AbiParam, Block, ExternalName, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc,
StackSlot, StackSlotData, StackSlotKind, TrapCode, Type, Value,
AbiParam, Block, FuncRef, Inst, InstBuilder, MemFlags, Signature, SourceLoc, StackSlot,
StackSlotData, StackSlotKind, TrapCode, Type, Value,
};
pub(crate) use cranelift_codegen::isa::{self, CallConv};
pub(crate) use cranelift_codegen::Context;
@ -251,7 +251,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
let mut flags_builder = settings::builder();
flags_builder.enable("is_pic").unwrap();
flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
flags_builder.set("enable_verifier", enable_verifier).unwrap();
flags_builder.set("regalloc_checker", enable_verifier).unwrap();
@ -279,6 +278,15 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
}
}
if target_triple.architecture == target_lexicon::Architecture::X86_64 {
// Windows depends on stack probes to grow the committed part of the stack
flags_builder.enable("enable_probestack").unwrap();
flags_builder.set("probestack_strategy", "inline").unwrap();
} else {
// __cranelift_probestack is not provided and inline stack probes are only supported on x86_64
flags_builder.set("enable_probestack", "false").unwrap();
}
let flags = settings::Flags::new(flags_builder);
let isa_builder = match sess.opts.cg.target_cpu.as_deref() {

2
compiler/rustc_codegen_cranelift/src/main_shim.rs
View file

@ -75,7 +75,7 @@ pub(crate) fn maybe_create_entry_wrapper(
let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig);
ctx.func.signature = cmain_sig;
{
let mut func_ctx = FunctionBuilderContext::new();
let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);

61
compiler/rustc_codegen_cranelift/src/num.rs
View file

@ -150,18 +150,12 @@ pub(crate) fn codegen_int_binop<'tcx>(
BinOp::BitXor => b.bxor(lhs, rhs),
BinOp::BitAnd => b.band(lhs, rhs),
BinOp::BitOr => b.bor(lhs, rhs),
BinOp::Shl => {
let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
fx.bcx.ins().ishl(lhs, actual_shift)
}
BinOp::Shl => b.ishl(lhs, rhs),
BinOp::Shr => {
let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
if signed {
fx.bcx.ins().sshr(lhs, actual_shift)
b.sshr(lhs, rhs)
} else {
fx.bcx.ins().ushr(lhs, actual_shift)
b.ushr(lhs, rhs)
}
}
// Compare binops handles by `codegen_binop`.
@ -279,22 +273,15 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
}
}
BinOp::Shl => {
let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
let masked_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
let val = fx.bcx.ins().ishl(lhs, masked_shift);
let val = fx.bcx.ins().ishl(lhs, rhs);
let ty = fx.bcx.func.dfg.value_type(val);
let max_shift = i64::from(ty.bits()) - 1;
let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
(val, has_overflow)
}
BinOp::Shr => {
let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
let masked_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
let val = if !signed {
fx.bcx.ins().ushr(lhs, masked_shift)
} else {
fx.bcx.ins().sshr(lhs, masked_shift)
};
let val =
if !signed { fx.bcx.ins().ushr(lhs, rhs) } else { fx.bcx.ins().sshr(lhs, rhs) };
let ty = fx.bcx.func.dfg.value_type(val);
let max_shift = i64::from(ty.bits()) - 1;
let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
@ -309,6 +296,42 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
CValue::by_val_pair(res, has_overflow, out_layout)
}
pub(crate) fn codegen_saturating_int_binop<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
lhs: CValue<'tcx>,
rhs: CValue<'tcx>,
) -> CValue<'tcx> {
assert_eq!(lhs.layout().ty, rhs.layout().ty);
let signed = type_sign(lhs.layout().ty);
let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs);
let (val, has_overflow) = checked_res.load_scalar_pair(fx);
let val = match (bin_op, signed) {
(BinOp::Add, false) => fx.bcx.ins().select(has_overflow, max, val),
(BinOp::Sub, false) => fx.bcx.ins().select(has_overflow, min, val),
(BinOp::Add, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
(BinOp::Sub, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
_ => unreachable!(),
};
CValue::by_val(val, lhs.layout())
}
pub(crate) fn codegen_float_binop<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,

40
compiler/rustc_codegen_cranelift/src/unsize.rs
View file

@ -25,7 +25,12 @@ pub(crate) fn unsized_info<'tcx>(
.bcx
.ins()
.iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64),
(&ty::Dynamic(ref data_a, ..), &ty::Dynamic(ref data_b, ..)) => {
(
&ty::Dynamic(ref data_a, _, src_dyn_kind),
&ty::Dynamic(ref data_b, _, target_dyn_kind),
) => {
assert_eq!(src_dyn_kind, target_dyn_kind);
let old_info =
old_info.expect("unsized_info: missing old info for trait upcasting coercion");
if data_a.principal_def_id() == data_b.principal_def_id() {
@ -101,6 +106,21 @@ fn unsize_ptr<'tcx>(
}
}
/// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type.
pub(crate) fn cast_to_dyn_star<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
src: Value,
src_ty_and_layout: TyAndLayout<'tcx>,
dst_ty: Ty<'tcx>,
old_info: Option<Value>,
) -> (Value, Value) {
assert!(
matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
"destination type must be a dyn*"
);
(src, unsized_info(fx, src_ty_and_layout.ty, dst_ty, old_info))
}
/// Coerce `src`, which is a reference to a value of type `src_ty`,
/// to a value of type `dst_ty` and store the result in `dst`
pub(crate) fn coerce_unsized_into<'tcx>(
@ -147,6 +167,24 @@ pub(crate) fn coerce_unsized_into<'tcx>(
}
}
pub(crate) fn coerce_dyn_star<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
src: CValue<'tcx>,
dst: CPlace<'tcx>,
) {
let (data, extra) = if let ty::Dynamic(_, _, ty::DynStar) = src.layout().ty.kind() {
let (data, vtable) = src.load_scalar_pair(fx);
(data, Some(vtable))
} else {
let data = src.load_scalar(fx);
(data, None)
};
let (data, vtable) = cast_to_dyn_star(fx, data, src.layout(), dst.layout().ty, extra);
dst.write_cvalue(fx, CValue::by_val_pair(data, vtable, dst.layout()));
}
// Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs
pub(crate) fn size_and_align_of_dst<'tcx>(

48
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -107,6 +107,50 @@ impl<'tcx> CValue<'tcx> {
}
}
// FIXME remove
// Forces the data value of a dyn* value to the stack and returns a pointer to it as well as the
// vtable pointer.
pub(crate) fn dyn_star_force_data_on_stack(
self,
fx: &mut FunctionCx<'_, '_, 'tcx>,
) -> (Value, Value) {
assert!(self.1.ty.is_dyn_star());
match self.0 {
CValueInner::ByRef(ptr, None) => {
let (a_scalar, b_scalar) = match self.1.abi {
Abi::ScalarPair(a, b) => (a, b),
_ => unreachable!("dyn_star_force_data_on_stack({:?})", self),
};
let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar);
let mut flags = MemFlags::new();
flags.set_notrap();
let vtable = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags);
(ptr.get_addr(fx), vtable)
}
CValueInner::ByValPair(data, vtable) => {
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (u32::try_from(fx.target_config.pointer_type().bytes()).unwrap() + 15)
/ 16
* 16,
});
let data_ptr = Pointer::stack_slot(stack_slot);
let mut flags = MemFlags::new();
flags.set_notrap();
data_ptr.store(fx, data, flags);
(data_ptr.get_addr(fx), vtable)
}
CValueInner::ByRef(_, Some(_)) | CValueInner::ByVal(_) => {
unreachable!("dyn_star_force_data_on_stack({:?})", self)
}
}
}
pub(crate) fn try_to_ptr(self) -> Option<(Pointer, Option<Value>)> {
match self.0 {
CValueInner::ByRef(ptr, meta) => Some((ptr, meta)),
@ -236,6 +280,10 @@ impl<'tcx> CValue<'tcx> {
crate::unsize::coerce_unsized_into(fx, self, dest);
}
pub(crate) fn coerce_dyn_star(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) {
crate::unsize::coerce_dyn_star(fx, self, dest);
}
/// If `ty` is signed, `const_val` must already be sign extended.
pub(crate) fn const_val(
fx: &mut FunctionCx<'_, '_, 'tcx>,

26
compiler/rustc_codegen_cranelift/src/vtable.rs
View file

@ -45,12 +45,26 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
arg: CValue<'tcx>,
idx: usize,
) -> (Value, Value) {
let (ptr, vtable) = if let Abi::ScalarPair(_, _) = arg.layout().abi {
arg.load_scalar_pair(fx)
} else {
let (ptr, vtable) = arg.try_to_ptr().unwrap();
(ptr.get_addr(fx), vtable.unwrap())
) -> (Pointer, Value) {
let (ptr, vtable) = 'block: {
if let ty::Ref(_, ty, _) = arg.layout().ty.kind() {
if ty.is_dyn_star() {
let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap().ty);
let dyn_star = CPlace::for_ptr(Pointer::new(arg.load_scalar(fx)), inner_layout);
let ptr = dyn_star.place_field(fx, mir::Field::new(0)).to_ptr();
let vtable =
dyn_star.place_field(fx, mir::Field::new(1)).to_cvalue(fx).load_scalar(fx);
break 'block (ptr, vtable);
}
}
if let Abi::ScalarPair(_, _) = arg.layout().abi {
let (ptr, vtable) = arg.load_scalar_pair(fx);
(Pointer::new(ptr), vtable)
} else {
let (ptr, vtable) = arg.try_to_ptr().unwrap();
(ptr, vtable.unwrap())
}
};
let usize_size = fx.layout_of(fx.tcx.types.usize).size.bytes();