bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '93a5433f17' into sync_cg_clif-2023-10-24

Browse source
This commit is contained in:
bjorn3 2023-10-24 12:22:23 +00:00
commit 484bc7fc88
14 changed files with 260 additions and 138 deletions
Download patch file Download diff file Expand all files Collapse all files

35
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -120,32 +120,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
args: &[Value],
) -> Cow<'_, [Value]> {
if self.tcx.sess.target.is_like_windows {
let (mut params, mut args): (Vec<_>, Vec<_>) =
params
.into_iter()
.zip(args)
.map(|(param, &arg)| {
if param.value_type == types::I128 {
let arg_ptr = Pointer::stack_slot(self.bcx.create_sized_stack_slot(
StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16 },
));
arg_ptr.store(self, arg, MemFlags::trusted());
(AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
} else {
(param, arg)
}
})
.unzip();
let (mut params, mut args): (Vec<_>, Vec<_>) = params
.into_iter()
.zip(args)
.map(|(param, &arg)| {
if param.value_type == types::I128 {
let arg_ptr = self.create_stack_slot(16, 16);
arg_ptr.store(self, arg, MemFlags::trusted());
(AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
} else {
(param, arg)
}
})
.unzip();
let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128;
if indirect_ret_val {
params.insert(0, AbiParam::new(self.pointer_type));
let ret_ptr =
Pointer::stack_slot(self.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
size: 16,
}));
let ret_ptr = self.create_stack_slot(16, 16);
args.insert(0, ret_ptr.get_addr(self));
self.lib_call_unadjusted(name, params, vec![], &args);
return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]);

11
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -189,16 +189,13 @@ pub(super) fn from_casted_value<'tcx>(
let abi_params = cast_target_to_abi_params(cast);
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
let ptr = fx.create_stack_slot(
// Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
// It may also be smaller for example when the type is a wrapper around an integer with a
// larger alignment than the integer.
size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
});
let ptr = Pointer::stack_slot(stack_slot);
std::cmp::max(abi_param_size, layout_size),
u32::try_from(layout.align.pref.bytes()).unwrap(),
);
let mut offset = 0;
let mut block_params_iter = block_params.iter().copied();
for param in abi_params {

6
compiler/rustc_codegen_cranelift/src/cast.rs
View file

@ -104,11 +104,7 @@ pub(crate) fn clif_int_or_float_cast(
&[from],
)[0];
// FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
size: 16,
});
let ret_ptr = Pointer::stack_slot(stack_slot);
let ret_ptr = fx.create_stack_slot(16, 16);
ret_ptr.store(fx, ret, MemFlags::trusted());
ret_ptr.load(fx, types::I128, MemFlags::trusted())
} else {

19
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -383,6 +383,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
})
}
pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer {
if align <= 16 {
let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (size + 15) / 16 * 16,
});
Pointer::stack_slot(stack_slot)
} else {
// Alignment is too big to handle using the above hack. Dynamically realign a stack slot
// instead. This wastes some space for the realignment.
let base_ptr = self.create_stack_slot(size + align, 16).get_addr(self);
let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align));
let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align));
Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset))
}
}
pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
if let Some(debug_context) = &mut self.cx.debug_context {
let (file, line, column) =

23
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -361,12 +361,26 @@ pub(crate) fn run_aot(
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<OngoingCodegen> {
// FIXME handle `-Ctarget-cpu=native`
let target_cpu = match tcx.sess.opts.cg.target_cpu {
Some(ref name) => name,
None => tcx.sess.target.cpu.as_ref(),
}
.to_owned();
let cgus = if tcx.sess.opts.output_types.should_codegen() {
tcx.collect_and_partition_mono_items(()).1
} else {
// If only `--emit metadata` is used, we shouldn't perform any codegen.
// Also `tcx.collect_and_partition_mono_items` may panic in that case.
&[]
return Box::new(OngoingCodegen {
modules: vec![],
allocator_module: None,
metadata_module: None,
metadata,
crate_info: CrateInfo::new(tcx, target_cpu),
concurrency_limiter: ConcurrencyLimiter::new(tcx.sess, 0),
});
};
if tcx.dep_graph.is_fully_enabled() {
@ -481,13 +495,6 @@ pub(crate) fn run_aot(
None
};
// FIXME handle `-Ctarget-cpu=native`
let target_cpu = match tcx.sess.opts.cg.target_cpu {
Some(ref name) => name,
None => tcx.sess.target.cpu.as_ref(),
}
.to_owned();
Box::new(OngoingCodegen {
modules,
allocator_module,

22
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -878,13 +878,7 @@ fn call_inline_asm<'tcx>(
inputs: Vec<(Size, Value)>,
outputs: Vec<(Size, CPlace<'tcx>)>,
) {
let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
size: u32::try_from(slot_size.bytes()).unwrap(),
});
if fx.clif_comments.enabled() {
fx.add_comment(stack_slot, "inline asm scratch slot");
}
let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);
let inline_asm_func = fx
.module
@ -904,15 +898,23 @@ fn call_inline_asm<'tcx>(
}
for (offset, value) in inputs {
fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
fx,
value,
MemFlags::trusted(),
);
}
let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
let stack_slot_addr = stack_slot.get_addr(fx);
fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
for (offset, place) in outputs {
let ty = fx.clif_type(place.layout().ty).unwrap();
let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
fx,
ty,
MemFlags::trusted(),
);
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
}
}

137
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
View file

@ -310,6 +310,143 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
let val = CValue::by_val_pair(cb_out, c, layout);
ret.write_cvalue(fx, val);
}
"llvm.x86.sse2.pavg.b" | "llvm.x86.sse2.pavg.w" => {
intrinsic_args!(fx, args => (a, b); intrinsic);
// FIXME use vector instructions when possible
simd_pair_for_each_lane(
fx,
a,
b,
ret,
&|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| {
// (a + b + 1) >> 1
let lane_ty = fx.bcx.func.dfg.value_type(a_lane);
let a_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), a_lane);
let b_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), b_lane);
let sum = fx.bcx.ins().iadd(a_lane, b_lane);
let num_plus_one = fx.bcx.ins().iadd_imm(sum, 1);
let res = fx.bcx.ins().ushr_imm(num_plus_one, 1);
fx.bcx.ins().ireduce(lane_ty, res)
},
);
}
"llvm.x86.sse2.psra.w" => {
intrinsic_args!(fx, args => (a, count); intrinsic);
let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlags::trusted());
let lane_ty = fx.clif_type(a.layout().ty.simd_size_and_type(fx.tcx).1).unwrap();
let max_count = fx.bcx.ins().iconst(types::I64, i64::from(lane_ty.bits() - 1));
let saturated_count = fx.bcx.ins().umin(count_lane, max_count);
// FIXME use vector instructions when possible
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, a_lane| {
fx.bcx.ins().sshr(a_lane, saturated_count)
});
}
"llvm.x86.sse2.psad.bw" => {
intrinsic_args!(fx, args => (a, b); intrinsic);
assert_eq!(a.layout(), b.layout());
let layout = a.layout();
let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
assert_eq!(lane_ty, fx.tcx.types.u8);
assert_eq!(ret_lane_ty, fx.tcx.types.u64);
assert_eq!(lane_count, ret_lane_count * 8);
let ret_lane_layout = fx.layout_of(fx.tcx.types.u64);
for out_lane_idx in 0..lane_count / 8 {
let mut lane_diff_acc = fx.bcx.ins().iconst(types::I64, 0);
for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 1 {
let a_lane = a.value_lane(fx, lane_idx).load_scalar(fx);
let b_lane = b.value_lane(fx, lane_idx).load_scalar(fx);
let lane_diff = fx.bcx.ins().isub(a_lane, b_lane);
let abs_lane_diff = fx.bcx.ins().iabs(lane_diff);
let abs_lane_diff = fx.bcx.ins().uextend(types::I64, abs_lane_diff);
lane_diff_acc = fx.bcx.ins().iadd(lane_diff_acc, abs_lane_diff);
}
let res_lane = CValue::by_val(lane_diff_acc, ret_lane_layout);
ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
}
}
"llvm.x86.ssse3.pmadd.ub.sw.128" => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
assert_eq!(lane_ty, fx.tcx.types.u8);
assert_eq!(ret_lane_ty, fx.tcx.types.i16);
assert_eq!(lane_count, ret_lane_count * 2);
let ret_lane_layout = fx.layout_of(fx.tcx.types.i16);
for out_lane_idx in 0..lane_count / 2 {
let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
let a_lane0 = fx.bcx.ins().uextend(types::I16, a_lane0);
let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
let b_lane0 = fx.bcx.ins().sextend(types::I16, b_lane0);
let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
let a_lane1 = fx.bcx.ins().uextend(types::I16, a_lane1);
let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
let b_lane1 = fx.bcx.ins().sextend(types::I16, b_lane1);
let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0);
let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1);
let (val, has_overflow) = fx.bcx.ins().sadd_overflow(mul0, mul1);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, mul1, 0);
let min = fx.bcx.ins().iconst(types::I16, i64::from(i16::MIN as u16));
let max = fx.bcx.ins().iconst(types::I16, i64::from(i16::MAX as u16));
let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
let res_lane = fx.bcx.ins().select(has_overflow, sat_val, val);
let res_lane = CValue::by_val(res_lane, ret_lane_layout);
ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
}
}
"llvm.x86.sse2.pmadd.wd" => {
intrinsic_args!(fx, args => (a, b); intrinsic);
assert_eq!(a.layout(), b.layout());
let layout = a.layout();
let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
assert_eq!(lane_ty, fx.tcx.types.i16);
assert_eq!(ret_lane_ty, fx.tcx.types.i32);
assert_eq!(lane_count, ret_lane_count * 2);
let ret_lane_layout = fx.layout_of(fx.tcx.types.i32);
for out_lane_idx in 0..lane_count / 2 {
let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
let a_lane0 = fx.bcx.ins().uextend(types::I32, a_lane0);
let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
let b_lane0 = fx.bcx.ins().sextend(types::I32, b_lane0);
let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
let a_lane1 = fx.bcx.ins().uextend(types::I32, a_lane1);
let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
let b_lane1 = fx.bcx.ins().sextend(types::I32, b_lane1);
let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0);
let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1);
let res_lane = fx.bcx.ins().iadd(mul0, mul1);
let res_lane = CValue::by_val(res_lane, ret_lane_layout);
ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
}
}
_ => {
fx.tcx
.sess

37
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -132,18 +132,11 @@ impl<'tcx> CValue<'tcx> {
(ptr.get_addr(fx), vtable)
}
CValueInner::ByValPair(data, vtable) => {
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (u32::try_from(fx.target_config.pointer_type().bytes()).unwrap() + 15)
/ 16
* 16,
});
let data_ptr = Pointer::stack_slot(stack_slot);
let mut flags = MemFlags::new();
flags.set_notrap();
data_ptr.store(fx, data, flags);
let data_ptr = fx.create_stack_slot(
u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(),
u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(),
);
data_ptr.store(fx, data, MemFlags::trusted());
(data_ptr.get_addr(fx), vtable)
}
@ -372,13 +365,11 @@ impl<'tcx> CPlace<'tcx> {
.fatal(format!("values of type {} are too big to store on the stack", layout.ty));
}
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
});
CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
let stack_slot = fx.create_stack_slot(
u32::try_from(layout.size.bytes()).unwrap(),
u32::try_from(layout.align.pref.bytes()).unwrap(),
);
CPlace { inner: CPlaceInner::Addr(stack_slot, None), layout }
}
pub(crate) fn new_var(
@ -543,13 +534,7 @@ impl<'tcx> CPlace<'tcx> {
_ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
_ if src_ty.is_vector() || dst_ty.is_vector() => {
// FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers.
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (src_ty.bytes() + 15) / 16 * 16,
});
let ptr = Pointer::stack_slot(stack_slot);
let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes());
ptr.store(fx, data, MemFlags::trusted());
ptr.load(fx, dst_ty, MemFlags::trusted())
}