bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '98ed962c7d' into master

Browse source
This commit is contained in:
Guillaume Gomez 2024-07-10 12:44:23 +02:00
commit 7cbe50e209
68 changed files with 2682 additions and 1135 deletions
Download patch file Download diff file Expand all files Collapse all files

33
compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
View file

@ -74,6 +74,10 @@ match name {
"llvm.amdgcn.cvt.sr.bf8.f32" => "__builtin_amdgcn_cvt_sr_bf8_f32",
"llvm.amdgcn.cvt.sr.fp8.f32" => "__builtin_amdgcn_cvt_sr_fp8_f32",
"llvm.amdgcn.dispatch.id" => "__builtin_amdgcn_dispatch_id",
"llvm.amdgcn.dot4.f32.bf8.bf8" => "__builtin_amdgcn_dot4_f32_bf8_bf8",
"llvm.amdgcn.dot4.f32.bf8.fp8" => "__builtin_amdgcn_dot4_f32_bf8_fp8",
"llvm.amdgcn.dot4.f32.fp8.bf8" => "__builtin_amdgcn_dot4_f32_fp8_bf8",
"llvm.amdgcn.dot4.f32.fp8.fp8" => "__builtin_amdgcn_dot4_f32_fp8_fp8",
"llvm.amdgcn.ds.add.gs.reg.rtn" => "__builtin_amdgcn_ds_add_gs_reg_rtn",
"llvm.amdgcn.ds.bpermute" => "__builtin_amdgcn_ds_bpermute",
"llvm.amdgcn.ds.fadd.v2bf16" => "__builtin_amdgcn_ds_atomic_fadd_v2bf16",
@ -2291,6 +2295,10 @@ match name {
"llvm.loongarch.csrxchg.d" => "__builtin_loongarch_csrxchg_d",
"llvm.loongarch.csrxchg.w" => "__builtin_loongarch_csrxchg_w",
"llvm.loongarch.dbar" => "__builtin_loongarch_dbar",
"llvm.loongarch.frecipe.d" => "__builtin_loongarch_frecipe_d",
"llvm.loongarch.frecipe.s" => "__builtin_loongarch_frecipe_s",
"llvm.loongarch.frsqrte.d" => "__builtin_loongarch_frsqrte_d",
"llvm.loongarch.frsqrte.s" => "__builtin_loongarch_frsqrte_s",
"llvm.loongarch.ibar" => "__builtin_loongarch_ibar",
"llvm.loongarch.iocsrrd.b" => "__builtin_loongarch_iocsrrd_b",
"llvm.loongarch.iocsrrd.d" => "__builtin_loongarch_iocsrrd_d",
@ -2529,6 +2537,8 @@ match name {
"llvm.loongarch.lasx.xvfnmsub.s" => "__builtin_lasx_xvfnmsub_s",
"llvm.loongarch.lasx.xvfrecip.d" => "__builtin_lasx_xvfrecip_d",
"llvm.loongarch.lasx.xvfrecip.s" => "__builtin_lasx_xvfrecip_s",
"llvm.loongarch.lasx.xvfrecipe.d" => "__builtin_lasx_xvfrecipe_d",
"llvm.loongarch.lasx.xvfrecipe.s" => "__builtin_lasx_xvfrecipe_s",
"llvm.loongarch.lasx.xvfrint.d" => "__builtin_lasx_xvfrint_d",
"llvm.loongarch.lasx.xvfrint.s" => "__builtin_lasx_xvfrint_s",
"llvm.loongarch.lasx.xvfrintrm.d" => "__builtin_lasx_xvfrintrm_d",
@ -2541,6 +2551,8 @@ match name {
"llvm.loongarch.lasx.xvfrintrz.s" => "__builtin_lasx_xvfrintrz_s",
"llvm.loongarch.lasx.xvfrsqrt.d" => "__builtin_lasx_xvfrsqrt_d",
"llvm.loongarch.lasx.xvfrsqrt.s" => "__builtin_lasx_xvfrsqrt_s",
"llvm.loongarch.lasx.xvfrsqrte.d" => "__builtin_lasx_xvfrsqrte_d",
"llvm.loongarch.lasx.xvfrsqrte.s" => "__builtin_lasx_xvfrsqrte_s",
"llvm.loongarch.lasx.xvfrstp.b" => "__builtin_lasx_xvfrstp_b",
"llvm.loongarch.lasx.xvfrstp.h" => "__builtin_lasx_xvfrstp_h",
"llvm.loongarch.lasx.xvfrstpi.b" => "__builtin_lasx_xvfrstpi_b",
@ -3255,6 +3267,8 @@ match name {
"llvm.loongarch.lsx.vfnmsub.s" => "__builtin_lsx_vfnmsub_s",
"llvm.loongarch.lsx.vfrecip.d" => "__builtin_lsx_vfrecip_d",
"llvm.loongarch.lsx.vfrecip.s" => "__builtin_lsx_vfrecip_s",
"llvm.loongarch.lsx.vfrecipe.d" => "__builtin_lsx_vfrecipe_d",
"llvm.loongarch.lsx.vfrecipe.s" => "__builtin_lsx_vfrecipe_s",
"llvm.loongarch.lsx.vfrint.d" => "__builtin_lsx_vfrint_d",
"llvm.loongarch.lsx.vfrint.s" => "__builtin_lsx_vfrint_s",
"llvm.loongarch.lsx.vfrintrm.d" => "__builtin_lsx_vfrintrm_d",
@ -3267,6 +3281,8 @@ match name {
"llvm.loongarch.lsx.vfrintrz.s" => "__builtin_lsx_vfrintrz_s",
"llvm.loongarch.lsx.vfrsqrt.d" => "__builtin_lsx_vfrsqrt_d",
"llvm.loongarch.lsx.vfrsqrt.s" => "__builtin_lsx_vfrsqrt_s",
"llvm.loongarch.lsx.vfrsqrte.d" => "__builtin_lsx_vfrsqrte_d",
"llvm.loongarch.lsx.vfrsqrte.s" => "__builtin_lsx_vfrsqrte_s",
"llvm.loongarch.lsx.vfrstp.b" => "__builtin_lsx_vfrstp_b",
"llvm.loongarch.lsx.vfrstp.h" => "__builtin_lsx_vfrstp_h",
"llvm.loongarch.lsx.vfrstpi.b" => "__builtin_lsx_vfrstpi_b",
@ -4434,6 +4450,7 @@ match name {
"llvm.nvvm.abs.bf16x2" => "__nvvm_abs_bf16x2",
"llvm.nvvm.abs.i" => "__nvvm_abs_i",
"llvm.nvvm.abs.ll" => "__nvvm_abs_ll",
"llvm.nvvm.activemask" => "__nvvm_activemask",
"llvm.nvvm.add.rm.d" => "__nvvm_add_rm_d",
"llvm.nvvm.add.rm.f" => "__nvvm_add_rm_f",
"llvm.nvvm.add.rm.ftz.f" => "__nvvm_add_rm_ftz_f",
@ -4522,6 +4539,7 @@ match name {
"llvm.nvvm.ex2.approx.d" => "__nvvm_ex2_approx_d",
"llvm.nvvm.ex2.approx.f" => "__nvvm_ex2_approx_f",
"llvm.nvvm.ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
"llvm.nvvm.exit" => "__nvvm_exit",
"llvm.nvvm.f2bf16.rn" => "__nvvm_f2bf16_rn",
"llvm.nvvm.f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
"llvm.nvvm.f2bf16.rz" => "__nvvm_f2bf16_rz",
@ -4722,8 +4740,11 @@ match name {
"llvm.nvvm.mul24.ui" => "__nvvm_mul24_ui",
"llvm.nvvm.mulhi.i" => "__nvvm_mulhi_i",
"llvm.nvvm.mulhi.ll" => "__nvvm_mulhi_ll",
"llvm.nvvm.mulhi.s" => "__nvvm_mulhi_s",
"llvm.nvvm.mulhi.ui" => "__nvvm_mulhi_ui",
"llvm.nvvm.mulhi.ull" => "__nvvm_mulhi_ull",
"llvm.nvvm.mulhi.us" => "__nvvm_mulhi_us",
"llvm.nvvm.nanosleep" => "__nvvm_nanosleep",
"llvm.nvvm.neg.bf16" => "__nvvm_neg_bf16",
"llvm.nvvm.neg.bf16x2" => "__nvvm_neg_bf16x2",
"llvm.nvvm.popc.i" => "__nvvm_popc_i",
@ -4783,6 +4804,7 @@ match name {
"llvm.nvvm.read.ptx.sreg.envreg7" => "__nvvm_read_ptx_sreg_envreg7",
"llvm.nvvm.read.ptx.sreg.envreg8" => "__nvvm_read_ptx_sreg_envreg8",
"llvm.nvvm.read.ptx.sreg.envreg9" => "__nvvm_read_ptx_sreg_envreg9",
"llvm.nvvm.read.ptx.sreg.globaltimer" => "__nvvm_read_ptx_sreg_globaltimer",
"llvm.nvvm.read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_gridid",
// [DUPLICATE]: "llvm.nvvm.read.ptx.sreg.gridid" => "__nvvm_read_ptx_sreg_",
"llvm.nvvm.read.ptx.sreg.laneid" => "__nvvm_read_ptx_sreg_laneid",
@ -4835,6 +4857,7 @@ match name {
"llvm.nvvm.redux.sync.umax" => "__nvvm_redux_sync_umax",
"llvm.nvvm.redux.sync.umin" => "__nvvm_redux_sync_umin",
"llvm.nvvm.redux.sync.xor" => "__nvvm_redux_sync_xor",
"llvm.nvvm.reflect" => "__nvvm_reflect",
"llvm.nvvm.rotate.b32" => "__nvvm_rotate_b32",
"llvm.nvvm.rotate.b64" => "__nvvm_rotate_b64",
"llvm.nvvm.rotate.right.b64" => "__nvvm_rotate_right_b64",
@ -4845,7 +4868,11 @@ match name {
"llvm.nvvm.rsqrt.approx.f" => "__nvvm_rsqrt_approx_f",
"llvm.nvvm.rsqrt.approx.ftz.f" => "__nvvm_rsqrt_approx_ftz_f",
"llvm.nvvm.sad.i" => "__nvvm_sad_i",
"llvm.nvvm.sad.ll" => "__nvvm_sad_ll",
"llvm.nvvm.sad.s" => "__nvvm_sad_s",
"llvm.nvvm.sad.ui" => "__nvvm_sad_ui",
"llvm.nvvm.sad.ull" => "__nvvm_sad_ull",
"llvm.nvvm.sad.us" => "__nvvm_sad_us",
"llvm.nvvm.saturate.d" => "__nvvm_saturate_d",
"llvm.nvvm.saturate.f" => "__nvvm_saturate_f",
"llvm.nvvm.saturate.ftz.f" => "__nvvm_saturate_ftz_f",
@ -5471,6 +5498,7 @@ match name {
"llvm.ppc.fctiwz" => "__builtin_ppc_fctiwz",
"llvm.ppc.fctudz" => "__builtin_ppc_fctudz",
"llvm.ppc.fctuwz" => "__builtin_ppc_fctuwz",
"llvm.ppc.fence" => "__builtin_ppc_fence",
"llvm.ppc.fmaf128.round.to.odd" => "__builtin_fmaf128_round_to_odd",
"llvm.ppc.fmsub" => "__builtin_ppc_fmsub",
"llvm.ppc.fmsubs" => "__builtin_ppc_fmsubs",
@ -5599,6 +5627,9 @@ match name {
"llvm.ppc.qpx.qvstfs" => "__builtin_qpx_qvstfs",
"llvm.ppc.qpx.qvstfsa" => "__builtin_qpx_qvstfsa",
"llvm.ppc.readflm" => "__builtin_readflm",
"llvm.ppc.rldimi" => "__builtin_ppc_rldimi",
"llvm.ppc.rlwimi" => "__builtin_ppc_rlwimi",
"llvm.ppc.rlwnm" => "__builtin_ppc_rlwnm",
"llvm.ppc.scalar.extract.expq" => "__builtin_vsx_scalar_extract_expq",
"llvm.ppc.scalar.insert.exp.qp" => "__builtin_vsx_scalar_insert_exp_qp",
"llvm.ppc.set.texasr" => "__builtin_set_texasr",
@ -5912,6 +5943,8 @@ match name {
"llvm.s390.vupllb" => "__builtin_s390_vupllb",
"llvm.s390.vupllf" => "__builtin_s390_vupllf",
"llvm.s390.vupllh" => "__builtin_s390_vupllh",
// spv
"llvm.spv.create.handle" => "__builtin_hlsl_create_handle",
// ve
"llvm.ve.vl.andm.MMM" => "__builtin_ve_vl_andm_MMM",
"llvm.ve.vl.andm.mmm" => "__builtin_ve_vl_andm_mmm",

17
compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs
View file

@ -15,7 +15,7 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
// Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
// arguments here.
if gcc_func.get_param_count() != args.len() {
match &*func_name {
match func_name {
// NOTE: the following intrinsics have a different number of parameters in LLVM and GCC.
"__builtin_ia32_prold512_mask"
| "__builtin_ia32_pmuldq512_mask"
@ -380,7 +380,7 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
_ => (),
}
} else {
match &*func_name {
match func_name {
"__builtin_ia32_rndscaless_mask_round" | "__builtin_ia32_rndscalesd_mask_round" => {
let new_args = args.to_vec();
let arg3_type = gcc_func.get_param_type(2);
@ -629,17 +629,22 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
#[cfg(feature = "master")]
pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
match name {
let gcc_name = match name {
"llvm.prefetch" => {
let gcc_name = "__builtin_prefetch";
let func = cx.context.get_builtin_function(gcc_name);
cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
return func;
}
_ => (),
}
let gcc_name = match name {
"llvm.aarch64.isb" => {
// FIXME: GCC doesn't support __builtin_arm_isb yet, check if this builtin is OK.
let gcc_name = "__atomic_thread_fence";
let func = cx.context.get_builtin_function(gcc_name);
cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
return func;
}
"llvm.x86.xgetbv" => "__builtin_ia32_xgetbv",
// NOTE: this doc specifies the equivalent GCC builtins: http://huonw.github.io/llvmint/llvmint/x86/index.html
"llvm.sqrt.v2f64" => "__builtin_ia32_sqrtpd",

69
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file

@ -91,7 +91,7 @@ fn get_simple_intrinsic<'gcc, 'tcx>(
sym::abort => "abort",
_ => return None,
};
Some(cx.context.get_builtin_function(&gcc_name))
Some(cx.context.get_builtin_function(gcc_name))
}
impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
@ -122,10 +122,17 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
let simple = get_simple_intrinsic(self, name);
// FIXME(tempdragon): Re-enable `clippy::suspicious_else_formatting` if the following issue is solved:
// https://github.com/rust-lang/rust-clippy/issues/12497
// and leave `else if use_integer_compare` to be placed "as is".
#[allow(clippy::suspicious_else_formatting)]
let llval = match name {
_ if simple.is_some() => {
// FIXME(antoyo): remove this cast when the API supports function.
let func = unsafe { std::mem::transmute(simple.expect("simple")) };
let func = unsafe {
std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(simple.expect("simple"))
};
self.call(
self.type_void(),
None,
@ -167,7 +174,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
sym::volatile_load | sym::unaligned_volatile_load => {
let tp_ty = fn_args.type_at(0);
let ptr = args[0].immediate();
let load = if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode {
let load = if let PassMode::Cast { cast: ref ty, pad_i32: _ } = fn_abi.ret.mode {
let gcc_ty = ty.gcc_type(self);
self.volatile_load(gcc_ty, ptr)
} else {
@ -213,12 +220,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
let after_block = func.new_block("after");
let arg = args[0].immediate();
let result = func.new_local(None, arg.get_type(), "zeros");
let result = func.new_local(None, self.u32_type, "zeros");
let zero = self.cx.gcc_zero(arg.get_type());
let cond = self.gcc_icmp(IntPredicate::IntEQ, arg, zero);
self.llbb().end_with_conditional(None, cond, then_block, else_block);
let zero_result = self.cx.gcc_uint(arg.get_type(), width);
let zero_result = self.cx.gcc_uint(self.u32_type, width);
then_block.add_assignment(None, result, zero_result);
then_block.end_with_jump(None, after_block);
@ -386,7 +393,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
};
if !fn_abi.ret.is_ignore() {
if let PassMode::Cast { cast: ty, .. } = &fn_abi.ret.mode {
if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode {
let ptr_llty = self.type_ptr_to(ty.gcc_type(self));
let ptr = self.pointercast(result.val.llval, ptr_llty);
self.store(llval, ptr, result.val.align);
@ -592,7 +599,7 @@ fn int_type_width_signed<'gcc, 'tcx>(
ty: Ty<'tcx>,
cx: &CodegenCx<'gcc, 'tcx>,
) -> Option<(u64, bool)> {
match ty.kind() {
match *ty.kind() {
ty::Int(t) => Some((
match t {
rustc_middle::ty::IntTy::Isize => u64::from(cx.tcx.sess.target.pointer_width),
@ -698,16 +705,17 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
fn count_leading_zeroes(&mut self, width: u64, arg: RValue<'gcc>) -> RValue<'gcc> {
// TODO(antoyo): use width?
let arg_type = arg.get_type();
let result_type = self.u32_type;
let count_leading_zeroes =
// TODO(antoyo): write a new function Type::is_compatible_with(&Type) and use it here
// instead of using is_uint().
if arg_type.is_uint(&self.cx) {
if arg_type.is_uint(self.cx) {
"__builtin_clz"
}
else if arg_type.is_ulong(&self.cx) {
else if arg_type.is_ulong(self.cx) {
"__builtin_clzl"
}
else if arg_type.is_ulonglong(&self.cx) {
else if arg_type.is_ulonglong(self.cx) {
"__builtin_clzll"
}
else if width == 128 {
@ -755,7 +763,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let res = self.context.new_array_access(self.location, result, index);
return self.gcc_int_cast(res.to_rvalue(), arg_type);
return self.gcc_int_cast(res.to_rvalue(), result_type);
}
else {
let count_leading_zeroes = self.context.get_builtin_function("__builtin_clzll");
@ -763,17 +771,18 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let diff = self.ulonglong_type.get_size() as i64 - arg_type.get_size() as i64;
let diff = self.context.new_rvalue_from_long(self.int_type, diff * 8);
let res = self.context.new_call(self.location, count_leading_zeroes, &[arg]) - diff;
return self.context.new_cast(self.location, res, arg_type);
return self.context.new_cast(self.location, res, result_type);
};
let count_leading_zeroes = self.context.get_builtin_function(count_leading_zeroes);
let res = self.context.new_call(self.location, count_leading_zeroes, &[arg]);
self.context.new_cast(self.location, res, arg_type)
self.context.new_cast(self.location, res, result_type)
}
fn count_trailing_zeroes(&mut self, _width: u64, arg: RValue<'gcc>) -> RValue<'gcc> {
let result_type = arg.get_type();
let arg = if result_type.is_signed(self.cx) {
let new_type = result_type.to_unsigned(self.cx);
let arg_type = arg.get_type();
let result_type = self.u32_type;
let arg = if arg_type.is_signed(self.cx) {
let new_type = arg_type.to_unsigned(self.cx);
self.gcc_int_cast(arg, new_type)
} else {
arg
@ -782,17 +791,17 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let (count_trailing_zeroes, expected_type) =
// TODO(antoyo): write a new function Type::is_compatible_with(&Type) and use it here
// instead of using is_uint().
if arg_type.is_uchar(&self.cx) || arg_type.is_ushort(&self.cx) || arg_type.is_uint(&self.cx) {
if arg_type.is_uchar(self.cx) || arg_type.is_ushort(self.cx) || arg_type.is_uint(self.cx) {
// NOTE: we don't need to & 0xFF for uchar because the result is undefined on zero.
("__builtin_ctz", self.cx.uint_type)
}
else if arg_type.is_ulong(&self.cx) {
else if arg_type.is_ulong(self.cx) {
("__builtin_ctzl", self.cx.ulong_type)
}
else if arg_type.is_ulonglong(&self.cx) {
else if arg_type.is_ulonglong(self.cx) {
("__builtin_ctzll", self.cx.ulonglong_type)
}
else if arg_type.is_u128(&self.cx) {
else if arg_type.is_u128(self.cx) {
// Adapted from the algorithm to count leading zeroes from: https://stackoverflow.com/a/28433850/389119
let array_type = self.context.new_array_type(None, arg_type, 3);
let result = self.current_func()
@ -863,18 +872,16 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
fn pop_count(&mut self, value: RValue<'gcc>) -> RValue<'gcc> {
// TODO(antoyo): use the optimized version with fewer operations.
let result_type = value.get_type();
let value_type = result_type.to_unsigned(self.cx);
let result_type = self.u32_type;
let arg_type = value.get_type();
let value_type = arg_type.to_unsigned(self.cx);
let value = if result_type.is_signed(self.cx) {
self.gcc_int_cast(value, value_type)
} else {
value
};
let value =
if arg_type.is_signed(self.cx) { self.gcc_int_cast(value, value_type) } else { value };
// only break apart 128-bit ints if they're not natively supported
// TODO(antoyo): remove this if/when native 128-bit integers land in libgccjit
if value_type.is_u128(&self.cx) && !self.cx.supports_128bit_integers {
if value_type.is_u128(self.cx) && !self.cx.supports_128bit_integers {
let sixty_four = self.gcc_int(value_type, 64);
let right_shift = self.gcc_lshr(value, sixty_four);
let high = self.gcc_int_cast(right_shift, self.cx.ulonglong_type);
@ -997,7 +1004,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
// Return `result_type`'s maximum or minimum value on overflow
// NOTE: convert the type to unsigned to have an unsigned shift.
let unsigned_type = result_type.to_unsigned(&self.cx);
let unsigned_type = result_type.to_unsigned(self.cx);
let shifted = self.gcc_lshr(
self.gcc_int_cast(lhs, unsigned_type),
self.gcc_int(unsigned_type, width as i64 - 1),
@ -1189,7 +1196,7 @@ fn codegen_gnu_try<'gcc>(
bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None);
});
let func = unsafe { std::mem::transmute(func) };
let func = unsafe { std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(func) };
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
@ -1263,7 +1270,7 @@ fn gen_fn<'a, 'gcc, 'tcx>(
// FIXME(eddyb) find a nicer way to do this.
cx.linkage.set(FunctionType::Internal);
let func = cx.declare_fn(name, fn_abi);
let func_val = unsafe { std::mem::transmute(func) };
let func_val = unsafe { std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(func) };
cx.set_frame_pointer_type(func_val);
cx.apply_target_cpu_attr(func_val);
let block = Builder::append_block(cx, func_val, "entry-block");

195
compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
View file

@ -13,6 +13,7 @@ use rustc_codegen_ssa::errors::InvalidMonomorphization;
use rustc_codegen_ssa::mir::operand::OperandRef;
use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods};
#[cfg(feature = "master")]
use rustc_hir as hir;
use rustc_middle::mir::BinOp;
use rustc_middle::span_bug;
@ -72,11 +73,11 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
let mask_ty = arg_tys[0];
let mut mask = match mask_ty.kind() {
let mut mask = match *mask_ty.kind() {
ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
ty::Array(elem, len)
if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
if matches!(*elem.kind(), ty::Uint(ty::UintTy::U8))
&& len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all())
== Some(expected_bytes) =>
{
@ -309,10 +310,9 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
})
.collect();
return Ok(bx.context.new_rvalue_from_vector(None, v_type, &elems));
} else {
// avoid the unnecessary truncation as an optimization.
return Ok(bx.context.new_bitcast(None, result, v_type));
}
// avoid the unnecessary truncation as an optimization.
return Ok(bx.context.new_bitcast(None, result, v_type));
}
// since gcc doesn't have vector shuffle methods available in non-patched builds, fallback to
// component-wise bitreverses if they're not available.
@ -342,11 +342,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
.map(|i| {
let index = bx.context.new_rvalue_from_long(bx.i32_type, i as i64);
let value = bx.extract_element(vector, index).to_rvalue();
if name == sym::simd_ctlz {
bx.count_leading_zeroes(value.get_type().get_size() as u64 * 8, value)
let value_type = value.get_type();
let element = if name == sym::simd_ctlz {
bx.count_leading_zeroes(value_type.get_size() as u64 * 8, value)
} else {
bx.count_trailing_zeroes(value.get_type().get_size() as u64 * 8, value)
}
bx.count_trailing_zeroes(value_type.get_size() as u64 * 8, value)
};
bx.context.new_cast(None, element, value_type)
})
.collect();
return Ok(bx.context.new_rvalue_from_vector(None, vector.get_type(), &elements));
@ -355,8 +357,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
if name == sym::simd_shuffle {
// Make sure this is actually an array, since typeck only checks the length-suffixed
// version of this intrinsic.
let n: u64 = match args[2].layout.ty.kind() {
ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
let n: u64 = match *args[2].layout.ty.kind() {
ty::Array(ty, len) if matches!(*ty.kind(), ty::Uint(ty::UintTy::U32)) => {
len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(
|| span_bug!(span, "could not evaluate shuffle index array length"),
)
@ -429,13 +431,148 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
m_len == v_len,
InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
);
match m_elem_ty.kind() {
match *m_elem_ty.kind() {
ty::Int(_) => {}
_ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }),
}
return Ok(bx.vector_select(args[0].immediate(), args[1].immediate(), args[2].immediate()));
}
if name == sym::simd_cast_ptr {
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
require!(
in_len == out_len,
InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len
}
);
match *in_elem.kind() {
ty::RawPtr(p_ty, _) => {
let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
});
require!(
metadata.is_unit(),
InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem }
);
}
_ => {
return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
}
}
match *out_elem.kind() {
ty::RawPtr(p_ty, _) => {
let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty)
});
require!(
metadata.is_unit(),
InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem }
);
}
_ => {
return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
}
}
let arg = args[0].immediate();
let elem_type = llret_ty.dyncast_vector().expect("vector return type").get_element_type();
let values: Vec<_> = (0..in_len)
.map(|i| {
let idx = bx.gcc_int(bx.usize_type, i as _);
let value = bx.extract_element(arg, idx);
bx.pointercast(value, elem_type)
})
.collect();
return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &values));
}
if name == sym::simd_expose_provenance {
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
require!(
in_len == out_len,
InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len
}
);
match *in_elem.kind() {
ty::RawPtr(_, _) => {}
_ => {
return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
}
}
match *out_elem.kind() {
ty::Uint(ty::UintTy::Usize) => {}
_ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }),
}
let arg = args[0].immediate();
let elem_type = llret_ty.dyncast_vector().expect("vector return type").get_element_type();
let values: Vec<_> = (0..in_len)
.map(|i| {
let idx = bx.gcc_int(bx.usize_type, i as _);
let value = bx.extract_element(arg, idx);
bx.ptrtoint(value, elem_type)
})
.collect();
return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &values));
}
if name == sym::simd_with_exposed_provenance {
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
require!(
in_len == out_len,
InvalidMonomorphization::ReturnLengthInputType {
span,
name,
in_len,
in_ty,
ret_ty,
out_len
}
);
match *in_elem.kind() {
ty::Uint(ty::UintTy::Usize) => {}
_ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }),
}
match *out_elem.kind() {
ty::RawPtr(_, _) => {}
_ => {
return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
}
}
let arg = args[0].immediate();
let elem_type = llret_ty.dyncast_vector().expect("vector return type").get_element_type();
let values: Vec<_> = (0..in_len)
.map(|i| {
let idx = bx.gcc_int(bx.usize_type, i as _);
let value = bx.extract_element(arg, idx);
bx.inttoptr(value, elem_type)
})
.collect();
return Ok(bx.context.new_rvalue_from_vector(bx.location, llret_ty, &values));
}
#[cfg(feature = "master")]
if name == sym::simd_cast || name == sym::simd_as {
require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
@ -462,13 +599,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
Unsupported,
}
let in_style = match in_elem.kind() {
let in_style = match *in_elem.kind() {
ty::Int(_) | ty::Uint(_) => Style::Int,
ty::Float(_) => Style::Float,
_ => Style::Unsupported,
};
let out_style = match out_elem.kind() {
let out_style = match *out_elem.kind() {
ty::Int(_) | ty::Uint(_) => Style::Int,
ty::Float(_) => Style::Float,
_ => Style::Unsupported,
@ -495,7 +632,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
macro_rules! arith_binary {
($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
$(if name == sym::$name {
match in_elem.kind() {
match *in_elem.kind() {
$($(ty::$p(_))|* => {
return Ok(bx.$call(args[0].immediate(), args[1].immediate()))
})*
@ -533,7 +670,6 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
let sign_shift = bx.context.new_rvalue_from_int(elem_type, elem_size as i32 - 1);
let one = bx.context.new_rvalue_one(elem_type);
let mut shift = 0;
for i in 0..in_len {
let elem =
bx.extract_element(vector, bx.context.new_rvalue_from_int(bx.int_type, i as i32));
@ -541,17 +677,16 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
let masked = shifted & one;
result = result
| (bx.context.new_cast(None, masked, result_type)
<< bx.context.new_rvalue_from_int(result_type, shift));
shift += 1;
<< bx.context.new_rvalue_from_int(result_type, i as i32));
}
match ret_ty.kind() {
match *ret_ty.kind() {
ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {
// Zero-extend iN to the bitmask type:
return Ok(result);
}
ty::Array(elem, len)
if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
if matches!(*elem.kind(), ty::Uint(ty::UintTy::U8))
&& len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all())
== Some(expected_bytes) =>
{
@ -590,7 +725,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
return Err(());
}};
}
let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
let (elem_ty_str, elem_ty) = if let ty::Float(ref f) = *in_elem.kind() {
let elem_ty = bx.cx.type_float_from_ty(*f);
match f.bit_width() {
32 => ("f", elem_ty),
@ -816,7 +951,9 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
let (pointer_count, underlying_ty) = match *element_ty1.kind() {
ty::RawPtr(p_ty, _) if p_ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)),
ty::RawPtr(p_ty, _) if p_ty == in_elem => {
(ptr_count(element_ty1), non_ptr(element_ty1))
}
_ => {
require!(
false,
@ -839,7 +976,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
// The element type of the third argument must be a signed integer type of any width:
let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
match element_ty2.kind() {
match *element_ty2.kind() {
ty::Int(_) => (),
_ => {
require!(
@ -955,7 +1092,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
assert_eq!(underlying_ty, non_ptr(element_ty0));
// The element type of the third argument must be a signed integer type of any width:
match element_ty2.kind() {
match *element_ty2.kind() {
ty::Int(_) => (),
_ => {
require!(
@ -1013,7 +1150,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
macro_rules! arith_unary {
($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
$(if name == sym::$name {
match in_elem.kind() {
match *in_elem.kind() {
$($(ty::$p(_))|* => {
return Ok(bx.$call(args[0].immediate()))
})*
@ -1137,7 +1274,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
ret_ty == in_elem,
InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
);
return match in_elem.kind() {
return match *in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_op(args[0].immediate(), $vec_op);
if $ordered {
@ -1206,7 +1343,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
ret_ty == in_elem,
InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
);
return match in_elem.kind() {
return match *in_elem.kind() {
ty::Int(_) | ty::Uint(_) => Ok(bx.$int_red(args[0].immediate())),
ty::Float(_) => Ok(bx.$float_red(args[0].immediate())),
_ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
@ -1235,7 +1372,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
);
args[0].immediate()
} else {
match in_elem.kind() {
match *in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {}
_ => return_error!(InvalidMonomorphization::UnsupportedSymbol {
span,
@ -1249,7 +1386,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
args[0].immediate()
};
return match in_elem.kind() {
return match *in_elem.kind() {
ty::Int(_) | ty::Uint(_) => {
let r = bx.vector_reduce_op(input, $op);
Ok(if !$boolean {