bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '3187d32079' into subtree-update

Browse source
This commit is contained in:
Guillaume Gomez 2024-09-27 22:00:17 +02:00
commit 325b70890a
42 changed files with 943 additions and 365 deletions
Download patch file Download diff file Expand all files Collapse all files

13
compiler/rustc_codegen_gcc/src/base.rs
View file

@ -128,8 +128,19 @@ pub fn compile_codegen_unit(
// NOTE: Rust relies on LLVM doing wrapping on overflow.
context.add_command_line_option("-fwrapv");
if let Some(model) = tcx.sess.code_model() {
use rustc_target::spec::CodeModel;
context.add_command_line_option(match model {
CodeModel::Tiny => "-mcmodel=tiny",
CodeModel::Small => "-mcmodel=small",
CodeModel::Kernel => "-mcmodel=kernel",
CodeModel::Medium => "-mcmodel=medium",
CodeModel::Large => "-mcmodel=large",
});
}
if tcx.sess.relocation_model() == rustc_target::spec::RelocModel::Static {
context.add_command_line_option("-mcmodel=kernel");
context.add_command_line_option("-fno-pie");
}

54
compiler/rustc_codegen_gcc/src/builder.rs
View file

@ -39,9 +39,6 @@ use crate::type_of::LayoutGccExt;
// TODO(antoyo)
type Funclet = ();
// TODO(antoyo): remove this variable.
static mut RETURN_VALUE_COUNT: usize = 0;
enum ExtremumOperation {
Max,
Min,
@ -50,13 +47,18 @@ enum ExtremumOperation {
pub struct Builder<'a: 'gcc, 'gcc, 'tcx> {
pub cx: &'a CodegenCx<'gcc, 'tcx>,
pub block: Block<'gcc>,
stack_var_count: Cell<usize>,
pub location: Option<Location<'gcc>>,
value_counter: Cell<u64>,
}
impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
fn with_cx(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Self {
Builder { cx, block, stack_var_count: Cell::new(0), location: None }
Builder { cx, block, location: None, value_counter: Cell::new(0) }
}
fn next_value_counter(&self) -> u64 {
self.value_counter.set(self.value_counter.get() + 1);
self.value_counter.get()
}
fn atomic_extremum(
@ -138,7 +140,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
) -> RValue<'gcc> {
let size = get_maybe_pointer_size(src);
let compare_exchange =
self.context.get_builtin_function(&format!("__atomic_compare_exchange_{}", size));
self.context.get_builtin_function(format!("__atomic_compare_exchange_{}", size));
let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
let failure_order = self.context.new_rvalue_from_int(self.i32_type, failure_order.to_gcc());
let weak = self.context.new_rvalue_from_int(self.bool_type, weak as i32);
@ -270,10 +272,12 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
actual_val.dereference(self.location).to_rvalue()
}
} else {
// FIXME: this condition seems wrong: it will pass when both types are not
// a vector.
assert!(
(!expected_ty.is_vector() || actual_ty.is_vector())
&& (expected_ty.is_vector() || !actual_ty.is_vector()),
"{:?} ({}) -> {:?} ({}), index: {:?}[{}]",
"{:?} (is vector: {}) -> {:?} (is vector: {}), Function: {:?}[{}]",
actual_ty,
actual_ty.is_vector(),
expected_ty,
@ -283,6 +287,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
);
// TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
// TODO: remove bitcast now that vector types can be compared?
// ==> We use bitcast to avoid having to do many manual casts from e.g. __m256i to __v32qi (in
// the case of _mm256_aesenc_epi128).
self.bitcast(actual_val, expected_ty)
}
} else {
@ -325,11 +331,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let void_type = self.context.new_type::<()>();
let current_func = self.block.get_function();
if return_type != void_type {
unsafe { RETURN_VALUE_COUNT += 1 };
let result = current_func.new_local(
self.location,
return_type,
&format!("returnValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("returnValue{}", self.next_value_counter()),
);
self.block.add_assignment(
self.location,
@ -341,7 +346,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
self.block
.add_eval(self.location, self.cx.context.new_call(self.location, func, &args));
// Return dummy value when not having return value.
self.context.new_rvalue_from_long(self.isize_type, 0)
self.context.new_rvalue_zero(self.isize_type)
}
}
@ -367,6 +372,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let args = {
let function_address_names = self.function_address_names.borrow();
let original_function_name = function_address_names.get(&func_ptr);
func_ptr = llvm::adjust_function(self.context, &func_name, func_ptr, args);
llvm::adjust_intrinsic_arguments(
self,
gcc_func,
@ -385,7 +391,6 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let current_func = self.block.get_function();
if return_type != void_type {
unsafe { RETURN_VALUE_COUNT += 1 };
let return_value = self.cx.context.new_call_through_ptr(self.location, func_ptr, &args);
let return_value = llvm::adjust_intrinsic_return_value(
self,
@ -398,7 +403,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let result = current_func.new_local(
self.location,
return_value.get_type(),
&format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("ptrReturnValue{}", self.next_value_counter()),
);
self.block.add_assignment(self.location, result, return_value);
result.to_rvalue()
@ -422,17 +427,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
self.cx.context.new_call_through_ptr(self.location, func_ptr, &args),
);
// Return dummy value when not having return value.
let result = current_func.new_local(
self.location,
self.isize_type,
"dummyValueThatShouldNeverBeUsed",
);
self.block.add_assignment(
self.location,
result,
self.context.new_rvalue_from_long(self.isize_type, 0),
);
result.to_rvalue()
self.context.new_rvalue_zero(self.isize_type)
}
}
@ -447,11 +442,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let return_type = self.context.new_type::<bool>();
let current_func = self.block.get_function();
// TODO(antoyo): return the new_call() directly? Since the overflow function has no side-effects.
unsafe { RETURN_VALUE_COUNT += 1 };
let result = current_func.new_local(
self.location,
return_type,
&format!("overflowReturnValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("overflowReturnValue{}", self.next_value_counter()),
);
self.block.add_assignment(
self.location,
@ -930,9 +924,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
fn alloca(&mut self, size: Size, align: Align) -> RValue<'gcc> {
let ty = self.cx.type_array(self.cx.type_i8(), size.bytes()).get_aligned(align.bytes());
// TODO(antoyo): It might be better to return a LValue, but fixing the rustc API is non-trivial.
self.stack_var_count.set(self.stack_var_count.get() + 1);
self.current_func()
.new_local(self.location, ty, &format!("stack_var_{}", self.stack_var_count.get()))
.new_local(self.location, ty, format!("stack_var_{}", self.next_value_counter()))
.get_address(self.location)
}
@ -955,11 +948,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
};
let ptr = self.context.new_cast(self.location, ptr, aligned_type.make_pointer());
let deref = ptr.dereference(self.location).to_rvalue();
unsafe { RETURN_VALUE_COUNT += 1 };
let loaded_value = function.new_local(
self.location,
aligned_type,
&format!("loadedValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("loadedValue{}", self.next_value_counter()),
);
block.add_assignment(self.location, loaded_value, deref);
loaded_value.to_rvalue()
@ -980,7 +972,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
// TODO(antoyo): use ty.
// TODO(antoyo): handle alignment.
let atomic_load =
self.context.get_builtin_function(&format!("__atomic_load_{}", size.bytes()));
self.context.get_builtin_function(format!("__atomic_load_{}", size.bytes()));
let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
let volatile_const_void_ptr_type =
@ -1136,7 +1128,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
) {
// TODO(antoyo): handle alignment.
let atomic_store =
self.context.get_builtin_function(&format!("__atomic_store_{}", size.bytes()));
self.context.get_builtin_function(format!("__atomic_store_{}", size.bytes()));
let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
let volatile_const_void_ptr_type =
self.context.new_type::<()>().make_volatile().make_pointer();

8
compiler/rustc_codegen_gcc/src/consts.rs
View file

@ -9,6 +9,7 @@ use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}
use rustc_middle::mir::interpret::{
self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint,
};
use rustc_middle::mir::mono::Linkage;
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::{self, Instance};
use rustc_middle::{bug, span_bug};
@ -258,7 +259,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
if !self.tcx.is_reachable_non_generic(def_id) {
#[cfg(feature = "master")]
global.add_string_attribute(VarAttribute::Visibility(Visibility::Hidden));
global.add_attribute(VarAttribute::Visibility(Visibility::Hidden));
}
global
@ -386,6 +387,11 @@ fn check_and_apply_linkage<'gcc, 'tcx>(
let global1 =
cx.declare_global_with_linkage(sym, cx.type_i8(), base::global_linkage_to_gcc(linkage));
if linkage == Linkage::ExternalWeak {
#[cfg(feature = "master")]
global1.add_attribute(VarAttribute::Weak);
}
// Declare an internal global `extern_with_linkage_foo` which
// is initialized with the address of `foo`. If `foo` is
// discarded during linking (for example, if `foo` has weak

35
compiler/rustc_codegen_gcc/src/context.rs
View file

@ -24,6 +24,7 @@ use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCA
use crate::callee::get_fn;
use crate::common::SignType;
#[cfg_attr(not(feature = "master"), allow(dead_code))]
pub struct CodegenCx<'gcc, 'tcx> {
pub codegen_unit: &'tcx CodegenUnit<'tcx>,
pub context: &'gcc Context<'gcc>,
@ -226,48 +227,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
"__builtin_umul_overflow",
"__builtin_usubll_overflow",
"__builtin_usub_overflow",
"sqrtf",
"sqrt",
"__builtin_powif",
"__builtin_powi",
"sinf",
"sin",
"cosf",
"cos",
"powf",
"pow",
"expf",
"exp",
"exp2f",
"exp2",
"logf",
"log",
"log10f",
"log10",
"log2f",
"log2",
"fmaf",
"fma",
"fabsf",
"fabs",
"fminf",
"fmin",
"fmaxf",
"fmax",
"copysignf",
"copysign",
"floorf",
"floor",
"ceilf",
"ceil",
"truncf",
"trunc",
"rintf",
"rint",
"nearbyintf",
"nearbyint",
"roundf",
"round",
];
for builtin in builtins.iter() {

4
compiler/rustc_codegen_gcc/src/debuginfo.rs
View file

@ -55,7 +55,7 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
}
/// Generate the `debug_context` in an MIR Body.
/// # Souce of Origin
/// # Source of Origin
/// Copied from `create_scope_map.rs` of rustc_codegen_llvm
fn compute_mir_scopes<'gcc, 'tcx>(
cx: &CodegenCx<'gcc, 'tcx>,
@ -90,7 +90,7 @@ fn compute_mir_scopes<'gcc, 'tcx>(
/// Update the `debug_context`, adding new scope to it,
/// if it's not added as is denoted in `instantiated`.
///
/// # Souce of Origin
/// # Source of Origin
/// Copied from `create_scope_map.rs` of rustc_codegen_llvm
/// FIXME(tempdragon/?): Add Scope Support Here.
fn make_mir_scope<'gcc, 'tcx>(

10
compiler/rustc_codegen_gcc/src/declare.rs
View file

@ -168,7 +168,15 @@ fn declare_raw_fn<'gcc>(
variadic: bool,
) -> Function<'gcc> {
if name.starts_with("llvm.") {
let intrinsic = llvm::intrinsic(name, cx);
let intrinsic = match name {
"llvm.fma.f16" => {
// fma is not a target builtin, but a normal builtin, so we handle it differently
// here.
cx.context.get_builtin_function("fma")
}
_ => llvm::intrinsic(name, cx),
};
cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic);
return intrinsic;
}

4
compiler/rustc_codegen_gcc/src/int.rs
View file

@ -6,8 +6,8 @@ use gccjit::{BinaryOp, ComparisonOp, FunctionType, Location, RValue, ToRValue, T
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
use rustc_codegen_ssa::traits::{BackendTypes, BaseTypeCodegenMethods, BuilderMethods, OverflowOp};
use rustc_middle::ty::{ParamEnv, Ty};
use rustc_target::abi::Endian;
use rustc_target::abi::call::{ArgAbi, ArgAttributes, Conv, FnAbi, PassMode};
use rustc_target::abi::Endian;
use rustc_target::spec;
use crate::builder::{Builder, ToGccComp};
@ -733,7 +733,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
// TODO(antoyo): check if it's faster to use string literals and a
// match instead of format!.
let bswap = self.cx.context.get_builtin_function(&format!("__builtin_bswap{}", width));
let bswap = self.cx.context.get_builtin_function(format!("__builtin_bswap{}", width));
// FIXME(antoyo): this cast should not be necessary. Remove
// when having proper sized integer types.
let param_type = bswap.get_param(0).to_rvalue().get_type();

364
compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
View file

@ -31,8 +31,11 @@ match name {
"llvm.AMDGPU.trig.preop.v2f64" => "__builtin_amdgpu_trig_preop",
"llvm.AMDGPU.trig.preop.v4f32" => "__builtin_amdgpu_trig_preop",
// aarch64
"llvm.aarch64.chkfeat" => "__builtin_arm_chkfeat",
"llvm.aarch64.dmb" => "__builtin_arm_dmb",
"llvm.aarch64.dsb" => "__builtin_arm_dsb",
"llvm.aarch64.gcspopm" => "__builtin_arm_gcspopm",
"llvm.aarch64.gcsss" => "__builtin_arm_gcsss",
"llvm.aarch64.isb" => "__builtin_arm_isb",
"llvm.aarch64.prefetch" => "__builtin_arm_prefetch",
"llvm.aarch64.sve.aesd" => "__builtin_sve_svaesd_u8",
@ -80,7 +83,6 @@ match name {
"llvm.amdgcn.dot4.f32.fp8.fp8" => "__builtin_amdgcn_dot4_f32_fp8_fp8",
"llvm.amdgcn.ds.add.gs.reg.rtn" => "__builtin_amdgcn_ds_add_gs_reg_rtn",
"llvm.amdgcn.ds.bpermute" => "__builtin_amdgcn_ds_bpermute",
"llvm.amdgcn.ds.fadd.v2bf16" => "__builtin_amdgcn_ds_atomic_fadd_v2bf16",
"llvm.amdgcn.ds.gws.barrier" => "__builtin_amdgcn_ds_gws_barrier",
"llvm.amdgcn.ds.gws.init" => "__builtin_amdgcn_ds_gws_init",
"llvm.amdgcn.ds.gws.sema.br" => "__builtin_amdgcn_ds_gws_sema_br",
@ -96,6 +98,7 @@ match name {
"llvm.amdgcn.fdot2.f16.f16" => "__builtin_amdgcn_fdot2_f16_f16",
"llvm.amdgcn.fdot2.f32.bf16" => "__builtin_amdgcn_fdot2_f32_bf16",
"llvm.amdgcn.fmul.legacy" => "__builtin_amdgcn_fmul_legacy",
"llvm.amdgcn.global.load.lds" => "__builtin_amdgcn_global_load_lds",
"llvm.amdgcn.groupstaticsize" => "__builtin_amdgcn_groupstaticsize",
"llvm.amdgcn.iglp.opt" => "__builtin_amdgcn_iglp_opt",
"llvm.amdgcn.implicit.buffer.ptr" => "__builtin_amdgcn_implicit_buffer_ptr",
@ -154,16 +157,11 @@ match name {
"llvm.amdgcn.mqsad.u32.u8" => "__builtin_amdgcn_mqsad_u32_u8",
"llvm.amdgcn.msad.u8" => "__builtin_amdgcn_msad_u8",
"llvm.amdgcn.perm" => "__builtin_amdgcn_perm",
"llvm.amdgcn.permlane16" => "__builtin_amdgcn_permlane16",
"llvm.amdgcn.permlane16.var" => "__builtin_amdgcn_permlane16_var",
"llvm.amdgcn.permlane64" => "__builtin_amdgcn_permlane64",
"llvm.amdgcn.permlanex16" => "__builtin_amdgcn_permlanex16",
"llvm.amdgcn.permlanex16.var" => "__builtin_amdgcn_permlanex16_var",
"llvm.amdgcn.qsad.pk.u16.u8" => "__builtin_amdgcn_qsad_pk_u16_u8",
"llvm.amdgcn.queue.ptr" => "__builtin_amdgcn_queue_ptr",
"llvm.amdgcn.rcp.legacy" => "__builtin_amdgcn_rcp_legacy",
"llvm.amdgcn.readfirstlane" => "__builtin_amdgcn_readfirstlane",
"llvm.amdgcn.readlane" => "__builtin_amdgcn_readlane",
"llvm.amdgcn.rsq.legacy" => "__builtin_amdgcn_rsq_legacy",
"llvm.amdgcn.s.barrier" => "__builtin_amdgcn_s_barrier",
"llvm.amdgcn.s.barrier.init" => "__builtin_amdgcn_s_barrier_init",
@ -192,6 +190,8 @@ match name {
"llvm.amdgcn.s.setreg" => "__builtin_amdgcn_s_setreg",
"llvm.amdgcn.s.sleep" => "__builtin_amdgcn_s_sleep",
"llvm.amdgcn.s.sleep.var" => "__builtin_amdgcn_s_sleep_var",
"llvm.amdgcn.s.ttracedata" => "__builtin_amdgcn_s_ttracedata",
"llvm.amdgcn.s.ttracedata.imm" => "__builtin_amdgcn_s_ttracedata_imm",
"llvm.amdgcn.s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready",
"llvm.amdgcn.s.waitcnt" => "__builtin_amdgcn_s_waitcnt",
"llvm.amdgcn.s.wakeup.barrier" => "__builtin_amdgcn_s_wakeup_barrier",
@ -227,7 +227,6 @@ match name {
"llvm.amdgcn.workgroup.id.x" => "__builtin_amdgcn_workgroup_id_x",
"llvm.amdgcn.workgroup.id.y" => "__builtin_amdgcn_workgroup_id_y",
"llvm.amdgcn.workgroup.id.z" => "__builtin_amdgcn_workgroup_id_z",
"llvm.amdgcn.writelane" => "__builtin_amdgcn_writelane",
// arm
"llvm.arm.cdp" => "__builtin_arm_cdp",
"llvm.arm.cdp2" => "__builtin_arm_cdp2",
@ -4536,10 +4535,18 @@ match name {
"llvm.nvvm.div.rz.d" => "__nvvm_div_rz_d",
"llvm.nvvm.div.rz.f" => "__nvvm_div_rz_f",
"llvm.nvvm.div.rz.ftz.f" => "__nvvm_div_rz_ftz_f",
"llvm.nvvm.e4m3x2.to.f16x2.rn" => "__nvvm_e4m3x2_to_f16x2_rn",
"llvm.nvvm.e4m3x2.to.f16x2.rn.relu" => "__nvvm_e4m3x2_to_f16x2_rn_relu",
"llvm.nvvm.e5m2x2.to.f16x2.rn" => "__nvvm_e5m2x2_to_f16x2_rn",
"llvm.nvvm.e5m2x2.to.f16x2.rn.relu" => "__nvvm_e5m2x2_to_f16x2_rn_relu",
"llvm.nvvm.ex2.approx.d" => "__nvvm_ex2_approx_d",
"llvm.nvvm.ex2.approx.f" => "__nvvm_ex2_approx_f",
"llvm.nvvm.ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
"llvm.nvvm.exit" => "__nvvm_exit",
"llvm.nvvm.f16x2.to.e4m3x2.rn" => "__nvvm_f16x2_to_e4m3x2_rn",
"llvm.nvvm.f16x2.to.e4m3x2.rn.relu" => "__nvvm_f16x2_to_e4m3x2_rn_relu",
"llvm.nvvm.f16x2.to.e5m2x2.rn" => "__nvvm_f16x2_to_e5m2x2_rn",
"llvm.nvvm.f16x2.to.e5m2x2.rn.relu" => "__nvvm_f16x2_to_e5m2x2_rn_relu",
"llvm.nvvm.f2bf16.rn" => "__nvvm_f2bf16_rn",
"llvm.nvvm.f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
"llvm.nvvm.f2bf16.rz" => "__nvvm_f2bf16_rz",
@ -4582,6 +4589,10 @@ match name {
"llvm.nvvm.fabs.d" => "__nvvm_fabs_d",
"llvm.nvvm.fabs.f" => "__nvvm_fabs_f",
"llvm.nvvm.fabs.ftz.f" => "__nvvm_fabs_ftz_f",
"llvm.nvvm.ff.to.e4m3x2.rn" => "__nvvm_ff_to_e4m3x2_rn",
"llvm.nvvm.ff.to.e4m3x2.rn.relu" => "__nvvm_ff_to_e4m3x2_rn_relu",
"llvm.nvvm.ff.to.e5m2x2.rn" => "__nvvm_ff_to_e5m2x2_rn",
"llvm.nvvm.ff.to.e5m2x2.rn.relu" => "__nvvm_ff_to_e5m2x2_rn_relu",
"llvm.nvvm.ff2bf16x2.rn" => "__nvvm_ff2bf16x2_rn",
"llvm.nvvm.ff2bf16x2.rn.relu" => "__nvvm_ff2bf16x2_rn_relu",
"llvm.nvvm.ff2bf16x2.rz" => "__nvvm_ff2bf16x2_rz",
@ -4866,6 +4877,7 @@ match name {
"llvm.nvvm.round.ftz.f" => "__nvvm_round_ftz_f",
"llvm.nvvm.rsqrt.approx.d" => "__nvvm_rsqrt_approx_d",
"llvm.nvvm.rsqrt.approx.f" => "__nvvm_rsqrt_approx_f",
"llvm.nvvm.rsqrt.approx.ftz.d" => "__nvvm_rsqrt_approx_ftz_d",
"llvm.nvvm.rsqrt.approx.ftz.f" => "__nvvm_rsqrt_approx_ftz_f",
"llvm.nvvm.sad.i" => "__nvvm_sad_i",
"llvm.nvvm.sad.ll" => "__nvvm_sad_ll",
@ -5164,6 +5176,8 @@ match name {
// ppc
"llvm.ppc.addex" => "__builtin_ppc_addex",
"llvm.ppc.addf128.round.to.odd" => "__builtin_addf128_round_to_odd",
"llvm.ppc.addg6s" => "__builtin_addg6s",
"llvm.ppc.addg6sd" => "__builtin_ppc_addg6s",
"llvm.ppc.altivec.crypto.vcipher" => "__builtin_altivec_crypto_vcipher",
"llvm.ppc.altivec.crypto.vcipherlast" => "__builtin_altivec_crypto_vcipherlast",
"llvm.ppc.altivec.crypto.vncipher" => "__builtin_altivec_crypto_vncipher",
@ -5461,6 +5475,10 @@ match name {
"llvm.ppc.bcdsub" => "__builtin_ppc_bcdsub",
"llvm.ppc.bcdsub.p" => "__builtin_ppc_bcdsub_p",
"llvm.ppc.bpermd" => "__builtin_bpermd",
"llvm.ppc.cbcdtd" => "__builtin_cbcdtd",
"llvm.ppc.cbcdtdd" => "__builtin_ppc_cbcdtd",
"llvm.ppc.cdtbcd" => "__builtin_cdtbcd",
"llvm.ppc.cdtbcdd" => "__builtin_ppc_cdtbcd",
"llvm.ppc.cfuged" => "__builtin_cfuged",
"llvm.ppc.cmpeqb" => "__builtin_ppc_cmpeqb",
"llvm.ppc.cmprb" => "__builtin_ppc_cmprb",
@ -5627,7 +5645,6 @@ match name {
"llvm.ppc.qpx.qvstfs" => "__builtin_qpx_qvstfs",
"llvm.ppc.qpx.qvstfsa" => "__builtin_qpx_qvstfsa",
"llvm.ppc.readflm" => "__builtin_readflm",
"llvm.ppc.rldimi" => "__builtin_ppc_rldimi",
"llvm.ppc.rlwimi" => "__builtin_ppc_rlwimi",
"llvm.ppc.rlwnm" => "__builtin_ppc_rlwnm",
"llvm.ppc.scalar.extract.expq" => "__builtin_vsx_scalar_extract_expq",
@ -7210,29 +7227,6 @@ match name {
"llvm.ve.vl.xorm.MMM" => "__builtin_ve_vl_xorm_MMM",
"llvm.ve.vl.xorm.mmm" => "__builtin_ve_vl_xorm_mmm",
// x86
"llvm.x86.3dnow.pavgusb" => "__builtin_ia32_pavgusb",
"llvm.x86.3dnow.pf2id" => "__builtin_ia32_pf2id",
"llvm.x86.3dnow.pfacc" => "__builtin_ia32_pfacc",
"llvm.x86.3dnow.pfadd" => "__builtin_ia32_pfadd",
"llvm.x86.3dnow.pfcmpeq" => "__builtin_ia32_pfcmpeq",
"llvm.x86.3dnow.pfcmpge" => "__builtin_ia32_pfcmpge",
"llvm.x86.3dnow.pfcmpgt" => "__builtin_ia32_pfcmpgt",
"llvm.x86.3dnow.pfmax" => "__builtin_ia32_pfmax",
"llvm.x86.3dnow.pfmin" => "__builtin_ia32_pfmin",
"llvm.x86.3dnow.pfmul" => "__builtin_ia32_pfmul",
"llvm.x86.3dnow.pfrcp" => "__builtin_ia32_pfrcp",
"llvm.x86.3dnow.pfrcpit1" => "__builtin_ia32_pfrcpit1",
"llvm.x86.3dnow.pfrcpit2" => "__builtin_ia32_pfrcpit2",
"llvm.x86.3dnow.pfrsqit1" => "__builtin_ia32_pfrsqit1",
"llvm.x86.3dnow.pfrsqrt" => "__builtin_ia32_pfrsqrt",
"llvm.x86.3dnow.pfsub" => "__builtin_ia32_pfsub",
"llvm.x86.3dnow.pfsubr" => "__builtin_ia32_pfsubr",
"llvm.x86.3dnow.pi2fd" => "__builtin_ia32_pi2fd",
"llvm.x86.3dnow.pmulhrw" => "__builtin_ia32_pmulhrw",
"llvm.x86.3dnowa.pf2iw" => "__builtin_ia32_pf2iw",
"llvm.x86.3dnowa.pfnacc" => "__builtin_ia32_pfnacc",
"llvm.x86.3dnowa.pfpnacc" => "__builtin_ia32_pfpnacc",
"llvm.x86.3dnowa.pi2fw" => "__builtin_ia32_pi2fw",
"llvm.x86.aadd32" => "__builtin_ia32_aadd32",
"llvm.x86.aadd64" => "__builtin_ia32_aadd64",
"llvm.x86.aand32" => "__builtin_ia32_aand32",
@ -7334,6 +7328,207 @@ match name {
"llvm.x86.avx.vtestz.ps.256" => "__builtin_ia32_vtestzps256",
"llvm.x86.avx.vzeroall" => "__builtin_ia32_vzeroall",
"llvm.x86.avx.vzeroupper" => "__builtin_ia32_vzeroupper",
"llvm.x86.avx10.mask.vcvt2ps2phx.128" => "__builtin_ia32_vcvt2ps2phx128_mask",
"llvm.x86.avx10.mask.vcvt2ps2phx.256" => "__builtin_ia32_vcvt2ps2phx256_mask",
"llvm.x86.avx10.mask.vcvt2ps2phx.512" => "__builtin_ia32_vcvt2ps2phx512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8128" => "__builtin_ia32_vcvtbiasph2bf8_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8256" => "__builtin_ia32_vcvtbiasph2bf8_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8512" => "__builtin_ia32_vcvtbiasph2bf8_512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8s128" => "__builtin_ia32_vcvtbiasph2bf8s_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8s256" => "__builtin_ia32_vcvtbiasph2bf8s_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8s512" => "__builtin_ia32_vcvtbiasph2bf8s_512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8128" => "__builtin_ia32_vcvtbiasph2hf8_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8256" => "__builtin_ia32_vcvtbiasph2hf8_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8512" => "__builtin_ia32_vcvtbiasph2hf8_512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8s128" => "__builtin_ia32_vcvtbiasph2hf8s_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8s256" => "__builtin_ia32_vcvtbiasph2hf8s_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8s512" => "__builtin_ia32_vcvtbiasph2hf8s_512_mask",
"llvm.x86.avx10.mask.vcvthf82ph128" => "__builtin_ia32_vcvthf8_2ph128_mask",
"llvm.x86.avx10.mask.vcvthf82ph256" => "__builtin_ia32_vcvthf8_2ph256_mask",
"llvm.x86.avx10.mask.vcvthf82ph512" => "__builtin_ia32_vcvthf8_2ph512_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8128" => "__builtin_ia32_vcvtneph2bf8_128_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8256" => "__builtin_ia32_vcvtneph2bf8_256_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8512" => "__builtin_ia32_vcvtneph2bf8_512_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8s128" => "__builtin_ia32_vcvtneph2bf8s_128_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8s256" => "__builtin_ia32_vcvtneph2bf8s_256_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8s512" => "__builtin_ia32_vcvtneph2bf8s_512_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8128" => "__builtin_ia32_vcvtneph2hf8_128_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8256" => "__builtin_ia32_vcvtneph2hf8_256_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8512" => "__builtin_ia32_vcvtneph2hf8_512_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8s128" => "__builtin_ia32_vcvtneph2hf8s_128_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8s256" => "__builtin_ia32_vcvtneph2hf8s_256_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8s512" => "__builtin_ia32_vcvtneph2hf8s_512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2dq256" => "__builtin_ia32_vcvtpd2dq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2ph256" => "__builtin_ia32_vcvtpd2ph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2ps256" => "__builtin_ia32_vcvtpd2ps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2qq256" => "__builtin_ia32_vcvtpd2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2udq256" => "__builtin_ia32_vcvtpd2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2uqq256" => "__builtin_ia32_vcvtpd2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2dq256" => "__builtin_ia32_vcvtph2dq256_round_mask",
"llvm.x86.avx10.mask.vcvtph2ibs128" => "__builtin_ia32_vcvtph2ibs128_mask",
"llvm.x86.avx10.mask.vcvtph2ibs256" => "__builtin_ia32_vcvtph2ibs256_mask",
"llvm.x86.avx10.mask.vcvtph2ibs512" => "__builtin_ia32_vcvtph2ibs512_mask",
"llvm.x86.avx10.mask.vcvtph2iubs128" => "__builtin_ia32_vcvtph2iubs128_mask",
"llvm.x86.avx10.mask.vcvtph2iubs256" => "__builtin_ia32_vcvtph2iubs256_mask",
"llvm.x86.avx10.mask.vcvtph2iubs512" => "__builtin_ia32_vcvtph2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2pd256" => "__builtin_ia32_vcvtph2pd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2psx256" => "__builtin_ia32_vcvtph2psx256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2qq256" => "__builtin_ia32_vcvtph2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2udq256" => "__builtin_ia32_vcvtph2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2uqq256" => "__builtin_ia32_vcvtph2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2uw256" => "__builtin_ia32_vcvtph2uw256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2w256" => "__builtin_ia32_vcvtph2w256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2dq256" => "__builtin_ia32_vcvtps2dq256_round_mask",
"llvm.x86.avx10.mask.vcvtps2ibs128" => "__builtin_ia32_vcvtps2ibs128_mask",
"llvm.x86.avx10.mask.vcvtps2ibs256" => "__builtin_ia32_vcvtps2ibs256_mask",
"llvm.x86.avx10.mask.vcvtps2ibs512" => "__builtin_ia32_vcvtps2ibs512_mask",
"llvm.x86.avx10.mask.vcvtps2iubs128" => "__builtin_ia32_vcvtps2iubs128_mask",
"llvm.x86.avx10.mask.vcvtps2iubs256" => "__builtin_ia32_vcvtps2iubs256_mask",
"llvm.x86.avx10.mask.vcvtps2iubs512" => "__builtin_ia32_vcvtps2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2pd256" => "__builtin_ia32_vcvtps2pd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2ph256" => "__builtin_ia32_vcvtps2ph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2phx256" => "__builtin_ia32_vcvtps2phx256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2qq256" => "__builtin_ia32_vcvtps2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2udq256" => "__builtin_ia32_vcvtps2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2uqq256" => "__builtin_ia32_vcvtps2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2dq256" => "__builtin_ia32_vcvttpd2dq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2qq256" => "__builtin_ia32_vcvttpd2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2udq256" => "__builtin_ia32_vcvttpd2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2uqq256" => "__builtin_ia32_vcvttpd2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2dq256" => "__builtin_ia32_vcvttph2dq256_round_mask",
"llvm.x86.avx10.mask.vcvttph2ibs128" => "__builtin_ia32_vcvttph2ibs128_mask",
"llvm.x86.avx10.mask.vcvttph2ibs256" => "__builtin_ia32_vcvttph2ibs256_mask",
"llvm.x86.avx10.mask.vcvttph2ibs512" => "__builtin_ia32_vcvttph2ibs512_mask",
"llvm.x86.avx10.mask.vcvttph2iubs128" => "__builtin_ia32_vcvttph2iubs128_mask",
"llvm.x86.avx10.mask.vcvttph2iubs256" => "__builtin_ia32_vcvttph2iubs256_mask",
"llvm.x86.avx10.mask.vcvttph2iubs512" => "__builtin_ia32_vcvttph2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2qq256" => "__builtin_ia32_vcvttph2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2udq256" => "__builtin_ia32_vcvttph2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2uqq256" => "__builtin_ia32_vcvttph2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2uw256" => "__builtin_ia32_vcvttph2uw256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2w256" => "__builtin_ia32_vcvttph2w256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2dq256" => "__builtin_ia32_vcvttps2dq256_round_mask",
"llvm.x86.avx10.mask.vcvttps2ibs128" => "__builtin_ia32_vcvttps2ibs128_mask",
"llvm.x86.avx10.mask.vcvttps2ibs256" => "__builtin_ia32_vcvttps2ibs256_mask",
"llvm.x86.avx10.mask.vcvttps2ibs512" => "__builtin_ia32_vcvttps2ibs512_mask",
"llvm.x86.avx10.mask.vcvttps2iubs128" => "__builtin_ia32_vcvttps2iubs128_mask",
"llvm.x86.avx10.mask.vcvttps2iubs256" => "__builtin_ia32_vcvttps2iubs256_mask",
"llvm.x86.avx10.mask.vcvttps2iubs512" => "__builtin_ia32_vcvttps2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2qq256" => "__builtin_ia32_vcvttps2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2udq256" => "__builtin_ia32_vcvttps2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2uqq256" => "__builtin_ia32_vcvttps2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfcmaddcph256" => "__builtin_ia32_vfcmaddcph256_round_mask3",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfcmulcph256" => "__builtin_ia32_vfcmulcph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfixupimmpd256" => "__builtin_ia32_vfixupimmpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfixupimmps256" => "__builtin_ia32_vfixupimmps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfmaddcph256" => "__builtin_ia32_vfmaddcph256_round_mask3",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfmulcph256" => "__builtin_ia32_vfmulcph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexppd256" => "__builtin_ia32_vgetexppd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexpph256" => "__builtin_ia32_vgetexpph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexpps256" => "__builtin_ia32_vgetexpps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantpd256" => "__builtin_ia32_vgetmantpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantph256" => "__builtin_ia32_vgetmantph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantps256" => "__builtin_ia32_vgetmantps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxpd.round" => "__builtin_ia32_vminmaxpd512_round_mask",
"llvm.x86.avx10.mask.vminmaxpd128" => "__builtin_ia32_vminmaxpd128_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxpd256.round" => "__builtin_ia32_vminmaxpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxph.round" => "__builtin_ia32_vminmaxph512_round_mask",
"llvm.x86.avx10.mask.vminmaxph128" => "__builtin_ia32_vminmaxph128_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxph256.round" => "__builtin_ia32_vminmaxph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxps.round" => "__builtin_ia32_vminmaxps512_round_mask",
"llvm.x86.avx10.mask.vminmaxps128" => "__builtin_ia32_vminmaxps128_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxps256.round" => "__builtin_ia32_vminmaxps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxsd.round" => "__builtin_ia32_vminmaxsd_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxsh.round" => "__builtin_ia32_vminmaxsh_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxss.round" => "__builtin_ia32_vminmaxss_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrangepd256" => "__builtin_ia32_vrangepd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrangeps256" => "__builtin_ia32_vrangeps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreducepd256" => "__builtin_ia32_vreducepd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreduceph256" => "__builtin_ia32_vreduceph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreduceps256" => "__builtin_ia32_vreduceps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscalepd256" => "__builtin_ia32_vrndscalepd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscaleph256" => "__builtin_ia32_vrndscaleph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscaleps256" => "__builtin_ia32_vrndscaleps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefpd256" => "__builtin_ia32_vscalefpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefph256" => "__builtin_ia32_vscalefph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefps256" => "__builtin_ia32_vscalefps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfcmaddcph256" => "__builtin_ia32_vfcmaddcph256_round_maskz",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfixupimmpd256" => "__builtin_ia32_vfixupimmpd256_round_maskz",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfixupimmps256" => "__builtin_ia32_vfixupimmps256_round_maskz",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfmaddcph256" => "__builtin_ia32_vfmaddcph256_round_maskz",
"llvm.x86.avx10.vaddpd256" => "__builtin_ia32_vaddpd256_round",
"llvm.x86.avx10.vaddph256" => "__builtin_ia32_vaddph256_round",
"llvm.x86.avx10.vaddps256" => "__builtin_ia32_vaddps256_round",
"llvm.x86.avx10.vcvtne2ph2bf8128" => "__builtin_ia32_vcvtne2ph2bf8_128",
"llvm.x86.avx10.vcvtne2ph2bf8256" => "__builtin_ia32_vcvtne2ph2bf8_256",
"llvm.x86.avx10.vcvtne2ph2bf8512" => "__builtin_ia32_vcvtne2ph2bf8_512",
"llvm.x86.avx10.vcvtne2ph2bf8s128" => "__builtin_ia32_vcvtne2ph2bf8s_128",
"llvm.x86.avx10.vcvtne2ph2bf8s256" => "__builtin_ia32_vcvtne2ph2bf8s_256",
"llvm.x86.avx10.vcvtne2ph2bf8s512" => "__builtin_ia32_vcvtne2ph2bf8s_512",
"llvm.x86.avx10.vcvtne2ph2hf8128" => "__builtin_ia32_vcvtne2ph2hf8_128",
"llvm.x86.avx10.vcvtne2ph2hf8256" => "__builtin_ia32_vcvtne2ph2hf8_256",
"llvm.x86.avx10.vcvtne2ph2hf8512" => "__builtin_ia32_vcvtne2ph2hf8_512",
"llvm.x86.avx10.vcvtne2ph2hf8s128" => "__builtin_ia32_vcvtne2ph2hf8s_128",
"llvm.x86.avx10.vcvtne2ph2hf8s256" => "__builtin_ia32_vcvtne2ph2hf8s_256",
"llvm.x86.avx10.vcvtne2ph2hf8s512" => "__builtin_ia32_vcvtne2ph2hf8s_512",
"llvm.x86.avx10.vcvtnebf162ibs128" => "__builtin_ia32_vcvtnebf162ibs128",
"llvm.x86.avx10.vcvtnebf162ibs256" => "__builtin_ia32_vcvtnebf162ibs256",
"llvm.x86.avx10.vcvtnebf162ibs512" => "__builtin_ia32_vcvtnebf162ibs512",
"llvm.x86.avx10.vcvtnebf162iubs128" => "__builtin_ia32_vcvtnebf162iubs128",
"llvm.x86.avx10.vcvtnebf162iubs256" => "__builtin_ia32_vcvtnebf162iubs256",
"llvm.x86.avx10.vcvtnebf162iubs512" => "__builtin_ia32_vcvtnebf162iubs512",
"llvm.x86.avx10.vcvttnebf162ibs128" => "__builtin_ia32_vcvttnebf162ibs128",
"llvm.x86.avx10.vcvttnebf162ibs256" => "__builtin_ia32_vcvttnebf162ibs256",
"llvm.x86.avx10.vcvttnebf162ibs512" => "__builtin_ia32_vcvttnebf162ibs512",
"llvm.x86.avx10.vcvttnebf162iubs128" => "__builtin_ia32_vcvttnebf162iubs128",
"llvm.x86.avx10.vcvttnebf162iubs256" => "__builtin_ia32_vcvttnebf162iubs256",
"llvm.x86.avx10.vcvttnebf162iubs512" => "__builtin_ia32_vcvttnebf162iubs512",
"llvm.x86.avx10.vdivpd256" => "__builtin_ia32_vdivpd256_round",
"llvm.x86.avx10.vdivph256" => "__builtin_ia32_vdivph256_round",
"llvm.x86.avx10.vdivps256" => "__builtin_ia32_vdivps256_round",
"llvm.x86.avx10.vdpphps.128" => "__builtin_ia32_vdpphps128",
"llvm.x86.avx10.vdpphps.256" => "__builtin_ia32_vdpphps256",
"llvm.x86.avx10.vdpphps.512" => "__builtin_ia32_vdpphps512",
"llvm.x86.avx10.vfmaddsubpd256" => "__builtin_ia32_vfmaddsubpd256_round",
"llvm.x86.avx10.vfmaddsubph256" => "__builtin_ia32_vfmaddsubph256_round",
"llvm.x86.avx10.vfmaddsubps256" => "__builtin_ia32_vfmaddsubps256_round",
"llvm.x86.avx10.vmaxpd256" => "__builtin_ia32_vmaxpd256_round",
"llvm.x86.avx10.vmaxph256" => "__builtin_ia32_vmaxph256_round",
"llvm.x86.avx10.vmaxps256" => "__builtin_ia32_vmaxps256_round",
"llvm.x86.avx10.vminmaxnepbf16128" => "__builtin_ia32_vminmaxnepbf16128",
"llvm.x86.avx10.vminmaxnepbf16256" => "__builtin_ia32_vminmaxnepbf16256",
"llvm.x86.avx10.vminmaxnepbf16512" => "__builtin_ia32_vminmaxnepbf16512",
"llvm.x86.avx10.vminmaxpd128" => "__builtin_ia32_vminmaxpd128",
"llvm.x86.avx10.vminmaxpd256" => "__builtin_ia32_vminmaxpd256",
"llvm.x86.avx10.vminmaxph128" => "__builtin_ia32_vminmaxph128",
"llvm.x86.avx10.vminmaxph256" => "__builtin_ia32_vminmaxph256",
"llvm.x86.avx10.vminmaxps128" => "__builtin_ia32_vminmaxps128",
"llvm.x86.avx10.vminmaxps256" => "__builtin_ia32_vminmaxps256",
"llvm.x86.avx10.vminpd256" => "__builtin_ia32_vminpd256_round",
"llvm.x86.avx10.vminph256" => "__builtin_ia32_vminph256_round",
"llvm.x86.avx10.vminps256" => "__builtin_ia32_vminps256_round",
"llvm.x86.avx10.vmpsadbw.512" => "__builtin_ia32_mpsadbw512",
"llvm.x86.avx10.vmulpd256" => "__builtin_ia32_vmulpd256_round",
"llvm.x86.avx10.vmulph256" => "__builtin_ia32_vmulph256_round",
"llvm.x86.avx10.vmulps256" => "__builtin_ia32_vmulps256_round",
"llvm.x86.avx10.vpdpbssd.512" => "__builtin_ia32_vpdpbssd512",
"llvm.x86.avx10.vpdpbssds.512" => "__builtin_ia32_vpdpbssds512",
"llvm.x86.avx10.vpdpbsud.512" => "__builtin_ia32_vpdpbsud512",
"llvm.x86.avx10.vpdpbsuds.512" => "__builtin_ia32_vpdpbsuds512",
"llvm.x86.avx10.vpdpbuud.512" => "__builtin_ia32_vpdpbuud512",
"llvm.x86.avx10.vpdpbuuds.512" => "__builtin_ia32_vpdpbuuds512",
"llvm.x86.avx10.vpdpwsud.512" => "__builtin_ia32_vpdpwsud512",
"llvm.x86.avx10.vpdpwsuds.512" => "__builtin_ia32_vpdpwsuds512",
"llvm.x86.avx10.vpdpwusd.512" => "__builtin_ia32_vpdpwusd512",
"llvm.x86.avx10.vpdpwusds.512" => "__builtin_ia32_vpdpwusds512",
"llvm.x86.avx10.vpdpwuud.512" => "__builtin_ia32_vpdpwuud512",
"llvm.x86.avx10.vpdpwuuds.512" => "__builtin_ia32_vpdpwuuds512",
"llvm.x86.avx10.vsqrtpd256" => "__builtin_ia32_vsqrtpd256_round",
"llvm.x86.avx10.vsqrtph256" => "__builtin_ia32_vsqrtph256_round",
"llvm.x86.avx10.vsqrtps256" => "__builtin_ia32_vsqrtps256_round",
"llvm.x86.avx10.vsubpd256" => "__builtin_ia32_vsubpd256_round",
"llvm.x86.avx10.vsubph256" => "__builtin_ia32_vsubph256_round",
"llvm.x86.avx10.vsubps256" => "__builtin_ia32_vsubps256_round",
"llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gatherd_d",
"llvm.x86.avx2.gather.d.d.256" => "__builtin_ia32_gatherd_d256",
"llvm.x86.avx2.gather.d.pd" => "__builtin_ia32_gatherd_pd",
@ -8738,10 +8933,10 @@ match name {
"llvm.x86.avx512.rcp14.ss" => "__builtin_ia32_rcp14ss_mask",
"llvm.x86.avx512.rcp28.pd" => "__builtin_ia32_rcp28pd_mask",
"llvm.x86.avx512.rcp28.ps" => "__builtin_ia32_rcp28ps_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
"llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
"llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
"llvm.x86.avx512.rndscale.sd" => "__builtin_ia32_rndscalesd",
"llvm.x86.avx512.rndscale.ss" => "__builtin_ia32_rndscaless",
"llvm.x86.avx512.rsqrt14.pd.128" => "__builtin_ia32_rsqrt14pd128_mask",
@ -8754,10 +8949,10 @@ match name {
"llvm.x86.avx512.rsqrt14.ss" => "__builtin_ia32_rsqrt14ss_mask",
"llvm.x86.avx512.rsqrt28.pd" => "__builtin_ia32_rsqrt28pd_mask",
"llvm.x86.avx512.rsqrt28.ps" => "__builtin_ia32_rsqrt28ps_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
"llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
"llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
"llvm.x86.avx512.scatter.dpd.512" => "__builtin_ia32_scattersiv8df",
"llvm.x86.avx512.scatter.dpi.512" => "__builtin_ia32_scattersiv16si",
"llvm.x86.avx512.scatter.dpq.512" => "__builtin_ia32_scattersiv8di",
@ -9082,75 +9277,6 @@ match name {
"llvm.x86.lwpval64" => "__builtin_ia32_lwpval64",
"llvm.x86.mmx.emms" => "__builtin_ia32_emms",
"llvm.x86.mmx.femms" => "__builtin_ia32_femms",
"llvm.x86.mmx.maskmovq" => "__builtin_ia32_maskmovq",
"llvm.x86.mmx.movnt.dq" => "__builtin_ia32_movntq",
"llvm.x86.mmx.packssdw" => "__builtin_ia32_packssdw",
"llvm.x86.mmx.packsswb" => "__builtin_ia32_packsswb",
"llvm.x86.mmx.packuswb" => "__builtin_ia32_packuswb",
"llvm.x86.mmx.padd.b" => "__builtin_ia32_paddb",
"llvm.x86.mmx.padd.d" => "__builtin_ia32_paddd",
"llvm.x86.mmx.padd.q" => "__builtin_ia32_paddq",
"llvm.x86.mmx.padd.w" => "__builtin_ia32_paddw",
"llvm.x86.mmx.padds.b" => "__builtin_ia32_paddsb",
"llvm.x86.mmx.padds.w" => "__builtin_ia32_paddsw",
"llvm.x86.mmx.paddus.b" => "__builtin_ia32_paddusb",
"llvm.x86.mmx.paddus.w" => "__builtin_ia32_paddusw",
"llvm.x86.mmx.palignr.b" => "__builtin_ia32_palignr",
"llvm.x86.mmx.pand" => "__builtin_ia32_pand",
"llvm.x86.mmx.pandn" => "__builtin_ia32_pandn",
"llvm.x86.mmx.pavg.b" => "__builtin_ia32_pavgb",
"llvm.x86.mmx.pavg.w" => "__builtin_ia32_pavgw",
"llvm.x86.mmx.pcmpeq.b" => "__builtin_ia32_pcmpeqb",
"llvm.x86.mmx.pcmpeq.d" => "__builtin_ia32_pcmpeqd",
"llvm.x86.mmx.pcmpeq.w" => "__builtin_ia32_pcmpeqw",
"llvm.x86.mmx.pcmpgt.b" => "__builtin_ia32_pcmpgtb",
"llvm.x86.mmx.pcmpgt.d" => "__builtin_ia32_pcmpgtd",
"llvm.x86.mmx.pcmpgt.w" => "__builtin_ia32_pcmpgtw",
"llvm.x86.mmx.pextr.w" => "__builtin_ia32_vec_ext_v4hi",
"llvm.x86.mmx.pinsr.w" => "__builtin_ia32_vec_set_v4hi",
"llvm.x86.mmx.pmadd.wd" => "__builtin_ia32_pmaddwd",
"llvm.x86.mmx.pmaxs.w" => "__builtin_ia32_pmaxsw",
"llvm.x86.mmx.pmaxu.b" => "__builtin_ia32_pmaxub",
"llvm.x86.mmx.pmins.w" => "__builtin_ia32_pminsw",
"llvm.x86.mmx.pminu.b" => "__builtin_ia32_pminub",
"llvm.x86.mmx.pmovmskb" => "__builtin_ia32_pmovmskb",
"llvm.x86.mmx.pmulh.w" => "__builtin_ia32_pmulhw",
"llvm.x86.mmx.pmulhu.w" => "__builtin_ia32_pmulhuw",
"llvm.x86.mmx.pmull.w" => "__builtin_ia32_pmullw",
"llvm.x86.mmx.pmulu.dq" => "__builtin_ia32_pmuludq",
"llvm.x86.mmx.por" => "__builtin_ia32_por",
"llvm.x86.mmx.psad.bw" => "__builtin_ia32_psadbw",
"llvm.x86.mmx.psll.d" => "__builtin_ia32_pslld",
"llvm.x86.mmx.psll.q" => "__builtin_ia32_psllq",
"llvm.x86.mmx.psll.w" => "__builtin_ia32_psllw",
"llvm.x86.mmx.pslli.d" => "__builtin_ia32_pslldi",
"llvm.x86.mmx.pslli.q" => "__builtin_ia32_psllqi",
"llvm.x86.mmx.pslli.w" => "__builtin_ia32_psllwi",
"llvm.x86.mmx.psra.d" => "__builtin_ia32_psrad",
"llvm.x86.mmx.psra.w" => "__builtin_ia32_psraw",
"llvm.x86.mmx.psrai.d" => "__builtin_ia32_psradi",
"llvm.x86.mmx.psrai.w" => "__builtin_ia32_psrawi",
"llvm.x86.mmx.psrl.d" => "__builtin_ia32_psrld",
"llvm.x86.mmx.psrl.q" => "__builtin_ia32_psrlq",
"llvm.x86.mmx.psrl.w" => "__builtin_ia32_psrlw",
"llvm.x86.mmx.psrli.d" => "__builtin_ia32_psrldi",
"llvm.x86.mmx.psrli.q" => "__builtin_ia32_psrlqi",
"llvm.x86.mmx.psrli.w" => "__builtin_ia32_psrlwi",
"llvm.x86.mmx.psub.b" => "__builtin_ia32_psubb",
"llvm.x86.mmx.psub.d" => "__builtin_ia32_psubd",
"llvm.x86.mmx.psub.q" => "__builtin_ia32_psubq",
"llvm.x86.mmx.psub.w" => "__builtin_ia32_psubw",
"llvm.x86.mmx.psubs.b" => "__builtin_ia32_psubsb",
"llvm.x86.mmx.psubs.w" => "__builtin_ia32_psubsw",
"llvm.x86.mmx.psubus.b" => "__builtin_ia32_psubusb",
"llvm.x86.mmx.psubus.w" => "__builtin_ia32_psubusw",
"llvm.x86.mmx.punpckhbw" => "__builtin_ia32_punpckhbw",
"llvm.x86.mmx.punpckhdq" => "__builtin_ia32_punpckhdq",
"llvm.x86.mmx.punpckhwd" => "__builtin_ia32_punpckhwd",
"llvm.x86.mmx.punpcklbw" => "__builtin_ia32_punpcklbw",
"llvm.x86.mmx.punpckldq" => "__builtin_ia32_punpckldq",
"llvm.x86.mmx.punpcklwd" => "__builtin_ia32_punpcklwd",
"llvm.x86.mmx.pxor" => "__builtin_ia32_pxor",
"llvm.x86.monitorx" => "__builtin_ia32_monitorx",
"llvm.x86.movdir64b" => "__builtin_ia32_movdir64b",
"llvm.x86.mwaitx" => "__builtin_ia32_mwaitx",
@ -9193,16 +9319,10 @@ match name {
"llvm.x86.sse.comile.ss" => "__builtin_ia32_comile",
"llvm.x86.sse.comilt.ss" => "__builtin_ia32_comilt",
"llvm.x86.sse.comineq.ss" => "__builtin_ia32_comineq",
"llvm.x86.sse.cvtpd2pi" => "__builtin_ia32_cvtpd2pi",
"llvm.x86.sse.cvtpi2pd" => "__builtin_ia32_cvtpi2pd",
"llvm.x86.sse.cvtpi2ps" => "__builtin_ia32_cvtpi2ps",
"llvm.x86.sse.cvtps2pi" => "__builtin_ia32_cvtps2pi",
"llvm.x86.sse.cvtsi2ss" => "__builtin_ia32_cvtsi2ss",
"llvm.x86.sse.cvtsi642ss" => "__builtin_ia32_cvtsi642ss",
"llvm.x86.sse.cvtss2si" => "__builtin_ia32_cvtss2si",
"llvm.x86.sse.cvtss2si64" => "__builtin_ia32_cvtss2si64",
"llvm.x86.sse.cvttpd2pi" => "__builtin_ia32_cvttpd2pi",
"llvm.x86.sse.cvttps2pi" => "__builtin_ia32_cvttps2pi",
"llvm.x86.sse.cvttss2si" => "__builtin_ia32_cvttss2si",
"llvm.x86.sse.cvttss2si64" => "__builtin_ia32_cvttss2si64",
"llvm.x86.sse.div.ss" => "__builtin_ia32_divss",
@ -9212,7 +9332,6 @@ match name {
"llvm.x86.sse.min.ss" => "__builtin_ia32_minss",
"llvm.x86.sse.movmsk.ps" => "__builtin_ia32_movmskps",
"llvm.x86.sse.mul.ss" => "__builtin_ia32_mulss",
"llvm.x86.sse.pshuf.w" => "__builtin_ia32_pshufw",
"llvm.x86.sse.rcp.ps" => "__builtin_ia32_rcpps",
"llvm.x86.sse.rcp.ss" => "__builtin_ia32_rcpss",
"llvm.x86.sse.rsqrt.ps" => "__builtin_ia32_rsqrtps",
@ -9398,35 +9517,20 @@ match name {
"llvm.x86.sse4a.insertqi" => "__builtin_ia32_insertqi",
"llvm.x86.sse4a.movnt.sd" => "__builtin_ia32_movntsd",
"llvm.x86.sse4a.movnt.ss" => "__builtin_ia32_movntss",
"llvm.x86.ssse3.pabs.b" => "__builtin_ia32_pabsb",
"llvm.x86.ssse3.pabs.b.128" => "__builtin_ia32_pabsb128",
"llvm.x86.ssse3.pabs.d" => "__builtin_ia32_pabsd",
"llvm.x86.ssse3.pabs.d.128" => "__builtin_ia32_pabsd128",
"llvm.x86.ssse3.pabs.w" => "__builtin_ia32_pabsw",
"llvm.x86.ssse3.pabs.w.128" => "__builtin_ia32_pabsw128",
"llvm.x86.ssse3.phadd.d" => "__builtin_ia32_phaddd",
"llvm.x86.ssse3.phadd.d.128" => "__builtin_ia32_phaddd128",
"llvm.x86.ssse3.phadd.sw" => "__builtin_ia32_phaddsw",
"llvm.x86.ssse3.phadd.sw.128" => "__builtin_ia32_phaddsw128",
"llvm.x86.ssse3.phadd.w" => "__builtin_ia32_phaddw",
"llvm.x86.ssse3.phadd.w.128" => "__builtin_ia32_phaddw128",
"llvm.x86.ssse3.phsub.d" => "__builtin_ia32_phsubd",
"llvm.x86.ssse3.phsub.d.128" => "__builtin_ia32_phsubd128",
"llvm.x86.ssse3.phsub.sw" => "__builtin_ia32_phsubsw",
"llvm.x86.ssse3.phsub.sw.128" => "__builtin_ia32_phsubsw128",
"llvm.x86.ssse3.phsub.w" => "__builtin_ia32_phsubw",
"llvm.x86.ssse3.phsub.w.128" => "__builtin_ia32_phsubw128",
"llvm.x86.ssse3.pmadd.ub.sw" => "__builtin_ia32_pmaddubsw",
"llvm.x86.ssse3.pmadd.ub.sw.128" => "__builtin_ia32_pmaddubsw128",
"llvm.x86.ssse3.pmul.hr.sw" => "__builtin_ia32_pmulhrsw",
"llvm.x86.ssse3.pmul.hr.sw.128" => "__builtin_ia32_pmulhrsw128",
"llvm.x86.ssse3.pshuf.b" => "__builtin_ia32_pshufb",
"llvm.x86.ssse3.pshuf.b.128" => "__builtin_ia32_pshufb128",
"llvm.x86.ssse3.psign.b" => "__builtin_ia32_psignb",
"llvm.x86.ssse3.psign.b.128" => "__builtin_ia32_psignb128",
"llvm.x86.ssse3.psign.d" => "__builtin_ia32_psignd",
"llvm.x86.ssse3.psign.d.128" => "__builtin_ia32_psignd128",
"llvm.x86.ssse3.psign.w" => "__builtin_ia32_psignw",
"llvm.x86.ssse3.psign.w.128" => "__builtin_ia32_psignw128",
"llvm.x86.sttilecfg" => "__builtin_ia32_tile_storeconfig",
"llvm.x86.stui" => "__builtin_ia32_stui",

383
compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs
View file

@ -1,11 +1,44 @@
use std::borrow::Cow;
use gccjit::{CType, Context};
use gccjit::{Function, FunctionPtrType, RValue, ToRValue, UnaryOp};
use rustc_codegen_ssa::traits::BuilderMethods;
use crate::builder::Builder;
use crate::context::CodegenCx;
#[cfg_attr(not(feature = "master"), allow(unused_variables))]
pub fn adjust_function<'gcc>(
context: &'gcc Context<'gcc>,
func_name: &str,
func_ptr: RValue<'gcc>,
args: &[RValue<'gcc>],
) -> RValue<'gcc> {
// FIXME: we should not need this hack: this is required because both _mm_fcmadd_sch
// and _mm_mask3_fcmadd_round_sch calls llvm.x86.avx512fp16.mask.vfcmadd.csh and we
// seem to need to map this one LLVM intrinsic to 2 different GCC builtins.
#[cfg(feature = "master")]
match func_name {
"__builtin_ia32_vfcmaddcsh_mask3_round" => {
if format!("{:?}", args[3]).ends_with("255") {
return context
.get_target_builtin_function("__builtin_ia32_vfcmaddcsh_mask_round")
.get_address(None);
}
}
"__builtin_ia32_vfmaddcsh_mask3_round" => {
if format!("{:?}", args[3]).ends_with("255") {
return context
.get_target_builtin_function("__builtin_ia32_vfmaddcsh_mask_round")
.get_address(None);
}
}
_ => (),
}
func_ptr
}
pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
builder: &Builder<'a, 'gcc, 'tcx>,
gcc_func: FunctionPtrType<'gcc>,
@ -13,6 +46,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
func_name: &str,
original_function_name: Option<&String>,
) -> Cow<'b, [RValue<'gcc>]> {
// TODO: this might not be a good way to workaround the missing tile builtins.
if func_name == "__builtin_trap" {
return vec![].into();
}
// Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
// arguments here.
if gcc_func.get_param_count() != args.len() {
@ -147,7 +185,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
| "__builtin_ia32_psrav16hi_mask"
| "__builtin_ia32_psrav8hi_mask"
| "__builtin_ia32_permvarhi256_mask"
| "__builtin_ia32_permvarhi128_mask" => {
| "__builtin_ia32_permvarhi128_mask"
| "__builtin_ia32_maxph128_mask"
| "__builtin_ia32_maxph256_mask"
| "__builtin_ia32_minph128_mask"
| "__builtin_ia32_minph256_mask" => {
let mut new_args = args.to_vec();
let arg3_type = gcc_func.get_param_type(2);
let vector_type = arg3_type.dyncast_vector().expect("vector type");
@ -182,7 +224,19 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
| "__builtin_ia32_vplzcntd_128_mask"
| "__builtin_ia32_vplzcntq_512_mask"
| "__builtin_ia32_vplzcntq_256_mask"
| "__builtin_ia32_vplzcntq_128_mask" => {
| "__builtin_ia32_vplzcntq_128_mask"
| "__builtin_ia32_cvtqq2pd128_mask"
| "__builtin_ia32_cvtqq2pd256_mask"
| "__builtin_ia32_cvtqq2ps256_mask"
| "__builtin_ia32_cvtuqq2pd128_mask"
| "__builtin_ia32_cvtuqq2pd256_mask"
| "__builtin_ia32_cvtuqq2ps256_mask"
| "__builtin_ia32_vcvtw2ph128_mask"
| "__builtin_ia32_vcvtw2ph256_mask"
| "__builtin_ia32_vcvtuw2ph128_mask"
| "__builtin_ia32_vcvtuw2ph256_mask"
| "__builtin_ia32_vcvtdq2ph256_mask"
| "__builtin_ia32_vcvtudq2ph256_mask" => {
let mut new_args = args.to_vec();
// Remove last arg as it doesn't seem to be used in GCC and is always false.
new_args.pop();
@ -281,7 +335,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
new_args.push(last_arg);
args = new_args.into();
}
"__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
"__builtin_ia32_vfmaddsubps512_mask"
| "__builtin_ia32_vfmaddsubpd512_mask"
| "__builtin_ia32_cmpsh_mask_round"
| "__builtin_ia32_vfmaddph512_mask"
| "__builtin_ia32_vfmaddsubph512_mask" => {
let mut new_args = args.to_vec();
let last_arg = new_args.pop().expect("last arg");
let arg4_type = gcc_func.get_param_type(3);
@ -304,9 +362,8 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
| "__builtin_ia32_vpermi2varpd128_mask"
| "__builtin_ia32_vpmadd52huq512_mask"
| "__builtin_ia32_vpmadd52luq512_mask"
| "__builtin_ia32_vpmadd52huq256_mask"
| "__builtin_ia32_vpmadd52luq256_mask"
| "__builtin_ia32_vpmadd52huq128_mask" => {
| "__builtin_ia32_vfmaddsubph128_mask"
| "__builtin_ia32_vfmaddsubph256_mask" => {
let mut new_args = args.to_vec();
let arg4_type = gcc_func.get_param_type(3);
let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
@ -355,7 +412,14 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
args = vec![new_args[1], new_args[0], new_args[2], minus_one].into();
}
"__builtin_ia32_xrstor" | "__builtin_ia32_xsavec" => {
"__builtin_ia32_xrstor"
| "__builtin_ia32_xrstor64"
| "__builtin_ia32_xsavec"
| "__builtin_ia32_xsavec64"
| "__builtin_ia32_xsave"
| "__builtin_ia32_xsave64"
| "__builtin_ia32_xsaveopt"
| "__builtin_ia32_xsaveopt64" => {
let new_args = args.to_vec();
let thirty_two = builder.context.new_rvalue_from_int(new_args[1].get_type(), 32);
let arg2 = new_args[1] << thirty_two | new_args[2];
@ -378,11 +442,76 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
);
args = vec![arg.get_address(None)].into();
}
"__builtin_ia32_cvtqq2pd512_mask"
| "__builtin_ia32_cvtqq2ps512_mask"
| "__builtin_ia32_cvtuqq2pd512_mask"
| "__builtin_ia32_cvtuqq2ps512_mask"
| "__builtin_ia32_sqrtph512_mask_round"
| "__builtin_ia32_vcvtw2ph512_mask_round"
| "__builtin_ia32_vcvtuw2ph512_mask_round"
| "__builtin_ia32_vcvtdq2ph512_mask_round"
| "__builtin_ia32_vcvtudq2ph512_mask_round"
| "__builtin_ia32_vcvtqq2ph512_mask_round"
| "__builtin_ia32_vcvtuqq2ph512_mask_round" => {
let mut old_args = args.to_vec();
let mut new_args = vec![];
new_args.push(old_args.swap_remove(0));
let arg2_type = gcc_func.get_param_type(1);
let vector_type = arg2_type.dyncast_vector().expect("vector type");
let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
let num_units = vector_type.get_num_units();
let first_arg =
builder.context.new_rvalue_from_vector(None, arg2_type, &vec![zero; num_units]);
new_args.push(first_arg);
let arg3_type = gcc_func.get_param_type(2);
let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1);
new_args.push(minus_one);
new_args.push(old_args.swap_remove(0));
args = new_args.into();
}
"__builtin_ia32_addph512_mask_round"
| "__builtin_ia32_subph512_mask_round"
| "__builtin_ia32_mulph512_mask_round"
| "__builtin_ia32_divph512_mask_round"
| "__builtin_ia32_maxph512_mask_round"
| "__builtin_ia32_minph512_mask_round" => {
let mut new_args = args.to_vec();
let last_arg = new_args.pop().expect("last arg");
let arg3_type = gcc_func.get_param_type(2);
let vector_type = arg3_type.dyncast_vector().expect("vector type");
let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
let num_units = vector_type.get_num_units();
let first_arg =
builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units]);
new_args.push(first_arg);
let arg4_type = gcc_func.get_param_type(3);
let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
new_args.push(minus_one);
new_args.push(last_arg);
args = new_args.into();
}
// NOTE: the LLVM intrinsics receive 3 floats, but the GCC builtin requires 3 vectors.
"__builtin_ia32_vfmaddsh3_mask" => {
let new_args = args.to_vec();
let arg1_type = gcc_func.get_param_type(0);
let arg2_type = gcc_func.get_param_type(1);
let arg3_type = gcc_func.get_param_type(2);
let arg4_type = gcc_func.get_param_type(3);
let a = builder.context.new_rvalue_from_vector(None, arg1_type, &[new_args[0]; 8]);
let b = builder.context.new_rvalue_from_vector(None, arg2_type, &[new_args[1]; 8]);
let c = builder.context.new_rvalue_from_vector(None, arg3_type, &[new_args[2]; 8]);
let arg4 = builder.context.new_rvalue_from_int(arg4_type, -1);
args = vec![a, b, c, arg4, new_args[3]].into();
}
_ => (),
}
} else {
match func_name {
"__builtin_ia32_rndscaless_mask_round" | "__builtin_ia32_rndscalesd_mask_round" => {
"__builtin_ia32_rndscaless_mask_round"
| "__builtin_ia32_rndscalesd_mask_round"
| "__builtin_ia32_reducesh_mask_round" => {
let new_args = args.to_vec();
let arg3_type = gcc_func.get_param_type(2);
let arg3 = builder.context.new_cast(None, new_args[4], arg3_type);
@ -390,7 +519,7 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
let arg4 = builder.context.new_bitcast(None, new_args[2], arg4_type);
args = vec![new_args[0], new_args[1], arg3, arg4, new_args[3], new_args[5]].into();
}
// NOTE: the LLVM intrinsic receives 3 floats, but the GCC builtin requires 3 vectors.
// NOTE: the LLVM intrinsics receive 3 floats, but the GCC builtin requires 3 vectors.
// FIXME: the intrinsics like _mm_mask_fmadd_sd should probably directly call the GCC
// intrinsic to avoid this.
"__builtin_ia32_vfmaddss3_round" => {
@ -473,6 +602,52 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
let new_args = args.to_vec();
args = vec![new_args[1], new_args[0], new_args[2]].into();
}
"__builtin_ia32_rangesd128_mask_round"
| "__builtin_ia32_rangess128_mask_round"
| "__builtin_ia32_reducesd_mask_round"
| "__builtin_ia32_reducess_mask_round" => {
let new_args = args.to_vec();
args = vec![
new_args[0],
new_args[1],
new_args[4],
new_args[2],
new_args[3],
new_args[5],
]
.into();
}
"__builtin_ia32_rndscalesh_mask_round" => {
let new_args = args.to_vec();
args = vec![
new_args[0],
new_args[1],
new_args[4],
new_args[2],
new_args[3],
new_args[5],
]
.into();
}
"fma" => {
let mut new_args = args.to_vec();
new_args[0] = builder.context.new_cast(None, new_args[0], builder.double_type);
new_args[1] = builder.context.new_cast(None, new_args[1], builder.double_type);
new_args[2] = builder.context.new_cast(None, new_args[2], builder.double_type);
args = new_args.into();
}
"__builtin_ia32_sqrtsh_mask_round"
| "__builtin_ia32_vcvtss2sh_mask_round"
| "__builtin_ia32_vcvtsd2sh_mask_round"
| "__builtin_ia32_vcvtsh2ss_mask_round"
| "__builtin_ia32_vcvtsh2sd_mask_round"
| "__builtin_ia32_rcpsh_mask"
| "__builtin_ia32_rsqrtsh_mask" => {
// The first two arguments are inverted, so swap them.
let mut new_args = args.to_vec();
new_args.swap(0, 1);
args = new_args.into();
}
_ => (),
}
}
@ -489,7 +664,9 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
orig_args: &[RValue<'gcc>],
) -> RValue<'gcc> {
match func_name {
"__builtin_ia32_vfmaddss3_round" | "__builtin_ia32_vfmaddsd3_round" => {
"__builtin_ia32_vfmaddss3_round"
| "__builtin_ia32_vfmaddsd3_round"
| "__builtin_ia32_vfmaddsh3_mask" => {
#[cfg(feature = "master")]
{
let zero = builder.context.new_rvalue_zero(builder.int_type);
@ -546,6 +723,10 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
success_variable.to_rvalue(),
]);
}
"fma" => {
let f16_type = builder.context.new_c_type(CType::Float16);
return_value = builder.context.new_cast(None, return_value, f16_type);
}
_ => (),
}
@ -779,7 +960,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
"llvm.x86.avx512.mask.cmp.b.256" => "__builtin_ia32_cmpb256_mask",
"llvm.x86.avx512.mask.cmp.b.128" => "__builtin_ia32_cmpb128_mask",
"llvm.x86.xrstor" => "__builtin_ia32_xrstor",
"llvm.x86.xrstor64" => "__builtin_ia32_xrstor64",
"llvm.x86.xsavec" => "__builtin_ia32_xsavec",
"llvm.x86.xsavec64" => "__builtin_ia32_xsavec64",
"llvm.x86.addcarry.32" => "__builtin_ia32_addcarryx_u32",
"llvm.x86.subborrow.32" => "__builtin_ia32_sbb_u32",
"llvm.x86.avx512.mask.compress.store.w.512" => "__builtin_ia32_compressstoreuhi512_mask",
@ -968,9 +1151,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
"llvm.x86.avx512.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128_mask",
"llvm.x86.avx512.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_mask",
"llvm.x86.avx512.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_mask",
"llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_mask",
"llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_mask",
"llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_mask",
"llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256",
"llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256",
"llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128",
"llvm.x86.avx512.vpdpwssd.512" => "__builtin_ia32_vpdpwssd_v16si",
"llvm.x86.avx512.vpdpwssd.256" => "__builtin_ia32_vpdpwssd_v8si",
"llvm.x86.avx512.vpdpwssd.128" => "__builtin_ia32_vpdpwssd_v4si",
@ -983,6 +1166,180 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
"llvm.x86.avx512.vpdpbusds.512" => "__builtin_ia32_vpdpbusds_v16si",
"llvm.x86.avx512.vpdpbusds.256" => "__builtin_ia32_vpdpbusds_v8si",
"llvm.x86.avx512.vpdpbusds.128" => "__builtin_ia32_vpdpbusds_v4si",
"llvm.x86.xsave" => "__builtin_ia32_xsave",
"llvm.x86.xsave64" => "__builtin_ia32_xsave64",
"llvm.x86.xsaveopt" => "__builtin_ia32_xsaveopt",
"llvm.x86.xsaveopt64" => "__builtin_ia32_xsaveopt64",
"llvm.x86.avx512.mask.loadu.w.512" => "__builtin_ia32_loaddquhi512_mask",
"llvm.x86.avx512.mask.loadu.b.512" => "__builtin_ia32_loaddquqi512_mask",
"llvm.x86.avx512.mask.loadu.w.256" => "__builtin_ia32_loaddquhi256_mask",
"llvm.x86.avx512.mask.loadu.b.256" => "__builtin_ia32_loaddquqi256_mask",
"llvm.x86.avx512.mask.loadu.w.128" => "__builtin_ia32_loaddquhi128_mask",
"llvm.x86.avx512.mask.loadu.b.128" => "__builtin_ia32_loaddquqi128_mask",
"llvm.x86.avx512.mask.storeu.w.512" => "__builtin_ia32_storedquhi512_mask",
"llvm.x86.avx512.mask.storeu.b.512" => "__builtin_ia32_storedquqi512_mask",
"llvm.x86.avx512.mask.storeu.w.256" => "__builtin_ia32_storedquhi256_mask",
"llvm.x86.avx512.mask.storeu.b.256" => "__builtin_ia32_storedquqi256_mask",
"llvm.x86.avx512.mask.storeu.w.128" => "__builtin_ia32_storedquhi128_mask",
"llvm.x86.avx512.mask.storeu.b.128" => "__builtin_ia32_storedquqi128_mask",
"llvm.x86.avx512.mask.expand.load.w.512" => "__builtin_ia32_expandloadhi512_mask",
"llvm.x86.avx512.mask.expand.load.w.256" => "__builtin_ia32_expandloadhi256_mask",
"llvm.x86.avx512.mask.expand.load.w.128" => "__builtin_ia32_expandloadhi128_mask",
"llvm.x86.avx512.mask.expand.load.b.512" => "__builtin_ia32_expandloadqi512_mask",
"llvm.x86.avx512.mask.expand.load.b.256" => "__builtin_ia32_expandloadqi256_mask",
"llvm.x86.avx512.mask.expand.load.b.128" => "__builtin_ia32_expandloadqi128_mask",
"llvm.x86.avx512.sitofp.round.v8f64.v8i64" => "__builtin_ia32_cvtqq2pd512_mask",
"llvm.x86.avx512.sitofp.round.v2f64.v2i64" => "__builtin_ia32_cvtqq2pd128_mask",
"llvm.x86.avx512.sitofp.round.v4f64.v4i64" => "__builtin_ia32_cvtqq2pd256_mask",
"llvm.x86.avx512.sitofp.round.v8f32.v8i64" => "__builtin_ia32_cvtqq2ps512_mask",
"llvm.x86.avx512.sitofp.round.v4f32.v4i64" => "__builtin_ia32_cvtqq2ps256_mask",
"llvm.x86.avx512.uitofp.round.v8f64.v8u64" => "__builtin_ia32_cvtuqq2pd512_mask",
"llvm.x86.avx512.uitofp.round.v2f64.v2u64" => "__builtin_ia32_cvtuqq2pd128_mask",
"llvm.x86.avx512.uitofp.round.v4f64.v4u64" => "__builtin_ia32_cvtuqq2pd256_mask",
"llvm.x86.avx512.uitofp.round.v8f32.v8u64" => "__builtin_ia32_cvtuqq2ps512_mask",
"llvm.x86.avx512.uitofp.round.v4f32.v4u64" => "__builtin_ia32_cvtuqq2ps256_mask",
"llvm.x86.avx512.mask.reduce.pd.512" => "__builtin_ia32_reducepd512_mask_round",
"llvm.x86.avx512.mask.reduce.ps.512" => "__builtin_ia32_reduceps512_mask_round",
"llvm.x86.avx512.mask.reduce.sd" => "__builtin_ia32_reducesd_mask_round",
"llvm.x86.avx512.mask.reduce.ss" => "__builtin_ia32_reducess_mask_round",
"llvm.x86.avx512.mask.loadu.d.256" => "__builtin_ia32_loaddqusi256_mask",
"llvm.x86.avx512.mask.loadu.q.256" => "__builtin_ia32_loaddqudi256_mask",
"llvm.x86.avx512.mask.loadu.ps.256" => "__builtin_ia32_loadups256_mask",
"llvm.x86.avx512.mask.loadu.pd.256" => "__builtin_ia32_loadupd256_mask",
"llvm.x86.avx512.mask.loadu.d.128" => "__builtin_ia32_loaddqusi128_mask",
"llvm.x86.avx512.mask.loadu.q.128" => "__builtin_ia32_loaddqudi128_mask",
"llvm.x86.avx512.mask.loadu.ps.128" => "__builtin_ia32_loadups128_mask",
"llvm.x86.avx512.mask.loadu.pd.128" => "__builtin_ia32_loadupd128_mask",
"llvm.x86.avx512.mask.load.d.512" => "__builtin_ia32_movdqa32load512_mask",
"llvm.x86.avx512.mask.load.q.512" => "__builtin_ia32_movdqa64load512_mask",
"llvm.x86.avx512.mask.load.ps.512" => "__builtin_ia32_loadaps512_mask",
"llvm.x86.avx512.mask.load.pd.512" => "__builtin_ia32_loadapd512_mask",
"llvm.x86.avx512.mask.load.d.256" => "__builtin_ia32_movdqa32load256_mask",
"llvm.x86.avx512.mask.load.q.256" => "__builtin_ia32_movdqa64load256_mask",
"llvm.x86.avx512fp16.mask.cmp.sh" => "__builtin_ia32_cmpsh_mask_round",
"llvm.x86.avx512fp16.vcomi.sh" => "__builtin_ia32_cmpsh_mask_round",
"llvm.x86.avx512fp16.add.ph.512" => "__builtin_ia32_addph512_mask_round",
"llvm.x86.avx512fp16.sub.ph.512" => "__builtin_ia32_subph512_mask_round",
"llvm.x86.avx512fp16.mul.ph.512" => "__builtin_ia32_mulph512_mask_round",
"llvm.x86.avx512fp16.div.ph.512" => "__builtin_ia32_divph512_mask_round",
"llvm.x86.avx512fp16.mask.vfmul.cph.512" => "__builtin_ia32_vfmulcph512_mask_round",
"llvm.x86.avx512fp16.mask.vfmul.csh" => "__builtin_ia32_vfmulcsh_mask_round",
"llvm.x86.avx512fp16.mask.vfcmul.cph.512" => "__builtin_ia32_vfcmulcph512_mask_round",
"llvm.x86.avx512fp16.mask.vfcmul.csh" => "__builtin_ia32_vfcmulcsh_mask_round",
"llvm.x86.avx512fp16.mask.vfmadd.cph.512" => "__builtin_ia32_vfmaddcph512_mask3_round",
"llvm.x86.avx512fp16.maskz.vfmadd.cph.512" => "__builtin_ia32_vfmaddcph512_maskz_round",
"llvm.x86.avx512fp16.mask.vfmadd.csh" => "__builtin_ia32_vfmaddcsh_mask3_round",
"llvm.x86.avx512fp16.maskz.vfmadd.csh" => "__builtin_ia32_vfmaddcsh_maskz_round",
"llvm.x86.avx512fp16.mask.vfcmadd.cph.512" => "__builtin_ia32_vfcmaddcph512_mask3_round",
"llvm.x86.avx512fp16.maskz.vfcmadd.cph.512" => "__builtin_ia32_vfcmaddcph512_maskz_round",
"llvm.x86.avx512fp16.mask.vfcmadd.csh" => "__builtin_ia32_vfcmaddcsh_mask3_round",
"llvm.x86.avx512fp16.maskz.vfcmadd.csh" => "__builtin_ia32_vfcmaddcsh_maskz_round",
"llvm.x86.avx512fp16.vfmadd.ph.512" => "__builtin_ia32_vfmaddph512_mask",
"llvm.x86.avx512fp16.vcvtsi642sh" => "__builtin_ia32_vcvtsi2sh64_round",
"llvm.x86.avx512fp16.vcvtusi642sh" => "__builtin_ia32_vcvtusi2sh64_round",
"llvm.x86.avx512fp16.vcvtsh2si64" => "__builtin_ia32_vcvtsh2si64_round",
"llvm.x86.avx512fp16.vcvtsh2usi64" => "__builtin_ia32_vcvtsh2usi64_round",
"llvm.x86.avx512fp16.vcvttsh2si64" => "__builtin_ia32_vcvttsh2si64_round",
"llvm.x86.avx512fp16.vcvttsh2usi64" => "__builtin_ia32_vcvttsh2usi64_round",
"llvm.x86.avx512.mask.load.ps.256" => "__builtin_ia32_loadaps256_mask",
"llvm.x86.avx512.mask.load.pd.256" => "__builtin_ia32_loadapd256_mask",
"llvm.x86.avx512.mask.load.d.128" => "__builtin_ia32_movdqa32load128_mask",
"llvm.x86.avx512.mask.load.q.128" => "__builtin_ia32_movdqa64load128_mask",
"llvm.x86.avx512.mask.load.ps.128" => "__builtin_ia32_loadaps128_mask",
"llvm.x86.avx512.mask.load.pd.128" => "__builtin_ia32_loadapd128_mask",
"llvm.x86.avx512.mask.storeu.d.256" => "__builtin_ia32_storedqusi256_mask",
"llvm.x86.avx512.mask.storeu.q.256" => "__builtin_ia32_storedqudi256_mask",
"llvm.x86.avx512.mask.storeu.ps.256" => "__builtin_ia32_storeups256_mask",
"llvm.x86.avx512.mask.storeu.pd.256" => "__builtin_ia32_storeupd256_mask",
"llvm.x86.avx512.mask.storeu.d.128" => "__builtin_ia32_storedqusi128_mask",
"llvm.x86.avx512.mask.storeu.q.128" => "__builtin_ia32_storedqudi128_mask",
"llvm.x86.avx512.mask.storeu.ps.128" => "__builtin_ia32_storeups128_mask",
"llvm.x86.avx512.mask.storeu.pd.128" => "__builtin_ia32_storeupd128_mask",
"llvm.x86.avx512.mask.store.d.512" => "__builtin_ia32_movdqa32store512_mask",
"llvm.x86.avx512.mask.store.q.512" => "__builtin_ia32_movdqa64store512_mask",
"llvm.x86.avx512.mask.store.ps.512" => "__builtin_ia32_storeaps512_mask",
"llvm.x86.avx512.mask.store.pd.512" => "__builtin_ia32_storeapd512_mask",
"llvm.x86.avx512.mask.store.d.256" => "__builtin_ia32_movdqa32store256_mask",
"llvm.x86.avx512.mask.store.q.256" => "__builtin_ia32_movdqa64store256_mask",
"llvm.x86.avx512.mask.store.ps.256" => "__builtin_ia32_storeaps256_mask",
"llvm.x86.avx512.mask.store.pd.256" => "__builtin_ia32_storeapd256_mask",
"llvm.x86.avx512.mask.store.d.128" => "__builtin_ia32_movdqa32store128_mask",
"llvm.x86.avx512.mask.store.q.128" => "__builtin_ia32_movdqa64store128_mask",
"llvm.x86.avx512.mask.store.ps.128" => "__builtin_ia32_storeaps128_mask",
"llvm.x86.avx512.mask.store.pd.128" => "__builtin_ia32_storeapd128_mask",
"llvm.x86.avx512fp16.vfmadd.f16" => "__builtin_ia32_vfmaddsh3_mask",
"llvm.x86.avx512fp16.vfmaddsub.ph.128" => "__builtin_ia32_vfmaddsubph128_mask",
"llvm.x86.avx512fp16.vfmaddsub.ph.256" => "__builtin_ia32_vfmaddsubph256_mask",
"llvm.x86.avx512fp16.vfmaddsub.ph.512" => "__builtin_ia32_vfmaddsubph512_mask",
"llvm.x86.avx512fp16.sqrt.ph.512" => "__builtin_ia32_sqrtph512_mask_round",
"llvm.x86.avx512fp16.mask.sqrt.sh" => "__builtin_ia32_sqrtsh_mask_round",
"llvm.x86.avx512fp16.max.ph.128" => "__builtin_ia32_maxph128_mask",
"llvm.x86.avx512fp16.max.ph.256" => "__builtin_ia32_maxph256_mask",
"llvm.x86.avx512fp16.max.ph.512" => "__builtin_ia32_maxph512_mask_round",
"llvm.x86.avx512fp16.min.ph.128" => "__builtin_ia32_minph128_mask",
"llvm.x86.avx512fp16.min.ph.256" => "__builtin_ia32_minph256_mask",
"llvm.x86.avx512fp16.min.ph.512" => "__builtin_ia32_minph512_mask_round",
"llvm.x86.avx512fp16.mask.getexp.sh" => "__builtin_ia32_getexpsh_mask_round",
"llvm.x86.avx512fp16.mask.rndscale.ph.128" => "__builtin_ia32_rndscaleph128_mask",
"llvm.x86.avx512fp16.mask.rndscale.ph.256" => "__builtin_ia32_rndscaleph256_mask",
"llvm.x86.avx512fp16.mask.rndscale.ph.512" => "__builtin_ia32_rndscaleph512_mask_round",
"llvm.x86.avx512fp16.mask.scalef.ph.512" => "__builtin_ia32_scalefph512_mask_round",
"llvm.x86.avx512fp16.mask.reduce.ph.512" => "__builtin_ia32_reduceph512_mask_round",
"llvm.x86.avx512fp16.mask.reduce.sh" => "__builtin_ia32_reducesh_mask_round",
"llvm.x86.avx512.sitofp.round.v8f16.v8i16" => "__builtin_ia32_vcvtw2ph128_mask",
"llvm.x86.avx512.sitofp.round.v16f16.v16i16" => "__builtin_ia32_vcvtw2ph256_mask",
"llvm.x86.avx512.sitofp.round.v32f16.v32i16" => "__builtin_ia32_vcvtw2ph512_mask_round",
"llvm.x86.avx512.uitofp.round.v8f16.v8u16" => "__builtin_ia32_vcvtuw2ph128_mask",
"llvm.x86.avx512.uitofp.round.v16f16.v16u16" => "__builtin_ia32_vcvtuw2ph256_mask",
"llvm.x86.avx512.uitofp.round.v32f16.v32u16" => "__builtin_ia32_vcvtuw2ph512_mask_round",
"llvm.x86.avx512.sitofp.round.v8f16.v8i32" => "__builtin_ia32_vcvtdq2ph256_mask",
"llvm.x86.avx512.sitofp.round.v16f16.v16i32" => "__builtin_ia32_vcvtdq2ph512_mask_round",
"llvm.x86.avx512fp16.vcvtsi2sh" => "__builtin_ia32_vcvtsi2sh32_round",
"llvm.x86.avx512.uitofp.round.v8f16.v8u32" => "__builtin_ia32_vcvtudq2ph256_mask",
"llvm.x86.avx512.uitofp.round.v16f16.v16u32" => "__builtin_ia32_vcvtudq2ph512_mask_round",
"llvm.x86.avx512fp16.vcvtusi2sh" => "__builtin_ia32_vcvtusi2sh32_round",
"llvm.x86.avx512.sitofp.round.v8f16.v8i64" => "__builtin_ia32_vcvtqq2ph512_mask_round",
"llvm.x86.avx512.uitofp.round.v8f16.v8u64" => "__builtin_ia32_vcvtuqq2ph512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtps2phx.512" => "__builtin_ia32_vcvtps2phx512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtpd2ph.512" => "__builtin_ia32_vcvtpd2ph512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2uw.512" => "__builtin_ia32_vcvtph2uw512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2w.512" => "__builtin_ia32_vcvttph2w512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2uw.512" => "__builtin_ia32_vcvttph2uw512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2dq.512" => "__builtin_ia32_vcvtph2dq512_mask_round",
"llvm.x86.avx512fp16.vcvtsh2si32" => "__builtin_ia32_vcvtsh2si32_round",
"llvm.x86.avx512fp16.mask.vcvtph2udq.512" => "__builtin_ia32_vcvtph2udq512_mask_round",
"llvm.x86.avx512fp16.vcvtsh2usi32" => "__builtin_ia32_vcvtsh2usi32_round",
"llvm.x86.avx512fp16.mask.vcvttph2dq.512" => "__builtin_ia32_vcvttph2dq512_mask_round",
"llvm.x86.avx512fp16.vcvttsh2si32" => "__builtin_ia32_vcvttsh2si32_round",
"llvm.x86.avx512fp16.mask.vcvttph2udq.512" => "__builtin_ia32_vcvttph2udq512_mask_round",
"llvm.x86.avx512fp16.vcvttsh2usi32" => "__builtin_ia32_vcvttsh2usi32_round",
"llvm.x86.avx512fp16.mask.vcvtph2qq.512" => "__builtin_ia32_vcvtph2qq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2uqq.512" => "__builtin_ia32_vcvtph2uqq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2qq.512" => "__builtin_ia32_vcvttph2qq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2uqq.512" => "__builtin_ia32_vcvttph2uqq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2psx.512" => "__builtin_ia32_vcvtph2psx512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2pd.512" => "__builtin_ia32_vcvtph2pd512_mask_round",
"llvm.x86.avx512fp16.mask.vfcmadd.cph.256" => "__builtin_ia32_vfcmaddcph256_mask3",
"llvm.x86.avx512fp16.mask.vfmadd.cph.256" => "__builtin_ia32_vfmaddcph256_mask3",
"llvm.x86.avx512fp16.mask.vfcmadd.cph.128" => "__builtin_ia32_vfcmaddcph128_mask3",
"llvm.x86.avx512fp16.mask.vfmadd.cph.128" => "__builtin_ia32_vfmaddcph128_mask3",
// TODO: support the tile builtins:
"llvm.x86.ldtilecfg" => "__builtin_trap",
"llvm.x86.sttilecfg" => "__builtin_trap",
"llvm.x86.tileloadd64" => "__builtin_trap",
"llvm.x86.tilerelease" => "__builtin_trap",
"llvm.x86.tilestored64" => "__builtin_trap",
"llvm.x86.tileloaddt164" => "__builtin_trap",
"llvm.x86.tilezero" => "__builtin_trap",
"llvm.x86.tdpbf16ps" => "__builtin_trap",
"llvm.x86.tdpbssd" => "__builtin_trap",
"llvm.x86.tdpbsud" => "__builtin_trap",
"llvm.x86.tdpbusd" => "__builtin_trap",
"llvm.x86.tdpbuud" => "__builtin_trap",
"llvm.x86.tdpfp16ps" => "__builtin_trap",
"llvm.x86.tcmmimfp16ps" => "__builtin_trap",
"llvm.x86.tcmmrlfp16ps" => "__builtin_trap",
// NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py
_ => include!("archs.rs"),

21
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file

@ -127,20 +127,13 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
// https://github.com/rust-lang/rust-clippy/issues/12497
// and leave `else if use_integer_compare` to be placed "as is".
#[allow(clippy::suspicious_else_formatting)]
let llval = match name {
let value = match name {
_ if simple.is_some() => {
// FIXME(antoyo): remove this cast when the API supports function.
let func = unsafe {
std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(simple.expect("simple"))
};
self.call(
self.type_void(),
None,
None,
let func = simple.expect("simple function");
self.cx.context.new_call(
self.location,
func,
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
None,
None,
)
}
sym::likely => self.expect(args[0].immediate(), true),
@ -383,7 +376,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
_ if name_str.starts_with("simd_") => {
match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
Ok(llval) => llval,
Ok(value) => value,
Err(()) => return Ok(()),
}
}
@ -396,9 +389,9 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode {
let ptr_llty = self.type_ptr_to(ty.gcc_type(self));
let ptr = self.pointercast(result.val.llval, ptr_llty);
self.store(llval, ptr, result.val.align);
self.store(value, ptr, result.val.align);
} else {
OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
.val
.store(self, result);
}

55
compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
View file

@ -198,7 +198,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
bx.context.new_bitcast(None, shuffled, v_type)
};
if name == sym::simd_bswap || name == sym::simd_bitreverse {
if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop) {
require!(
bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
@ -209,6 +209,22 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
return Ok(simd_bswap(bx, args[0].immediate()));
}
let simd_ctpop = |bx: &mut Builder<'a, 'gcc, 'tcx>, vector: RValue<'gcc>| -> RValue<'gcc> {
let mut vector_elements = vec![];
let elem_ty = bx.element_type(llret_ty);
for i in 0..in_len {
let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
let element = bx.extract_element(vector, index).to_rvalue();
let result = bx.context.new_cast(None, bx.pop_count(element), elem_ty);
vector_elements.push(result);
}
bx.context.new_rvalue_from_vector(None, llret_ty, &vector_elements)
};
if name == sym::simd_ctpop {
return Ok(simd_ctpop(bx, args[0].immediate()));
}
// We use a different algorithm from non-vector bitreverse to take advantage of most
// processors' vector shuffle units. It works like this:
// 1. Generate pre-reversed low and high nibbles as a vector.
@ -718,11 +734,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
return Err(());
}};
}
let (elem_ty_str, elem_ty) = if let ty::Float(ref f) = *in_elem.kind() {
let (elem_ty_str, elem_ty, cast_type) = if let ty::Float(ref f) = *in_elem.kind() {
let elem_ty = bx.cx.type_float_from_ty(*f);
match f.bit_width() {
32 => ("f", elem_ty),
64 => ("", elem_ty),
16 => ("", elem_ty, Some(bx.cx.double_type)),
32 => ("f", elem_ty, None),
64 => ("", elem_ty, None),
_ => {
return_error!(InvalidMonomorphization::FloatingPointVector {
span,
@ -758,10 +775,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
_ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
};
let builtin_name = format!("{}{}", intr_name, elem_ty_str);
let funcs = bx.cx.functions.borrow();
let function = funcs
.get(&builtin_name)
.unwrap_or_else(|| panic!("unable to find builtin function {}", builtin_name));
let function = bx.context.get_builtin_function(builtin_name);
// TODO(antoyo): add platform-specific behavior here for architectures that have these
// intrinsics as instructions (for instance, gpus)
@ -769,17 +783,28 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
for i in 0..in_len {
let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
// we have to treat fpowi specially, since fpowi's second argument is always an i32
let arguments = if name == sym::simd_fpowi {
vec![
let mut arguments = vec![];
if name == sym::simd_fpowi {
arguments = vec![
bx.extract_element(args[0].immediate(), index).to_rvalue(),
args[1].immediate(),
]
];
} else {
args.iter()
.map(|arg| bx.extract_element(arg.immediate(), index).to_rvalue())
.collect()
for arg in args {
let mut element = bx.extract_element(arg.immediate(), index).to_rvalue();
// FIXME: it would probably be better to not have casts here and use the proper
// instructions.
if let Some(typ) = cast_type {
element = bx.context.new_cast(None, element, typ);
}
arguments.push(element);
}
};
vector_elements.push(bx.context.new_call(None, *function, &arguments));
let mut result = bx.context.new_call(None, function, &arguments);
if cast_type.is_some() {
result = bx.context.new_cast(None, result, elem_ty);
}
vector_elements.push(result);
}
let c = bx.context.new_rvalue_from_vector(None, vec_ty, &vector_elements);
Ok(c)

2
compiler/rustc_codegen_gcc/src/lib.rs
View file

@ -363,7 +363,7 @@ impl Deref for SyncContext {
unsafe impl Send for SyncContext {}
// FIXME(antoyo): that shouldn't be Sync. Parallel compilation is currently disabled with "-Zno-parallel-llvm".
// TODO: disable it here by returing false in CodegenBackend::supports_parallel().
// TODO: disable it here by returning false in CodegenBackend::supports_parallel().
unsafe impl Sync for SyncContext {}
impl WriteBackendMethods for GccCodegenBackend {

2
compiler/rustc_codegen_gcc/src/mono_item.rs
View file

@ -37,7 +37,7 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
let global = self.define_global(symbol_name, gcc_type, is_tls, attrs.link_section);
#[cfg(feature = "master")]
global.add_string_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
global.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
// TODO(antoyo): set linkage.
self.instances.borrow_mut().insert(instance, global);