bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #107269 - bjorn3:sync_cg_clif-2023-01-24, r=bjorn3

Browse source 
Sync rustc_codegen_cranelift

For cg_clif itself there have been a couple of bug fixes since the last sync, a Cranelift update and implemented all remaining simd platform intrinsics used by `std::simd`. (`std::arch` still misses a lot though) Most of the diff is from reworking of the cg_clif build system though.

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler
This commit is contained in:
bors 2023-01-27 00:03:09 +00:00
commit a2d002afe7
47 changed files with 1393 additions and 1111 deletions
Download patch file Download diff file Expand all files Collapse all files

37
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -7,6 +7,7 @@ mod returning;
use cranelift_module::ModuleError;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_session::Session;
use rustc_target::abi::call::{Conv, FnAbi};
use rustc_target::spec::abi::Abi;
@ -22,7 +23,7 @@ fn clif_sig_from_fn_abi<'tcx>(
default_call_conv: CallConv,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
) -> Signature {
let call_conv = conv_to_call_conv(fn_abi.conv, default_call_conv);
let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv);
let inputs = fn_abi.args.iter().map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()).flatten();
@ -33,24 +34,32 @@ fn clif_sig_from_fn_abi<'tcx>(
Signature { params, returns, call_conv }
}
pub(crate) fn conv_to_call_conv(c: Conv, default_call_conv: CallConv) -> CallConv {
pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv {
match c {
Conv::Rust | Conv::C => default_call_conv,
Conv::RustCold => CallConv::Cold,
Conv::X86_64SysV => CallConv::SystemV,
Conv::X86_64Win64 => CallConv::WindowsFastcall,
Conv::ArmAapcs
| Conv::CCmseNonSecureCall
| Conv::Msp430Intr
// Should already get a back compat warning
Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => {
default_call_conv
}
Conv::X86Intr => sess.fatal("x86-interrupt call conv not yet implemented"),
Conv::ArmAapcs => sess.fatal("aapcs call conv not yet implemented"),
Conv::CCmseNonSecureCall => {
sess.fatal("C-cmse-nonsecure-call call conv is not yet implemented");
}
Conv::Msp430Intr
| Conv::PtxKernel
| Conv::X86Fastcall
| Conv::X86Intr
| Conv::X86Stdcall
| Conv::X86ThisCall
| Conv::X86VectorCall
| Conv::AmdGpuKernel
| Conv::AvrInterrupt
| Conv::AvrNonBlockingInterrupt => todo!("{:?}", c),
| Conv::AvrNonBlockingInterrupt => {
unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
}
}
}
@ -161,6 +170,12 @@ fn make_local_place<'tcx>(
layout: TyAndLayout<'tcx>,
is_ssa: bool,
) -> CPlace<'tcx> {
if layout.is_unsized() {
fx.tcx.sess.span_fatal(
fx.mir.local_decls[local].source_info.span,
"unsized locals are not yet supported",
);
}
let place = if is_ssa {
if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi {
CPlace::new_var_pair(fx, local, layout)

42
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -113,6 +113,8 @@ pub(crate) fn codegen_fn<'tcx>(
};
tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
fx.bcx.seal_all_blocks();
fx.bcx.finalize();
// Recover all necessary data from fx, before accessing func will prevent future access to it.
let symbol_name = fx.symbol_name;
@ -303,6 +305,9 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
let source_info = bb_data.terminator().source_info;
fx.set_debug_loc(source_info);
let _print_guard =
crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind));
match &bb_data.terminator().kind {
TerminatorKind::Goto { target } => {
if let TerminatorKind::Return = fx.mir[*target].terminator().kind {
@ -464,7 +469,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
*destination,
);
}
TerminatorKind::Resume | TerminatorKind::Abort => {
TerminatorKind::Abort => {
codegen_panic_cannot_unwind(fx, source_info);
}
TerminatorKind::Resume => {
// FIXME implement unwinding
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
@ -487,9 +495,6 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
}
};
}
fx.bcx.seal_all_blocks();
fx.bcx.finalize();
}
fn codegen_stmt<'tcx>(
@ -932,7 +937,28 @@ pub(crate) fn codegen_panic<'tcx>(
codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span);
}
pub(crate) fn codegen_panic_inner<'tcx>(
pub(crate) fn codegen_panic_nounwind<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
msg_str: &str,
source_info: mir::SourceInfo,
) {
let msg_ptr = fx.anonymous_str(msg_str);
let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
let args = [msg_ptr, msg_len];
codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span);
}
pub(crate) fn codegen_panic_cannot_unwind<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
source_info: mir::SourceInfo,
) {
let args = [];
codegen_panic_inner(fx, rustc_hir::LangItem::PanicCannotUnwind, &args, source_info.span);
}
fn codegen_panic_inner<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
lang_item: rustc_hir::LangItem,
args: &[Value],
@ -949,11 +975,7 @@ pub(crate) fn codegen_panic_inner<'tcx>(
fx.lib_call(
&*symbol_name,
vec![
AbiParam::new(fx.pointer_type),
AbiParam::new(fx.pointer_type),
AbiParam::new(fx.pointer_type),
],
args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),
vec![],
args,
);

9
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -168,6 +168,15 @@ pub(crate) fn codegen_icmp_imm(
}
}
pub(crate) fn codegen_bitcast(fx: &mut FunctionCx<'_, '_, '_>, dst_ty: Type, val: Value) -> Value {
let mut flags = MemFlags::new();
flags.set_endianness(match fx.tcx.data_layout.endian {
rustc_target::abi::Endian::Big => cranelift_codegen::ir::Endianness::Big,
rustc_target::abi::Endian::Little => cranelift_codegen::ir::Endianness::Little,
});
fx.bcx.ins().bitcast(dst_ty, flags, val)
}
pub(crate) fn type_zero_value(bcx: &mut FunctionBuilder<'_>, ty: Type) -> Value {
if ty == types::I128 {
let zero = bcx.ins().iconst(types::I64, 0);

248
compiler/rustc_codegen_cranelift/src/cranelift_native.rs Normal file
View file

@ -0,0 +1,248 @@
// Vendored from https://github.com/bytecodealliance/wasmtime/blob/b58a197d33f044193c3d608010f5e6ec394ac07e/cranelift/native/src/lib.rs
// which is licensed as
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// unlike rustc_codegen_cranelift itself. Also applies a small change to remove #![cfg_attr] that
// rust's CI complains about and to fix formatting to match rustc.
// FIXME revert back to the external crate with Cranelift 0.93
#![allow(warnings)]
//! Performs autodetection of the host for the purposes of running
//! Cranelift to generate code to run on the same machine.
#![deny(missing_docs, trivial_numeric_casts, unused_extern_crates, unstable_features)]
#![warn(unused_import_braces)]
use cranelift_codegen::isa;
use target_lexicon::Triple;
/// Return an `isa` builder configured for the current host
/// machine, or `Err(())` if the host machine is not supported
/// in the current configuration.
pub fn builder() -> Result<isa::Builder, &'static str> {
builder_with_options(true)
}
/// Return an `isa` builder configured for the current host
/// machine, or `Err(())` if the host machine is not supported
/// in the current configuration.
///
/// Selects the given backend variant specifically; this is
/// useful when more than oen backend exists for a given target
/// (e.g., on x86-64).
pub fn builder_with_options(infer_native_flags: bool) -> Result<isa::Builder, &'static str> {
let mut isa_builder = isa::lookup(Triple::host()).map_err(|err| match err {
isa::LookupError::SupportDisabled => "support for architecture disabled at compile time",
isa::LookupError::Unsupported => "unsupported architecture",
})?;
#[cfg(target_arch = "x86_64")]
{
use cranelift_codegen::settings::Configurable;
if !std::is_x86_feature_detected!("sse2") {
return Err("x86 support requires SSE2");
}
if !infer_native_flags {
return Ok(isa_builder);
}
// These are temporarily enabled by default (see #3810 for
// more) so that a default-constructed `Flags` can work with
// default Wasmtime features. Otherwise, the user must
// explicitly use native flags or turn these on when on x86-64
// platforms to avoid a configuration panic. In order for the
// "enable if detected" logic below to work, we must turn them
// *off* (differing from the default) and then re-enable below
// if present.
isa_builder.set("has_sse3", "false").unwrap();
isa_builder.set("has_ssse3", "false").unwrap();
isa_builder.set("has_sse41", "false").unwrap();
isa_builder.set("has_sse42", "false").unwrap();
if std::is_x86_feature_detected!("sse3") {
isa_builder.enable("has_sse3").unwrap();
}
if std::is_x86_feature_detected!("ssse3") {
isa_builder.enable("has_ssse3").unwrap();
}
if std::is_x86_feature_detected!("sse4.1") {
isa_builder.enable("has_sse41").unwrap();
}
if std::is_x86_feature_detected!("sse4.2") {
isa_builder.enable("has_sse42").unwrap();
}
if std::is_x86_feature_detected!("popcnt") {
isa_builder.enable("has_popcnt").unwrap();
}
if std::is_x86_feature_detected!("avx") {
isa_builder.enable("has_avx").unwrap();
}
if std::is_x86_feature_detected!("avx2") {
isa_builder.enable("has_avx2").unwrap();
}
if std::is_x86_feature_detected!("fma") {
isa_builder.enable("has_fma").unwrap();
}
if std::is_x86_feature_detected!("bmi1") {
isa_builder.enable("has_bmi1").unwrap();
}
if std::is_x86_feature_detected!("bmi2") {
isa_builder.enable("has_bmi2").unwrap();
}
if std::is_x86_feature_detected!("avx512bitalg") {
isa_builder.enable("has_avx512bitalg").unwrap();
}
if std::is_x86_feature_detected!("avx512dq") {
isa_builder.enable("has_avx512dq").unwrap();
}
if std::is_x86_feature_detected!("avx512f") {
isa_builder.enable("has_avx512f").unwrap();
}
if std::is_x86_feature_detected!("avx512vl") {
isa_builder.enable("has_avx512vl").unwrap();
}
if std::is_x86_feature_detected!("avx512vbmi") {
isa_builder.enable("has_avx512vbmi").unwrap();
}
if std::is_x86_feature_detected!("lzcnt") {
isa_builder.enable("has_lzcnt").unwrap();
}
}
#[cfg(target_arch = "aarch64")]
{
use cranelift_codegen::settings::Configurable;
if !infer_native_flags {
return Ok(isa_builder);
}
if std::arch::is_aarch64_feature_detected!("lse") {
isa_builder.enable("has_lse").unwrap();
}
if std::arch::is_aarch64_feature_detected!("paca") {
isa_builder.enable("has_pauth").unwrap();
}
if cfg!(target_os = "macos") {
// Pointer authentication is always available on Apple Silicon.
isa_builder.enable("sign_return_address").unwrap();
// macOS enforces the use of the B key for return addresses.
isa_builder.enable("sign_return_address_with_bkey").unwrap();
}
}
// There is no is_s390x_feature_detected macro yet, so for now
// we use getauxval from the libc crate directly.
#[cfg(all(target_arch = "s390x", target_os = "linux"))]
{
use cranelift_codegen::settings::Configurable;
if !infer_native_flags {
return Ok(isa_builder);
}
let v = unsafe { libc::getauxval(libc::AT_HWCAP) };
const HWCAP_S390X_VXRS_EXT2: libc::c_ulong = 32768;
if (v & HWCAP_S390X_VXRS_EXT2) != 0 {
isa_builder.enable("has_vxrs_ext2").unwrap();
// There is no separate HWCAP bit for mie2, so assume
// that any machine with vxrs_ext2 also has mie2.
isa_builder.enable("has_mie2").unwrap();
}
}
// `is_riscv_feature_detected` is nightly only for now, use
// getauxval from the libc crate directly as a temporary measure.
#[cfg(all(target_arch = "riscv64", target_os = "linux"))]
{
use cranelift_codegen::settings::Configurable;
if !infer_native_flags {
return Ok(isa_builder);
}
let v = unsafe { libc::getauxval(libc::AT_HWCAP) };
const HWCAP_RISCV_EXT_A: libc::c_ulong = 1 << (b'a' - b'a');
const HWCAP_RISCV_EXT_C: libc::c_ulong = 1 << (b'c' - b'a');
const HWCAP_RISCV_EXT_D: libc::c_ulong = 1 << (b'd' - b'a');
const HWCAP_RISCV_EXT_F: libc::c_ulong = 1 << (b'f' - b'a');
const HWCAP_RISCV_EXT_M: libc::c_ulong = 1 << (b'm' - b'a');
const HWCAP_RISCV_EXT_V: libc::c_ulong = 1 << (b'v' - b'a');
if (v & HWCAP_RISCV_EXT_A) != 0 {
isa_builder.enable("has_a").unwrap();
}
if (v & HWCAP_RISCV_EXT_C) != 0 {
isa_builder.enable("has_c").unwrap();
}
if (v & HWCAP_RISCV_EXT_D) != 0 {
isa_builder.enable("has_d").unwrap();
}
if (v & HWCAP_RISCV_EXT_F) != 0 {
isa_builder.enable("has_f").unwrap();
// TODO: There doesn't seem to be a bit associated with this extension
// rust enables it with the `f` extension:
// https://github.com/rust-lang/stdarch/blob/790411f93c4b5eada3c23abb4c9a063fb0b24d99/crates/std_detect/src/detect/os/linux/riscv.rs#L43
isa_builder.enable("has_zicsr").unwrap();
}
if (v & HWCAP_RISCV_EXT_M) != 0 {
isa_builder.enable("has_m").unwrap();
}
if (v & HWCAP_RISCV_EXT_V) != 0 {
isa_builder.enable("has_v").unwrap();
}
// TODO: ZiFencei does not have a bit associated with it
// TODO: Zbkb does not have a bit associated with it
}
// squelch warnings about unused mut/variables on some platforms.
drop(&mut isa_builder);
drop(infer_native_flags);
Ok(isa_builder)
}
#[cfg(test)]
mod tests {
use super::builder;
use cranelift_codegen::isa::CallConv;
use cranelift_codegen::settings;
#[test]
fn test() {
if let Ok(isa_builder) = builder() {
let flag_builder = settings::builder();
let isa = isa_builder.finish(settings::Flags::new(flag_builder)).unwrap();
if cfg!(all(target_os = "macos", target_arch = "aarch64")) {
assert_eq!(isa.default_call_conv(), CallConv::AppleAarch64);
} else if cfg!(any(unix, target_os = "nebulet")) {
assert_eq!(isa.default_call_conv(), CallConv::SystemV);
} else if cfg!(windows) {
assert_eq!(isa.default_call_conv(), CallConv::WindowsFastcall);
}
if cfg!(target_pointer_width = "64") {
assert_eq!(isa.pointer_bits(), 64);
} else if cfg!(target_pointer_width = "32") {
assert_eq!(isa.pointer_bits(), 32);
} else if cfg!(target_pointer_width = "16") {
assert_eq!(isa.pointer_bits(), 16);
}
}
}
}
/// Version number of this crate.
pub const VERSION: &str = env!("CARGO_PKG_VERSION");

14
compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
View file

@ -20,6 +20,14 @@ use indexmap::IndexSet;
pub(crate) use emit::{DebugReloc, DebugRelocName};
pub(crate) use unwind::UnwindContext;
pub(crate) fn producer() -> String {
format!(
"cg_clif (rustc {}, cranelift {})",
rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
cranelift_codegen::VERSION,
)
}
pub(crate) struct DebugContext {
endian: RunTimeEndian,
@ -57,11 +65,7 @@ impl DebugContext {
let mut dwarf = DwarfUnit::new(encoding);
let producer = format!(
"cg_clif (rustc {}, cranelift {})",
rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
cranelift_codegen::VERSION,
);
let producer = producer();
let comp_dir = tcx
.sess
.opts

18
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -108,6 +108,8 @@ impl OngoingCodegen {
self.concurrency_limiter.finished();
sess.abort_if_errors();
(
CodegenResults {
modules,
@ -169,10 +171,22 @@ fn emit_cgu(
fn emit_module(
output_filenames: &OutputFilenames,
prof: &SelfProfilerRef,
object: cranelift_object::object::write::Object<'_>,
mut object: cranelift_object::object::write::Object<'_>,
kind: ModuleKind,
name: String,
) -> Result<CompiledModule, String> {
if object.format() == cranelift_object::object::BinaryFormat::Elf {
let comment_section = object.add_section(
Vec::new(),
b".comment".to_vec(),
cranelift_object::object::SectionKind::OtherString,
);
let mut producer = vec![0];
producer.extend(crate::debuginfo::producer().as_bytes());
producer.push(0);
object.set_section_data(comment_section, producer, 1);
}
let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name));
let mut file = match File::create(&tmp_file) {
Ok(file) => file,
@ -399,8 +413,6 @@ pub(crate) fn run_aot(
.collect::<Vec<_>>()
});
tcx.sess.abort_if_errors();
let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
let created_alloc_shim =

4
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
View file

@ -33,8 +33,8 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
// cast float to int
let a_lane = match lane_ty {
types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
types::F32 => codegen_bitcast(fx, types::I32, a_lane),
types::F64 => codegen_bitcast(fx, types::I64, a_lane),
_ => a_lane,
};

11
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -201,7 +201,7 @@ fn bool_to_zero_or_max_uint<'tcx>(
let mut res = fx.bcx.ins().bmask(int_ty, val);
if ty.is_float() {
res = fx.bcx.ins().bitcast(ty, res);
res = codegen_bitcast(fx, ty, res);
}
res
@ -241,10 +241,9 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
substs,
args,
destination,
target,
source_info.span,
);
let ret_block = fx.get_block(target);
fx.bcx.ins().jump(ret_block, &[]);
} else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
let ret_block = fx.get_block(target);
fx.bcx.ins().jump(ret_block, &[]);
@ -651,7 +650,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let layout = fx.layout_of(substs.type_at(0));
if layout.abi.is_uninhabited() {
with_no_trimmed_paths!({
crate::base::codegen_panic(
crate::base::codegen_panic_nounwind(
fx,
&format!("attempted to instantiate uninhabited type `{}`", layout.ty),
source_info,
@ -664,7 +663,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
&& !fx.tcx.permits_zero_init(fx.param_env().and(layout))
{
with_no_trimmed_paths!({
crate::base::codegen_panic(
crate::base::codegen_panic_nounwind(
fx,
&format!(
"attempted to zero-initialize type `{}`, which is invalid",
@ -680,7 +679,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
&& !fx.tcx.permits_uninit_init(fx.param_env().and(layout))
{
with_no_trimmed_paths!({
crate::base::codegen_panic(
crate::base::codegen_panic_nounwind(
fx,
&format!(
"attempted to leave type `{}` uninitialized, which is invalid",

122
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -24,6 +24,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
_substs: SubstsRef<'tcx>,
args: &[mir::Operand<'tcx>],
ret: CPlace<'tcx>,
target: BasicBlock,
span: Span,
) {
match intrinsic {
@ -277,16 +278,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
} else {
fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
let trap_block = fx.bcx.create_block();
let dummy_block = fx.bcx.create_block();
let true_ = fx.bcx.ins().iconst(types::I8, 1);
fx.bcx.ins().brnz(true_, trap_block, &[]);
fx.bcx.ins().jump(dummy_block, &[]);
let ret_block = fx.get_block(target);
fx.bcx.ins().jump(ret_block, &[]);
fx.bcx.switch_to_block(trap_block);
crate::trap::trap_unimplemented(
fx,
"Index argument for `simd_extract` is not a constant",
);
fx.bcx.switch_to_block(dummy_block);
return;
};
@ -770,11 +770,119 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
});
}
// simd_arith_offset
// simd_scatter
// simd_gather
sym::simd_expose_addr | sym::simd_from_exposed_addr | sym::simd_cast_ptr => {
intrinsic_args!(fx, args => (arg); intrinsic);
ret.write_cvalue_transmute(fx, arg);
}
sym::simd_arith_offset => {
intrinsic_args!(fx, args => (ptr, offset); intrinsic);
let (lane_count, ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
let pointee_ty = ptr_lane_ty.builtin_deref(true).unwrap().ty;
let pointee_size = fx.layout_of(pointee_ty).size.bytes();
let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
let ret_lane_layout = fx.layout_of(ret_lane_ty);
assert_eq!(lane_count, ret_lane_count);
for lane_idx in 0..lane_count {
let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
let offset_lane = offset.value_lane(fx, lane_idx).load_scalar(fx);
let ptr_diff = if pointee_size != 1 {
fx.bcx.ins().imul_imm(offset_lane, pointee_size as i64)
} else {
offset_lane
};
let res_lane = fx.bcx.ins().iadd(ptr_lane, ptr_diff);
let res_lane = CValue::by_val(res_lane, ret_lane_layout);
ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
}
}
sym::simd_gather => {
intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
assert_eq!(val_lane_count, ptr_lane_count);
assert_eq!(val_lane_count, mask_lane_count);
assert_eq!(val_lane_count, ret_lane_count);
let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
let ret_lane_layout = fx.layout_of(ret_lane_ty);
for lane_idx in 0..ptr_lane_count {
let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
let if_enabled = fx.bcx.create_block();
let if_disabled = fx.bcx.create_block();
let next = fx.bcx.create_block();
let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
fx.bcx.ins().jump(if_disabled, &[]);
fx.bcx.seal_block(if_enabled);
fx.bcx.seal_block(if_disabled);
fx.bcx.switch_to_block(if_enabled);
let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0);
fx.bcx.ins().jump(next, &[res]);
fx.bcx.switch_to_block(if_disabled);
fx.bcx.ins().jump(next, &[val_lane]);
fx.bcx.seal_block(next);
fx.bcx.switch_to_block(next);
fx.bcx.ins().nop();
ret.place_lane(fx, lane_idx)
.write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout));
}
}
sym::simd_scatter => {
intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
assert_eq!(val_lane_count, ptr_lane_count);
assert_eq!(val_lane_count, mask_lane_count);
for lane_idx in 0..ptr_lane_count {
let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx);
let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
let if_enabled = fx.bcx.create_block();
let next = fx.bcx.create_block();
fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
fx.bcx.ins().jump(next, &[]);
fx.bcx.seal_block(if_enabled);
fx.bcx.switch_to_block(if_enabled);
fx.bcx.ins().store(MemFlags::trusted(), val_lane, ptr_lane, 0);
fx.bcx.ins().jump(next, &[]);
fx.bcx.seal_block(next);
fx.bcx.switch_to_block(next);
}
}
_ => {
fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
fx.tcx.sess.span_err(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
// Prevent verifier error
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
}
let ret_block = fx.get_block(target);
fx.bcx.ins().jump(ret_block, &[]);
}

8
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -57,6 +57,8 @@ mod compiler_builtins;
mod concurrency_limiter;
mod config;
mod constant;
// FIXME revert back to the external crate with Cranelift 0.93
mod cranelift_native;
mod debuginfo;
mod discriminant;
mod driver;
@ -278,12 +280,14 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
}
}
if target_triple.architecture == target_lexicon::Architecture::X86_64 {
if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::X86_64 =
target_triple.architecture
{
// Windows depends on stack probes to grow the committed part of the stack
flags_builder.enable("enable_probestack").unwrap();
flags_builder.set("probestack_strategy", "inline").unwrap();
} else {
// __cranelift_probestack is not provided and inline stack probes are only supported on x86_64
// __cranelift_probestack is not provided and inline stack probes are only supported on AArch64 and x86_64
flags_builder.set("enable_probestack", "false").unwrap();
}

15
compiler/rustc_codegen_cranelift/src/main_shim.rs
View file

@ -64,13 +64,20 @@ pub(crate) fn maybe_create_entry_wrapper(
],
returns: vec![AbiParam::new(m.target_config().pointer_type() /*isize*/)],
call_conv: crate::conv_to_call_conv(
tcx.sess,
tcx.sess.target.options.entry_abi,
m.target_config().default_call_conv,
),
};
let entry_name = tcx.sess.target.options.entry_name.as_ref();
let cmain_func_id = m.declare_function(entry_name, Linkage::Export, &cmain_sig).unwrap();
let cmain_func_id = match m.declare_function(entry_name, Linkage::Export, &cmain_sig) {
Ok(func_id) => func_id,
Err(err) => {
tcx.sess
.fatal(&format!("entry symbol `{entry_name}` declared multiple times: {err}"));
}
};
let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
@ -162,7 +169,11 @@ pub(crate) fn maybe_create_entry_wrapper(
bcx.seal_all_blocks();
bcx.finalize();
}
m.define_function(cmain_func_id, &mut ctx).unwrap();
if let Err(err) = m.define_function(cmain_func_id, &mut ctx) {
tcx.sess.fatal(&format!("entry symbol `{entry_name}` defined multiple times: {err}"));
}
unwind_context.add_function(cmain_func_id, &ctx, m.isa());
}
}

4
compiler/rustc_codegen_cranelift/src/optimize/peephole.rs
View file

@ -7,7 +7,7 @@ use cranelift_frontend::FunctionBuilder;
/// otherwise return the given value and false.
pub(crate) fn maybe_unwrap_bool_not(bcx: &mut FunctionBuilder<'_>, arg: Value) -> (Value, bool) {
if let ValueDef::Result(arg_inst, 0) = bcx.func.dfg.value_def(arg) {
match bcx.func.dfg[arg_inst] {
match bcx.func.dfg.insts[arg_inst] {
// This is the lowering of `Rvalue::Not`
InstructionData::IntCompareImm {
opcode: Opcode::IcmpImm,
@ -34,7 +34,7 @@ pub(crate) fn maybe_known_branch_taken(
return None;
};
match bcx.func.dfg[arg_inst] {
match bcx.func.dfg.insts[arg_inst] {
InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => {
if test_zero {
Some(imm.bits() == 0)

4
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -514,8 +514,8 @@ impl<'tcx> CPlace<'tcx> {
(types::I32, types::F32)
| (types::F32, types::I32)
| (types::I64, types::F64)
| (types::F64, types::I64) => fx.bcx.ins().bitcast(dst_ty, data),
_ if src_ty.is_vector() && dst_ty.is_vector() => fx.bcx.ins().bitcast(dst_ty, data),
| (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data),
_ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
_ if src_ty.is_vector() || dst_ty.is_vector() => {
// FIXME do something more efficient for transmutes between vectors and integers.
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {