bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '97e5045493' into sync_cg_clif-2021-12-20

Browse source
This commit is contained in:
bjorn3 2021-12-20 18:56:35 +01:00
commit 3426a730ff
36 changed files with 1017 additions and 532 deletions
Download patch file Download diff file Expand all files Collapse all files

14
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -18,11 +18,11 @@ pub(crate) use self::returning::codegen_return;
fn clif_sig_from_fn_abi<'tcx>(
tcx: TyCtxt<'tcx>,
triple: &target_lexicon::Triple,
default_call_conv: CallConv,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
) -> Signature {
let call_conv = match fn_abi.conv {
Conv::Rust | Conv::C => CallConv::triple_default(triple),
Conv::Rust | Conv::C => default_call_conv,
Conv::X86_64SysV => CallConv::SystemV,
Conv::X86_64Win64 => CallConv::WindowsFastcall,
Conv::ArmAapcs
@ -55,7 +55,7 @@ pub(crate) fn get_function_sig<'tcx>(
assert!(!inst.substs.needs_infer());
clif_sig_from_fn_abi(
tcx,
triple,
CallConv::triple_default(triple),
&RevealAllLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()),
)
}
@ -91,7 +91,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
returns: Vec<AbiParam>,
args: &[Value],
) -> &[Value] {
let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
let call_inst = self.bcx.ins().call(func_ref, args);
@ -420,7 +420,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
}
let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0].value, idx);
let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
(CallTarget::Indirect(sig, method), Some(ptr))
@ -440,7 +440,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
}
let func = codegen_operand(fx, func).load_scalar(fx);
let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
(CallTarget::Indirect(sig, func), None)
@ -531,7 +531,7 @@ pub(crate) fn codegen_drop<'tcx>(
let fn_abi =
RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty());
let sig = clif_sig_from_fn_abi(fx.tcx, fx.triple(), &fn_abi);
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
}

5
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -117,7 +117,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
PassMode::Cast(cast) => cast_target_to_abi_params(cast),
PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
if on_stack {
let size = u32::try_from(self.layout.size.bytes()).unwrap();
// Abi requires aligning struct size to pointer size
let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi);
let size = u32::try_from(size.bytes()).unwrap();
smallvec![apply_arg_attrs_to_abi_param(
AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),),
attrs
@ -204,7 +206,6 @@ pub(super) fn from_casted_value<'tcx>(
// It may also be smaller for example when the type is a wrapper around an integer with a
// larger alignment than the integer.
size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
offset: None,
});
let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
let mut offset = 0;

10
compiler/rustc_codegen_cranelift/src/archive.rs
View file

@ -1,7 +1,6 @@
//! Creation of ar archives like for the lib and staticlib crate type
use std::collections::BTreeMap;
use std::convert::TryFrom;
use std::fs::File;
use std::io::{self, Read, Seek};
use std::path::{Path, PathBuf};
@ -10,7 +9,7 @@ use rustc_codegen_ssa::back::archive::ArchiveBuilder;
use rustc_session::Session;
use object::read::archive::ArchiveFile;
use object::{Object, ObjectSymbol, ReadCache, SymbolKind};
use object::{Object, ObjectSymbol, ReadCache};
#[derive(Debug)]
enum ArchiveEntry {
@ -150,12 +149,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
object
.symbols()
.filter_map(|symbol| {
if symbol.is_undefined()
|| symbol.is_local()
|| symbol.kind() != SymbolKind::Data
&& symbol.kind() != SymbolKind::Text
&& symbol.kind() != SymbolKind::Tls
{
if symbol.is_undefined() || symbol.is_local() {
None
} else {
symbol.name().map(|name| name.as_bytes().to_vec()).ok()

18
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -49,13 +49,15 @@ pub(crate) fn codegen_fn<'tcx>(
(0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
// Make FunctionCx
let pointer_type = module.target_config().pointer_type();
let target_config = module.target_config();
let pointer_type = target_config.pointer_type();
let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
let mut fx = FunctionCx {
cx,
module,
tcx,
target_config,
pointer_type,
constants_cx: ConstantCx::new(),
@ -72,8 +74,6 @@ pub(crate) fn codegen_fn<'tcx>(
clif_comments,
source_info_set: indexmap::IndexSet::new(),
next_ssa_var: 0,
inline_asm_index: 0,
};
let arg_uninhabited = fx
@ -204,7 +204,6 @@ pub(crate) fn verify_func(
tcx.sess.err(&format!("{:?}", err));
let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
&func,
None,
Some(Box::new(writer)),
err,
);
@ -296,9 +295,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
AssertKind::BoundsCheck { ref len, ref index } => {
let len = codegen_operand(fx, len).load_scalar(fx);
let index = codegen_operand(fx, index).load_scalar(fx);
let location = fx
.get_caller_location(source_info.span)
.load_scalar(fx);
let location = fx.get_caller_location(source_info.span).load_scalar(fx);
codegen_panic_inner(
fx,
@ -681,7 +678,7 @@ fn codegen_stmt<'tcx>(
// FIXME use emit_small_memset where possible
let addr = lval.to_ptr().get_addr(fx);
let val = operand.load_scalar(fx);
fx.bcx.call_memset(fx.module.target_config(), addr, val, times);
fx.bcx.call_memset(fx.target_config, addr, val, times);
} else {
let loop_block = fx.bcx.create_block();
let loop_block2 = fx.bcx.create_block();
@ -754,8 +751,7 @@ fn codegen_stmt<'tcx>(
NullOp::AlignOf => layout.align.abi.bytes(),
NullOp::Box => unreachable!(),
};
let val =
CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into());
lval.write_cvalue(fx, val);
}
Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() {
@ -803,7 +799,7 @@ fn codegen_stmt<'tcx>(
let elem_size: u64 = pointee.size.bytes();
let bytes =
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes);
fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
}
}
}

8
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -1,3 +1,4 @@
use cranelift_codegen::isa::TargetFrontendConfig;
use rustc_index::vec::IndexVec;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
@ -235,6 +236,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
pub(crate) module: &'m mut dyn Module,
pub(crate) tcx: TyCtxt<'tcx>,
pub(crate) target_config: TargetFrontendConfig, // Cached from module
pub(crate) pointer_type: Type, // Cached from module
pub(crate) constants_cx: ConstantCx,
@ -255,8 +257,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
/// This should only be accessed by `CPlace::new_var`.
pub(crate) next_ssa_var: u32,
pub(crate) inline_asm_index: u32,
}
impl<'tcx> LayoutOfHelpers<'tcx> for FunctionCx<'_, '_, 'tcx> {
@ -359,10 +359,6 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
crate::constant::codegen_const_value(self, const_loc, self.tcx.caller_location_ty())
}
pub(crate) fn triple(&self) -> &target_lexicon::Triple {
self.module.isa().triple()
}
pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value {
let mut data_ctx = DataContext::new();
data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice());

2
compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
View file

@ -69,8 +69,6 @@ impl WriterRelocate {
/// Perform the collected relocations to be usable for JIT usage.
#[cfg(feature = "jit")]
pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
use std::convert::TryInto;
for reloc in self.relocs.drain(..) {
match reloc.name {
super::DebugRelocName::Section(_) => unreachable!(),

44
compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
View file

@ -10,7 +10,7 @@ use crate::prelude::*;
use rustc_index::vec::IndexVec;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::{LabelValueLoc, StackSlots, ValueLabel, ValueLoc};
use cranelift_codegen::ir::{LabelValueLoc, ValueLabel};
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::ValueLocRange;
@ -67,7 +67,12 @@ impl<'tcx> DebugContext<'tcx> {
rustc_interface::util::version_str().unwrap_or("unknown version"),
cranelift_codegen::VERSION,
);
let comp_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped).into_owned();
let comp_dir = tcx
.sess
.opts
.working_dir
.to_string_lossy(FileNameDisplayPreference::Remapped)
.into_owned();
let (name, file_info) = match tcx.sess.local_crate_source_file.clone() {
Some(path) => {
let name = path.to_string_lossy().into_owned();
@ -250,7 +255,7 @@ impl<'tcx> DebugContext<'tcx> {
// FIXME make it more reliable and implement scopes before re-enabling this.
if false {
let value_labels_ranges = context.build_value_labels_ranges(isa).unwrap();
let value_labels_ranges = std::collections::HashMap::new(); // FIXME
for (local, _local_decl) in mir.local_decls.iter_enumerated() {
let ty = self.tcx.subst_and_normalize_erasing_regions(
@ -264,7 +269,6 @@ impl<'tcx> DebugContext<'tcx> {
self,
isa,
symbol,
context,
&local_map,
&value_labels_ranges,
Place { local, projection: ty::List::empty() },
@ -283,7 +287,6 @@ fn place_location<'tcx>(
debug_context: &mut DebugContext<'tcx>,
isa: &dyn TargetIsa,
symbol: usize,
context: &Context,
local_map: &IndexVec<mir::Local, CPlace<'tcx>>,
#[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap<
ValueLabel,
@ -306,12 +309,7 @@ fn place_location<'tcx>(
addend: i64::from(value_loc_range.start),
},
end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
data: translate_loc(
isa,
value_loc_range.loc,
&context.func.stack_slots,
)
.unwrap(),
data: translate_loc(isa, value_loc_range.loc).unwrap(),
})
.collect(),
);
@ -340,34 +338,14 @@ fn place_location<'tcx>(
AttributeValue::Exprloc(Expression::new())
// For PointerBase::Stack:
//AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot), &context.func.stack_slots).unwrap())
//AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot)).unwrap())
}
}
}
// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
fn translate_loc(
isa: &dyn TargetIsa,
loc: LabelValueLoc,
stack_slots: &StackSlots,
) -> Option<Expression> {
fn translate_loc(isa: &dyn TargetIsa, loc: LabelValueLoc) -> Option<Expression> {
match loc {
LabelValueLoc::ValueLoc(ValueLoc::Reg(reg)) => {
let machine_reg = isa.map_dwarf_register(reg).unwrap();
let mut expr = Expression::new();
expr.op_reg(gimli::Register(machine_reg));
Some(expr)
}
LabelValueLoc::ValueLoc(ValueLoc::Stack(ss)) => {
if let Some(ss_offset) = stack_slots[ss].offset {
let mut expr = Expression::new();
expr.op_breg(X86_64::RBP, i64::from(ss_offset) + 16);
Some(expr)
} else {
None
}
}
LabelValueLoc::ValueLoc(ValueLoc::Unassigned) => unreachable!(),
LabelValueLoc::Reg(reg) => {
let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
let mut expr = Expression::new();

2
compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
View file

@ -1,5 +1,3 @@
use std::convert::{TryFrom, TryInto};
use rustc_data_structures::fx::FxHashMap;
use cranelift_module::FuncId;

5
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -4,6 +4,7 @@
use std::path::PathBuf;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_metadata::EncodedMetadata;
@ -123,6 +124,7 @@ fn module_codegen(
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
super::predefine_mono_items(tcx, &mut module, &mono_items);
for (mono_item, _) in mono_items {
@ -277,7 +279,8 @@ pub(crate) fn run_aot(
let tmp_file =
tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
let obj = crate::metadata::new_metadata_object(tcx, &metadata_cgu_name, &metadata);
let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name);
if let Err(err) = std::fs::write(&tmp_file, obj) {
tcx.sess.fatal(&format!("error writing metadata object file: {}", err));

26
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -3,7 +3,7 @@
use std::cell::RefCell;
use std::ffi::CString;
use std::lazy::{Lazy, SyncOnceCell};
use std::lazy::SyncOnceCell;
use std::os::raw::{c_char, c_int};
use std::sync::{mpsc, Mutex};
@ -11,6 +11,7 @@ use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::Session;
use rustc_span::Symbol;
use cranelift_jit::{JITBuilder, JITModule};
@ -23,7 +24,7 @@ struct JitState {
}
thread_local! {
static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
static LAZY_JIT_STATE: RefCell<Option<JitState>> = const { RefCell::new(None) };
}
/// The Sender owned by the rustc thread
@ -50,12 +51,11 @@ impl UnsafeMessage {
fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> {
thread_local! {
/// The Sender owned by the local thread
static LOCAL_MESSAGE_SENDER: Lazy<mpsc::Sender<UnsafeMessage>> = Lazy::new(||
static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> =
GLOBAL_MESSAGE_SENDER
.get().unwrap()
.lock().unwrap()
.clone()
);
.clone();
}
LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self))
}
@ -76,7 +76,13 @@ fn create_jit_module<'tcx>(
jit_builder.symbols(imported_symbols);
let mut jit_module = JITModule::new(jit_builder);
let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false);
let mut cx = crate::CodegenCx::new(
tcx,
backend_config.clone(),
jit_module.isa(),
false,
Symbol::intern("dummy_cgu_name"),
);
crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
crate::main_shim::maybe_create_entry_wrapper(
@ -246,7 +252,13 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
jit_module.prepare_for_function_redefine(func_id).unwrap();
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
let mut cx = crate::CodegenCx::new(
tcx,
backend_config,
jit_module.isa(),
false,
Symbol::intern("dummy_cgu_name"),
);
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
assert!(cx.global_asm.is_empty());

653
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -6,6 +6,7 @@ use std::fmt::Write;
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_middle::mir::InlineAsmOperand;
use rustc_span::Symbol;
use rustc_target::asm::*;
pub(crate) fn codegen_inline_asm<'tcx>(
@ -17,10 +18,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
) {
// FIXME add .eh_frame unwind info directives
if template.is_empty() {
// Black box
return;
} else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::User(1));
return;
@ -41,8 +39,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
assert_eq!(operands.len(), 4);
let (leaf, eax_place) = match operands[1] {
InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax));
assert_eq!(
reg,
InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax))
);
(
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place.unwrap()),
@ -64,8 +64,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
};
let (sub_leaf, ecx_place) = match operands[2] {
InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx));
assert_eq!(
reg,
InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx))
);
(
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place.unwrap()),
@ -75,8 +77,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
};
let edx_place = match operands[3] {
InlineAsmOperand::Out { reg, late: true, place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx));
assert_eq!(
reg,
InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx))
);
crate::base::codegen_place(fx, place.unwrap())
}
_ => unreachable!(),
@ -96,60 +100,59 @@ pub(crate) fn codegen_inline_asm<'tcx>(
crate::trap::trap_unimplemented(fx, "Alloca is not supported");
}
let mut slot_size = Size::from_bytes(0);
let mut clobbered_regs = Vec::new();
let mut inputs = Vec::new();
let mut outputs = Vec::new();
let mut new_slot = |reg_class: InlineAsmRegClass| {
let reg_size = reg_class
.supported_types(InlineAsmArch::X86_64)
.iter()
.map(|(ty, _)| ty.size())
.max()
.unwrap();
let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap();
slot_size = slot_size.align_to(align);
let offset = slot_size;
slot_size += reg_size;
offset
let mut asm_gen = InlineAssemblyGenerator {
tcx: fx.tcx,
arch: fx.tcx.sess.asm_arch.unwrap(),
template,
operands,
options,
registers: Vec::new(),
stack_slots_clobber: Vec::new(),
stack_slots_input: Vec::new(),
stack_slots_output: Vec::new(),
stack_slot_size: Size::from_bytes(0),
};
asm_gen.allocate_registers();
asm_gen.allocate_stack_slots();
// FIXME overlap input and output slots to save stack space
for operand in operands {
let inline_asm_index = fx.cx.inline_asm_index.get();
fx.cx.inline_asm_index.set(inline_asm_index + 1);
let asm_name = format!(
"__inline_asm_{}_n{}",
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
inline_asm_index
);
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
fx.cx.global_asm.push_str(&generated_asm);
for (i, operand) in operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg, ref value } => {
let reg = expect_reg(reg);
clobbered_regs.push((reg, new_slot(reg.reg_class())));
InlineAsmOperand::In { reg: _, ref value } => {
inputs.push((
reg,
new_slot(reg.reg_class()),
asm_gen.stack_slots_input[i].unwrap(),
crate::base::codegen_operand(fx, value).load_scalar(fx),
));
}
InlineAsmOperand::Out { reg, late: _, place } => {
let reg = expect_reg(reg);
clobbered_regs.push((reg, new_slot(reg.reg_class())));
InlineAsmOperand::Out { reg: _, late: _, place } => {
if let Some(place) = place {
outputs.push((
reg,
new_slot(reg.reg_class()),
asm_gen.stack_slots_output[i].unwrap(),
crate::base::codegen_place(fx, place),
));
}
}
InlineAsmOperand::InOut { reg, late: _, ref in_value, out_place } => {
let reg = expect_reg(reg);
clobbered_regs.push((reg, new_slot(reg.reg_class())));
InlineAsmOperand::InOut { reg: _, late: _, ref in_value, out_place } => {
inputs.push((
reg,
new_slot(reg.reg_class()),
asm_gen.stack_slots_input[i].unwrap(),
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
));
if let Some(out_place) = out_place {
outputs.push((
reg,
new_slot(reg.reg_class()),
asm_gen.stack_slots_output[i].unwrap(),
crate::base::codegen_place(fx, out_place),
));
}
@ -160,110 +163,474 @@ pub(crate) fn codegen_inline_asm<'tcx>(
}
}
let inline_asm_index = fx.inline_asm_index;
fx.inline_asm_index += 1;
let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index);
let generated_asm = generate_asm_wrapper(
&asm_name,
InlineAsmArch::X86_64,
options,
template,
clobbered_regs,
&inputs,
&outputs,
);
fx.cx.global_asm.push_str(&generated_asm);
call_inline_asm(fx, &asm_name, slot_size, inputs, outputs);
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
}
fn generate_asm_wrapper(
asm_name: &str,
struct InlineAssemblyGenerator<'a, 'tcx> {
tcx: TyCtxt<'tcx>,
arch: InlineAsmArch,
template: &'a [InlineAsmTemplatePiece],
operands: &'a [InlineAsmOperand<'tcx>],
options: InlineAsmOptions,
template: &[InlineAsmTemplatePiece],
clobbered_regs: Vec<(InlineAsmReg, Size)>,
inputs: &[(InlineAsmReg, Size, Value)],
outputs: &[(InlineAsmReg, Size, CPlace<'_>)],
) -> String {
let mut generated_asm = String::new();
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
writeln!(generated_asm, "{}:", asm_name).unwrap();
registers: Vec<Option<InlineAsmReg>>,
stack_slots_clobber: Vec<Option<Size>>,
stack_slots_input: Vec<Option<Size>>,
stack_slots_output: Vec<Option<Size>>,
stack_slot_size: Size,
}
generated_asm.push_str(".intel_syntax noprefix\n");
generated_asm.push_str(" push rbp\n");
generated_asm.push_str(" mov rbp,rdi\n");
impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
fn allocate_registers(&mut self) {
let sess = self.tcx.sess;
let map = allocatable_registers(
self.arch,
|feature| sess.target_features.contains(&Symbol::intern(feature)),
&sess.target,
);
let mut allocated = FxHashMap::<_, (bool, bool)>::default();
let mut regs = vec![None; self.operands.len()];
// Save clobbered registers
if !options.contains(InlineAsmOptions::NORETURN) {
// FIXME skip registers saved by the calling convention
for &(reg, offset) in &clobbered_regs {
save_register(&mut generated_asm, arch, reg, offset);
}
}
// Write input registers
for &(reg, offset, _value) in inputs {
restore_register(&mut generated_asm, arch, reg, offset);
}
if options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push_str(".att_syntax\n");
}
// The actual inline asm
for piece in template {
match piece {
InlineAsmTemplatePiece::String(s) => {
generated_asm.push_str(s);
// Add explicit registers to the allocated set.
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
regs[i] = Some(reg);
allocated.entry(reg).or_default().0 = true;
}
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(reg), late: true, ..
} => {
regs[i] = Some(reg);
allocated.entry(reg).or_default().1 = true;
}
InlineAsmOperand::Out { reg: InlineAsmRegOrRegClass::Reg(reg), .. }
| InlineAsmOperand::InOut { reg: InlineAsmRegOrRegClass::Reg(reg), .. } => {
regs[i] = Some(reg);
allocated.insert(reg, (true, true));
}
_ => (),
}
InlineAsmTemplatePiece::Placeholder { operand_idx: _, modifier: _, span: _ } => todo!(),
}
}
generated_asm.push('\n');
if options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push_str(".intel_syntax noprefix\n");
}
if !options.contains(InlineAsmOptions::NORETURN) {
// Read output registers
for &(reg, offset, _place) in outputs {
save_register(&mut generated_asm, arch, reg, offset);
}
// Restore clobbered registers
for &(reg, offset) in clobbered_regs.iter().rev() {
restore_register(&mut generated_asm, arch, reg, offset);
// Allocate out/inout/inlateout registers first because they are more constrained.
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::RegClass(class),
late: false,
..
}
| InlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::RegClass(class), ..
} => {
let mut alloc_reg = None;
for &reg in &map[&class] {
let mut used = false;
reg.overlapping_regs(|r| {
if allocated.contains_key(&r) {
used = true;
}
});
if !used {
alloc_reg = Some(reg);
break;
}
}
let reg = alloc_reg.expect("cannot allocate registers");
regs[i] = Some(reg);
allocated.insert(reg, (true, true));
}
_ => (),
}
}
generated_asm.push_str(" pop rbp\n");
generated_asm.push_str(" ret\n");
} else {
generated_asm.push_str(" ud2\n");
// Allocate in/lateout.
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg: InlineAsmRegOrRegClass::RegClass(class), .. } => {
let mut alloc_reg = None;
for &reg in &map[&class] {
let mut used = false;
reg.overlapping_regs(|r| {
if allocated.get(&r).copied().unwrap_or_default().0 {
used = true;
}
});
if !used {
alloc_reg = Some(reg);
break;
}
}
let reg = alloc_reg.expect("cannot allocate registers");
regs[i] = Some(reg);
allocated.entry(reg).or_default().0 = true;
}
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::RegClass(class),
late: true,
..
} => {
let mut alloc_reg = None;
for &reg in &map[&class] {
let mut used = false;
reg.overlapping_regs(|r| {
if allocated.get(&r).copied().unwrap_or_default().1 {
used = true;
}
});
if !used {
alloc_reg = Some(reg);
break;
}
}
let reg = alloc_reg.expect("cannot allocate registers");
regs[i] = Some(reg);
allocated.entry(reg).or_default().1 = true;
}
_ => (),
}
}
self.registers = regs;
}
generated_asm.push_str(".att_syntax\n");
writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
generated_asm.push_str(".text\n");
generated_asm.push_str("\n\n");
fn allocate_stack_slots(&mut self) {
let mut slot_size = Size::from_bytes(0);
let mut slots_clobber = vec![None; self.operands.len()];
let mut slots_input = vec![None; self.operands.len()];
let mut slots_output = vec![None; self.operands.len()];
generated_asm
let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| {
let reg_size =
reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap();
let align = rustc_target::abi::Align::from_bytes(reg_size.bytes()).unwrap();
let offset = slot_size.align_to(align);
*slot_size = offset + reg_size;
offset
};
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
// Allocate stack slots for saving clobbered registers
let abi_clobber = InlineAsmClobberAbi::parse(
self.arch,
|feature| self.tcx.sess.target_features.contains(&Symbol::intern(feature)),
&self.tcx.sess.target,
Symbol::intern("C"),
)
.unwrap()
.clobbered_regs();
for (i, reg) in self.registers.iter().enumerate().filter_map(|(i, r)| r.map(|r| (i, r))) {
let mut need_save = true;
// If the register overlaps with a register clobbered by function call, then
// we don't need to save it.
for r in abi_clobber {
r.overlapping_regs(|r| {
if r == reg {
need_save = false;
}
});
if !need_save {
break;
}
}
if need_save {
slots_clobber[i] = Some(new_slot(reg.reg_class()));
}
}
// Allocate stack slots for inout
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::InOut { reg, out_place: Some(_), .. } => {
let slot = new_slot(reg.reg_class());
slots_input[i] = Some(slot);
slots_output[i] = Some(slot);
}
_ => (),
}
}
let slot_size_before_input = slot_size;
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
// Allocate stack slots for input
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::In { reg, .. }
| InlineAsmOperand::InOut { reg, out_place: None, .. } => {
slots_input[i] = Some(new_slot(reg.reg_class()));
}
_ => (),
}
}
// Reset slot size to before input so that input and output operands can overlap
// and save some memory.
let slot_size_after_input = slot_size;
slot_size = slot_size_before_input;
let mut new_slot = |x| new_slot_fn(&mut slot_size, x);
// Allocate stack slots for output
for (i, operand) in self.operands.iter().enumerate() {
match *operand {
InlineAsmOperand::Out { reg, place: Some(_), .. } => {
slots_output[i] = Some(new_slot(reg.reg_class()));
}
_ => (),
}
}
slot_size = slot_size.max(slot_size_after_input);
self.stack_slots_clobber = slots_clobber;
self.stack_slots_input = slots_input;
self.stack_slots_output = slots_output;
self.stack_slot_size = slot_size;
}
fn generate_asm_wrapper(&self, asm_name: &str) -> String {
let mut generated_asm = String::new();
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
writeln!(generated_asm, "{}:", asm_name).unwrap();
let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
if is_x86 {
generated_asm.push_str(".intel_syntax noprefix\n");
}
Self::prologue(&mut generated_asm, self.arch);
// Save clobbered registers
if !self.options.contains(InlineAsmOptions::NORETURN) {
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
}
// Write input registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_input.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push_str(".att_syntax\n");
}
// The actual inline asm
for piece in self.template {
match piece {
InlineAsmTemplatePiece::String(s) => {
generated_asm.push_str(s);
}
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
if self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push('%');
}
self.registers[*operand_idx]
.unwrap()
.emit(&mut generated_asm, self.arch, *modifier)
.unwrap();
}
}
}
generated_asm.push('\n');
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push_str(".intel_syntax noprefix\n");
}
if !self.options.contains(InlineAsmOptions::NORETURN) {
// Read output registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_output.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
// Restore clobbered registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
Self::epilogue(&mut generated_asm, self.arch);
} else {
Self::epilogue_noreturn(&mut generated_asm, self.arch);
}
if is_x86 {
generated_asm.push_str(".att_syntax\n");
}
writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
generated_asm.push_str(".text\n");
generated_asm.push_str("\n\n");
generated_asm
}
fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
match arch {
InlineAsmArch::X86 => {
generated_asm.push_str(" push ebp\n");
generated_asm.push_str(" mov ebp,[esp+8]\n");
}
InlineAsmArch::X86_64 => {
generated_asm.push_str(" push rbp\n");
generated_asm.push_str(" mov rbp,rdi\n");
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" addi sp, sp, -8\n");
generated_asm.push_str(" sw ra, 4(sp)\n");
generated_asm.push_str(" sw s0, 0(sp)\n");
generated_asm.push_str(" mv s0, a0\n");
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" addi sp, sp, -16\n");
generated_asm.push_str(" sd ra, 8(sp)\n");
generated_asm.push_str(" sd s0, 0(sp)\n");
generated_asm.push_str(" mv s0, a0\n");
}
_ => unimplemented!("prologue for {:?}", arch),
}
}
fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
match arch {
InlineAsmArch::X86 => {
generated_asm.push_str(" pop ebp\n");
generated_asm.push_str(" ret\n");
}
InlineAsmArch::X86_64 => {
generated_asm.push_str(" pop rbp\n");
generated_asm.push_str(" ret\n");
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" lw s0, 0(sp)\n");
generated_asm.push_str(" lw ra, 4(sp)\n");
generated_asm.push_str(" addi sp, sp, 8\n");
generated_asm.push_str(" ret\n");
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" ld s0, 0(sp)\n");
generated_asm.push_str(" ld ra, 8(sp)\n");
generated_asm.push_str(" addi sp, sp, 16\n");
generated_asm.push_str(" ret\n");
}
_ => unimplemented!("epilogue for {:?}", arch),
}
}
fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
match arch {
InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
generated_asm.push_str(" ud2\n");
}
InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
generated_asm.push_str(" ebreak\n");
}
_ => unimplemented!("epilogue_noreturn for {:?}", arch),
}
}
fn save_register(
generated_asm: &mut String,
arch: InlineAsmArch,
reg: InlineAsmReg,
offset: Size,
) {
match arch {
InlineAsmArch::X86 => {
write!(generated_asm, " mov [ebp+0x{:x}], ", offset.bytes()).unwrap();
reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
generated_asm.push('\n');
}
InlineAsmArch::X86_64 => {
write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
generated_asm.push('\n');
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" sw ");
reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" sd ");
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
}
_ => unimplemented!("save_register for {:?}", arch),
}
}
fn restore_register(
generated_asm: &mut String,
arch: InlineAsmArch,
reg: InlineAsmReg,
offset: Size,
) {
match arch {
InlineAsmArch::X86 => {
generated_asm.push_str(" mov ");
reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
writeln!(generated_asm, ", [ebp+0x{:x}]", offset.bytes()).unwrap();
}
InlineAsmArch::X86_64 => {
generated_asm.push_str(" mov ");
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" lw ");
reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" ld ");
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
}
_ => unimplemented!("restore_register for {:?}", arch),
}
}
}
fn call_inline_asm<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
asm_name: &str,
slot_size: Size,
inputs: Vec<(InlineAsmReg, Size, Value)>,
outputs: Vec<(InlineAsmReg, Size, CPlace<'tcx>)>,
inputs: Vec<(Size, Value)>,
outputs: Vec<(Size, CPlace<'tcx>)>,
) {
let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
offset: None,
size: u32::try_from(slot_size.bytes()).unwrap(),
});
if fx.clif_comments.enabled() {
@ -287,50 +654,16 @@ fn call_inline_asm<'tcx>(
fx.add_comment(inline_asm_func, asm_name);
}
for (_reg, offset, value) in inputs {
for (offset, value) in inputs {
fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
}
let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
for (_reg, offset, place) in outputs {
for (offset, place) in outputs {
let ty = fx.clif_type(place.layout().ty).unwrap();
let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
}
}
fn expect_reg(reg_or_class: InlineAsmRegOrRegClass) -> InlineAsmReg {
match reg_or_class {
InlineAsmRegOrRegClass::Reg(reg) => reg,
InlineAsmRegOrRegClass::RegClass(class) => unimplemented!("{:?}", class),
}
}
fn save_register(generated_asm: &mut String, arch: InlineAsmArch, reg: InlineAsmReg, offset: Size) {
match arch {
InlineAsmArch::X86_64 => {
write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
generated_asm.push('\n');
}
_ => unimplemented!("save_register for {:?}", arch),
}
}
fn restore_register(
generated_asm: &mut String,
arch: InlineAsmArch,
reg: InlineAsmReg,
offset: Size,
) {
match arch {
InlineAsmArch::X86_64 => {
generated_asm.push_str(" mov ");
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
}
_ => unimplemented!("restore_register for {:?}", arch),
}
}

12
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -503,10 +503,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
if intrinsic == sym::copy_nonoverlapping {
// FIXME emit_small_memcpy
fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
} else {
// FIXME emit_small_memmove
fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
}
};
// NOTE: the volatile variants have src and dst swapped
@ -522,10 +522,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
// FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
if intrinsic == sym::volatile_copy_nonoverlapping_memory {
// FIXME emit_small_memcpy
fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
} else {
// FIXME emit_small_memmove
fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
}
};
size_of_val, <T> (c ptr) {
@ -673,7 +673,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let dst_ptr = dst.load_scalar(fx);
// FIXME make the memset actually volatile when switching to emit_small_memset
// FIXME use emit_small_memset
fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
};
ctlz | ctlz_nonzero, <T> (v arg) {
// FIXME trap on `ctlz_nonzero` with zero arg.
@ -1067,7 +1067,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
kw.Try, (v f, v data, v _catch_fn) {
// FIXME once unwinding is supported, change this to actually catch panics
let f_sig = fx.bcx.func.import_signature(Signature {
call_conv: CallConv::triple_default(fx.triple()),
call_conv: fx.target_config.default_call_conv,
params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
returns: vec![],
});

53
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -67,7 +67,34 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
_ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap();
// If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
// If there is no suffix, use the index array length.
let n: u16 = if intrinsic == sym::simd_shuffle {
// Make sure this is actually an array, since typeck only checks the length-suffixed
// version of this intrinsic.
let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
match idx_ty.kind() {
ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
span_bug!(span, "could not evaluate shuffle index array length")
}).try_into().unwrap()
}
_ => {
fx.tcx.sess.span_err(
span,
&format!(
"simd_shuffle index must be an array of `u32`, got `{}`",
idx_ty,
),
);
// Prevent verifier error
crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
return;
}
}
} else {
intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
};
assert_eq!(x.layout(), y.layout());
let layout = x.layout();
@ -378,27 +405,27 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_reduce_min, (c v) {
// FIXME support floats
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
let lt = fx.bcx.ins().icmp(if layout.ty.is_signed() {
IntCC::SignedLessThan
} else {
IntCC::UnsignedLessThan
}, a, b);
let lt = match layout.ty.kind() {
ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
_ => unreachable!(),
};
fx.bcx.ins().select(lt, a, b)
});
};
simd_reduce_max, (c v) {
// FIXME support floats
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
let gt = fx.bcx.ins().icmp(if layout.ty.is_signed() {
IntCC::SignedGreaterThan
} else {
IntCC::UnsignedGreaterThan
}, a, b);
let gt = match layout.ty.kind() {
ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
_ => unreachable!(),
};
fx.bcx.ins().select(gt, a, b)
});
};

31
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -4,7 +4,6 @@
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
extern crate snap;
#[macro_use]
extern crate rustc_middle;
extern crate rustc_ast;
@ -26,6 +25,7 @@ extern crate rustc_target;
extern crate rustc_driver;
use std::any::Any;
use std::cell::Cell;
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
@ -34,6 +34,7 @@ use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_session::config::OutputFilenames;
use rustc_session::Session;
use rustc_span::Symbol;
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::settings::{self, Configurable};
@ -59,7 +60,6 @@ mod inline_asm;
mod intrinsics;
mod linkage;
mod main_shim;
mod metadata;
mod num;
mod optimize;
mod pointer;
@ -71,9 +71,7 @@ mod value_and_place;
mod vtable;
mod prelude {
pub(crate) use std::convert::{TryFrom, TryInto};
pub(crate) use rustc_span::{Span, FileNameDisplayPreference};
pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
pub(crate) use rustc_middle::bug;
@ -125,9 +123,11 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
struct CodegenCx<'tcx> {
tcx: TyCtxt<'tcx>,
global_asm: String,
inline_asm_index: Cell<usize>,
cached_context: Context,
debug_context: Option<DebugContext<'tcx>>,
unwind_context: UnwindContext,
cgu_name: Symbol,
}
impl<'tcx> CodegenCx<'tcx> {
@ -136,6 +136,7 @@ impl<'tcx> CodegenCx<'tcx> {
backend_config: BackendConfig,
isa: &dyn TargetIsa,
debug_info: bool,
cgu_name: Symbol,
) -> Self {
assert_eq!(pointer_ty(tcx), isa.pointer_type());
@ -145,9 +146,11 @@ impl<'tcx> CodegenCx<'tcx> {
CodegenCx {
tcx,
global_asm: String::new(),
inline_asm_index: Cell::new(0),
cached_context: Context::new(),
debug_context,
unwind_context,
cgu_name,
}
}
}
@ -269,19 +272,16 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
let flags = settings::Flags::new(flags_builder);
let variant = cranelift_codegen::isa::BackendVariant::MachInst;
let isa_builder = match sess.opts.cg.target_cpu.as_deref() {
Some("native") => {
let builder = cranelift_native::builder_with_options(variant, true).unwrap();
let builder = cranelift_native::builder_with_options(true).unwrap();
builder
}
Some(value) => {
let mut builder =
cranelift_codegen::isa::lookup_variant(target_triple.clone(), variant)
.unwrap_or_else(|err| {
sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
});
cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
});
if let Err(_) = builder.enable(value) {
sess.fatal("the specified target cpu isn't currently supported by Cranelift.");
}
@ -289,10 +289,9 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
}
None => {
let mut builder =
cranelift_codegen::isa::lookup_variant(target_triple.clone(), variant)
.unwrap_or_else(|err| {
sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
});
cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| {
sess.fatal(&format!("can't compile for {}: {}", target_triple, err));
});
if target_triple.architecture == target_lexicon::Architecture::X86_64 {
// Don't use "haswell" as the default, as it implies `has_lzcnt`.
// macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.

76
compiler/rustc_codegen_cranelift/src/metadata.rs
View file

@ -1,76 +0,0 @@
//! Writing of the rustc metadata for dylibs
use object::write::{Object, StandardSegment, Symbol, SymbolSection};
use object::{SectionKind, SymbolFlags, SymbolKind, SymbolScope};
use rustc_metadata::EncodedMetadata;
use rustc_middle::ty::TyCtxt;
// Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112
pub(crate) fn new_metadata_object(
tcx: TyCtxt<'_>,
cgu_name: &str,
metadata: &EncodedMetadata,
) -> Vec<u8> {
use snap::write::FrameEncoder;
use std::io::Write;
let mut compressed = rustc_metadata::METADATA_HEADER.to_vec();
FrameEncoder::new(&mut compressed).write_all(metadata.raw_data()).unwrap();
let triple = crate::target_triple(tcx.sess);
let binary_format = match triple.binary_format {
target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
target_lexicon::BinaryFormat::Coff => object::BinaryFormat::Coff,
target_lexicon::BinaryFormat::Macho => object::BinaryFormat::MachO,
binary_format => tcx.sess.fatal(&format!("binary format {} is unsupported", binary_format)),
};
let architecture = match triple.architecture {
target_lexicon::Architecture::Aarch64(_) => object::Architecture::Aarch64,
target_lexicon::Architecture::Arm(_) => object::Architecture::Arm,
target_lexicon::Architecture::Avr => object::Architecture::Avr,
target_lexicon::Architecture::Hexagon => object::Architecture::Hexagon,
target_lexicon::Architecture::Mips32(_) => object::Architecture::Mips,
target_lexicon::Architecture::Mips64(_) => object::Architecture::Mips64,
target_lexicon::Architecture::Msp430 => object::Architecture::Msp430,
target_lexicon::Architecture::Powerpc => object::Architecture::PowerPc,
target_lexicon::Architecture::Powerpc64 => object::Architecture::PowerPc64,
target_lexicon::Architecture::Powerpc64le => todo!(),
target_lexicon::Architecture::Riscv32(_) => object::Architecture::Riscv32,
target_lexicon::Architecture::Riscv64(_) => object::Architecture::Riscv64,
target_lexicon::Architecture::S390x => object::Architecture::S390x,
target_lexicon::Architecture::Sparc64 => object::Architecture::Sparc64,
target_lexicon::Architecture::Sparcv9 => object::Architecture::Sparc64,
target_lexicon::Architecture::X86_32(_) => object::Architecture::I386,
target_lexicon::Architecture::X86_64 => object::Architecture::X86_64,
architecture => {
tcx.sess.fatal(&format!("target architecture {:?} is unsupported", architecture,))
}
};
let endian = match triple.endianness().unwrap() {
target_lexicon::Endianness::Little => object::Endianness::Little,
target_lexicon::Endianness::Big => object::Endianness::Big,
};
let mut object = Object::new(binary_format, architecture, endian);
object.add_file_symbol(cgu_name.as_bytes().to_vec());
let segment = object.segment_name(StandardSegment::Data).to_vec();
let section_id = object.add_section(segment, b".rustc".to_vec(), SectionKind::Data);
let offset = object.append_section_data(section_id, &compressed, 1);
// For MachO and probably PE this is necessary to prevent the linker from throwing away the
// .rustc section. For ELF this isn't necessary, but it also doesn't harm.
object.add_symbol(Symbol {
name: rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx).into_bytes(),
value: offset,
size: compressed.len() as u64,
kind: SymbolKind::Data,
scope: SymbolScope::Dynamic,
weak: false,
section: SymbolSection::Section(section_id),
flags: SymbolFlags::None,
});
object.write().unwrap()
}

13
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -57,7 +57,7 @@ use std::io::Write;
use cranelift_codegen::{
entity::SecondaryMap,
ir::{entities::AnyEntity, function::DisplayFunctionAnnotations},
ir::entities::AnyEntity,
write::{FuncWriter, PlainWriter},
};
@ -129,7 +129,6 @@ impl FuncWriter for &'_ CommentWriter {
&mut self,
w: &mut dyn fmt::Write,
func: &Function,
reg_info: Option<&isa::RegInfo>,
) -> Result<bool, fmt::Error> {
for comment in &self.global_comments {
if !comment.is_empty() {
@ -142,7 +141,7 @@ impl FuncWriter for &'_ CommentWriter {
writeln!(w)?;
}
self.super_preamble(w, func, reg_info)
self.super_preamble(w, func)
}
fn write_entity_definition(
@ -165,11 +164,10 @@ impl FuncWriter for &'_ CommentWriter {
&mut self,
w: &mut dyn fmt::Write,
func: &Function,
isa: Option<&dyn isa::TargetIsa>,
block: Block,
indent: usize,
) -> fmt::Result {
PlainWriter.write_block_header(w, func, isa, block, indent)
PlainWriter.write_block_header(w, func, block, indent)
}
fn write_instruction(
@ -177,11 +175,10 @@ impl FuncWriter for &'_ CommentWriter {
w: &mut dyn fmt::Write,
func: &Function,
aliases: &SecondaryMap<Value, Vec<Value>>,
isa: Option<&dyn isa::TargetIsa>,
inst: Inst,
indent: usize,
) -> fmt::Result {
PlainWriter.write_instruction(w, func, aliases, isa, inst, indent)?;
PlainWriter.write_instruction(w, func, aliases, inst, indent)?;
if let Some(comment) = self.entity_comments.get(&inst.into()) {
writeln!(w, "; {}", comment.replace('\n', "\n; "))?;
}
@ -249,7 +246,6 @@ pub(crate) fn write_clif_file<'tcx>(
&mut clif_comments,
&mut clif,
&context.func,
&DisplayFunctionAnnotations { isa: Some(isa), value_ranges: None },
)
.unwrap();
@ -278,7 +274,6 @@ impl fmt::Debug for FunctionCx<'_, '_, '_> {
&mut &self.clif_comments,
&mut clif,
&self.bcx.func,
&DisplayFunctionAnnotations::default(),
)
.unwrap();
writeln!(f, "\n{}", clif)

2
compiler/rustc_codegen_cranelift/src/trap.rs
View file

@ -9,7 +9,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
"puts",
Linkage::Import,
&Signature {
call_conv: CallConv::triple_default(fx.triple()),
call_conv: fx.target_config.default_call_conv,
params: vec![AbiParam::new(fx.pointer_type)],
returns: vec![AbiParam::new(types::I32)],
},

62
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -329,7 +329,6 @@ impl<'tcx> CPlace<'tcx> {
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
offset: None,
});
CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
}
@ -472,7 +471,6 @@ impl<'tcx> CPlace<'tcx> {
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
size: (src_ty.bytes() + 15) / 16 * 16,
offset: None,
});
let ptr = Pointer::stack_slot(stack_slot);
ptr.store(fx, data, MemFlags::trusted());
@ -512,6 +510,26 @@ impl<'tcx> CPlace<'tcx> {
let dst_layout = self.layout();
let to_ptr = match self.inner {
CPlaceInner::Var(_local, var) => {
if let ty::Array(element, len) = dst_layout.ty.kind() {
// Can only happen for vector types
let len =
u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
let vector_ty = fx.clif_type(element).unwrap().by(len).unwrap();
let data = match from.0 {
CValueInner::ByRef(ptr, None) => {
let mut flags = MemFlags::new();
flags.set_notrap();
ptr.load(fx, vector_ty, flags)
}
CValueInner::ByVal(_)
| CValueInner::ByValPair(_, _)
| CValueInner::ByRef(_, Some(_)) => bug!("array should be ByRef"),
};
fx.bcx.def_var(var, data);
return;
}
let data = CValue(from.0, dst_layout).load_scalar(fx);
let dst_ty = fx.clif_type(self.layout().ty).unwrap();
transmute_value(fx, var, data, dst_ty);
@ -583,7 +601,7 @@ impl<'tcx> CPlace<'tcx> {
let src_align = src_layout.align.abi.bytes() as u8;
let dst_align = dst_layout.align.abi.bytes() as u8;
fx.bcx.emit_small_memory_copy(
fx.module.target_config(),
fx.target_config,
to_addr,
from_addr,
size,
@ -605,14 +623,39 @@ impl<'tcx> CPlace<'tcx> {
let layout = self.layout();
match self.inner {
CPlaceInner::Var(local, var) => {
if let Abi::Vector { .. } = layout.abi {
CPlaceInner::Var(local, var) => match layout.ty.kind() {
ty::Array(_, _) => {
// Can only happen for vector types
return CPlace {
inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()),
layout: layout.field(fx, field.as_u32().try_into().unwrap()),
};
}
}
ty::Adt(adt_def, substs) if layout.ty.is_simd() => {
let f0_ty = adt_def.non_enum_variant().fields[0].ty(fx.tcx, substs);
match f0_ty.kind() {
ty::Array(_, _) => {
assert_eq!(field.as_u32(), 0);
return CPlace {
inner: CPlaceInner::Var(local, var),
layout: layout.field(fx, field.as_u32().try_into().unwrap()),
};
}
_ => {
return CPlace {
inner: CPlaceInner::VarLane(
local,
var,
field.as_u32().try_into().unwrap(),
),
layout: layout.field(fx, field.as_u32().try_into().unwrap()),
};
}
}
}
_ => {}
},
CPlaceInner::VarPair(local, var1, var2) => {
let layout = layout.field(&*fx, field.index());
@ -629,7 +672,12 @@ impl<'tcx> CPlace<'tcx> {
let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field);
if field_layout.is_unsized() {
CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
if let ty::Foreign(_) = field_layout.ty.kind() {
assert!(extra.is_none());
CPlace::for_ptr(field_ptr, field_layout)
} else {
CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
}
} else {
CPlace::for_ptr(field_ptr, field_layout)
}