Merge commit 'def04540a4' into sync_cg_clif-2023-11-16
This commit is contained in:
bjorn3
commit
4db95ce3cb
15 changed files with 472 additions and 191 deletions
|
|
@ -10,7 +10,7 @@ use target_lexicon::BinaryFormat;
|
|||
|
||||
use crate::prelude::*;
|
||||
|
||||
enum CInlineAsmOperand<'tcx> {
|
||||
pub(crate) enum CInlineAsmOperand<'tcx> {
|
||||
In {
|
||||
reg: InlineAsmRegOrRegClass,
|
||||
value: Value,
|
||||
|
|
@ -34,7 +34,7 @@ enum CInlineAsmOperand<'tcx> {
|
|||
},
|
||||
}
|
||||
|
||||
pub(crate) fn codegen_inline_asm<'tcx>(
|
||||
pub(crate) fn codegen_inline_asm_terminator<'tcx>(
|
||||
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
||||
span: Span,
|
||||
template: &[InlineAsmTemplatePiece],
|
||||
|
|
@ -42,8 +42,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
|
|||
options: InlineAsmOptions,
|
||||
destination: Option<mir::BasicBlock>,
|
||||
) {
|
||||
// FIXME add .eh_frame unwind info directives
|
||||
|
||||
// Used by panic_abort on Windows, but uses a syntax which only happens to work with
|
||||
// asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
|
||||
// the LLVM backend.
|
||||
|
|
@ -135,15 +133,33 @@ pub(crate) fn codegen_inline_asm<'tcx>(
|
|||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
let mut outputs = Vec::new();
|
||||
codegen_inline_asm_inner(fx, template, &operands, options);
|
||||
|
||||
match destination {
|
||||
Some(destination) => {
|
||||
let destination_block = fx.get_block(destination);
|
||||
fx.bcx.ins().jump(destination_block, &[]);
|
||||
}
|
||||
None => {
|
||||
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn codegen_inline_asm_inner<'tcx>(
|
||||
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
||||
template: &[InlineAsmTemplatePiece],
|
||||
operands: &[CInlineAsmOperand<'tcx>],
|
||||
options: InlineAsmOptions,
|
||||
) {
|
||||
// FIXME add .eh_frame unwind info directives
|
||||
|
||||
let mut asm_gen = InlineAssemblyGenerator {
|
||||
tcx: fx.tcx,
|
||||
arch: fx.tcx.sess.asm_arch.unwrap(),
|
||||
enclosing_def_id: fx.instance.def_id(),
|
||||
template,
|
||||
operands: &operands,
|
||||
operands,
|
||||
options,
|
||||
registers: Vec::new(),
|
||||
stack_slots_clobber: Vec::new(),
|
||||
|
|
@ -165,6 +181,8 @@ pub(crate) fn codegen_inline_asm<'tcx>(
|
|||
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
|
||||
fx.cx.global_asm.push_str(&generated_asm);
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
let mut outputs = Vec::new();
|
||||
for (i, operand) in operands.iter().enumerate() {
|
||||
match operand {
|
||||
CInlineAsmOperand::In { reg: _, value } => {
|
||||
|
|
@ -186,16 +204,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
|
|||
}
|
||||
|
||||
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
|
||||
|
||||
match destination {
|
||||
Some(destination) => {
|
||||
let destination_block = fx.get_block(destination);
|
||||
fx.bcx.ins().jump(destination_block, &[]);
|
||||
}
|
||||
None => {
|
||||
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct InlineAssemblyGenerator<'a, 'tcx> {
|
||||
|
|
@ -637,8 +645,21 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
|
|||
) {
|
||||
match arch {
|
||||
InlineAsmArch::X86_64 => {
|
||||
write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
||||
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
|
||||
match reg {
|
||||
InlineAsmReg::X86(reg)
|
||||
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
||||
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
||||
{
|
||||
// rustc emits x0 rather than xmm0
|
||||
write!(generated_asm, " movups [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
||||
write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32)
|
||||
.unwrap();
|
||||
}
|
||||
_ => {
|
||||
write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
|
||||
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
|
||||
}
|
||||
}
|
||||
generated_asm.push('\n');
|
||||
}
|
||||
InlineAsmArch::AArch64 => {
|
||||
|
|
@ -663,8 +684,24 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
|
|||
) {
|
||||
match arch {
|
||||
InlineAsmArch::X86_64 => {
|
||||
generated_asm.push_str(" mov ");
|
||||
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
|
||||
match reg {
|
||||
InlineAsmReg::X86(reg)
|
||||
if reg as u32 >= X86InlineAsmReg::xmm0 as u32
|
||||
&& reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
|
||||
{
|
||||
// rustc emits x0 rather than xmm0
|
||||
write!(
|
||||
generated_asm,
|
||||
" movups xmm{}",
|
||||
reg as u32 - X86InlineAsmReg::xmm0 as u32
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
_ => {
|
||||
generated_asm.push_str(" mov ");
|
||||
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap()
|
||||
}
|
||||
}
|
||||
writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
|
||||
}
|
||||
InlineAsmArch::AArch64 => {
|
||||
|
|
@ -720,7 +757,12 @@ fn call_inline_asm<'tcx>(
|
|||
fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
|
||||
|
||||
for (offset, place) in outputs {
|
||||
let ty = fx.clif_type(place.layout().ty).unwrap();
|
||||
let ty = if place.layout().ty.is_simd() {
|
||||
let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
|
||||
fx.clif_type(lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
|
||||
} else {
|
||||
fx.clif_type(place.layout().ty).unwrap()
|
||||
};
|
||||
let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
|
||||
fx,
|
||||
ty,
|
||||
|
|
@ -729,83 +771,3 @@ fn call_inline_asm<'tcx>(
|
|||
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn codegen_xgetbv<'tcx>(
|
||||
fx: &mut FunctionCx<'_, '_, 'tcx>,
|
||||
xcr_no: Value,
|
||||
ret: CPlace<'tcx>,
|
||||
) {
|
||||
// FIXME add .eh_frame unwind info directives
|
||||
|
||||
let operands = vec![
|
||||
CInlineAsmOperand::In {
|
||||
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
|
||||
value: xcr_no,
|
||||
},
|
||||
CInlineAsmOperand::Out {
|
||||
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
|
||||
late: true,
|
||||
place: Some(ret),
|
||||
},
|
||||
CInlineAsmOperand::Out {
|
||||
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
|
||||
late: true,
|
||||
place: None,
|
||||
},
|
||||
];
|
||||
let options = InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM;
|
||||
|
||||
let mut inputs = Vec::new();
|
||||
let mut outputs = Vec::new();
|
||||
|
||||
let mut asm_gen = InlineAssemblyGenerator {
|
||||
tcx: fx.tcx,
|
||||
arch: fx.tcx.sess.asm_arch.unwrap(),
|
||||
enclosing_def_id: fx.instance.def_id(),
|
||||
template: &[InlineAsmTemplatePiece::String(
|
||||
"
|
||||
xgetbv
|
||||
// out = rdx << 32 | rax
|
||||
shl rdx, 32
|
||||
or rax, rdx
|
||||
"
|
||||
.to_string(),
|
||||
)],
|
||||
operands: &operands,
|
||||
options,
|
||||
registers: Vec::new(),
|
||||
stack_slots_clobber: Vec::new(),
|
||||
stack_slots_input: Vec::new(),
|
||||
stack_slots_output: Vec::new(),
|
||||
stack_slot_size: Size::from_bytes(0),
|
||||
};
|
||||
asm_gen.allocate_registers();
|
||||
asm_gen.allocate_stack_slots();
|
||||
|
||||
let inline_asm_index = fx.cx.inline_asm_index.get();
|
||||
fx.cx.inline_asm_index.set(inline_asm_index + 1);
|
||||
let asm_name = format!(
|
||||
"__inline_asm_{}_n{}",
|
||||
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
|
||||
inline_asm_index
|
||||
);
|
||||
|
||||
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
|
||||
fx.cx.global_asm.push_str(&generated_asm);
|
||||
|
||||
for (i, operand) in operands.iter().enumerate() {
|
||||
match operand {
|
||||
CInlineAsmOperand::In { reg: _, value } => {
|
||||
inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
|
||||
}
|
||||
CInlineAsmOperand::Out { reg: _, late: _, place } => {
|
||||
if let Some(place) = place {
|
||||
outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
|
||||
}
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue