Implement asm! codegen
This commit is contained in:
Amanieu d'Antras
parent
342a64caef
commit
abed45ff9f
8 changed files with 608 additions and 17 deletions
|
|
@ -1,14 +1,21 @@
|
|||
use crate::builder::Builder;
|
||||
use crate::context::CodegenCx;
|
||||
use crate::llvm;
|
||||
use crate::type_::Type;
|
||||
use crate::type_of::LayoutLlvmExt;
|
||||
use crate::value::Value;
|
||||
|
||||
use rustc_ast::ast::LlvmAsmDialect;
|
||||
use rustc_codegen_ssa::mir::operand::OperandValue;
|
||||
use rustc_codegen_ssa::mir::place::PlaceRef;
|
||||
use rustc_codegen_ssa::traits::*;
|
||||
use rustc_data_structures::fx::FxHashMap;
|
||||
use rustc_hir as hir;
|
||||
use rustc_middle::span_bug;
|
||||
use rustc_middle::ty::layout::TyAndLayout;
|
||||
use rustc_span::Span;
|
||||
use rustc_target::abi::*;
|
||||
use rustc_target::asm::*;
|
||||
|
||||
use libc::{c_char, c_uint};
|
||||
use log::debug;
|
||||
|
|
@ -40,7 +47,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
|
|||
indirect_outputs.push(operand.immediate());
|
||||
}
|
||||
} else {
|
||||
output_types.push(place.layout.llvm_type(self.cx()));
|
||||
output_types.push(place.layout.llvm_type(self.cx));
|
||||
}
|
||||
}
|
||||
if !indirect_outputs.is_empty() {
|
||||
|
|
@ -89,6 +96,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
|
|||
ia.volatile,
|
||||
ia.alignstack,
|
||||
ia.dialect,
|
||||
span,
|
||||
);
|
||||
if r.is_none() {
|
||||
return false;
|
||||
|
|
@ -102,22 +110,210 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
|
|||
OperandValue::Immediate(v).store(self, place);
|
||||
}
|
||||
|
||||
// Store mark in a metadata node so we can map LLVM errors
|
||||
// back to source locations. See #17552.
|
||||
unsafe {
|
||||
let key = "srcloc";
|
||||
let kind = llvm::LLVMGetMDKindIDInContext(
|
||||
self.llcx,
|
||||
key.as_ptr() as *const c_char,
|
||||
key.len() as c_uint,
|
||||
);
|
||||
|
||||
let val: &'ll Value = self.const_i32(span.ctxt().outer_expn().as_u32() as i32);
|
||||
|
||||
llvm::LLVMSetMetadata(r, kind, llvm::LLVMMDNodeInContext(self.llcx, &val, 1));
|
||||
true
|
||||
}
|
||||
|
||||
true
|
||||
fn codegen_inline_asm(
|
||||
&mut self,
|
||||
template: &[InlineAsmTemplatePiece],
|
||||
operands: &[InlineAsmOperandRef<'tcx, Self>],
|
||||
options: InlineAsmOptions,
|
||||
span: Span,
|
||||
) {
|
||||
let asm_arch = self.tcx.sess.asm_arch.unwrap();
|
||||
|
||||
// Collect the types of output operands
|
||||
let mut constraints = vec![];
|
||||
let mut output_types = vec![];
|
||||
let mut op_idx = FxHashMap::default();
|
||||
for (idx, op) in operands.iter().enumerate() {
|
||||
match *op {
|
||||
InlineAsmOperandRef::Out { reg, late, place } => {
|
||||
let ty = if let Some(place) = place {
|
||||
llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
|
||||
} else {
|
||||
// If the output is discarded, we don't really care what
|
||||
// type is used. We're just using this to tell LLVM to
|
||||
// reserve the register.
|
||||
dummy_output_type(self.cx, reg.reg_class())
|
||||
};
|
||||
output_types.push(ty);
|
||||
op_idx.insert(idx, constraints.len());
|
||||
let prefix = if late { "=" } else { "=&" };
|
||||
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
|
||||
}
|
||||
InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
|
||||
let ty = if let Some(ref out_place) = out_place {
|
||||
llvm_fixup_output_type(self.cx, reg.reg_class(), &out_place.layout)
|
||||
} else {
|
||||
// LLVM required tied operands to have the same type,
|
||||
// so we just use the type of the input.
|
||||
llvm_fixup_output_type(self.cx, reg.reg_class(), &in_value.layout)
|
||||
};
|
||||
output_types.push(ty);
|
||||
op_idx.insert(idx, constraints.len());
|
||||
let prefix = if late { "=" } else { "=&" };
|
||||
constraints.push(format!("{}{}", prefix, reg_to_llvm(reg)));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect input operands
|
||||
let mut inputs = vec![];
|
||||
for (idx, op) in operands.iter().enumerate() {
|
||||
match *op {
|
||||
InlineAsmOperandRef::In { reg, value } => {
|
||||
let value =
|
||||
llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
|
||||
inputs.push(value);
|
||||
op_idx.insert(idx, constraints.len());
|
||||
constraints.push(reg_to_llvm(reg));
|
||||
}
|
||||
InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => {
|
||||
let value = llvm_fixup_input(
|
||||
self,
|
||||
in_value.immediate(),
|
||||
reg.reg_class(),
|
||||
&in_value.layout,
|
||||
);
|
||||
inputs.push(value);
|
||||
constraints.push(format!("{}", op_idx[&idx]));
|
||||
}
|
||||
InlineAsmOperandRef::SymFn { instance } => {
|
||||
inputs.push(self.cx.get_fn(instance));
|
||||
op_idx.insert(idx, constraints.len());
|
||||
constraints.push("s".to_string());
|
||||
}
|
||||
InlineAsmOperandRef::SymStatic { def_id } => {
|
||||
inputs.push(self.cx.get_static(def_id));
|
||||
op_idx.insert(idx, constraints.len());
|
||||
constraints.push("s".to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Build the template string
|
||||
let mut template_str = String::new();
|
||||
for piece in template {
|
||||
match *piece {
|
||||
InlineAsmTemplatePiece::String(ref s) => {
|
||||
if s.contains('$') {
|
||||
for c in s.chars() {
|
||||
if c == '$' {
|
||||
template_str.push_str("$$");
|
||||
} else {
|
||||
template_str.push(c);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
template_str.push_str(s)
|
||||
}
|
||||
}
|
||||
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
|
||||
match operands[operand_idx] {
|
||||
InlineAsmOperandRef::In { reg, .. }
|
||||
| InlineAsmOperandRef::Out { reg, .. }
|
||||
| InlineAsmOperandRef::InOut { reg, .. } => {
|
||||
let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
|
||||
if let Some(modifier) = modifier {
|
||||
template_str.push_str(&format!(
|
||||
"${{{}:{}}}",
|
||||
op_idx[&operand_idx], modifier
|
||||
));
|
||||
} else {
|
||||
template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
|
||||
}
|
||||
}
|
||||
InlineAsmOperandRef::Const { ref string } => {
|
||||
// Const operands get injected directly into the template
|
||||
template_str.push_str(string);
|
||||
}
|
||||
InlineAsmOperandRef::SymFn { .. }
|
||||
| InlineAsmOperandRef::SymStatic { .. } => {
|
||||
// Only emit the raw symbol name
|
||||
template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
|
||||
match asm_arch {
|
||||
InlineAsmArch::AArch64 | InlineAsmArch::Arm => {
|
||||
constraints.push("~{cc}".to_string());
|
||||
}
|
||||
InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
|
||||
constraints.extend_from_slice(&[
|
||||
"~{dirflag}".to_string(),
|
||||
"~{fpsr}".to_string(),
|
||||
"~{flags}".to_string(),
|
||||
]);
|
||||
}
|
||||
InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {}
|
||||
}
|
||||
}
|
||||
if !options.contains(InlineAsmOptions::NOMEM) {
|
||||
// This is actually ignored by LLVM, but it's probably best to keep
|
||||
// it just in case. LLVM instead uses the ReadOnly/ReadNone
|
||||
// attributes on the call instruction to optimize.
|
||||
constraints.push("~{memory}".to_string());
|
||||
}
|
||||
let volatile = !options.contains(InlineAsmOptions::PURE);
|
||||
let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
|
||||
let output_type = match &output_types[..] {
|
||||
[] => self.type_void(),
|
||||
[ty] => ty,
|
||||
tys => self.type_struct(&tys, false),
|
||||
};
|
||||
let dialect = match asm_arch {
|
||||
InlineAsmArch::X86 | InlineAsmArch::X86_64 => LlvmAsmDialect::Intel,
|
||||
_ => LlvmAsmDialect::Att,
|
||||
};
|
||||
let result = inline_asm_call(
|
||||
self,
|
||||
&template_str,
|
||||
&constraints.join(","),
|
||||
&inputs,
|
||||
output_type,
|
||||
volatile,
|
||||
alignstack,
|
||||
dialect,
|
||||
span,
|
||||
)
|
||||
.unwrap_or_else(|| span_bug!(span, "LLVM asm constraint validation failed"));
|
||||
|
||||
if options.contains(InlineAsmOptions::PURE) {
|
||||
if options.contains(InlineAsmOptions::NOMEM) {
|
||||
llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result);
|
||||
} else if options.contains(InlineAsmOptions::READONLY) {
|
||||
llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result);
|
||||
}
|
||||
} else {
|
||||
if options.contains(InlineAsmOptions::NOMEM) {
|
||||
llvm::Attribute::InaccessibleMemOnly
|
||||
.apply_callsite(llvm::AttributePlace::Function, result);
|
||||
} else {
|
||||
// LLVM doesn't have an attribute to represent ReadOnly + SideEffect
|
||||
}
|
||||
}
|
||||
|
||||
// Write results to outputs
|
||||
for (idx, op) in operands.iter().enumerate() {
|
||||
if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
|
||||
| InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
|
||||
{
|
||||
let value = if output_types.len() == 1 {
|
||||
result
|
||||
} else {
|
||||
self.extract_value(result, op_idx[&idx] as u64)
|
||||
};
|
||||
let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout);
|
||||
OperandValue::Immediate(value).store(self, place);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -138,7 +334,8 @@ fn inline_asm_call(
|
|||
output: &'ll llvm::Type,
|
||||
volatile: bool,
|
||||
alignstack: bool,
|
||||
dia: ::rustc_ast::ast::LlvmAsmDialect,
|
||||
dia: LlvmAsmDialect,
|
||||
span: Span,
|
||||
) -> Option<&'ll Value> {
|
||||
let volatile = if volatile { llvm::True } else { llvm::False };
|
||||
let alignstack = if alignstack { llvm::True } else { llvm::False };
|
||||
|
|
@ -168,10 +365,261 @@ fn inline_asm_call(
|
|||
alignstack,
|
||||
llvm::AsmDialect::from_generic(dia),
|
||||
);
|
||||
Some(bx.call(v, inputs, None))
|
||||
let call = bx.call(v, inputs, None);
|
||||
|
||||
// Store mark in a metadata node so we can map LLVM errors
|
||||
// back to source locations. See #17552.
|
||||
let key = "srcloc";
|
||||
let kind = llvm::LLVMGetMDKindIDInContext(
|
||||
bx.llcx,
|
||||
key.as_ptr() as *const c_char,
|
||||
key.len() as c_uint,
|
||||
);
|
||||
|
||||
let val: &'ll Value = bx.const_i32(span.ctxt().outer_expn().as_u32() as i32);
|
||||
llvm::LLVMSetMetadata(call, kind, llvm::LLVMMDNodeInContext(bx.llcx, &val, 1));
|
||||
|
||||
Some(call)
|
||||
} else {
|
||||
// LLVM has detected an issue with our constraints, bail out
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a register class to an LLVM constraint code.
|
||||
fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String {
|
||||
match reg {
|
||||
InlineAsmRegOrRegClass::Reg(reg) => format!("{{{}}}", reg.name()),
|
||||
InlineAsmRegOrRegClass::RegClass(reg) => match reg {
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => "l",
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w",
|
||||
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
|
||||
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
|
||||
| InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk",
|
||||
}
|
||||
.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a modifier into LLVM's equivalent modifier.
|
||||
fn modifier_to_llvm(
|
||||
arch: InlineAsmArch,
|
||||
reg: InlineAsmRegClass,
|
||||
modifier: Option<char>,
|
||||
) -> Option<char> {
|
||||
match reg {
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
|
||||
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
|
||||
if modifier == Some('v') { None } else { modifier }
|
||||
}
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => None,
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
|
||||
if modifier.is_none() {
|
||||
Some('q')
|
||||
} else {
|
||||
modifier
|
||||
}
|
||||
}
|
||||
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
|
||||
| InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
|
||||
| InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
|
||||
None if arch == InlineAsmArch::X86_64 => Some('q'),
|
||||
None => Some('k'),
|
||||
Some('l') => Some('b'),
|
||||
Some('h') => Some('h'),
|
||||
Some('x') => Some('w'),
|
||||
Some('e') => Some('k'),
|
||||
Some('r') => Some('q'),
|
||||
_ => unreachable!(),
|
||||
},
|
||||
InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg)
|
||||
| InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg)
|
||||
| InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
|
||||
(X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
|
||||
(X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
|
||||
(X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
|
||||
(_, Some('x')) => Some('x'),
|
||||
(_, Some('y')) => Some('t'),
|
||||
(_, Some('z')) => Some('g'),
|
||||
_ => unreachable!(),
|
||||
},
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Type to use for outputs that are discarded. It doesn't really matter what
|
||||
/// the type is, as long as it is valid for the constraint code.
|
||||
fn dummy_output_type(cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass) -> &'ll Type {
|
||||
match reg {
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
|
||||
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
|
||||
cx.type_vector(cx.type_i64(), 2)
|
||||
}
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg_thumb) => cx.type_i32(),
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
|
||||
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
|
||||
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
|
||||
cx.type_vector(cx.type_i64(), 2)
|
||||
}
|
||||
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
|
||||
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
|
||||
| InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
|
||||
| InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg)
|
||||
| InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
|
||||
InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
|
||||
/// the equivalent integer type.
|
||||
fn llvm_asm_scalar_type(cx: &CodegenCx<'ll, 'tcx>, scalar: &Scalar) -> &'ll Type {
|
||||
match scalar.value {
|
||||
Primitive::Int(Integer::I8, _) => cx.type_i8(),
|
||||
Primitive::Int(Integer::I16, _) => cx.type_i16(),
|
||||
Primitive::Int(Integer::I32, _) => cx.type_i32(),
|
||||
Primitive::Int(Integer::I64, _) => cx.type_i64(),
|
||||
Primitive::F32 => cx.type_f32(),
|
||||
Primitive::F64 => cx.type_f64(),
|
||||
Primitive::Pointer => cx.type_isize(),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fix up an input value to work around LLVM bugs.
|
||||
fn llvm_fixup_input(
|
||||
bx: &mut Builder<'a, 'll, 'tcx>,
|
||||
mut value: &'ll Value,
|
||||
reg: InlineAsmRegClass,
|
||||
layout: &TyAndLayout<'tcx>,
|
||||
) -> &'ll Value {
|
||||
match (reg, &layout.abi) {
|
||||
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
|
||||
if let Primitive::Int(Integer::I8, _) = s.value {
|
||||
let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
|
||||
bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
|
||||
let elem_ty = llvm_asm_scalar_type(bx.cx, s);
|
||||
let count = 16 / layout.size.bytes();
|
||||
let vec_ty = bx.cx.type_vector(elem_ty, count);
|
||||
if let Primitive::Pointer = s.value {
|
||||
value = bx.ptrtoint(value, bx.cx.type_isize());
|
||||
}
|
||||
bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
|
||||
}
|
||||
(
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
|
||||
Abi::Vector { element, count },
|
||||
) if layout.size.bytes() == 8 => {
|
||||
let elem_ty = llvm_asm_scalar_type(bx.cx, element);
|
||||
let vec_ty = bx.cx.type_vector(elem_ty, *count);
|
||||
let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
|
||||
bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
|
||||
}
|
||||
_ => value,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fix up an output value to work around LLVM bugs.
|
||||
fn llvm_fixup_output(
|
||||
bx: &mut Builder<'a, 'll, 'tcx>,
|
||||
mut value: &'ll Value,
|
||||
reg: InlineAsmRegClass,
|
||||
layout: &TyAndLayout<'tcx>,
|
||||
) -> &'ll Value {
|
||||
match (reg, &layout.abi) {
|
||||
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
|
||||
if let Primitive::Int(Integer::I8, _) = s.value {
|
||||
bx.extract_element(value, bx.const_i32(0))
|
||||
} else {
|
||||
value
|
||||
}
|
||||
}
|
||||
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
|
||||
value = bx.extract_element(value, bx.const_i32(0));
|
||||
if let Primitive::Pointer = s.value {
|
||||
value = bx.inttoptr(value, layout.llvm_type(bx.cx));
|
||||
}
|
||||
value
|
||||
}
|
||||
(
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
|
||||
Abi::Vector { element, count },
|
||||
) if layout.size.bytes() == 8 => {
|
||||
let elem_ty = llvm_asm_scalar_type(bx.cx, element);
|
||||
let vec_ty = bx.cx.type_vector(elem_ty, *count * 2);
|
||||
let indices: Vec<_> = (0..*count).map(|x| bx.const_i32(x as i32)).collect();
|
||||
bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
|
||||
}
|
||||
_ => value,
|
||||
}
|
||||
}
|
||||
|
||||
/// Output type to use for llvm_fixup_output.
|
||||
fn llvm_fixup_output_type(
|
||||
cx: &CodegenCx<'ll, 'tcx>,
|
||||
reg: InlineAsmRegClass,
|
||||
layout: &TyAndLayout<'tcx>,
|
||||
) -> &'ll Type {
|
||||
match (reg, &layout.abi) {
|
||||
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
|
||||
if let Primitive::Int(Integer::I8, _) = s.value {
|
||||
cx.type_vector(cx.type_i8(), 8)
|
||||
} else {
|
||||
layout.llvm_type(cx)
|
||||
}
|
||||
}
|
||||
(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
|
||||
let elem_ty = llvm_asm_scalar_type(cx, s);
|
||||
let count = 16 / layout.size.bytes();
|
||||
cx.type_vector(elem_ty, count)
|
||||
}
|
||||
(
|
||||
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
|
||||
Abi::Vector { element, count },
|
||||
) if layout.size.bytes() == 8 => {
|
||||
let elem_ty = llvm_asm_scalar_type(cx, element);
|
||||
cx.type_vector(elem_ty, count * 2)
|
||||
}
|
||||
_ => layout.llvm_type(cx),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,6 +124,8 @@ pub enum Attribute {
|
|||
NonLazyBind = 23,
|
||||
OptimizeNone = 24,
|
||||
ReturnsTwice = 25,
|
||||
ReadNone = 26,
|
||||
InaccessibleMemOnly = 27,
|
||||
}
|
||||
|
||||
/// LLVMIntPredicate
|
||||
|
|
|
|||
|
|
@ -358,7 +358,8 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
|
|||
| TerminatorKind::SwitchInt { .. }
|
||||
| TerminatorKind::Yield { .. }
|
||||
| TerminatorKind::FalseEdges { .. }
|
||||
| TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
|
||||
| TerminatorKind::FalseUnwind { .. }
|
||||
| TerminatorKind::InlineAsm { .. } => { /* nothing to do */ }
|
||||
TerminatorKind::Call { cleanup: unwind, .. }
|
||||
| TerminatorKind::Assert { cleanup: unwind, .. }
|
||||
| TerminatorKind::DropAndReplace { unwind, .. }
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use crate::meth;
|
|||
use crate::traits::*;
|
||||
use crate::MemFlags;
|
||||
|
||||
use rustc_ast::ast;
|
||||
use rustc_hir::lang_items;
|
||||
use rustc_index::vec::Idx;
|
||||
use rustc_middle::mir;
|
||||
|
|
@ -914,6 +915,98 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
|
|||
mir::TerminatorKind::FalseEdges { .. } | mir::TerminatorKind::FalseUnwind { .. } => {
|
||||
bug!("borrowck false edges in codegen")
|
||||
}
|
||||
|
||||
mir::TerminatorKind::InlineAsm { template, ref operands, options, ref destination } => {
|
||||
let span = terminator.source_info.span;
|
||||
|
||||
let operands: Vec<_> = operands
|
||||
.iter()
|
||||
.map(|op| match *op {
|
||||
mir::InlineAsmOperand::In { reg, ref value } => {
|
||||
let value = self.codegen_operand(&mut bx, value);
|
||||
InlineAsmOperandRef::In { reg, value }
|
||||
}
|
||||
mir::InlineAsmOperand::Out { reg, late, ref place } => {
|
||||
let place =
|
||||
place.map(|place| self.codegen_place(&mut bx, place.as_ref()));
|
||||
InlineAsmOperandRef::Out { reg, late, place }
|
||||
}
|
||||
mir::InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
|
||||
let in_value = self.codegen_operand(&mut bx, in_value);
|
||||
let out_place = out_place
|
||||
.map(|out_place| self.codegen_place(&mut bx, out_place.as_ref()));
|
||||
InlineAsmOperandRef::InOut { reg, late, in_value, out_place }
|
||||
}
|
||||
mir::InlineAsmOperand::Const { ref value } => {
|
||||
if let mir::Operand::Constant(constant) = value {
|
||||
let const_value =
|
||||
self.eval_mir_constant(constant).unwrap_or_else(|_| {
|
||||
span_bug!(span, "asm const cannot be resolved")
|
||||
});
|
||||
let ty = constant.literal.ty;
|
||||
let value = const_value
|
||||
.try_to_bits_for_ty(bx.tcx(), ty::ParamEnv::reveal_all(), ty)
|
||||
.unwrap_or_else(|| {
|
||||
span_bug!(span, "asm const has non-scalar value")
|
||||
});
|
||||
let string = match ty.kind {
|
||||
ty::Uint(_) => value.to_string(),
|
||||
ty::Int(int_ty) => {
|
||||
match int_ty.normalize(bx.tcx().sess.target.ptr_width) {
|
||||
ast::IntTy::I8 => (value as i8).to_string(),
|
||||
ast::IntTy::I16 => (value as i16).to_string(),
|
||||
ast::IntTy::I32 => (value as i32).to_string(),
|
||||
ast::IntTy::I64 => (value as i64).to_string(),
|
||||
ast::IntTy::I128 => (value as i128).to_string(),
|
||||
ast::IntTy::Isize => unreachable!(),
|
||||
}
|
||||
}
|
||||
ty::Float(ast::FloatTy::F32) => {
|
||||
f32::from_bits(value as u32).to_string()
|
||||
}
|
||||
ty::Float(ast::FloatTy::F64) => {
|
||||
f64::from_bits(value as u64).to_string()
|
||||
}
|
||||
_ => span_bug!(span, "asm const has bad type {}", ty),
|
||||
};
|
||||
InlineAsmOperandRef::Const { string }
|
||||
} else {
|
||||
span_bug!(span, "asm const is not a constant");
|
||||
}
|
||||
}
|
||||
mir::InlineAsmOperand::SymFn { ref value } => {
|
||||
if let ty::FnDef(def_id, substs) = value.literal.ty.kind {
|
||||
let instance = ty::Instance::resolve(
|
||||
bx.tcx(),
|
||||
ty::ParamEnv::reveal_all(),
|
||||
def_id,
|
||||
substs,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
InlineAsmOperandRef::SymFn { instance }
|
||||
} else {
|
||||
span_bug!(span, "invalid type for asm sym (fn)");
|
||||
}
|
||||
}
|
||||
mir::InlineAsmOperand::SymStatic { ref value } => {
|
||||
if let Some(def_id) = value.check_static_ptr(bx.tcx()) {
|
||||
InlineAsmOperandRef::SymStatic { def_id }
|
||||
} else {
|
||||
span_bug!(span, "invalid type for asm sym (static)");
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
bx.codegen_inline_asm(template, &operands, options, span);
|
||||
|
||||
if let Some(target) = destination {
|
||||
helper.funclet_br(self, &mut bx, *target);
|
||||
} else {
|
||||
bx.unreachable();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,39 @@
|
|||
use super::BackendTypes;
|
||||
use crate::mir::operand::OperandRef;
|
||||
use crate::mir::place::PlaceRef;
|
||||
use rustc_hir::def_id::DefId;
|
||||
use rustc_hir::{GlobalAsm, LlvmInlineAsmInner};
|
||||
use rustc_middle::ty::Instance;
|
||||
use rustc_span::Span;
|
||||
use rustc_target::asm::{InlineAsmOptions, InlineAsmRegOrRegClass, InlineAsmTemplatePiece};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
|
||||
In {
|
||||
reg: InlineAsmRegOrRegClass,
|
||||
value: OperandRef<'tcx, B::Value>,
|
||||
},
|
||||
Out {
|
||||
reg: InlineAsmRegOrRegClass,
|
||||
late: bool,
|
||||
place: Option<PlaceRef<'tcx, B::Value>>,
|
||||
},
|
||||
InOut {
|
||||
reg: InlineAsmRegOrRegClass,
|
||||
late: bool,
|
||||
in_value: OperandRef<'tcx, B::Value>,
|
||||
out_place: Option<PlaceRef<'tcx, B::Value>>,
|
||||
},
|
||||
Const {
|
||||
string: String,
|
||||
},
|
||||
SymFn {
|
||||
instance: Instance<'tcx>,
|
||||
},
|
||||
SymStatic {
|
||||
def_id: DefId,
|
||||
},
|
||||
}
|
||||
|
||||
pub trait AsmBuilderMethods<'tcx>: BackendTypes {
|
||||
/// Take an inline assembly expression and splat it out via LLVM
|
||||
|
|
@ -12,6 +44,15 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
|
|||
inputs: Vec<Self::Value>,
|
||||
span: Span,
|
||||
) -> bool;
|
||||
|
||||
/// Take an inline assembly expression and splat it out via LLVM
|
||||
fn codegen_inline_asm(
|
||||
&mut self,
|
||||
template: &[InlineAsmTemplatePiece],
|
||||
operands: &[InlineAsmOperandRef<'tcx, Self>],
|
||||
options: InlineAsmOptions,
|
||||
span: Span,
|
||||
);
|
||||
}
|
||||
|
||||
pub trait AsmMethods {
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ mod type_;
|
|||
mod write;
|
||||
|
||||
pub use self::abi::AbiBuilderMethods;
|
||||
pub use self::asm::{AsmBuilderMethods, AsmMethods};
|
||||
pub use self::asm::{AsmBuilderMethods, AsmMethods, InlineAsmOperandRef};
|
||||
pub use self::backend::{Backend, BackendTypes, CodegenBackend, ExtraBackendMethods};
|
||||
pub use self::builder::{BuilderMethods, OverflowOp};
|
||||
pub use self::consts::ConstMethods;
|
||||
|
|
|
|||
|
|
@ -203,6 +203,10 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
|
|||
return Attribute::OptimizeNone;
|
||||
case ReturnsTwice:
|
||||
return Attribute::ReturnsTwice;
|
||||
case ReadNone:
|
||||
return Attribute::ReadNone;
|
||||
case InaccessibleMemOnly:
|
||||
return Attribute::InaccessibleMemOnly;
|
||||
}
|
||||
report_fatal_error("bad AttributeKind");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,6 +82,8 @@ enum LLVMRustAttribute {
|
|||
NonLazyBind = 23,
|
||||
OptimizeNone = 24,
|
||||
ReturnsTwice = 25,
|
||||
ReadNone = 26,
|
||||
InaccessibleMemOnly = 27,
|
||||
};
|
||||
|
||||
typedef struct OpaqueRustString *RustStringRef;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue