bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Merge commit 'b385428e3d' into subtree-update_cg_gcc_2024-03-05

Browse source
This commit is contained in:
Guillaume Gomez 2024-03-05 19:58:36 +01:00
commit 0d359efbe6
76 changed files with 7183 additions and 4278 deletions
Download patch file Download diff file Expand all files Collapse all files

223
compiler/rustc_codegen_gcc/src/asm.rs
View file

@ -2,7 +2,10 @@ use gccjit::{LValue, RValue, ToRValue, Type};
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_codegen_ssa::mir::operand::OperandValue;
use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::{AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef, InlineAsmOperandRef};
use rustc_codegen_ssa::traits::{
AsmBuilderMethods, AsmMethods, BaseTypeMethods, BuilderMethods, GlobalAsmOperandRef,
InlineAsmOperandRef,
};
use rustc_middle::{bug, ty::Instance};
use rustc_span::Span;
@ -11,11 +14,10 @@ use rustc_target::asm::*;
use std::borrow::Cow;
use crate::builder::Builder;
use crate::callee::get_fn;
use crate::context::CodegenCx;
use crate::errors::UnwindingInlineAsm;
use crate::type_of::LayoutGccExt;
use crate::callee::get_fn;
// Rust asm! and GCC Extended Asm semantics differ substantially.
//
@ -68,7 +70,6 @@ use crate::callee::get_fn;
const ATT_SYNTAX_INS: &str = ".att_syntax noprefix\n\t";
const INTEL_SYNTAX_INS: &str = "\n\t.intel_syntax noprefix";
struct AsmOutOperand<'a, 'tcx, 'gcc> {
rust_idx: usize,
constraint: &'a str,
@ -76,13 +77,13 @@ struct AsmOutOperand<'a, 'tcx, 'gcc> {
readwrite: bool,
tmp_var: LValue<'gcc>,
out_place: Option<PlaceRef<'tcx, RValue<'gcc>>>
out_place: Option<PlaceRef<'tcx, RValue<'gcc>>>,
}
struct AsmInOperand<'a, 'tcx> {
rust_idx: usize,
constraint: Cow<'a, str>,
val: RValue<'tcx>
val: RValue<'tcx>,
}
impl AsmOutOperand<'_, '_, '_> {
@ -95,23 +96,28 @@ impl AsmOutOperand<'_, '_, '_> {
res.push('&');
}
res.push_str(&self.constraint);
res.push_str(self.constraint);
res
}
}
enum ConstraintOrRegister {
Constraint(&'static str),
Register(&'static str)
Register(&'static str),
}
impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
fn codegen_inline_asm(
&mut self,
template: &[InlineAsmTemplatePiece],
rust_operands: &[InlineAsmOperandRef<'tcx, Self>],
options: InlineAsmOptions,
span: &[Span],
instance: Instance<'_>,
_dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
) {
if options.contains(InlineAsmOptions::MAY_UNWIND) {
self.sess().dcx()
.create_err(UnwindingInlineAsm { span: span[0] })
.emit();
self.sess().dcx().create_err(UnwindingInlineAsm { span: span[0] }).emit();
return;
}
@ -157,32 +163,40 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
use ConstraintOrRegister::*;
let (constraint, ty) = match (reg_to_gcc(reg), place) {
(Constraint(constraint), Some(place)) => (constraint, place.layout.gcc_type(self.cx)),
(Constraint(constraint), Some(place)) => {
(constraint, place.layout.gcc_type(self.cx))
}
// When `reg` is a class and not an explicit register but the out place is not specified,
// we need to create an unused output variable to assign the output to. This var
// needs to be of a type that's "compatible" with the register class, but specific type
// doesn't matter.
(Constraint(constraint), None) => (constraint, dummy_output_type(self.cx, reg.reg_class())),
(Constraint(constraint), None) => {
(constraint, dummy_output_type(self.cx, reg.reg_class()))
}
(Register(_), Some(_)) => {
// left for the next pass
continue
},
continue;
}
(Register(reg_name), None) => {
// `clobber_abi` can add lots of clobbers that are not supported by the target,
// such as AVX-512 registers, so we just ignore unsupported registers
let is_target_supported = reg.reg_class().supported_types(asm_arch).iter()
.any(|&(_, feature)| {
if let Some(feature) = feature {
self.tcx.asm_target_features(instance.def_id()).contains(&feature)
} else {
true // Register class is unconditionally supported
}
});
let is_target_supported =
reg.reg_class().supported_types(asm_arch).iter().any(
|&(_, feature)| {
if let Some(feature) = feature {
self.tcx
.asm_target_features(instance.def_id())
.contains(&feature)
} else {
true // Register class is unconditionally supported
}
},
);
if is_target_supported && !clobbers.contains(&reg_name) {
clobbers.push(reg_name);
}
continue
continue;
}
};
@ -193,7 +207,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
late,
readwrite: false,
tmp_var,
out_place: place
out_place: place,
});
}
@ -202,23 +216,22 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
inputs.push(AsmInOperand {
constraint: Cow::Borrowed(constraint),
rust_idx,
val: value.immediate()
val: value.immediate(),
});
}
else {
} else {
// left for the next pass
continue
continue;
}
}
InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
let constraint = if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) {
constraint
}
else {
// left for the next pass
continue
};
let constraint =
if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) {
constraint
} else {
// left for the next pass
continue;
};
// Rustc frontend guarantees that input and output types are "compatible",
// so we can just use input var's type for the output variable.
@ -249,7 +262,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
inputs.push(AsmInOperand {
constraint,
rust_idx,
val: in_value.immediate()
val: in_value.immediate(),
});
}
}
@ -267,7 +280,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
InlineAsmOperandRef::SymStatic { def_id } => {
// TODO(@Amanieu): Additional mangling is needed on
// some targets to add a leading underscore (Mach-O).
constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
constants_len +=
self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
}
}
}
@ -280,10 +294,9 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
let out_place = if let Some(place) = place {
place
}
else {
} else {
// processed in the previous pass
continue
continue;
};
let ty = out_place.layout.gcc_type(self.cx);
@ -291,12 +304,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
tmp_var.set_register_name(reg_name);
outputs.push(AsmOutOperand {
constraint: "r".into(),
constraint: "r",
rust_idx,
late,
readwrite: false,
tmp_var,
out_place: Some(out_place)
out_place: Some(out_place),
});
}
@ -314,7 +327,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
inputs.push(AsmInOperand {
constraint: "r".into(),
rust_idx,
val: reg_var.to_rvalue()
val: reg_var.to_rvalue(),
});
}
@ -330,7 +343,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
tmp_var.set_register_name(reg_name);
outputs.push(AsmOutOperand {
constraint: "r".into(),
constraint: "r",
rust_idx,
late,
readwrite: false,
@ -342,7 +355,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
inputs.push(AsmInOperand {
constraint,
rust_idx,
val: in_value.immediate()
val: in_value.immediate(),
});
}
@ -373,7 +386,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
// 3. Build the template string
let mut template_str = String::with_capacity(estimate_template_length(template, constants_len, att_dialect));
let mut template_str =
String::with_capacity(estimate_template_length(template, constants_len, att_dialect));
if att_dialect {
template_str.push_str(ATT_SYNTAX_INS);
}
@ -383,16 +397,15 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
InlineAsmTemplatePiece::String(ref string) => {
for char in string.chars() {
// TODO(antoyo): might also need to escape | if rustc doesn't do it.
let escaped_char =
match char {
'%' => "%%",
'{' => "%{",
'}' => "%}",
_ => {
template_str.push(char);
continue;
},
};
let escaped_char = match char {
'%' => "%%",
'{' => "%{",
'}' => "%}",
_ => {
template_str.push(char);
continue;
}
};
template_str.push_str(escaped_char);
}
}
@ -408,9 +421,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
};
match rust_operands[operand_idx] {
InlineAsmOperandRef::Out { reg, .. } => {
InlineAsmOperandRef::Out { reg, .. } => {
let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
let gcc_index = outputs.iter()
let gcc_index = outputs
.iter()
.position(|op| operand_idx == op.rust_idx)
.expect("wrong rust index");
push_to_template(modifier, gcc_index);
@ -418,7 +432,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
InlineAsmOperandRef::In { reg, .. } => {
let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
let in_gcc_index = inputs.iter()
let in_gcc_index = inputs
.iter()
.position(|op| operand_idx == op.rust_idx)
.expect("wrong rust index");
let gcc_index = in_gcc_index + outputs.len();
@ -429,7 +444,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
// The input register is tied to the output, so we can just use the index of the output register
let gcc_index = outputs.iter()
let gcc_index = outputs
.iter()
.position(|op| operand_idx == op.rust_idx)
.expect("wrong rust index");
push_to_template(modifier, gcc_index);
@ -496,7 +512,8 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
}
if options.contains(InlineAsmOptions::NORETURN) {
let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) };
let builtin_unreachable: RValue<'gcc> =
unsafe { std::mem::transmute(builtin_unreachable) };
self.call(self.type_void(), None, None, builtin_unreachable, &[], None);
}
@ -517,19 +534,23 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
}
}
fn estimate_template_length(template: &[InlineAsmTemplatePiece], constants_len: usize, att_dialect: bool) -> usize {
let len: usize = template.iter().map(|piece| {
match *piece {
InlineAsmTemplatePiece::String(ref string) => {
string.len()
fn estimate_template_length(
template: &[InlineAsmTemplatePiece],
constants_len: usize,
att_dialect: bool,
) -> usize {
let len: usize = template
.iter()
.map(|piece| {
match *piece {
InlineAsmTemplatePiece::String(ref string) => string.len(),
InlineAsmTemplatePiece::Placeholder { .. } => {
// '%' + 1 char modifier + 1 char index
3
}
}
InlineAsmTemplatePiece::Placeholder { .. } => {
// '%' + 1 char modifier + 1 char index
3
}
}
})
.sum();
})
.sum();
// increase it by 5% to account for possible '%' signs that'll be duplicated
// I pulled the number out of blue, but should be fair enough
@ -562,7 +583,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
_ => unimplemented!(),
}
},
}
// They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html
InlineAsmRegOrRegClass::RegClass(reg) => match reg {
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
@ -610,7 +631,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
| InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
unreachable!("clobber-only")
},
}
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
@ -637,7 +658,7 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
InlineAsmRegClass::Err => unreachable!(),
}
},
};
ConstraintOrRegister::Constraint(constraint)
@ -653,7 +674,7 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
unimplemented!()
}
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg)=> cx.type_i32(),
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
| InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
@ -686,7 +707,7 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
| InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
unreachable!("clobber-only")
},
}
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => cx.type_f32(),
@ -704,9 +725,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
bug!("LLVM backend does not support SPIR-V")
},
}
InlineAsmRegClass::S390x(
S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr
S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
) => cx.type_i32(),
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
InlineAsmRegClass::Err => unreachable!(),
@ -714,7 +735,13 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
}
impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
fn codegen_global_asm(&self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span]) {
fn codegen_global_asm(
&self,
template: &[InlineAsmTemplatePiece],
operands: &[GlobalAsmOperandRef<'tcx>],
options: InlineAsmOptions,
_line_spans: &[Span],
) {
let asm_arch = self.tcx.sess.asm_arch.unwrap();
// Default to Intel syntax on x86
@ -732,15 +759,17 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
let mut index = 0;
while index < string.len() {
// NOTE: gcc does not allow inline comment, so remove them.
let comment_index = string[index..].find("//")
let comment_index = string[index..]
.find("//")
.map(|comment_index| comment_index + index)
.unwrap_or(string.len());
template_str.push_str(&string[index..comment_index]);
index = string[comment_index..].find('\n')
index = string[comment_index..]
.find('\n')
.map(|index| index + comment_index)
.unwrap_or(string.len());
}
},
}
InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
match operands[operand_idx] {
GlobalAsmOperandRef::Const { ref string } => {
@ -782,14 +811,22 @@ impl<'gcc, 'tcx> AsmMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
}
}
fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option<char>) -> Option<char> {
fn modifier_to_gcc(
arch: InlineAsmArch,
reg: InlineAsmRegClass,
modifier: Option<char>,
) -> Option<char> {
// The modifiers can be retrieved from
// https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers
match reg {
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
if modifier == Some('v') { None } else { modifier }
if modifier == Some('v') {
None
} else {
modifier
}
}
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
unreachable!("clobber-only")
@ -821,7 +858,13 @@ fn modifier_to_gcc(arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option
}
InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
| InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
None => if arch == InlineAsmArch::X86_64 { Some('q') } else { Some('k') },
None => {
if arch == InlineAsmArch::X86_64 {
Some('q')
} else {
Some('k')
}
}
Some('l') => Some('b'),
Some('h') => Some('h'),
Some('x') => Some('w'),