bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '81dc066758' into sync_cg_clif-2023-10-09

Browse source
This commit is contained in:
bjorn3 2023-10-09 08:52:46 +00:00
commit 169055f2ff
47 changed files with 1230 additions and 734 deletions
Download patch file Download diff file Expand all files Collapse all files

6
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -6,6 +6,7 @@ mod returning;
use std::borrow::Cow;
use cranelift_codegen::ir::{AbiParam, SigRef};
use cranelift_module::ModuleError;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::ty::layout::FnAbiOf;
@ -13,12 +14,9 @@ use rustc_session::Session;
use rustc_target::abi::call::{Conv, FnAbi};
use rustc_target::spec::abi::Abi;
use cranelift_codegen::ir::{AbiParam, SigRef};
use self::pass_mode::*;
use crate::prelude::*;
pub(crate) use self::returning::codegen_return;
use crate::prelude::*;
fn clif_sig_from_fn_abi<'tcx>(
tcx: TyCtxt<'tcx>,

6
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -1,14 +1,14 @@
//! Argument passing
use crate::prelude::*;
use crate::value_and_place::assert_assignable;
use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose};
use rustc_target::abi::call::{
ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind,
};
use smallvec::{smallvec, SmallVec};
use crate::prelude::*;
use crate::value_and_place::assert_assignable;
pub(super) trait ArgAbiExt<'tcx> {
fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>;
fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>);

4
compiler/rustc_codegen_cranelift/src/abi/returning.rs
View file

@ -1,10 +1,10 @@
//! Return value handling
use crate::prelude::*;
use rustc_target::abi::call::{ArgAbi, PassMode};
use smallvec::{smallvec, SmallVec};
use crate::prelude::*;
/// Return a place where the return value of the current function can be written to. If necessary
/// this adds an extra parameter pointing to where the return value needs to be stored.
pub(super) fn codegen_return_param<'tcx>(

4
compiler/rustc_codegen_cranelift/src/allocator.rs
View file

@ -1,8 +1,6 @@
//! Allocator shim
// Adapted from rustc
use crate::prelude::*;
use rustc_ast::expand::allocator::{
alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy,
ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE,
@ -10,6 +8,8 @@ use rustc_ast::expand::allocator::{
use rustc_codegen_ssa::base::allocator_kind_for_codegen;
use rustc_session::config::OomStrategy;
use crate::prelude::*;
/// Returns whether an allocator shim was created
pub(crate) fn codegen(
tcx: TyCtxt<'_>,

4
compiler/rustc_codegen_cranelift/src/analyze.rs
View file

@ -1,11 +1,11 @@
//! SSA analysis
use crate::prelude::*;
use rustc_index::IndexVec;
use rustc_middle::mir::StatementKind::*;
use rustc_middle::ty::Ty;
use crate::prelude::*;
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) enum SsaKind {
NotSsa,

7
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -1,15 +1,14 @@
//! Codegen of a single function
use cranelift_codegen::ir::UserFuncName;
use cranelift_codegen::CodegenError;
use cranelift_module::ModuleError;
use rustc_ast::InlineAsmOptions;
use rustc_index::IndexVec;
use rustc_middle::ty::adjustment::PointerCoercion;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
use cranelift_codegen::ir::UserFuncName;
use cranelift_codegen::CodegenError;
use cranelift_module::ModuleError;
use crate::constant::ConstantCx;
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;

4
compiler/rustc_codegen_cranelift/src/cast.rs
View file

@ -129,8 +129,8 @@ pub(crate) fn clif_int_or_float_cast(
let (min, max) = match (to_ty, to_signed) {
(types::I8, false) => (0, i64::from(u8::MAX)),
(types::I16, false) => (0, i64::from(u16::MAX)),
(types::I8, true) => (i64::from(i8::MIN), i64::from(i8::MAX)),
(types::I16, true) => (i64::from(i16::MIN), i64::from(i16::MAX)),
(types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)),
(types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)),
_ => unreachable!(),
};
let min_val = fx.bcx.ins().iconst(types::I32, min);

13
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -1,6 +1,5 @@
use cranelift_codegen::isa::TargetFrontendConfig;
use gimli::write::FileId;
use rustc_data_structures::sync::Lrc;
use rustc_index::IndexVec;
use rustc_middle::ty::layout::{
@ -204,9 +203,9 @@ pub(crate) fn type_min_max_value(
(types::I8, false) | (types::I16, false) | (types::I32, false) | (types::I64, false) => {
0i64
}
(types::I8, true) => i64::from(i8::MIN),
(types::I16, true) => i64::from(i16::MIN),
(types::I32, true) => i64::from(i32::MIN),
(types::I8, true) => i64::from(i8::MIN as u8),
(types::I16, true) => i64::from(i16::MIN as u16),
(types::I32, true) => i64::from(i32::MIN as u32),
(types::I64, true) => i64::MIN,
_ => unreachable!(),
};
@ -216,9 +215,9 @@ pub(crate) fn type_min_max_value(
(types::I16, false) => i64::from(u16::MAX),
(types::I32, false) => i64::from(u32::MAX),
(types::I64, false) => u64::MAX as i64,
(types::I8, true) => i64::from(i8::MAX),
(types::I16, true) => i64::from(i16::MAX),
(types::I32, true) => i64::from(i32::MAX),
(types::I8, true) => i64::from(i8::MAX as u8),
(types::I16, true) => i64::from(i16::MAX as u16),
(types::I32, true) => i64::from(i32::MAX as u32),
(types::I64, true) => i64::MAX,
_ => unreachable!(),
};

3
compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
View file

@ -1,8 +1,7 @@
use std::sync::{Arc, Condvar, Mutex};
use rustc_session::Session;
use jobserver::HelperThread;
use rustc_session::Session;
// FIXME don't panic when a worker thread panics

8
compiler/rustc_codegen_cranelift/src/constant.rs
View file

@ -1,12 +1,11 @@
//! Handling of `static`s, `const`s and promoted allocations
use cranelift_module::*;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::interpret::{read_target_uint, AllocId, GlobalAlloc, Scalar};
use rustc_middle::mir::ConstValue;
use cranelift_module::*;
use crate::prelude::*;
pub(crate) struct ConstantCx {
@ -101,7 +100,7 @@ pub(crate) fn codegen_const_value<'tcx>(
if fx.clif_type(layout.ty).is_some() {
return CValue::const_val(fx, layout, int);
} else {
let raw_val = int.to_bits(int.size()).unwrap();
let raw_val = int.size().truncate(int.to_bits(int.size()).unwrap());
let val = match int.size().bytes() {
1 => fx.bcx.ins().iconst(types::I8, raw_val as i64),
2 => fx.bcx.ins().iconst(types::I16, raw_val as i64),
@ -187,8 +186,7 @@ pub(crate) fn codegen_const_value<'tcx>(
ConstValue::Slice { data, meta } => {
let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data);
let ptr = pointer_for_allocation(fx, alloc_id).get_addr(fx);
// FIXME: the `try_from` here can actually fail, e.g. for very long ZST slices.
let len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(meta).unwrap());
let len = fx.bcx.ins().iconst(fx.pointer_type, meta as i64);
CValue::by_val_pair(ptr, len, layout)
}
}

3
compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
View file

@ -1,10 +1,9 @@
//! Write the debuginfo into an object file.
use cranelift_object::ObjectProduct;
use rustc_data_structures::fx::FxHashMap;
use gimli::write::{Address, AttributeValue, EndianVec, Result, Sections, Writer};
use gimli::{RunTimeEndian, SectionId};
use rustc_data_structures::fx::FxHashMap;
use super::object::WriteDebugInfo;
use super::DebugContext;

16
compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
View file

@ -3,20 +3,18 @@
use std::ffi::OsStr;
use std::path::{Component, Path};
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
use cranelift_codegen::binemit::CodeOffset;
use cranelift_codegen::MachSrcLoc;
use gimli::write::{
Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
};
use rustc_data_structures::sync::Lrc;
use rustc_span::{
FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm,
};
use cranelift_codegen::binemit::CodeOffset;
use cranelift_codegen::MachSrcLoc;
use gimli::write::{
Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
};
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
// OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`.
fn split_path_dir_and_file(path: &Path) -> (&Path, &OsStr) {

8
compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
View file

@ -5,11 +5,8 @@ mod line_info;
mod object;
mod unwind;
use crate::prelude::*;
use cranelift_codegen::ir::Endianness;
use cranelift_codegen::isa::TargetIsa;
use gimli::write::{
Address, AttributeValue, DwarfUnit, FileId, LineProgram, LineString, Range, RangeList,
UnitEntryId,
@ -17,8 +14,9 @@ use gimli::write::{
use gimli::{Encoding, Format, LineEncoding, RunTimeEndian};
use indexmap::IndexSet;
pub(crate) use emit::{DebugReloc, DebugRelocName};
pub(crate) use unwind::UnwindContext;
pub(crate) use self::emit::{DebugReloc, DebugRelocName};
pub(crate) use self::unwind::UnwindContext;
use crate::prelude::*;
pub(crate) fn producer() -> String {
format!(

7
compiler/rustc_codegen_cranelift/src/debuginfo/object.rs
View file

@ -1,12 +1,9 @@
use rustc_data_structures::fx::FxHashMap;
use cranelift_module::FuncId;
use cranelift_object::ObjectProduct;
use gimli::SectionId;
use object::write::{Relocation, StandardSegment};
use object::{RelocationEncoding, SectionKind};
use gimli::SectionId;
use rustc_data_structures::fx::FxHashMap;
use crate::debuginfo::{DebugReloc, DebugRelocName};

4
compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
View file

@ -1,15 +1,13 @@
//! Unwind info generation (`.eh_frame`)
use crate::prelude::*;
use cranelift_codegen::ir::Endianness;
use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa};
use cranelift_object::ObjectProduct;
use gimli::write::{Address, CieId, EhFrame, FrameTable, Section};
use gimli::RunTimeEndian;
use super::object::WriteDebugInfo;
use crate::prelude::*;
pub(crate) struct UnwindContext {
endian: RunTimeEndian,

3
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -6,6 +6,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::thread::JoinHandle;
use cranelift_object::{ObjectBuilder, ObjectModule};
use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
use rustc_data_structures::profiling::SelfProfilerRef;
@ -17,8 +18,6 @@ use rustc_session::cgu_reuse_tracker::CguReuse;
use rustc_session::config::{DebugInfo, OutputFilenames, OutputType};
use rustc_session::Session;
use cranelift_object::{ObjectBuilder, ObjectModule};
use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
use crate::global_asm::GlobalAsmConfig;
use crate::{prelude::*, BackendConfig};

3
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -6,13 +6,12 @@ use std::ffi::CString;
use std::os::raw::{c_char, c_int};
use std::sync::{mpsc, Mutex, OnceLock};
use cranelift_jit::{JITBuilder, JITModule};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::Session;
use rustc_span::Symbol;
use cranelift_jit::{JITBuilder, JITModule};
use crate::{prelude::*, BackendConfig};
use crate::{CodegenCx, CodegenMode};

23
compiler/rustc_codegen_cranelift/src/global_asm.rs
View file

@ -9,16 +9,22 @@ use std::sync::Arc;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_hir::{InlineAsmOperand, ItemId};
use rustc_session::config::{OutputFilenames, OutputType};
use rustc_target::asm::InlineAsmArch;
use crate::prelude::*;
pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) {
let item = tcx.hir().item(item_id);
if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
global_asm.push_str("\n.intel_syntax noprefix\n");
} else {
global_asm.push_str("\n.att_syntax\n");
let is_x86 =
matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64);
if is_x86 {
if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
global_asm.push_str("\n.intel_syntax noprefix\n");
} else {
global_asm.push_str("\n.att_syntax\n");
}
}
for piece in asm.template {
match *piece {
@ -65,7 +71,11 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
}
}
}
global_asm.push_str("\n.att_syntax\n\n");
global_asm.push('\n');
if is_x86 {
global_asm.push_str(".att_syntax\n\n");
}
} else {
bug!("Expected GlobalAsm found {:?}", item);
}
@ -115,11 +125,12 @@ pub(crate) fn compile_global_asm(
}
// Remove all LLVM style comments
let global_asm = global_asm
let mut global_asm = global_asm
.lines()
.map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
.collect::<Vec<_>>()
.join("\n");
global_asm.push('\n');
let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));

140
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -1,13 +1,14 @@
//! Codegen of `asm!` invocations.
use crate::prelude::*;
use std::fmt::Write;
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_middle::mir::InlineAsmOperand;
use rustc_span::sym;
use rustc_target::asm::*;
use target_lexicon::BinaryFormat;
use crate::prelude::*;
enum CInlineAsmOperand<'tcx> {
In {
@ -43,7 +44,9 @@ pub(crate) fn codegen_inline_asm<'tcx>(
) {
// FIXME add .eh_frame unwind info directives
if !template.is_empty() {
if !template.is_empty()
&& (cfg!(not(feature = "inline_asm")) || fx.tcx.sess.target.is_like_windows)
{
// Used by panic_abort
if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
fx.bcx.ins().trap(TrapCode::User(1));
@ -589,11 +592,29 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
}
fn generate_asm_wrapper(&self, asm_name: &str) -> String {
let binary_format = crate::target_triple(self.tcx.sess).binary_format;
let mut generated_asm = String::new();
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
writeln!(generated_asm, "{}:", asm_name).unwrap();
match binary_format {
BinaryFormat::Elf => {
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
writeln!(generated_asm, ".type {},@function", asm_name).unwrap();
writeln!(generated_asm, ".section .text.{},\"ax\",@progbits", asm_name).unwrap();
writeln!(generated_asm, "{}:", asm_name).unwrap();
}
BinaryFormat::Macho => {
writeln!(generated_asm, ".globl _{}", asm_name).unwrap();
writeln!(generated_asm, "_{}:", asm_name).unwrap();
}
BinaryFormat::Coff => {
writeln!(generated_asm, ".globl {}", asm_name).unwrap();
writeln!(generated_asm, "{}:", asm_name).unwrap();
}
_ => self
.tcx
.sess
.fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
}
let is_x86 = matches!(self.arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
@ -690,8 +711,19 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
if is_x86 {
generated_asm.push_str(".att_syntax\n");
}
writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
generated_asm.push_str(".text\n");
match binary_format {
BinaryFormat::Elf => {
writeln!(generated_asm, ".size {name}, .-{name}", name = asm_name).unwrap();
generated_asm.push_str(".text\n");
}
BinaryFormat::Macho | BinaryFormat::Coff => {}
_ => self
.tcx
.sess
.fatal(format!("Unsupported binary format for inline asm: {binary_format:?}")),
}
generated_asm.push_str("\n\n");
generated_asm
@ -699,25 +731,19 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
fn prologue(generated_asm: &mut String, arch: InlineAsmArch) {
match arch {
InlineAsmArch::X86 => {
generated_asm.push_str(" push ebp\n");
generated_asm.push_str(" mov ebp,[esp+8]\n");
}
InlineAsmArch::X86_64 => {
generated_asm.push_str(" push rbp\n");
generated_asm.push_str(" mov rbp,rdi\n");
generated_asm.push_str(" mov rbp,rsp\n");
generated_asm.push_str(" push rbx\n"); // rbx is callee saved
// rbx is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
generated_asm.push_str(" mov rbx,rdi\n");
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" addi sp, sp, -8\n");
generated_asm.push_str(" sw ra, 4(sp)\n");
generated_asm.push_str(" sw s0, 0(sp)\n");
generated_asm.push_str(" mv s0, a0\n");
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" addi sp, sp, -16\n");
generated_asm.push_str(" sd ra, 8(sp)\n");
generated_asm.push_str(" sd s0, 0(sp)\n");
generated_asm.push_str(" mv s0, a0\n");
InlineAsmArch::AArch64 => {
generated_asm.push_str(" stp fp, lr, [sp, #-32]!\n");
generated_asm.push_str(" mov fp, sp\n");
generated_asm.push_str(" str x19, [sp, #24]\n"); // x19 is callee saved
// x19 is reserved by LLVM for the "base pointer", so rustc doesn't allow using it
generated_asm.push_str(" mov x19, x0\n");
}
_ => unimplemented!("prologue for {:?}", arch),
}
@ -725,24 +751,14 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
fn epilogue(generated_asm: &mut String, arch: InlineAsmArch) {
match arch {
InlineAsmArch::X86 => {
generated_asm.push_str(" pop ebp\n");
generated_asm.push_str(" ret\n");
}
InlineAsmArch::X86_64 => {
generated_asm.push_str(" pop rbx\n");
generated_asm.push_str(" pop rbp\n");
generated_asm.push_str(" ret\n");
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" lw s0, 0(sp)\n");
generated_asm.push_str(" lw ra, 4(sp)\n");
generated_asm.push_str(" addi sp, sp, 8\n");
generated_asm.push_str(" ret\n");
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" ld s0, 0(sp)\n");
generated_asm.push_str(" ld ra, 8(sp)\n");
generated_asm.push_str(" addi sp, sp, 16\n");
InlineAsmArch::AArch64 => {
generated_asm.push_str(" ldr x19, [sp, #24]\n");
generated_asm.push_str(" ldp fp, lr, [sp], #32\n");
generated_asm.push_str(" ret\n");
}
_ => unimplemented!("epilogue for {:?}", arch),
@ -751,11 +767,11 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
fn epilogue_noreturn(generated_asm: &mut String, arch: InlineAsmArch) {
match arch {
InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
InlineAsmArch::X86_64 => {
generated_asm.push_str(" ud2\n");
}
InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
generated_asm.push_str(" ebreak\n");
InlineAsmArch::AArch64 => {
generated_asm.push_str(" brk #0x1");
}
_ => unimplemented!("epilogue_noreturn for {:?}", arch),
}
@ -768,25 +784,15 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
offset: Size,
) {
match arch {
InlineAsmArch::X86 => {
write!(generated_asm, " mov [ebp+0x{:x}], ", offset.bytes()).unwrap();
reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
generated_asm.push('\n');
}
InlineAsmArch::X86_64 => {
write!(generated_asm, " mov [rbp+0x{:x}], ", offset.bytes()).unwrap();
write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
generated_asm.push('\n');
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" sw ");
reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" sd ");
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
InlineAsmArch::AArch64 => {
generated_asm.push_str(" str ");
reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
}
_ => unimplemented!("save_register for {:?}", arch),
}
@ -799,25 +805,15 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
offset: Size,
) {
match arch {
InlineAsmArch::X86 => {
generated_asm.push_str(" mov ");
reg.emit(generated_asm, InlineAsmArch::X86, None).unwrap();
writeln!(generated_asm, ", [ebp+0x{:x}]", offset.bytes()).unwrap();
}
InlineAsmArch::X86_64 => {
generated_asm.push_str(" mov ");
reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
writeln!(generated_asm, ", [rbp+0x{:x}]", offset.bytes()).unwrap();
writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
}
InlineAsmArch::RiscV32 => {
generated_asm.push_str(" lw ");
reg.emit(generated_asm, InlineAsmArch::RiscV32, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
}
InlineAsmArch::RiscV64 => {
generated_asm.push_str(" ld ");
reg.emit(generated_asm, InlineAsmArch::RiscV64, None).unwrap();
writeln!(generated_asm, ", 0x{:x}(s0)", offset.bytes()).unwrap();
InlineAsmArch::AArch64 => {
generated_asm.push_str(" ldr ");
reg.emit(generated_asm, InlineAsmArch::AArch64, None).unwrap();
writeln!(generated_asm, ", [x19, 0x{:x}]", offset.bytes()).unwrap();
}
_ => unimplemented!("restore_register for {:?}", arch),
}

4
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View file

@ -1,10 +1,10 @@
//! Emulate LLVM intrinsics
use rustc_middle::ty::GenericArgsRef;
use crate::intrinsics::*;
use crate::prelude::*;
use rustc_middle::ty::GenericArgsRef;
pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: &str,

39
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
View file

@ -1,10 +1,10 @@
//! Emulate AArch64 LLVM intrinsics
use rustc_middle::ty::GenericArgsRef;
use crate::intrinsics::*;
use crate::prelude::*;
use rustc_middle::ty::GenericArgsRef;
pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: &str,
@ -156,6 +156,41 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
});
}
// FIXME generalize vector types
"llvm.aarch64.neon.tbl1.v16i8" => {
intrinsic_args!(fx, args => (t, idx); intrinsic);
let zero = fx.bcx.ins().iconst(types::I8, 0);
for i in 0..16 {
let idx_lane = idx.value_lane(fx, i).load_scalar(fx);
let is_zero =
fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, idx_lane, 16);
let t_idx = fx.bcx.ins().uextend(fx.pointer_type, idx_lane);
let t_lane = t.value_lane_dyn(fx, t_idx).load_scalar(fx);
let res = fx.bcx.ins().select(is_zero, zero, t_lane);
ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted());
}
}
// FIXME generalize vector types
"llvm.aarch64.neon.umaxp.v16i8" => {
intrinsic_args!(fx, args => (a, b); intrinsic);
// FIXME add helper for horizontal pairwise operations
for i in 0..8 {
let lane1 = a.value_lane(fx, i * 2).load_scalar(fx);
let lane2 = a.value_lane(fx, i * 2 + 1).load_scalar(fx);
let res = fx.bcx.ins().umax(lane1, lane2);
ret.place_lane(fx, i).to_ptr().store(fx, res, MemFlags::trusted());
}
for i in 0..8 {
let lane1 = b.value_lane(fx, i * 2).load_scalar(fx);
let lane2 = b.value_lane(fx, i * 2 + 1).load_scalar(fx);
let res = fx.bcx.ins().umax(lane1, lane2);
ret.place_lane(fx, 8 + i).to_ptr().store(fx, res, MemFlags::trusted());
}
}
/*
_ if intrinsic.starts_with("llvm.aarch64.neon.sshl.v")
|| intrinsic.starts_with("llvm.aarch64.neon.sqshl.v")

10
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
View file

@ -1,10 +1,10 @@
//! Emulate x86 LLVM intrinsics
use rustc_middle::ty::GenericArgsRef;
use crate::intrinsics::*;
use crate::prelude::*;
use rustc_middle::ty::GenericArgsRef;
pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: &str,
@ -74,8 +74,10 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
};
let x = codegen_operand(fx, x);
let y = codegen_operand(fx, y);
let kind = crate::constant::mir_operand_get_const_val(fx, kind)
.expect("llvm.x86.sse2.cmp.* kind not const");
let kind = match kind {
Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0,
Operand::Copy(_) | Operand::Move(_) => unreachable!("{kind:?}"),
};
let flt_cc = match kind
.try_to_bits(Size::from_bytes(1))

7
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -18,17 +18,16 @@ mod llvm_aarch64;
mod llvm_x86;
mod simd;
pub(crate) use cpuid::codegen_cpuid_call;
pub(crate) use llvm::codegen_llvm_intrinsic_call;
use cranelift_codegen::ir::AtomicRmwOp;
use rustc_middle::ty;
use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement};
use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
use rustc_middle::ty::GenericArgsRef;
use rustc_span::symbol::{kw, sym, Symbol};
pub(crate) use self::cpuid::codegen_cpuid_call;
pub(crate) use self::llvm::codegen_llvm_intrinsic_call;
use crate::prelude::*;
use cranelift_codegen::ir::AtomicRmwOp;
fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
bug!("wrong number of args for intrinsic {}", intrinsic);

39
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -148,7 +148,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let total_len = lane_count * 2;
let indexes =
idx.iter().map(|idx| idx.unwrap_leaf().try_to_u16().unwrap()).collect::<Vec<u16>>();
idx.iter().map(|idx| idx.unwrap_leaf().try_to_u32().unwrap()).collect::<Vec<u32>>();
for &idx in &indexes {
assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
@ -216,8 +216,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let indexes = {
use rustc_middle::mir::interpret::*;
let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
.expect("simd_shuffle idx not const");
let idx_const = match idx {
Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0,
Operand::Copy(_) | Operand::Move(_) => unreachable!("{idx:?}"),
};
let idx_bytes = match idx_const {
ConstValue::Indirect { alloc_id, offset } => {
@ -343,7 +345,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
ret.write_cvalue(fx, ret_lane);
}
sym::simd_neg => {
sym::simd_neg
| sym::simd_bswap
| sym::simd_bitreverse
| sym::simd_ctlz
| sym::simd_cttz => {
intrinsic_args!(fx, args => (a); intrinsic);
if !a.layout().ty.is_simd() {
@ -351,16 +357,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
return;
}
simd_for_each_lane(
fx,
a,
ret,
&|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
ty::Int(_) => fx.bcx.ins().ineg(lane),
ty::Float(_) => fx.bcx.ins().fneg(lane),
_ => unreachable!(),
},
);
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| match (
lane_ty.kind(),
intrinsic,
) {
(ty::Int(_), sym::simd_neg) => fx.bcx.ins().ineg(lane),
(ty::Float(_), sym::simd_neg) => fx.bcx.ins().fneg(lane),
(ty::Uint(ty::UintTy::U8) | ty::Int(ty::IntTy::I8), sym::simd_bswap) => lane,
(ty::Uint(_) | ty::Int(_), sym::simd_bswap) => fx.bcx.ins().bswap(lane),
(ty::Uint(_) | ty::Int(_), sym::simd_bitreverse) => fx.bcx.ins().bitrev(lane),
(ty::Uint(_) | ty::Int(_), sym::simd_ctlz) => fx.bcx.ins().clz(lane),
(ty::Uint(_) | ty::Int(_), sym::simd_cttz) => fx.bcx.ins().ctz(lane),
_ => unreachable!(),
});
}
sym::simd_add

39
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -29,6 +29,8 @@ use std::any::Any;
use std::cell::{Cell, RefCell};
use std::sync::Arc;
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::settings::{self, Configurable};
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
use rustc_data_structures::profiling::SelfProfilerRef;
@ -39,9 +41,6 @@ use rustc_session::config::OutputFilenames;
use rustc_session::Session;
use rustc_span::Symbol;
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::settings::{self, Configurable};
pub use crate::config::*;
use crate::prelude::*;
@ -76,22 +75,6 @@ mod value_and_place;
mod vtable;
mod prelude {
pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
pub(crate) use rustc_middle::bug;
pub(crate) use rustc_middle::mir::{self, *};
pub(crate) use rustc_middle::ty::layout::{self, LayoutOf, TyAndLayout};
pub(crate) use rustc_middle::ty::{
self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
TypeFoldable, TypeVisitableExt, UintTy,
};
pub(crate) use rustc_target::abi::{Abi, FieldIdx, Scalar, Size, VariantIdx, FIRST_VARIANT};
pub(crate) use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
pub(crate) use rustc_index::Idx;
pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
pub(crate) use cranelift_codegen::ir::function::Function;
pub(crate) use cranelift_codegen::ir::types;
@ -103,6 +86,18 @@ mod prelude {
pub(crate) use cranelift_codegen::Context;
pub(crate) use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
pub(crate) use cranelift_module::{self, DataDescription, FuncId, Linkage, Module};
pub(crate) use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
pub(crate) use rustc_hir::def_id::{DefId, LOCAL_CRATE};
pub(crate) use rustc_index::Idx;
pub(crate) use rustc_middle::bug;
pub(crate) use rustc_middle::mir::{self, *};
pub(crate) use rustc_middle::ty::layout::{self, LayoutOf, TyAndLayout};
pub(crate) use rustc_middle::ty::{
self, FloatTy, Instance, InstanceDef, IntTy, ParamEnv, Ty, TyCtxt, TypeAndMut,
TypeFoldable, TypeVisitableExt, UintTy,
};
pub(crate) use rustc_span::{FileNameDisplayPreference, Span};
pub(crate) use rustc_target::abi::{Abi, FieldIdx, Scalar, Size, VariantIdx, FIRST_VARIANT};
pub(crate) use crate::abi::*;
pub(crate) use crate::base::{codegen_operand, codegen_place};
@ -263,9 +258,9 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar
let preserve_frame_pointer = sess.target.options.frame_pointer
!= rustc_target::spec::FramePointer::MayOmit
|| matches!(sess.opts.cg.force_frame_pointers, Some(true));
if preserve_frame_pointer {
flags_builder.set("preserve_frame_pointers", "true").unwrap();
}
flags_builder
.set("preserve_frame_pointers", if preserve_frame_pointer { "true" } else { "false" })
.unwrap();
let tls_model = match target_triple.binary_format {
BinaryFormat::Elf => "elf_gd",

5
compiler/rustc_codegen_cranelift/src/pointer.rs
View file

@ -1,11 +1,10 @@
//! Defines [`Pointer`] which is used to improve the quality of the generated clif ir for pointer
//! operations.
use crate::prelude::*;
use cranelift_codegen::ir::immediates::Offset32;
use rustc_target::abi::Align;
use cranelift_codegen::ir::immediates::Offset32;
use crate::prelude::*;
/// A pointer pointing either to a certain address, a certain stack slot or nothing.
#[derive(Copy, Clone, Debug)]

22
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -63,8 +63,8 @@ use cranelift_codegen::{
ir::entities::AnyEntity,
write::{FuncWriter, PlainWriter},
};
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
use rustc_session::config::{OutputFilenames, OutputType};
use crate::prelude::*;
@ -80,15 +80,17 @@ impl CommentWriter {
pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
let enabled = should_write_ir(tcx);
let global_comments = if enabled {
vec![
format!("symbol {}", tcx.symbol_name(instance).name),
format!("instance {:?}", instance),
format!(
"abi {:?}",
RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty())
),
String::new(),
]
with_no_trimmed_paths!({
vec![
format!("symbol {}", tcx.symbol_name(instance).name),
format!("instance {:?}", instance),
format!(
"abi {:?}",
RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty())
),
String::new(),
]
})
} else {
vec![]
};

10
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -1,11 +1,10 @@
//! Definition of [`CValue`] and [`CPlace`]
use crate::prelude::*;
use rustc_middle::ty::FnSig;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::immediates::Offset32;
use rustc_middle::ty::FnSig;
use crate::prelude::*;
fn codegen_field<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
@ -310,7 +309,8 @@ impl<'tcx> CValue<'tcx> {
fx.bcx.ins().iconcat(lsb, msb)
}
ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => {
fx.bcx.ins().iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64)
let raw_val = const_val.size().truncate(const_val.to_bits(layout.size).unwrap());
fx.bcx.ins().iconst(clif_ty, raw_val as i64)
}
ty::Float(FloatTy::F32) => {
fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))