bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit 'e9d1a0a7b0' into sync_cg_clif-2022-08-24

Browse source
This commit is contained in:
bjorn3 2022-08-24 18:40:58 +02:00
commit 7b9c8c8781
46 changed files with 2292 additions and 1259 deletions
Download patch file Download diff file Expand all files Collapse all files

6
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -23,7 +23,7 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam {
(RegKind::Integer, 9..=16) => types::I128,
(RegKind::Float, 4) => types::F32,
(RegKind::Float, 8) => types::F64,
(RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(),
(RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(),
_ => unreachable!("{:?}", reg),
};
AbiParam::new(clif_ty)
@ -184,7 +184,7 @@ pub(super) fn from_casted_value<'tcx>(
let abi_params = cast_target_to_abi_params(cast);
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
@ -193,7 +193,7 @@ pub(super) fn from_casted_value<'tcx>(
// larger alignment than the integer.
size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
});
let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
let ptr = Pointer::stack_slot(stack_slot);
let mut offset = 0;
let mut block_params_iter = block_params.iter().copied();
for param in abi_params {

227
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -6,21 +6,43 @@ use rustc_middle::ty::adjustment::PointerCast;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
use indexmap::IndexSet;
use crate::constant::ConstantCx;
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
use crate::pretty_clif::CommentWriter;
pub(crate) fn codegen_fn<'tcx>(
cx: &mut crate::CodegenCx<'tcx>,
pub(crate) struct CodegenedFunction {
symbol_name: String,
func_id: FuncId,
func: Function,
clif_comments: CommentWriter,
func_debug_cx: Option<FunctionDebugContext>,
}
#[cfg_attr(not(feature = "jit"), allow(dead_code))]
pub(crate) fn codegen_and_compile_fn<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
cached_context: &mut Context,
module: &mut dyn Module,
instance: Instance<'tcx>,
) {
let tcx = cx.tcx;
let _inst_guard =
crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance);
compile_fn(cx, cached_context, module, codegened_func);
}
pub(crate) fn codegen_fn<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
cached_func: Function,
module: &mut dyn Module,
instance: Instance<'tcx>,
) -> CodegenedFunction {
debug_assert!(!instance.substs.needs_infer());
let mir = tcx.instance_mir(instance.def);
@ -34,15 +56,14 @@ pub(crate) fn codegen_fn<'tcx>(
});
// Declare function
let symbol_name = tcx.symbol_name(instance);
let symbol_name = tcx.symbol_name(instance).name.to_string();
let sig = get_function_sig(tcx, module.isa().triple(), instance);
let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap();
cx.cached_context.clear();
let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
// Make the FunctionBuilder
let mut func_ctx = FunctionBuilderContext::new();
let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new());
let mut func = cached_func;
func.clear();
func.name = ExternalName::user(0, func_id.as_u32());
func.signature = sig;
func.collect_debug_info();
@ -59,6 +80,12 @@ pub(crate) fn codegen_fn<'tcx>(
let pointer_type = target_config.pointer_type();
let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context {
Some(debug_context.define_function(tcx, &symbol_name, mir.span))
} else {
None
};
let mut fx = FunctionCx {
cx,
module,
@ -66,6 +93,7 @@ pub(crate) fn codegen_fn<'tcx>(
target_config,
pointer_type,
constants_cx: ConstantCx::new(),
func_debug_cx,
instance,
symbol_name,
@ -78,81 +106,48 @@ pub(crate) fn codegen_fn<'tcx>(
caller_location: None, // set by `codegen_fn_prelude`
clif_comments,
source_info_set: indexmap::IndexSet::new(),
last_source_file: None,
next_ssa_var: 0,
};
let arg_uninhabited = fx
.mir
.args_iter()
.any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
if !crate::constant::check_constants(&mut fx) {
fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
} else if arg_uninhabited {
fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
} else {
tcx.sess.time("codegen clif ir", || {
tcx.sess
.time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
codegen_fn_content(&mut fx);
});
}
tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
// Recover all necessary data from fx, before accessing func will prevent future access to it.
let instance = fx.instance;
let symbol_name = fx.symbol_name;
let clif_comments = fx.clif_comments;
let source_info_set = fx.source_info_set;
let local_map = fx.local_map;
let func_debug_cx = fx.func_debug_cx;
fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
crate::pretty_clif::write_clif_file(
tcx,
"unopt",
module.isa(),
instance,
&func,
&clif_comments,
);
if cx.should_write_ir {
crate::pretty_clif::write_clif_file(
tcx.output_filenames(()),
&symbol_name,
"unopt",
module.isa(),
&func,
&clif_comments,
);
}
// Verify function
verify_func(tcx, &clif_comments, &func);
compile_fn(
cx,
module,
instance,
symbol_name.name,
func_id,
func,
clif_comments,
source_info_set,
local_map,
);
CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
}
fn compile_fn<'tcx>(
cx: &mut crate::CodegenCx<'tcx>,
pub(crate) fn compile_fn(
cx: &mut crate::CodegenCx,
cached_context: &mut Context,
module: &mut dyn Module,
instance: Instance<'tcx>,
symbol_name: &str,
func_id: FuncId,
func: Function,
mut clif_comments: CommentWriter,
source_info_set: IndexSet<SourceInfo>,
local_map: IndexVec<mir::Local, CPlace<'tcx>>,
codegened_func: CodegenedFunction,
) {
let tcx = cx.tcx;
let clif_comments = codegened_func.clif_comments;
// Store function in context
let context = &mut cx.cached_context;
let context = cached_context;
context.clear();
context.func = func;
context.func = codegened_func.func;
// If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
// instruction, which doesn't have an encoding.
@ -164,17 +159,6 @@ fn compile_fn<'tcx>(
// invalidate it when it would change.
context.domtree.clear();
// Perform rust specific optimizations
tcx.sess.time("optimize clif ir", || {
crate::optimize::optimize_function(
tcx,
module.isa(),
instance,
context,
&mut clif_comments,
);
});
#[cfg(any())] // This is never true
let _clif_guard = {
use std::fmt::Write;
@ -203,46 +187,44 @@ fn compile_fn<'tcx>(
};
// Define function
tcx.sess.time("define function", || {
context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
module.define_function(func_id, context).unwrap();
cx.profiler.verbose_generic_activity("define function").run(|| {
context.want_disasm = cx.should_write_ir;
module.define_function(codegened_func.func_id, context).unwrap();
});
// Write optimized function to file for debugging
crate::pretty_clif::write_clif_file(
tcx,
"opt",
module.isa(),
instance,
&context.func,
&clif_comments,
);
if cx.should_write_ir {
// Write optimized function to file for debugging
crate::pretty_clif::write_clif_file(
&cx.output_filenames,
&codegened_func.symbol_name,
"opt",
module.isa(),
&context.func,
&clif_comments,
);
if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
crate::pretty_clif::write_ir_file(
tcx,
|| format!("{}.vcode", tcx.symbol_name(instance).name),
|file| file.write_all(disasm.as_bytes()),
)
if let Some(disasm) = &context.compiled_code().unwrap().disasm {
crate::pretty_clif::write_ir_file(
&cx.output_filenames,
&format!("{}.vcode", codegened_func.symbol_name),
|file| file.write_all(disasm.as_bytes()),
)
}
}
// Define debuginfo for function
let isa = module.isa();
let debug_context = &mut cx.debug_context;
let unwind_context = &mut cx.unwind_context;
tcx.sess.time("generate debug info", || {
cx.profiler.verbose_generic_activity("generate debug info").run(|| {
if let Some(debug_context) = debug_context {
debug_context.define_function(
instance,
func_id,
symbol_name,
isa,
codegened_func.func_debug_cx.unwrap().finalize(
debug_context,
codegened_func.func_id,
context,
&source_info_set,
local_map,
);
}
unwind_context.add_function(func_id, &context, isa);
unwind_context.add_function(codegened_func.func_id, &context, isa);
});
}
@ -268,7 +250,27 @@ pub(crate) fn verify_func(
});
}
fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
if !crate::constant::check_constants(fx) {
fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
// compilation should have been aborted
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
return;
}
let arg_uninhabited = fx
.mir
.args_iter()
.any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
if arg_uninhabited {
fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
return;
}
fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block));
for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
let block = fx.get_block(bb);
fx.bcx.switch_to_block(block);
@ -457,17 +459,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
template,
operands,
*options,
*destination,
);
match *destination {
Some(destination) => {
let destination_block = fx.get_block(destination);
fx.bcx.ins().jump(destination_block, &[]);
}
None => {
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
}
}
TerminatorKind::Resume | TerminatorKind::Abort => {
// FIXME implement unwinding
@ -711,9 +704,7 @@ fn codegen_stmt<'tcx>(
Rvalue::Discriminant(place) => {
let place = codegen_place(fx, place);
let value = place.to_cvalue(fx);
let discr =
crate::discriminant::codegen_get_discriminant(fx, value, dest_layout);
lval.write_cvalue(fx, discr);
crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout);
}
Rvalue::Repeat(ref operand, times) => {
let operand = codegen_operand(fx, operand);

46
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -1,14 +1,18 @@
use cranelift_codegen::isa::TargetFrontendConfig;
use gimli::write::FileId;
use rustc_data_structures::sync::Lrc;
use rustc_index::vec::IndexVec;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
};
use rustc_middle::ty::SymbolName;
use rustc_span::SourceFile;
use rustc_target::abi::call::FnAbi;
use rustc_target::abi::{Integer, Primitive};
use rustc_target::spec::{HasTargetSpec, Target};
use crate::constant::ConstantCx;
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@ -74,7 +78,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
_ => unreachable!(),
};
match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
// Cranelift currently only implements icmp for 128bit vectors.
Some(vector_ty) if vector_ty.bits() == 128 => vector_ty,
_ => return None,
@ -232,15 +236,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
}
pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
pub(crate) cx: &'clif mut crate::CodegenCx,
pub(crate) module: &'m mut dyn Module,
pub(crate) tcx: TyCtxt<'tcx>,
pub(crate) target_config: TargetFrontendConfig, // Cached from module
pub(crate) pointer_type: Type, // Cached from module
pub(crate) constants_cx: ConstantCx,
pub(crate) func_debug_cx: Option<FunctionDebugContext>,
pub(crate) instance: Instance<'tcx>,
pub(crate) symbol_name: SymbolName<'tcx>,
pub(crate) symbol_name: String,
pub(crate) mir: &'tcx Body<'tcx>,
pub(crate) fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>,
@ -252,7 +257,11 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
pub(crate) caller_location: Option<CValue<'tcx>>,
pub(crate) clif_comments: crate::pretty_clif::CommentWriter,
pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>,
/// Last accessed source file and it's debuginfo file id.
///
/// For optimization purposes only
pub(crate) last_source_file: Option<(Lrc<SourceFile>, FileId)>,
/// This should only be accessed by `CPlace::new_var`.
pub(crate) next_ssa_var: u32,
@ -336,8 +345,31 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
}
pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
let (index, _) = self.source_info_set.insert_full(source_info);
self.bcx.set_srcloc(SourceLoc::new(index as u32));
if let Some(debug_context) = &mut self.cx.debug_context {
let (file, line, column) =
DebugContext::get_span_loc(self.tcx, self.mir.span, source_info.span);
// add_source_file is very slow.
// Optimize for the common case of the current file not being changed.
let mut cached_file_id = None;
if let Some((ref last_source_file, last_file_id)) = self.last_source_file {
// If the allocations are not equal, the files may still be equal, but that
// doesn't matter, as this is just an optimization.
if rustc_data_structures::sync::Lrc::ptr_eq(last_source_file, &file) {
cached_file_id = Some(last_file_id);
}
}
let file_id = if let Some(file_id) = cached_file_id {
file_id
} else {
debug_context.add_source_file(&file)
};
let source_loc =
self.func_debug_cx.as_mut().unwrap().add_dbg_loc(file_id, line, column);
self.bcx.set_srcloc(source_loc);
}
}
// Note: must be kept in sync with get_caller_location from cg_ssa

168
compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs Normal file
View file

@ -0,0 +1,168 @@
use std::sync::{Arc, Condvar, Mutex};
use rustc_session::Session;
use jobserver::HelperThread;
// FIXME don't panic when a worker thread panics
pub(super) struct ConcurrencyLimiter {
helper_thread: Option<HelperThread>,
state: Arc<Mutex<state::ConcurrencyLimiterState>>,
available_token_condvar: Arc<Condvar>,
}
impl ConcurrencyLimiter {
pub(super) fn new(sess: &Session, pending_jobs: usize) -> Self {
let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs)));
let available_token_condvar = Arc::new(Condvar::new());
let state_helper = state.clone();
let available_token_condvar_helper = available_token_condvar.clone();
let helper_thread = sess
.jobserver
.clone()
.into_helper_thread(move |token| {
let mut state = state_helper.lock().unwrap();
state.add_new_token(token.unwrap());
available_token_condvar_helper.notify_one();
})
.unwrap();
ConcurrencyLimiter {
helper_thread: Some(helper_thread),
state,
available_token_condvar: Arc::new(Condvar::new()),
}
}
pub(super) fn acquire(&mut self) -> ConcurrencyLimiterToken {
let mut state = self.state.lock().unwrap();
loop {
state.assert_invariants();
if state.try_start_job() {
return ConcurrencyLimiterToken {
state: self.state.clone(),
available_token_condvar: self.available_token_condvar.clone(),
};
}
self.helper_thread.as_mut().unwrap().request_token();
state = self.available_token_condvar.wait(state).unwrap();
}
}
pub(super) fn job_already_done(&mut self) {
let mut state = self.state.lock().unwrap();
state.job_already_done();
}
}
impl Drop for ConcurrencyLimiter {
fn drop(&mut self) {
//
self.helper_thread.take();
// Assert that all jobs have finished
let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap();
state.assert_done();
}
}
#[derive(Debug)]
pub(super) struct ConcurrencyLimiterToken {
state: Arc<Mutex<state::ConcurrencyLimiterState>>,
available_token_condvar: Arc<Condvar>,
}
impl Drop for ConcurrencyLimiterToken {
fn drop(&mut self) {
let mut state = self.state.lock().unwrap();
state.job_finished();
self.available_token_condvar.notify_one();
}
}
mod state {
use jobserver::Acquired;
#[derive(Debug)]
pub(super) struct ConcurrencyLimiterState {
pending_jobs: usize,
active_jobs: usize,
// None is used to represent the implicit token, Some to represent explicit tokens
tokens: Vec<Option<Acquired>>,
}
impl ConcurrencyLimiterState {
pub(super) fn new(pending_jobs: usize) -> Self {
ConcurrencyLimiterState { pending_jobs, active_jobs: 0, tokens: vec![None] }
}
pub(super) fn assert_invariants(&self) {
// There must be no excess active jobs
assert!(self.active_jobs <= self.pending_jobs);
// There may not be more active jobs than there are tokens
assert!(self.active_jobs <= self.tokens.len());
}
pub(super) fn assert_done(&self) {
assert_eq!(self.pending_jobs, 0);
assert_eq!(self.active_jobs, 0);
}
pub(super) fn add_new_token(&mut self, token: Acquired) {
self.tokens.push(Some(token));
self.drop_excess_capacity();
}
pub(super) fn try_start_job(&mut self) -> bool {
if self.active_jobs < self.tokens.len() {
// Using existing token
self.job_started();
return true;
}
false
}
pub(super) fn job_started(&mut self) {
self.assert_invariants();
self.active_jobs += 1;
self.drop_excess_capacity();
self.assert_invariants();
}
pub(super) fn job_finished(&mut self) {
self.assert_invariants();
self.pending_jobs -= 1;
self.active_jobs -= 1;
self.assert_invariants();
self.drop_excess_capacity();
self.assert_invariants();
}
pub(super) fn job_already_done(&mut self) {
self.assert_invariants();
self.pending_jobs -= 1;
self.assert_invariants();
self.drop_excess_capacity();
self.assert_invariants();
}
fn drop_excess_capacity(&mut self) {
self.assert_invariants();
// Drop all tokens that can never be used anymore
self.tokens.truncate(std::cmp::max(self.pending_jobs, 1));
// Keep some excess tokens to satisfy requests faster
const MAX_EXTRA_CAPACITY: usize = 2;
self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1));
self.assert_invariants();
}
}
}

2
compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
View file

@ -9,7 +9,7 @@ use gimli::{RunTimeEndian, SectionId};
use super::object::WriteDebugInfo;
use super::DebugContext;
impl DebugContext<'_> {
impl DebugContext {
pub(crate) fn emit(&mut self, product: &mut ObjectProduct) {
let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone());
let root = self.dwarf.unit.root();

211
compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
View file

@ -3,8 +3,10 @@
use std::ffi::OsStr;
use std::path::{Component, Path};
use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
use rustc_data_structures::sync::Lrc;
use rustc_span::{
FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm,
};
@ -14,7 +16,6 @@ use cranelift_codegen::MachSrcLoc;
use gimli::write::{
Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
UnitEntryId,
};
// OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`.
@ -47,9 +48,9 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
}
}
pub(crate) const MD5_LEN: usize = 16;
const MD5_LEN: usize = 16;
pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
if hash.kind == SourceFileHashAlgorithm::Md5 {
let mut buf = [0u8; MD5_LEN];
buf.copy_from_slice(hash.hash_bytes());
@ -59,160 +60,132 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
}
}
fn line_program_add_file(
line_program: &mut LineProgram,
line_strings: &mut LineStringTable,
file: &SourceFile,
) -> FileId {
match &file.name {
FileName::Real(path) => {
let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available());
let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
let file_name = osstr_as_utf8_bytes(file_name);
impl DebugContext {
pub(crate) fn get_span_loc(
tcx: TyCtxt<'_>,
function_span: Span,
span: Span,
) -> (Lrc<SourceFile>, u64, u64) {
// Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
// In order to have a good line stepping behavior in debugger, we overwrite debug
// locations of macro expansions with that of the outermost expansion site
// (unless the crate is being compiled with `-Z debug-macros`).
let span = if !span.from_expansion() || tcx.sess.opts.unstable_opts.debug_macros {
span
} else {
// Walk up the macro expansion chain until we reach a non-expanded span.
// We also stop at the function body level because no line stepping can occur
// at the level above that.
rustc_span::hygiene::walk_chain(span, function_span.ctxt())
};
let dir_id = if !dir_name.is_empty() {
let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
line_program.add_directory(dir_name)
} else {
line_program.default_directory()
};
let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
match tcx.sess.source_map().lookup_line(span.lo()) {
Ok(SourceFileAndLine { sf: file, line }) => {
let line_pos = file.line_begin_pos(span.lo());
let info = make_file_info(file.src_hash);
line_program.file_has_md5 &= info.is_some();
line_program.add_file(file_name, dir_id, info)
(
file,
u64::try_from(line).unwrap() + 1,
u64::from((span.lo() - line_pos).to_u32()) + 1,
)
}
Err(file) => (file, 0, 0),
}
// FIXME give more appropriate file names
filename => {
let dir_id = line_program.default_directory();
let dummy_file_name = LineString::new(
filename.prefer_remapped().to_string().into_bytes(),
line_program.encoding(),
line_strings,
);
line_program.add_file(dummy_file_name, dir_id, None)
}
pub(crate) fn add_source_file(&mut self, source_file: &SourceFile) -> FileId {
let line_program: &mut LineProgram = &mut self.dwarf.unit.line_program;
let line_strings: &mut LineStringTable = &mut self.dwarf.line_strings;
match &source_file.name {
FileName::Real(path) => {
let (dir_path, file_name) =
split_path_dir_and_file(path.remapped_path_if_available());
let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
let file_name = osstr_as_utf8_bytes(file_name);
let dir_id = if !dir_name.is_empty() {
let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
line_program.add_directory(dir_name)
} else {
line_program.default_directory()
};
let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
let info = make_file_info(source_file.src_hash);
line_program.file_has_md5 &= info.is_some();
line_program.add_file(file_name, dir_id, info)
}
// FIXME give more appropriate file names
filename => {
let dir_id = line_program.default_directory();
let dummy_file_name = LineString::new(
filename.prefer_remapped().to_string().into_bytes(),
line_program.encoding(),
line_strings,
);
line_program.add_file(dummy_file_name, dir_id, None)
}
}
}
}
impl<'tcx> DebugContext<'tcx> {
pub(super) fn emit_location(&mut self, entry_id: UnitEntryId, span: Span) {
let loc = self.tcx.sess.source_map().lookup_char_pos(span.lo());
let file_id = line_program_add_file(
&mut self.dwarf.unit.line_program,
&mut self.dwarf.line_strings,
&loc.file,
);
let entry = self.dwarf.unit.get_mut(entry_id);
entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
impl FunctionDebugContext {
pub(crate) fn add_dbg_loc(&mut self, file_id: FileId, line: u64, column: u64) -> SourceLoc {
let (index, _) = self.source_loc_set.insert_full((file_id, line, column));
SourceLoc::new(u32::try_from(index).unwrap())
}
pub(super) fn create_debug_lines(
&mut self,
debug_context: &mut DebugContext,
symbol: usize,
entry_id: UnitEntryId,
context: &Context,
function_span: Span,
source_info_set: &indexmap::IndexSet<SourceInfo>,
) -> CodeOffset {
let tcx = self.tcx;
let line_program = &mut self.dwarf.unit.line_program;
let create_row_for_span =
|debug_context: &mut DebugContext, source_loc: (FileId, u64, u64)| {
let (file_id, line, col) = source_loc;
let line_strings = &mut self.dwarf.line_strings;
let mut last_span = None;
let mut last_file = None;
let mut create_row_for_span = |line_program: &mut LineProgram, span: Span| {
if let Some(last_span) = last_span {
if span == last_span {
line_program.generate_row();
return;
}
}
last_span = Some(span);
// Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
// In order to have a good line stepping behavior in debugger, we overwrite debug
// locations of macro expansions with that of the outermost expansion site
// (unless the crate is being compiled with `-Z debug-macros`).
let span = if !span.from_expansion() || tcx.sess.opts.unstable_opts.debug_macros {
span
} else {
// Walk up the macro expansion chain until we reach a non-expanded span.
// We also stop at the function body level because no line stepping can occur
// at the level above that.
rustc_span::hygiene::walk_chain(span, function_span.ctxt())
debug_context.dwarf.unit.line_program.row().file = file_id;
debug_context.dwarf.unit.line_program.row().line = line;
debug_context.dwarf.unit.line_program.row().column = col;
debug_context.dwarf.unit.line_program.generate_row();
};
let (file, line, col) = match tcx.sess.source_map().lookup_line(span.lo()) {
Ok(SourceFileAndLine { sf: file, line }) => {
let line_pos = file.line_begin_pos(span.lo());
(
file,
u64::try_from(line).unwrap() + 1,
u64::from((span.lo() - line_pos).to_u32()) + 1,
)
}
Err(file) => (file, 0, 0),
};
// line_program_add_file is very slow.
// Optimize for the common case of the current file not being changed.
let current_file_changed = if let Some(last_file) = &last_file {
// If the allocations are not equal, then the files may still be equal, but that
// is not a problem, as this is just an optimization.
!rustc_data_structures::sync::Lrc::ptr_eq(last_file, &file)
} else {
true
};
if current_file_changed {
let file_id = line_program_add_file(line_program, line_strings, &file);
line_program.row().file = file_id;
last_file = Some(file);
}
line_program.row().line = line;
line_program.row().column = col;
line_program.generate_row();
};
line_program.begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
debug_context
.dwarf
.unit
.line_program
.begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
let mut func_end = 0;
let mcr = context.mach_compile_result.as_ref().unwrap();
let mcr = context.compiled_code().unwrap();
for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
line_program.row().address_offset = u64::from(start);
debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start);
if !loc.is_default() {
let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
create_row_for_span(line_program, source_info.span);
let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap();
create_row_for_span(debug_context, source_loc);
} else {
create_row_for_span(line_program, function_span);
create_row_for_span(debug_context, self.function_source_loc);
}
func_end = end;
}
line_program.end_sequence(u64::from(func_end));
debug_context.dwarf.unit.line_program.end_sequence(u64::from(func_end));
let func_end = mcr.buffer.total_size();
assert_ne!(func_end, 0);
let entry = self.dwarf.unit.get_mut(entry_id);
let entry = debug_context.dwarf.unit.get_mut(self.entry_id);
entry.set(
gimli::DW_AT_low_pc,
AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
);
entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
self.emit_location(entry_id, function_span);
func_end
}
}

284
compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
View file

@ -7,35 +7,34 @@ mod unwind;
use crate::prelude::*;
use rustc_index::vec::IndexVec;
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::ir::{Endianness, LabelValueLoc, ValueLabel};
use cranelift_codegen::ir::Endianness;
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::ValueLocRange;
use gimli::write::{
Address, AttributeValue, DwarfUnit, Expression, LineProgram, LineString, Location,
LocationList, Range, RangeList, UnitEntryId,
Address, AttributeValue, DwarfUnit, FileId, LineProgram, LineString, Range, RangeList,
UnitEntryId,
};
use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, X86_64};
use gimli::{Encoding, Format, LineEncoding, RunTimeEndian};
use indexmap::IndexSet;
pub(crate) use emit::{DebugReloc, DebugRelocName};
pub(crate) use unwind::UnwindContext;
pub(crate) struct DebugContext<'tcx> {
tcx: TyCtxt<'tcx>,
pub(crate) struct DebugContext {
endian: RunTimeEndian,
dwarf: DwarfUnit,
unit_range_list: RangeList,
types: FxHashMap<Ty<'tcx>, UnitEntryId>,
}
impl<'tcx> DebugContext<'tcx> {
pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self {
pub(crate) struct FunctionDebugContext {
entry_id: UnitEntryId,
function_source_loc: (FileId, u64, u64),
source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>,
}
impl DebugContext {
pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa) -> Self {
let encoding = Encoding {
format: Format::Dwarf32,
// FIXME this should be configurable
@ -101,127 +100,18 @@ impl<'tcx> DebugContext<'tcx> {
root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
}
DebugContext {
tcx,
endian,
dwarf,
unit_range_list: RangeList(Vec::new()),
types: FxHashMap::default(),
}
}
fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
if let Some(type_id) = self.types.get(&ty) {
return *type_id;
}
let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag);
let primitive = |dwarf: &mut DwarfUnit, ate| {
let type_id = new_entry(dwarf, gimli::DW_TAG_base_type);
let type_entry = dwarf.unit.get_mut(type_id);
type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate));
type_id
};
let name = format!("{}", ty);
let layout = self.tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap();
let type_id = match ty.kind() {
ty::Bool => primitive(&mut self.dwarf, gimli::DW_ATE_boolean),
ty::Char => primitive(&mut self.dwarf, gimli::DW_ATE_UTF),
ty::Uint(_) => primitive(&mut self.dwarf, gimli::DW_ATE_unsigned),
ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed),
ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float),
ty::Ref(_, pointee_ty, _mutbl)
| ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => {
let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type);
// Ensure that type is inserted before recursing to avoid duplicates
self.types.insert(ty, type_id);
let pointee = self.dwarf_ty(*pointee_ty);
let type_entry = self.dwarf.unit.get_mut(type_id);
//type_entry.set(gimli::DW_AT_mutable, AttributeValue::Flag(mutbl == rustc_hir::Mutability::Mut));
type_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(pointee));
type_id
}
ty::Adt(adt_def, _substs) if adt_def.is_struct() && !layout.is_unsized() => {
let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type);
// Ensure that type is inserted before recursing to avoid duplicates
self.types.insert(ty, type_id);
let variant = adt_def.non_enum_variant();
for (field_idx, field_def) in variant.fields.iter().enumerate() {
let field_offset = layout.fields.offset(field_idx);
let field_layout = layout.field(
&layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() },
field_idx,
);
let field_type = self.dwarf_ty(field_layout.ty);
let field_id = self.dwarf.unit.add(type_id, gimli::DW_TAG_member);
let field_entry = self.dwarf.unit.get_mut(field_id);
field_entry.set(
gimli::DW_AT_name,
AttributeValue::String(field_def.name.as_str().to_string().into_bytes()),
);
field_entry.set(
gimli::DW_AT_data_member_location,
AttributeValue::Udata(field_offset.bytes()),
);
field_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(field_type));
}
type_id
}
_ => new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type),
};
let type_entry = self.dwarf.unit.get_mut(type_id);
type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
self.types.insert(ty, type_id);
type_id
}
fn define_local(&mut self, scope: UnitEntryId, name: String, ty: Ty<'tcx>) -> UnitEntryId {
let dw_ty = self.dwarf_ty(ty);
let var_id = self.dwarf.unit.add(scope, gimli::DW_TAG_variable);
let var_entry = self.dwarf.unit.get_mut(var_id);
var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
var_id
DebugContext { endian, dwarf, unit_range_list: RangeList(Vec::new()) }
}
pub(crate) fn define_function(
&mut self,
instance: Instance<'tcx>,
func_id: FuncId,
tcx: TyCtxt<'_>,
name: &str,
isa: &dyn TargetIsa,
context: &Context,
source_info_set: &indexmap::IndexSet<SourceInfo>,
local_map: IndexVec<mir::Local, CPlace<'tcx>>,
) {
let symbol = func_id.as_u32() as usize;
let mir = self.tcx.instance_mir(instance.def);
function_span: Span,
) -> FunctionDebugContext {
let (file, line, column) = DebugContext::get_span_loc(tcx, function_span, function_span);
let file_id = self.add_source_file(&file);
// FIXME: add to appropriate scope instead of root
let scope = self.dwarf.unit.root();
@ -233,14 +123,35 @@ impl<'tcx> DebugContext<'tcx> {
entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set);
entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line));
entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(column));
self.unit_range_list.0.push(Range::StartLength {
FunctionDebugContext {
entry_id,
function_source_loc: (file_id, line, column),
source_loc_set: IndexSet::new(),
}
}
}
impl FunctionDebugContext {
pub(crate) fn finalize(
mut self,
debug_context: &mut DebugContext,
func_id: FuncId,
context: &Context,
) {
let symbol = func_id.as_u32() as usize;
let end = self.create_debug_lines(debug_context, symbol, context);
debug_context.unit_range_list.0.push(Range::StartLength {
begin: Address::Symbol { symbol, addend: 0 },
length: u64::from(end),
});
let func_entry = self.dwarf.unit.get_mut(entry_id);
let func_entry = debug_context.dwarf.unit.get_mut(self.entry_id);
// Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
func_entry.set(
gimli::DW_AT_low_pc,
@ -248,110 +159,5 @@ impl<'tcx> DebugContext<'tcx> {
);
// Using Udata for DW_AT_high_pc requires at least DWARF4
func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
// FIXME make it more reliable and implement scopes before re-enabling this.
if false {
let value_labels_ranges = std::collections::HashMap::new(); // FIXME
for (local, _local_decl) in mir.local_decls.iter_enumerated() {
let ty = self.tcx.subst_and_normalize_erasing_regions(
instance.substs,
ty::ParamEnv::reveal_all(),
mir.local_decls[local].ty,
);
let var_id = self.define_local(entry_id, format!("{:?}", local), ty);
let location = place_location(
self,
isa,
symbol,
&local_map,
&value_labels_ranges,
Place { local, projection: ty::List::empty() },
);
let var_entry = self.dwarf.unit.get_mut(var_id);
var_entry.set(gimli::DW_AT_location, location);
}
}
// FIXME create locals for all entries in mir.var_debug_info
}
}
fn place_location<'tcx>(
debug_context: &mut DebugContext<'tcx>,
isa: &dyn TargetIsa,
symbol: usize,
local_map: &IndexVec<mir::Local, CPlace<'tcx>>,
#[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap<
ValueLabel,
Vec<ValueLocRange>,
>,
place: Place<'tcx>,
) -> AttributeValue {
assert!(place.projection.is_empty()); // FIXME implement them
match local_map[place.local].inner() {
CPlaceInner::Var(_local, var) => {
let value_label = cranelift_codegen::ir::ValueLabel::new(var.index());
if let Some(value_loc_ranges) = value_labels_ranges.get(&value_label) {
let loc_list = LocationList(
value_loc_ranges
.iter()
.map(|value_loc_range| Location::StartEnd {
begin: Address::Symbol {
symbol,
addend: i64::from(value_loc_range.start),
},
end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
data: translate_loc(isa, value_loc_range.loc).unwrap(),
})
.collect(),
);
let loc_list_id = debug_context.dwarf.unit.locations.add(loc_list);
AttributeValue::LocationListRef(loc_list_id)
} else {
// FIXME set value labels for unused locals
AttributeValue::Exprloc(Expression::new())
}
}
CPlaceInner::VarPair(_, _, _) => {
// FIXME implement this
AttributeValue::Exprloc(Expression::new())
}
CPlaceInner::VarLane(_, _, _) => {
// FIXME implement this
AttributeValue::Exprloc(Expression::new())
}
CPlaceInner::Addr(_, _) => {
// FIXME implement this (used by arguments and returns)
AttributeValue::Exprloc(Expression::new())
// For PointerBase::Stack:
//AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot)).unwrap())
}
}
}
// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
fn translate_loc(isa: &dyn TargetIsa, loc: LabelValueLoc) -> Option<Expression> {
match loc {
LabelValueLoc::Reg(reg) => {
let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
let mut expr = Expression::new();
expr.op_reg(gimli::Register(machine_reg));
Some(expr)
}
LabelValueLoc::SPOffset(offset) => {
let mut expr = Expression::new();
expr.op_breg(X86_64::RSP, offset);
Some(expr)
}
}
}

20
compiler/rustc_codegen_cranelift/src/discriminant.rs
View file

@ -62,16 +62,14 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
pub(crate) fn codegen_get_discriminant<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
dest: CPlace<'tcx>,
value: CValue<'tcx>,
dest_layout: TyAndLayout<'tcx>,
) -> CValue<'tcx> {
) {
let layout = value.layout();
if layout.abi == Abi::Uninhabited {
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached);
// Return a dummy value
return CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout);
if layout.abi.is_uninhabited() {
return;
}
let (tag_scalar, tag_field, tag_encoding) = match &layout.variants {
@ -89,7 +87,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
} else {
ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap()
};
return CValue::const_val(fx, dest_layout, discr_val);
let res = CValue::const_val(fx, dest_layout, discr_val);
dest.write_cvalue(fx, res);
return;
}
Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
(tag, *tag_field, tag_encoding)
@ -110,7 +110,8 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
_ => false,
};
let val = clif_intcast(fx, tag, cast_to, signed);
CValue::by_val(val, dest_layout)
let res = CValue::by_val(val, dest_layout);
dest.write_cvalue(fx, res);
}
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
// Rebase from niche values to discriminants, and check
@ -170,7 +171,8 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
CValue::by_val(discr, dest_layout)
let res = CValue::by_val(discr, dest_layout);
dest.write_cvalue(fx, res);
}
}
}

558
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -1,33 +1,129 @@
//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
//! standalone executable.
use std::fs::File;
use std::path::PathBuf;
use std::sync::Arc;
use std::thread::JoinHandle;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
use rustc_session::cgu_reuse_tracker::CguReuse;
use rustc_session::config::{DebugInfo, OutputType};
use rustc_session::config::{DebugInfo, OutputFilenames, OutputType};
use rustc_session::Session;
use cranelift_codegen::isa::TargetIsa;
use cranelift_object::{ObjectBuilder, ObjectModule};
use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
use crate::global_asm::GlobalAsmConfig;
use crate::{prelude::*, BackendConfig};
struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
struct ModuleCodegenResult {
module_regular: CompiledModule,
module_global_asm: Option<CompiledModule>,
existing_work_product: Option<(WorkProductId, WorkProduct)>,
}
impl<HCX> HashStable<HCX> for ModuleCodegenResult {
enum OngoingModuleCodegen {
Sync(Result<ModuleCodegenResult, String>),
Async(JoinHandle<Result<ModuleCodegenResult, String>>),
}
impl<HCX> HashStable<HCX> for OngoingModuleCodegen {
fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) {
// do nothing
}
}
fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
pub(crate) struct OngoingCodegen {
modules: Vec<OngoingModuleCodegen>,
allocator_module: Option<CompiledModule>,
metadata_module: Option<CompiledModule>,
metadata: EncodedMetadata,
crate_info: CrateInfo,
concurrency_limiter: ConcurrencyLimiter,
}
impl OngoingCodegen {
pub(crate) fn join(
self,
sess: &Session,
backend_config: &BackendConfig,
) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) {
let mut work_products = FxHashMap::default();
let mut modules = vec![];
for module_codegen in self.modules {
let module_codegen_result = match module_codegen {
OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result,
OngoingModuleCodegen::Async(join_handle) => match join_handle.join() {
Ok(module_codegen_result) => module_codegen_result,
Err(panic) => std::panic::resume_unwind(panic),
},
};
let module_codegen_result = match module_codegen_result {
Ok(module_codegen_result) => module_codegen_result,
Err(err) => sess.fatal(&err),
};
let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } =
module_codegen_result;
if let Some((work_product_id, work_product)) = existing_work_product {
work_products.insert(work_product_id, work_product);
} else {
let work_product = if backend_config.disable_incr_cache {
None
} else if let Some(module_global_asm) = &module_global_asm {
rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
sess,
&module_regular.name,
&[
("o", &module_regular.object.as_ref().unwrap()),
("asm.o", &module_global_asm.object.as_ref().unwrap()),
],
)
} else {
rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
sess,
&module_regular.name,
&[("o", &module_regular.object.as_ref().unwrap())],
)
};
if let Some((work_product_id, work_product)) = work_product {
work_products.insert(work_product_id, work_product);
}
}
modules.push(module_regular);
if let Some(module_global_asm) = module_global_asm {
modules.push(module_global_asm);
}
}
drop(self.concurrency_limiter);
(
CodegenResults {
modules,
allocator_module: self.allocator_module,
metadata_module: self.metadata_module,
metadata: self.metadata,
crate_info: self.crate_info,
},
work_products,
)
}
}
fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule {
let isa = crate::build_isa(sess, backend_config);
let mut builder =
ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
// Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
@ -37,15 +133,15 @@ fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectM
ObjectModule::new(builder)
}
fn emit_module(
tcx: TyCtxt<'_>,
backend_config: &BackendConfig,
fn emit_cgu(
output_filenames: &OutputFilenames,
prof: &SelfProfilerRef,
name: String,
kind: ModuleKind,
module: ObjectModule,
debug: Option<DebugContext<'_>>,
debug: Option<DebugContext>,
unwind_context: UnwindContext,
) -> ModuleCodegenResult {
global_asm_object_file: Option<PathBuf>,
) -> Result<ModuleCodegenResult, String> {
let mut product = module.finish();
if let Some(mut debug) = debug {
@ -54,134 +150,191 @@ fn emit_module(
unwind_context.emit(&mut product);
let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name));
let obj = product.object.write().unwrap();
let module_regular =
emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?;
tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap());
Ok(ModuleCodegenResult {
module_regular,
module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule {
name: format!("{name}.asm"),
kind: ModuleKind::Regular,
object: Some(global_asm_object_file),
dwarf_object: None,
bytecode: None,
}),
existing_work_product: None,
})
}
if let Err(err) = std::fs::write(&tmp_file, obj) {
tcx.sess.fatal(&format!("error writing object file: {}", err));
}
let work_product = if backend_config.disable_incr_cache {
None
} else {
rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
tcx.sess,
&name,
&[("o", &tmp_file)],
)
fn emit_module(
output_filenames: &OutputFilenames,
prof: &SelfProfilerRef,
object: cranelift_object::object::write::Object<'_>,
kind: ModuleKind,
name: String,
) -> Result<CompiledModule, String> {
let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name));
let mut file = match File::create(&tmp_file) {
Ok(file) => file,
Err(err) => return Err(format!("error creating object file: {}", err)),
};
ModuleCodegenResult(
CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None },
work_product,
)
if let Err(err) = object.write_stream(&mut file) {
return Err(format!("error writing object file: {}", err));
}
prof.artifact_size("object_file", &*name, file.metadata().unwrap().len());
Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None })
}
fn reuse_workproduct_for_cgu(
tcx: TyCtxt<'_>,
cgu: &CodegenUnit<'_>,
work_products: &mut FxHashMap<WorkProductId, WorkProduct>,
) -> CompiledModule {
) -> Result<ModuleCodegenResult, String> {
let work_product = cgu.previous_work_product(tcx);
let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
let source_file = rustc_incremental::in_incr_comp_dir_sess(
let obj_out_regular =
tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
let source_file_regular = rustc_incremental::in_incr_comp_dir_sess(
&tcx.sess,
&work_product.saved_files.get("o").expect("no saved object file in work product"),
);
if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) {
tcx.sess.err(&format!(
if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) {
return Err(format!(
"unable to copy {} to {}: {}",
source_file.display(),
obj_out.display(),
source_file_regular.display(),
obj_out_regular.display(),
err
));
}
let obj_out_global_asm =
crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm");
let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") {
let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o);
if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm)
{
return Err(format!(
"unable to copy {} to {}: {}",
source_file_regular.display(),
obj_out_regular.display(),
err
));
}
true
} else {
false
};
work_products.insert(cgu.work_product_id(), work_product);
CompiledModule {
name: cgu.name().to_string(),
kind: ModuleKind::Regular,
object: Some(obj_out),
dwarf_object: None,
bytecode: None,
}
Ok(ModuleCodegenResult {
module_regular: CompiledModule {
name: cgu.name().to_string(),
kind: ModuleKind::Regular,
object: Some(obj_out_regular),
dwarf_object: None,
bytecode: None,
},
module_global_asm: if has_global_asm {
Some(CompiledModule {
name: cgu.name().to_string(),
kind: ModuleKind::Regular,
object: Some(obj_out_global_asm),
dwarf_object: None,
bytecode: None,
})
} else {
None
},
existing_work_product: Some((cgu.work_product_id(), work_product)),
})
}
fn module_codegen(
tcx: TyCtxt<'_>,
(backend_config, cgu_name): (BackendConfig, rustc_span::Symbol),
) -> ModuleCodegenResult {
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
(backend_config, global_asm_config, cgu_name, token): (
BackendConfig,
Arc<GlobalAsmConfig>,
rustc_span::Symbol,
ConcurrencyLimiterToken,
),
) -> OngoingModuleCodegen {
let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || {
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let isa = crate::build_isa(tcx.sess, &backend_config);
let mut module = make_module(tcx.sess, isa, cgu_name.as_str().to_string());
let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
let mut cx = crate::CodegenCx::new(
tcx,
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
super::predefine_mono_items(tcx, &mut module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
cx.tcx
.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst));
}
MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id),
MonoItem::GlobalAsm(item_id) => {
let item = cx.tcx.hir().item(item_id);
if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
cx.global_asm.push_str("\n.intel_syntax noprefix\n");
} else {
cx.global_asm.push_str("\n.att_syntax\n");
}
for piece in asm.template {
match *piece {
InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s),
InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
}
}
cx.global_asm.push_str("\n.att_syntax\n\n");
} else {
bug!("Expected GlobalAsm found {:?}", item);
let mut cx = crate::CodegenCx::new(
tcx,
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
super::predefine_mono_items(tcx, &mut module, &mono_items);
let mut codegened_functions = vec![];
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
tcx.sess.time("codegen fn", || {
let codegened_function = crate::base::codegen_fn(
tcx,
&mut cx,
Function::new(),
&mut module,
inst,
);
codegened_functions.push(codegened_function);
});
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(tcx, &mut module, def_id)
}
MonoItem::GlobalAsm(item_id) => {
crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
}
}
}
}
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut module,
&mut cx.unwind_context,
false,
cgu.is_primary(),
);
let debug_context = cx.debug_context;
let unwind_context = cx.unwind_context;
let codegen_result = tcx.sess.time("write object file", || {
emit_module(
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&backend_config,
cgu.name().as_str().to_string(),
ModuleKind::Regular,
module,
debug_context,
unwind_context,
)
&mut module,
&mut cx.unwind_context,
false,
cgu.is_primary(),
);
let cgu_name = cgu.name().as_str().to_owned();
(cgu_name, cx, module, codegened_functions)
});
codegen_global_asm(tcx, cgu.name().as_str(), &cx.global_asm);
OngoingModuleCodegen::Async(std::thread::spawn(move || {
cx.profiler.clone().verbose_generic_activity("compile functions").run(|| {
let mut cached_context = Context::new();
for codegened_func in codegened_functions {
crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
}
});
codegen_result
let global_asm_object_file =
cx.profiler.verbose_generic_activity("compile assembly").run(|| {
crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
})?;
let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| {
emit_cgu(
&global_asm_config.output_filenames,
&cx.profiler,
cgu_name,
module,
cx.debug_context,
cx.unwind_context,
global_asm_object_file,
)
});
std::mem::drop(token);
codegen_result
}))
}
pub(crate) fn run_aot(
@ -189,9 +342,7 @@ pub(crate) fn run_aot(
backend_config: BackendConfig,
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
let mut work_products = FxHashMap::default();
) -> Box<OngoingCodegen> {
let cgus = if tcx.sess.opts.output_types.should_codegen() {
tcx.collect_and_partition_mono_items(()).1
} else {
@ -206,62 +357,69 @@ pub(crate) fn run_aot(
}
}
let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));
let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
cgus.iter()
.map(|cgu| {
let cgu_reuse = determine_cgu_reuse(tcx, cgu);
let cgu_reuse = if backend_config.disable_incr_cache {
CguReuse::No
} else {
determine_cgu_reuse(tcx, cgu)
};
tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);
match cgu_reuse {
_ if backend_config.disable_incr_cache => {}
CguReuse::No => {}
CguReuse::PreLto => {
return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products);
CguReuse::No => {
let dep_node = cgu.codegen_dep_node(tcx);
tcx.dep_graph
.with_task(
dep_node,
tcx,
(
backend_config.clone(),
global_asm_config.clone(),
cgu.name(),
concurrency_limiter.acquire(),
),
module_codegen,
Some(rustc_middle::dep_graph::hash_result),
)
.0
}
CguReuse::PreLto => unreachable!(),
CguReuse::PostLto => {
concurrency_limiter.job_already_done();
OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu))
}
CguReuse::PostLto => unreachable!(),
}
let dep_node = cgu.codegen_dep_node(tcx);
let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
dep_node,
tcx,
(backend_config.clone(), cgu.name()),
module_codegen,
Some(rustc_middle::dep_graph::hash_result),
);
if let Some((id, product)) = work_product {
work_products.insert(id, product);
}
module
})
.collect::<Vec<_>>()
});
tcx.sess.abort_if_errors();
let isa = crate::build_isa(tcx.sess, &backend_config);
let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string());
assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
let created_alloc_shim =
crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
let allocator_module = if created_alloc_shim {
let ModuleCodegenResult(module, work_product) = emit_module(
tcx,
&backend_config,
"allocator_shim".to_string(),
let mut product = allocator_module.finish();
allocator_unwind_context.emit(&mut product);
match emit_module(
tcx.output_filenames(()),
&tcx.sess.prof,
product.object,
ModuleKind::Allocator,
allocator_module,
None,
allocator_unwind_context,
);
if let Some((id, product)) = work_product {
work_products.insert(id, product);
"allocator_shim".to_owned(),
) {
Ok(allocator_module) => Some(allocator_module),
Err(err) => tcx.sess.fatal(err),
}
Some(module)
} else {
None
};
@ -308,102 +466,14 @@ pub(crate) fn run_aot(
}
.to_owned();
Box::new((
CodegenResults {
modules,
allocator_module,
metadata_module,
metadata,
crate_info: CrateInfo::new(tcx, target_cpu),
},
work_products,
))
}
fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
use std::io::Write;
use std::process::{Command, Stdio};
if global_asm.is_empty() {
return;
}
if cfg!(not(feature = "inline_asm"))
|| tcx.sess.target.is_like_osx
|| tcx.sess.target.is_like_windows
{
if global_asm.contains("__rust_probestack") {
return;
}
// FIXME fix linker error on macOS
if cfg!(not(feature = "inline_asm")) {
tcx.sess.fatal(
"asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
);
} else {
tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
}
}
let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as");
let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld");
// Remove all LLVM style comments
let global_asm = global_asm
.lines()
.map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
.collect::<Vec<_>>()
.join("\n");
let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name));
// Assemble `global_asm`
let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
let mut child = Command::new(assembler)
.arg("-o")
.arg(&global_asm_object_file)
.stdin(Stdio::piped())
.spawn()
.expect("Failed to spawn `as`.");
child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
let status = child.wait().expect("Failed to wait for `as`.");
if !status.success() {
tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm));
}
// Link the global asm and main object file together
let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main");
std::fs::rename(&output_object_file, &main_object_file).unwrap();
let status = Command::new(linker)
.arg("-r") // Create a new object file
.arg("-o")
.arg(output_object_file)
.arg(&main_object_file)
.arg(&global_asm_object_file)
.status()
.unwrap();
if !status.success() {
tcx.sess.fatal(&format!(
"Failed to link `{}` and `{}` together",
main_object_file.display(),
global_asm_object_file.display(),
));
}
std::fs::remove_file(global_asm_object_file).unwrap();
std::fs::remove_file(main_object_file).unwrap();
}
fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf {
let mut new_filename = path.file_stem().unwrap().to_owned();
new_filename.push(postfix);
if let Some(extension) = path.extension() {
new_filename.push(".");
new_filename.push(extension);
}
path.set_file_name(new_filename);
path
Box::new(OngoingCodegen {
modules,
allocator_module,
metadata_module,
metadata,
crate_info: CrateInfo::new(tcx, target_cpu),
concurrency_limiter,
})
}
// Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953
@ -432,5 +502,5 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
cgu.name()
);
if tcx.try_mark_green(&dep_node) { CguReuse::PreLto } else { CguReuse::No }
if tcx.try_mark_green(&dep_node) { CguReuse::PostLto } else { CguReuse::No }
}

49
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -61,11 +61,11 @@ impl UnsafeMessage {
}
}
fn create_jit_module<'tcx>(
tcx: TyCtxt<'tcx>,
fn create_jit_module(
tcx: TyCtxt<'_>,
backend_config: &BackendConfig,
hotswap: bool,
) -> (JITModule, CodegenCx<'tcx>) {
) -> (JITModule, CodegenCx) {
let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info);
@ -111,6 +111,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
&backend_config,
matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
);
let mut cached_context = Context::new();
let (_, cgus) = tcx.collect_and_partition_mono_items(());
let mono_items = cgus
@ -128,11 +129,19 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
cx.tcx.sess.time("codegen fn", || {
crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
tcx.sess.time("codegen fn", || {
crate::base::codegen_and_compile_fn(
tcx,
&mut cx,
&mut cached_context,
&mut jit_module,
inst,
)
});
}
CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
CodegenMode::JitLazy => {
codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst)
}
},
MonoItem::Static(def_id) => {
crate::constant::codegen_static(tcx, &mut jit_module, def_id);
@ -259,7 +268,15 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
false,
Symbol::intern("dummy_cgu_name"),
);
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
tcx.sess.time("codegen fn", || {
crate::base::codegen_and_compile_fn(
tcx,
&mut cx,
&mut Context::new(),
jit_module,
instance,
)
});
assert!(cx.global_asm.is_empty());
jit_module.finalize_definitions();
@ -334,9 +351,13 @@ fn load_imported_symbols_for_jit(
imported_symbols
}
fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
let tcx = cx.tcx;
fn codegen_shim<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut CodegenCx,
cached_context: &mut Context,
module: &mut JITModule,
inst: Instance<'tcx>,
) {
let pointer_type = module.target_config().pointer_type();
let name = tcx.symbol_name(inst).name;
@ -357,8 +378,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
)
.unwrap();
cx.cached_context.clear();
let trampoline = &mut cx.cached_context.func;
let context = cached_context;
context.clear();
let trampoline = &mut context.func;
trampoline.signature = sig.clone();
let mut builder_ctx = FunctionBuilderContext::new();
@ -381,5 +403,6 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
trampoline_builder.ins().return_(&ret_vals);
module.define_function(func_id, &mut cx.cached_context).unwrap();
module.define_function(func_id, context).unwrap();
cx.unwind_context.add_function(func_id, context, module.isa());
}

114
compiler/rustc_codegen_cranelift/src/global_asm.rs Normal file
View file

@ -0,0 +1,114 @@
//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
//! standalone executable.
use std::io::Write;
use std::path::PathBuf;
use std::process::{Command, Stdio};
use std::sync::Arc;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_hir::ItemId;
use rustc_session::config::{OutputFilenames, OutputType};
use crate::prelude::*;
pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) {
let item = tcx.hir().item(item_id);
if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
global_asm.push_str("\n.intel_syntax noprefix\n");
} else {
global_asm.push_str("\n.att_syntax\n");
}
for piece in asm.template {
match *piece {
InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
}
}
global_asm.push_str("\n.att_syntax\n\n");
} else {
bug!("Expected GlobalAsm found {:?}", item);
}
}
#[derive(Debug)]
pub(crate) struct GlobalAsmConfig {
asm_enabled: bool,
assembler: PathBuf,
pub(crate) output_filenames: Arc<OutputFilenames>,
}
impl GlobalAsmConfig {
pub(crate) fn new(tcx: TyCtxt<'_>) -> Self {
let asm_enabled = cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows;
GlobalAsmConfig {
asm_enabled,
assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"),
output_filenames: tcx.output_filenames(()).clone(),
}
}
}
pub(crate) fn compile_global_asm(
config: &GlobalAsmConfig,
cgu_name: &str,
global_asm: &str,
) -> Result<Option<PathBuf>, String> {
if global_asm.is_empty() {
return Ok(None);
}
if !config.asm_enabled {
if global_asm.contains("__rust_probestack") {
return Ok(None);
}
// FIXME fix linker error on macOS
if cfg!(not(feature = "inline_asm")) {
return Err(
"asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift"
.to_owned(),
);
} else {
return Err("asm! and global_asm! are not yet supported on Windows".to_owned());
}
}
// Remove all LLVM style comments
let global_asm = global_asm
.lines()
.map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
.collect::<Vec<_>>()
.join("\n");
let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));
// Assemble `global_asm`
let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
let mut child = Command::new(&config.assembler)
.arg("-o")
.arg(&global_asm_object_file)
.stdin(Stdio::piped())
.spawn()
.expect("Failed to spawn `as`.");
child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
let status = child.wait().expect("Failed to wait for `as`.");
if !status.success() {
return Err(format!("Failed to assemble `{}`", global_asm));
}
Ok(Some(global_asm_object_file))
}
pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf {
let mut new_filename = path.file_stem().unwrap().to_owned();
new_filename.push(postfix);
if let Some(extension) = path.extension() {
new_filename.push(".");
new_filename.push(extension);
}
path.set_file_name(new_filename);
path
}

176
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -15,15 +15,19 @@ pub(crate) fn codegen_inline_asm<'tcx>(
template: &[InlineAsmTemplatePiece],
operands: &[InlineAsmOperand<'tcx>],
options: InlineAsmOptions,
destination: Option<mir::BasicBlock>,
) {
// FIXME add .eh_frame unwind info directives
if !template.is_empty() {
// Used by panic_abort
if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::User(1));
fx.bcx.ins().trap(TrapCode::User(1));
return;
} else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
}
// Used by stdarch
if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
&& matches!(
template[1],
InlineAsmTemplatePiece::Placeholder {
@ -47,51 +51,46 @@ pub(crate) fn codegen_inline_asm<'tcx>(
{
assert_eq!(operands.len(), 4);
let (leaf, eax_place) = match operands[1] {
InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
assert_eq!(
reg,
InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax))
);
(
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place.unwrap()),
)
}
InlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
late: true,
ref in_value,
out_place: Some(out_place),
} => (
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place),
),
_ => unreachable!(),
};
let ebx_place = match operands[0] {
InlineAsmOperand::Out { reg, late: true, place } => {
assert_eq!(
reg,
InlineAsmOperand::Out {
reg:
InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
X86InlineAsmRegClass::reg
))
);
crate::base::codegen_place(fx, place.unwrap())
}
X86InlineAsmRegClass::reg,
)),
late: true,
place: Some(place),
} => crate::base::codegen_place(fx, place),
_ => unreachable!(),
};
let (sub_leaf, ecx_place) = match operands[2] {
InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
assert_eq!(
reg,
InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx))
);
(
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place.unwrap()),
)
}
InlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
late: true,
ref in_value,
out_place: Some(out_place),
} => (
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place),
),
_ => unreachable!(),
};
let edx_place = match operands[3] {
InlineAsmOperand::Out { reg, late: true, place } => {
assert_eq!(
reg,
InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx))
);
crate::base::codegen_place(fx, place.unwrap())
}
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
late: true,
place: Some(place),
} => crate::base::codegen_place(fx, place),
_ => unreachable!(),
};
@ -101,12 +100,99 @@ pub(crate) fn codegen_inline_asm<'tcx>(
ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
let destination_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(destination_block, &[]);
return;
} else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
}
// Used by compiler-builtins
if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
// ___chkstk, ___chkstk_ms and __alloca are only used on Windows
crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
return;
} else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
crate::trap::trap_unimplemented(fx, "Alloca is not supported");
return;
}
// Used by measureme
if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string())
&& template[1] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string())
&& matches!(
template[3],
InlineAsmTemplatePiece::Placeholder {
operand_idx: 0,
modifier: Some('r'),
span: _
}
)
&& template[4] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[5] == InlineAsmTemplatePiece::String("cpuid".to_string())
&& template[6] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[7] == InlineAsmTemplatePiece::String("mov ".to_string())
&& matches!(
template[8],
InlineAsmTemplatePiece::Placeholder {
operand_idx: 0,
modifier: Some('r'),
span: _
}
)
&& template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string())
{
let destination_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(destination_block, &[]);
return;
} else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) {
// Return zero dummy values for all performance counters
match operands[0] {
InlineAsmOperand::In {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
value: _,
} => {}
_ => unreachable!(),
};
let lo = match operands[1] {
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
late: true,
place: Some(place),
} => crate::base::codegen_place(fx, place),
_ => unreachable!(),
};
let hi = match operands[2] {
InlineAsmOperand::Out {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
late: true,
place: Some(place),
} => crate::base::codegen_place(fx, place),
_ => unreachable!(),
};
let u32_layout = fx.layout_of(fx.tcx.types.u32);
let zero = fx.bcx.ins().iconst(types::I32, 0);
lo.write_cvalue(fx, CValue::by_val(zero, u32_layout));
hi.write_cvalue(fx, CValue::by_val(zero, u32_layout));
let destination_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(destination_block, &[]);
return;
} else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string())
&& matches!(
template[1],
InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ }
)
&& template[2] == InlineAsmTemplatePiece::String(", (".to_string())
&& matches!(
template[3],
InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ }
)
&& template[4] == InlineAsmTemplatePiece::String(")".to_string())
{
let destination_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(destination_block, &[]);
return;
}
}
@ -175,6 +261,16 @@ pub(crate) fn codegen_inline_asm<'tcx>(
}
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
match destination {
Some(destination) => {
let destination_block = fx.get_block(destination);
fx.bcx.ins().jump(destination_block, &[]);
}
None => {
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
}
}
struct InlineAssemblyGenerator<'a, 'tcx> {
@ -637,7 +733,7 @@ fn call_inline_asm<'tcx>(
inputs: Vec<(Size, Value)>,
outputs: Vec<(Size, CPlace<'tcx>)>,
) {
let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData {
let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
size: u32::try_from(slot_size.bytes()).unwrap(),
});

2
compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
View file

@ -62,7 +62,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
fx.bcx.switch_to_block(unsupported_leaf);
crate::trap::trap_unreachable(
crate::trap::trap_unimplemented(
fx,
"__cpuid_count arch intrinsic doesn't yet support specified leaf",
);

1
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View file

@ -139,6 +139,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
.sess
.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
crate::trap::trap_unimplemented(fx, intrinsic);
return;
}
}

47
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -44,7 +44,7 @@ fn report_atomic_type_validation_error<'tcx>(
),
);
// Prevent verifier error
crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
@ -53,7 +53,7 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
_ => unreachable!(),
};
match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
// Cranelift currently only implements icmp for 128bit vectors.
Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
_ => None,
@ -301,7 +301,44 @@ fn codegen_float_intrinsic_call<'tcx>(
_ => unreachable!(),
};
let res = fx.easy_call(name, &args, ty);
let layout = fx.layout_of(ty);
let res = match intrinsic {
sym::fmaf32 | sym::fmaf64 => {
let a = args[0].load_scalar(fx);
let b = args[1].load_scalar(fx);
let c = args[2].load_scalar(fx);
CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
}
sym::copysignf32 | sym::copysignf64 => {
let a = args[0].load_scalar(fx);
let b = args[1].load_scalar(fx);
CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
}
sym::fabsf32
| sym::fabsf64
| sym::floorf32
| sym::floorf64
| sym::ceilf32
| sym::ceilf64
| sym::truncf32
| sym::truncf64 => {
let a = args[0].load_scalar(fx);
let val = match intrinsic {
sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
_ => unreachable!(),
};
CValue::by_val(val, layout)
}
// These intrinsics aren't supported natively by Cranelift.
// Lower them to a libcall.
_ => fx.easy_call(name, &args, ty),
};
ret.write_cvalue(fx, res);
true
@ -818,8 +855,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
let ret_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(ret_block, &[]);
return;
} else {
fx.tcx
@ -851,8 +886,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
let ret_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(ret_block, &[]);
return;
} else {
fx.tcx

33
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -14,7 +14,7 @@ fn report_simd_type_validation_error(
) {
fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
// Prevent verifier error
crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
pub(super) fn codegen_simd_intrinsic_call<'tcx>(
@ -157,7 +157,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
),
);
// Prevent verifier error
crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
return;
}
}
@ -274,12 +274,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
idx_const
} else {
fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
let res = crate::trap::trap_unimplemented_ret_value(
let trap_block = fx.bcx.create_block();
let dummy_block = fx.bcx.create_block();
let true_ = fx.bcx.ins().iconst(types::I8, 1);
fx.bcx.ins().brnz(true_, trap_block, &[]);
fx.bcx.ins().jump(dummy_block, &[]);
fx.bcx.switch_to_block(trap_block);
crate::trap::trap_unimplemented(
fx,
ret.layout(),
"Index argument for `simd_extract` is not a constant",
);
ret.write_cvalue(fx, res);
fx.bcx.switch_to_block(dummy_block);
return;
};
@ -392,21 +397,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let layout = a.layout();
let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
let res_lane_layout = fx.layout_of(lane_ty);
for lane in 0..lane_count {
let a_lane = a.value_lane(fx, lane);
let b_lane = b.value_lane(fx, lane);
let c_lane = c.value_lane(fx, lane);
let a_lane = a.value_lane(fx, lane).load_scalar(fx);
let b_lane = b.value_lane(fx, lane).load_scalar(fx);
let c_lane = c.value_lane(fx, lane).load_scalar(fx);
let res_lane = match lane_ty.kind() {
ty::Float(FloatTy::F32) => {
fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty)
}
ty::Float(FloatTy::F64) => {
fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty)
}
_ => unreachable!(),
};
let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
let res_lane = CValue::by_val(res_lane, res_lane_layout);
ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
}

55
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -4,6 +4,7 @@
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
extern crate jobserver;
#[macro_use]
extern crate rustc_middle;
extern crate rustc_ast;
@ -25,10 +26,12 @@ extern crate rustc_target;
extern crate rustc_driver;
use std::any::Any;
use std::cell::Cell;
use std::cell::{Cell, RefCell};
use std::sync::Arc;
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_errors::ErrorGuaranteed;
use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
@ -51,11 +54,13 @@ mod cast;
mod codegen_i128;
mod common;
mod compiler_builtins;
mod concurrency_limiter;
mod config;
mod constant;
mod debuginfo;
mod discriminant;
mod driver;
mod global_asm;
mod inline_asm;
mod intrinsics;
mod linkage;
@ -119,19 +124,20 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
/// The codegen context holds any information shared between the codegen of individual functions
/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
struct CodegenCx<'tcx> {
tcx: TyCtxt<'tcx>,
struct CodegenCx {
profiler: SelfProfilerRef,
output_filenames: Arc<OutputFilenames>,
should_write_ir: bool,
global_asm: String,
inline_asm_index: Cell<usize>,
cached_context: Context,
debug_context: Option<DebugContext<'tcx>>,
debug_context: Option<DebugContext>,
unwind_context: UnwindContext,
cgu_name: Symbol,
}
impl<'tcx> CodegenCx<'tcx> {
impl CodegenCx {
fn new(
tcx: TyCtxt<'tcx>,
tcx: TyCtxt<'_>,
backend_config: BackendConfig,
isa: &dyn TargetIsa,
debug_info: bool,
@ -147,10 +153,11 @@ impl<'tcx> CodegenCx<'tcx> {
None
};
CodegenCx {
tcx,
profiler: tcx.prof.clone(),
output_filenames: tcx.output_filenames(()).clone(),
should_write_ir: crate::pretty_clif::should_write_ir(tcx),
global_asm: String::new(),
inline_asm_index: Cell::new(0),
cached_context: Context::new(),
debug_context,
unwind_context,
cgu_name,
@ -159,7 +166,7 @@ impl<'tcx> CodegenCx<'tcx> {
}
pub struct CraneliftCodegenBackend {
pub config: Option<BackendConfig>,
pub config: RefCell<Option<BackendConfig>>,
}
impl CodegenBackend for CraneliftCodegenBackend {
@ -169,6 +176,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
Lto::No | Lto::ThinLocal => {}
Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
}
let mut config = self.config.borrow_mut();
if config.is_none() {
let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args)
.unwrap_or_else(|err| sess.fatal(&err));
*config = Some(new_config);
}
}
fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> {
@ -186,15 +200,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
need_metadata_module: bool,
) -> Box<dyn Any> {
tcx.sess.abort_if_errors();
let config = if let Some(config) = self.config.clone() {
config
} else {
if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() {
tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif");
}
BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
.unwrap_or_else(|err| tcx.sess.fatal(&err))
};
let config = self.config.borrow().clone().unwrap();
match config.codegen_mode {
CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
@ -210,12 +216,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
fn join_codegen(
&self,
ongoing_codegen: Box<dyn Any>,
_sess: &Session,
sess: &Session,
_outputs: &OutputFilenames,
) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> {
Ok(*ongoing_codegen
.downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>()
.unwrap())
Ok(ongoing_codegen
.downcast::<driver::aot::OngoingCodegen>()
.unwrap()
.join(sess, self.config.borrow().as_ref().unwrap()))
}
fn link(
@ -312,5 +319,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
#[no_mangle]
pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
Box::new(CraneliftCodegenBackend { config: None })
Box::new(CraneliftCodegenBackend { config: RefCell::new(None) })
}

17
compiler/rustc_codegen_cranelift/src/optimize/mod.rs
View file

@ -1,20 +1,3 @@
//! Various optimizations specific to cg_clif
use cranelift_codegen::isa::TargetIsa;
use crate::prelude::*;
pub(crate) mod peephole;
pub(crate) fn optimize_function<'tcx>(
tcx: TyCtxt<'tcx>,
isa: &dyn TargetIsa,
instance: Instance<'tcx>,
ctx: &mut Context,
clif_comments: &mut crate::pretty_clif::CommentWriter,
) {
// FIXME classify optimizations over opt levels once we have more
crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx.func, &*clif_comments);
crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
}

61
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -62,7 +62,7 @@ use cranelift_codegen::{
};
use rustc_middle::ty::layout::FnAbiOf;
use rustc_session::config::OutputType;
use rustc_session::config::{OutputFilenames, OutputType};
use crate::prelude::*;
@ -205,15 +205,11 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
}
pub(crate) fn write_ir_file(
tcx: TyCtxt<'_>,
name: impl FnOnce() -> String,
output_filenames: &OutputFilenames,
name: &str,
write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
) {
if !should_write_ir(tcx) {
return;
}
let clif_output_dir = tcx.output_filenames(()).with_extension("clif");
let clif_output_dir = output_filenames.with_extension("clif");
match std::fs::create_dir(&clif_output_dir) {
Ok(()) => {}
@ -221,44 +217,43 @@ pub(crate) fn write_ir_file(
res @ Err(_) => res.unwrap(),
}
let clif_file_name = clif_output_dir.join(name());
let clif_file_name = clif_output_dir.join(name);
let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
if let Err(err) = res {
tcx.sess.warn(&format!("error writing ir file: {}", err));
// Using early_warn as no Session is available here
rustc_session::early_warn(
rustc_session::config::ErrorOutputType::default(),
&format!("error writing ir file: {}", err),
);
}
}
pub(crate) fn write_clif_file<'tcx>(
tcx: TyCtxt<'tcx>,
pub(crate) fn write_clif_file(
output_filenames: &OutputFilenames,
symbol_name: &str,
postfix: &str,
isa: &dyn cranelift_codegen::isa::TargetIsa,
instance: Instance<'tcx>,
func: &cranelift_codegen::ir::Function,
mut clif_comments: &CommentWriter,
) {
// FIXME work around filename too long errors
write_ir_file(
tcx,
|| format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
|file| {
let mut clif = String::new();
cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func)
.unwrap();
write_ir_file(output_filenames, &format!("{}.{}.clif", symbol_name, postfix), |file| {
let mut clif = String::new();
cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func).unwrap();
for flag in isa.flags().iter() {
writeln!(file, "set {}", flag)?;
}
write!(file, "target {}", isa.triple().architecture.to_string())?;
for isa_flag in isa.isa_flags().iter() {
write!(file, " {}", isa_flag)?;
}
writeln!(file, "\n")?;
writeln!(file)?;
file.write_all(clif.as_bytes())?;
Ok(())
},
);
for flag in isa.flags().iter() {
writeln!(file, "set {}", flag)?;
}
write!(file, "target {}", isa.triple().architecture.to_string())?;
for isa_flag in isa.isa_flags().iter() {
write!(file, " {}", isa_flag)?;
}
writeln!(file, "\n")?;
writeln!(file)?;
file.write_all(clif.as_bytes())?;
Ok(())
});
}
impl fmt::Debug for FunctionCx<'_, '_, '_> {

6
compiler/rustc_codegen_cranelift/src/toolchain.rs
View file

@ -8,10 +8,8 @@ use rustc_session::Session;
/// Tries to infer the path of a binary for the target toolchain from the linker name.
pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf {
let (mut linker, _linker_flavor) = linker_and_flavor(sess);
let linker_file_name = linker
.file_name()
.and_then(|name| name.to_str())
.unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker"));
let linker_file_name =
linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8");
if linker_file_name == "ld.lld" {
if tool != "ld" {

25
compiler/rustc_codegen_cranelift/src/trap.rs
View file

@ -25,33 +25,10 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
fx.bcx.ins().call(puts, &[msg_ptr]);
}
/// Use this for example when a function call should never return. This will fill the current block,
/// so you can **not** add instructions to it afterwards.
///
/// Trap code: user65535
pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen.
/// Unlike `trap_unreachable` this will not fill the current block, so you **must** add instructions
/// to it afterwards.
///
/// Trap code: user65535
pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
}
/// Like `trap_unimplemented` but returns a fake value of the specified type.
///
/// Trap code: user65535
pub(crate) fn trap_unimplemented_ret_value<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
dest_layout: TyAndLayout<'tcx>,
msg: impl AsRef<str>,
) -> CValue<'tcx> {
trap_unimplemented(fx, msg);
CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout)
fx.bcx.ins().trap(TrapCode::User(!0));
}

10
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -122,7 +122,7 @@ impl<'tcx> CValue<'tcx> {
let clif_ty = match layout.abi {
Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar),
Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element)
.by(u16::try_from(count).unwrap())
.by(u32::try_from(count).unwrap())
.unwrap(),
_ => unreachable!("{:?}", layout.ty),
};
@ -330,7 +330,7 @@ impl<'tcx> CPlace<'tcx> {
.fatal(&format!("values of type {} are too big to store on the stack", layout.ty));
}
let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
@ -472,7 +472,7 @@ impl<'tcx> CPlace<'tcx> {
}
_ if src_ty.is_vector() || dst_ty.is_vector() => {
// FIXME do something more efficient for transmutes between vectors and integers.
let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
@ -519,7 +519,7 @@ impl<'tcx> CPlace<'tcx> {
if let ty::Array(element, len) = dst_layout.ty.kind() {
// Can only happen for vector types
let len =
u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
u32::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
let data = match from.0 {
@ -614,7 +614,7 @@ impl<'tcx> CPlace<'tcx> {
dst_align,
src_align,
true,
MemFlags::trusted(),
flags,
);
}
CValueInner::ByRef(_, Some(_)) => todo!(),