bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Sync rustc_codegen_cranelift 'ddd4ce2553'

Browse source
This commit is contained in:
Erin Power 2021-04-30 14:49:58 +02:00
commit 505aa48f24
43 changed files with 784 additions and 1243 deletions
Download patch file Download diff file Expand all files Collapse all files

15
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -63,16 +63,16 @@ pub(crate) fn import_function<'tcx>(
module: &mut dyn Module,
inst: Instance<'tcx>,
) -> FuncId {
let name = tcx.symbol_name(inst).name.to_string();
let name = tcx.symbol_name(inst).name;
let sig = get_function_sig(tcx, module.isa().triple(), inst);
module.declare_function(&name, Linkage::Import, &sig).unwrap()
module.declare_function(name, Linkage::Import, &sig).unwrap()
}
impl<'tcx> FunctionCx<'_, '_, 'tcx> {
/// Instance must be monomorphized
pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef {
let func_id = import_function(self.tcx, self.cx.module, inst);
let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
let func_id = import_function(self.tcx, self.module, inst);
let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
if self.clif_comments.enabled() {
self.add_comment(func_ref, format!("{:?}", inst));
@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
args: &[Value],
) -> &[Value] {
let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) };
let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
let call_inst = self.bcx.ins().call(func_ref, args);
if self.clif_comments.enabled() {
self.add_comment(call_inst, format!("easy_call {}", name));
@ -295,7 +295,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
pub(crate) fn codegen_terminator_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
span: Span,
current_block: Block,
func: &Operand<'tcx>,
args: &[Operand<'tcx>],
destination: Option<(Place<'tcx>, BasicBlock)>,
@ -357,7 +356,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
.map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD))
.unwrap_or(false);
if is_cold {
fx.cold_blocks.insert(current_block);
// FIXME Mark current_block block as cold once Cranelift supports it
}
// Unpack arguments tuple for closures

4
compiler/rustc_codegen_cranelift/src/allocator.rs
View file

@ -11,7 +11,7 @@ use rustc_span::symbol::sym;
pub(crate) fn codegen(
tcx: TyCtxt<'_>,
module: &mut impl Module,
unwind_context: &mut UnwindContext<'_>,
unwind_context: &mut UnwindContext,
) -> bool {
let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| {
use rustc_middle::middle::dependency_format::Linkage;
@ -29,7 +29,7 @@ pub(crate) fn codegen(
fn codegen_inner(
module: &mut impl Module,
unwind_context: &mut UnwindContext<'_>,
unwind_context: &mut UnwindContext,
kind: AllocatorKind,
) {
let usize_ty = module.target_config().pointer_type();

20
compiler/rustc_codegen_cranelift/src/backend.rs
View file

@ -5,23 +5,23 @@ use std::convert::{TryFrom, TryInto};
use rustc_data_structures::fx::FxHashMap;
use rustc_session::Session;
use cranelift_codegen::isa::TargetIsa;
use cranelift_module::FuncId;
use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
use object::write::*;
use object::{RelocationEncoding, SectionKind, SymbolFlags};
use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct};
use gimli::SectionId;
use crate::debuginfo::{DebugReloc, DebugRelocName};
pub(crate) trait WriteMetadata {
fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, is_like_osx: bool);
fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>);
}
impl WriteMetadata for object::write::Object {
fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>, _is_like_osx: bool) {
fn add_rustc_section(&mut self, symbol_name: String, data: Vec<u8>) {
let segment = self.segment_name(object::write::StandardSegment::Data).to_vec();
let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data);
let offset = self.append_section_data(section_id, &data, 1);
@ -113,7 +113,7 @@ impl WriteDebugInfo for ObjectProduct {
}
pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec<u8> {
let triple = crate::build_isa(sess).triple().clone();
let triple = crate::target_triple(sess);
let binary_format = match triple.binary_format {
target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf,
@ -141,13 +141,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object
metadata_object.write().unwrap()
}
pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule {
let mut builder = ObjectBuilder::new(
crate::build_isa(sess),
name + ".o",
cranelift_module::default_libcall_names(),
)
.unwrap();
pub(crate) fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
let mut builder =
ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
// Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
// is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections
// can easily double the amount of time necessary to perform linking.

146
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -6,9 +6,14 @@ use rustc_middle::ty::adjustment::PointerCast;
use rustc_middle::ty::layout::FnAbiExt;
use rustc_target::abi::call::FnAbi;
use crate::constant::ConstantCx;
use crate::prelude::*;
pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
pub(crate) fn codegen_fn<'tcx>(
cx: &mut crate::CodegenCx<'tcx>,
module: &mut dyn Module,
instance: Instance<'tcx>,
) {
let tcx = cx.tcx;
let _inst_guard =
@ -18,9 +23,9 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
let mir = tcx.instance_mir(instance.def);
// Declare function
let name = tcx.symbol_name(instance).name.to_string();
let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
let symbol_name = tcx.symbol_name(instance);
let sig = get_function_sig(tcx, module.isa().triple(), instance);
let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap();
cx.cached_context.clear();
@ -39,15 +44,19 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
(0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
// Make FunctionCx
let pointer_type = cx.module.target_config().pointer_type();
let pointer_type = module.target_config().pointer_type();
let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
let mut fx = FunctionCx {
cx,
module,
tcx,
pointer_type,
vtables: FxHashMap::default(),
constants_cx: ConstantCx::new(),
instance,
symbol_name,
mir,
fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])),
@ -55,7 +64,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
block_map,
local_map: IndexVec::with_capacity(mir.local_decls.len()),
caller_location: None, // set by `codegen_fn_prelude`
cold_blocks: EntitySet::new(),
clif_comments,
source_info_set: indexmap::IndexSet::new(),
@ -90,7 +98,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
let mut clif_comments = fx.clif_comments;
let source_info_set = fx.source_info_set;
let local_map = fx.local_map;
let cold_blocks = fx.cold_blocks;
fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
// Store function in context
let context = &mut cx.cached_context;
@ -103,21 +112,15 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
// Perform rust specific optimizations
tcx.sess.time("optimize clif ir", || {
crate::optimize::optimize_function(
tcx,
instance,
context,
&cold_blocks,
&mut clif_comments,
);
crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments);
});
// If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
// instruction, which doesn't have an encoding.
context.compute_cfg();
context.compute_domtree();
context.eliminate_unreachable_code(cx.module.isa()).unwrap();
context.dce(cx.module.isa()).unwrap();
context.eliminate_unreachable_code(module.isa()).unwrap();
context.dce(module.isa()).unwrap();
// Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
// invalidate it when it would change.
context.domtree.clear();
@ -125,7 +128,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
// Define function
let module = &mut cx.module;
tcx.sess.time("define function", || {
module
.define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
@ -136,7 +138,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
crate::pretty_clif::write_clif_file(
tcx,
"opt",
Some(cx.module.isa()),
Some(module.isa()),
instance,
&context,
&clif_comments,
@ -145,13 +147,13 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
crate::pretty_clif::write_ir_file(
tcx,
&format!("{}.vcode", tcx.symbol_name(instance).name),
|| format!("{}.vcode", tcx.symbol_name(instance).name),
|file| file.write_all(disasm.as_bytes()),
)
}
// Define debuginfo for function
let isa = cx.module.isa();
let isa = module.isa();
let debug_context = &mut cx.debug_context;
let unwind_context = &mut cx.unwind_context;
tcx.sess.time("generate debug info", || {
@ -159,7 +161,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In
debug_context.define_function(
instance,
func_id,
&name,
symbol_name.name,
isa,
context,
&source_info_set,
@ -205,9 +207,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
// Unwinding after panicking is not supported
continue;
// FIXME once unwinding is supported uncomment next lines
// // Unwinding is unlikely to happen, so mark cleanup block's as cold.
// fx.cold_blocks.insert(block);
// FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do
// so for cleanup blocks.
}
fx.bcx.ins().nop();
@ -262,7 +263,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
let target = fx.get_block(*target);
let failure = fx.bcx.create_block();
fx.cold_blocks.insert(failure);
// FIXME Mark failure block as cold once Cranelift supports it
if *expected {
fx.bcx.ins().brz(cond, failure, &[]);
@ -355,14 +356,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
from_hir_call: _,
} => {
fx.tcx.sess.time("codegen call", || {
crate::abi::codegen_terminator_call(
fx,
*fn_span,
block,
func,
args,
*destination,
)
crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination)
});
}
TerminatorKind::InlineAsm {
@ -664,7 +658,7 @@ fn codegen_stmt<'tcx>(
// FIXME use emit_small_memset where possible
let addr = lval.to_ptr().get_addr(fx);
let val = operand.load_scalar(fx);
fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times);
fx.bcx.call_memset(fx.module.target_config(), addr, val, times);
} else {
let loop_block = fx.bcx.create_block();
let loop_block2 = fx.bcx.create_block();
@ -750,85 +744,15 @@ fn codegen_stmt<'tcx>(
| StatementKind::AscribeUserType(..) => {}
StatementKind::LlvmInlineAsm(asm) => {
use rustc_span::symbol::Symbol;
let LlvmInlineAsm { asm, outputs, inputs } = &**asm;
let rustc_hir::LlvmInlineAsmInner {
asm: asm_code, // Name
outputs: output_names, // Vec<LlvmInlineAsmOutput>
inputs: input_names, // Vec<Name>
clobbers, // Vec<Name>
volatile, // bool
alignstack, // bool
dialect: _,
asm_str_style: _,
} = asm;
match asm_code.as_str().trim() {
match asm.asm.asm.as_str().trim() {
"" => {
// Black box
}
"mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => {
assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]);
assert_eq!(output_names.len(), 4);
for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() {
assert_eq!(&output_names[i].constraint.as_str(), c);
assert!(!output_names[i].is_rw);
assert!(!output_names[i].is_indirect);
}
assert_eq!(clobbers, &[]);
assert!(!volatile);
assert!(!alignstack);
assert_eq!(inputs.len(), 2);
let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax
let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx
let (eax, ebx, ecx, edx) =
crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf);
assert_eq!(outputs.len(), 4);
codegen_place(fx, outputs[0])
.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
codegen_place(fx, outputs[1])
.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
codegen_place(fx, outputs[2])
.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
codegen_place(fx, outputs[3])
.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
}
"xgetbv" => {
assert_eq!(input_names, &[Symbol::intern("{ecx}")]);
assert_eq!(output_names.len(), 2);
for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() {
assert_eq!(&output_names[i].constraint.as_str(), c);
assert!(!output_names[i].is_rw);
assert!(!output_names[i].is_indirect);
}
assert_eq!(clobbers, &[]);
assert!(!volatile);
assert!(!alignstack);
crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported");
}
// ___chkstk, ___chkstk_ms and __alloca are only used on Windows
_ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => {
crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
}
_ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => {
crate::trap::trap_unimplemented(fx, "Alloca is not supported");
}
// Used in sys::windows::abort_internal
"int $$0x29" => {
crate::trap::trap_unimplemented(fx, "Windows abort");
}
_ => fx
.tcx
.sess
.span_fatal(stmt.source_info.span, "Inline assembly is not supported"),
_ => fx.tcx.sess.span_fatal(
stmt.source_info.span,
"Legacy `llvm_asm!` inline assembly is not supported. \
Try using the new `asm!` instead.",
),
}
}
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
@ -844,7 +768,7 @@ fn codegen_stmt<'tcx>(
let elem_size: u64 = pointee.size.bytes();
let bytes =
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes);
}
}
}

20
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -1,8 +1,10 @@
use rustc_index::vec::IndexVec;
use rustc_middle::ty::SymbolName;
use rustc_target::abi::call::FnAbi;
use rustc_target::abi::{Integer, Primitive};
use rustc_target::spec::{HasTargetSpec, Target};
use crate::constant::ConstantCx;
use crate::prelude::*;
pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@ -226,12 +228,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
}
}
pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>,
pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
pub(crate) module: &'m mut dyn Module,
pub(crate) tcx: TyCtxt<'tcx>,
pub(crate) pointer_type: Type, // Cached from module
pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>,
pub(crate) constants_cx: ConstantCx,
pub(crate) instance: Instance<'tcx>,
pub(crate) symbol_name: SymbolName<'tcx>,
pub(crate) mir: &'tcx Body<'tcx>,
pub(crate) fn_abi: Option<FnAbi<'tcx, Ty<'tcx>>>,
@ -242,9 +248,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> {
/// When `#[track_caller]` is used, the implicit caller location is stored in this variable.
pub(crate) caller_location: Option<CValue<'tcx>>,
/// See [`crate::optimize::code_layout`] for more information.
pub(crate) cold_blocks: EntitySet<Block>,
pub(crate) clif_comments: crate::pretty_clif::CommentWriter,
pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>,
@ -339,7 +342,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
}
pub(crate) fn triple(&self) -> &target_lexicon::Triple {
self.cx.module.isa().triple()
self.module.isa().triple()
}
pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value {
@ -352,15 +355,14 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
let mut data_ctx = DataContext::new();
data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice());
let msg_id = self
.cx
.module
.declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false)
.unwrap();
// Ignore DuplicateDefinition error, as the data will be the same
let _ = self.cx.module.define_data(msg_id, &data_ctx);
let _ = self.module.define_data(msg_id, &data_ctx);
let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func);
if self.clif_comments.enabled() {
self.add_comment(local_msg_id, msg);
}

107
compiler/rustc_codegen_cranelift/src/config.rs Normal file
View file

@ -0,0 +1,107 @@
use std::env;
use std::str::FromStr;
fn bool_env_var(key: &str) -> bool {
env::var(key).as_ref().map(|val| &**val) == Ok("1")
}
/// The mode to use for compilation.
#[derive(Copy, Clone, Debug)]
pub enum CodegenMode {
/// AOT compile the crate. This is the default.
Aot,
/// JIT compile and execute the crate.
Jit,
/// JIT compile and execute the crate, but only compile functions the first time they are used.
JitLazy,
}
impl FromStr for CodegenMode {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"aot" => Ok(CodegenMode::Aot),
"jit" => Ok(CodegenMode::Jit),
"jit-lazy" => Ok(CodegenMode::JitLazy),
_ => Err(format!("Unknown codegen mode `{}`", s)),
}
}
}
/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars.
#[derive(Clone, Debug)]
pub struct BackendConfig {
/// Should the crate be AOT compiled or JIT executed.
///
/// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`.
pub codegen_mode: CodegenMode,
/// When JIT mode is enable pass these arguments to the program.
///
/// Defaults to the value of `CG_CLIF_JIT_ARGS`.
pub jit_args: Vec<String>,
/// Display the time it took to perform codegen for a crate.
///
/// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise.
/// Can be set using `-Cllvm-args=display_cg_time=...`.
pub display_cg_time: bool,
/// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run
/// once before passing the clif ir to Cranelift for compilation.
///
/// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is
/// compiled with debug assertions enabled or false otherwise. Can be set using
/// `-Cllvm-args=enable_verifier=...`.
pub enable_verifier: bool,
/// Don't cache object files in the incremental cache. Useful during development of cg_clif
/// to make it possible to use incremental mode for all analyses performed by rustc without
/// caching object files when their content should have been changed by a change to cg_clif.
///
/// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false
/// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`.
pub disable_incr_cache: bool,
}
impl Default for BackendConfig {
fn default() -> Self {
BackendConfig {
codegen_mode: CodegenMode::Aot,
jit_args: {
let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
args.split(' ').map(|arg| arg.to_string()).collect()
},
display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"),
enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),
}
}
}
impl BackendConfig {
/// Parse the configuration passed in using `-Cllvm-args`.
pub fn from_opts(opts: &[String]) -> Result<Self, String> {
fn parse_bool(name: &str, value: &str) -> Result<bool, String> {
value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name))
}
let mut config = BackendConfig::default();
for opt in opts {
if let Some((name, value)) = opt.split_once('=') {
match name {
"mode" => config.codegen_mode = value.parse()?,
"display_cg_time" => config.display_cg_time = parse_bool(name, value)?,
"enable_verifier" => config.enable_verifier = parse_bool(name, value)?,
"disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?,
_ => return Err(format!("Unknown option `{}`", name)),
}
} else {
return Err(format!("Invalid option `{}`", opt));
}
}
Ok(config)
}
}

67
compiler/rustc_codegen_cranelift/src/constant.rs
View file

@ -2,7 +2,7 @@
use rustc_span::DUMMY_SP;
use rustc_data_structures::fx::FxHashSet;
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_errors::ErrorReported;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::interpret::{
@ -15,10 +15,10 @@ use cranelift_module::*;
use crate::prelude::*;
#[derive(Default)]
pub(crate) struct ConstantCx {
todo: Vec<TodoItem>,
done: FxHashSet<DataId>,
anon_allocs: FxHashMap<AllocId, DataId>,
}
#[derive(Copy, Clone, Debug)]
@ -28,6 +28,10 @@ enum TodoItem {
}
impl ConstantCx {
pub(crate) fn new() -> Self {
ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() }
}
pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) {
//println!("todo {:?}", self.todo);
define_all_allocs(tcx, module, &mut self);
@ -74,8 +78,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
all_constants_ok
}
pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) {
pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) {
let mut constants_cx = ConstantCx::new();
constants_cx.todo.push(TodoItem::Static(def_id));
constants_cx.finalize(tcx, module);
}
pub(crate) fn codegen_tls_ref<'tcx>(
@ -83,8 +89,8 @@ pub(crate) fn codegen_tls_ref<'tcx>(
def_id: DefId,
layout: TyAndLayout<'tcx>,
) -> CValue<'tcx> {
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("tls {:?}", def_id));
}
@ -97,8 +103,8 @@ fn codegen_static_ref<'tcx>(
def_id: DefId,
layout: TyAndLayout<'tcx>,
) -> CPlace<'tcx> {
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", def_id));
}
@ -182,28 +188,31 @@ pub(crate) fn codegen_const_value<'tcx>(
let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id);
let base_addr = match alloc_kind {
Some(GlobalAlloc::Memory(alloc)) => {
fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
let data_id =
data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
let data_id = data_id_for_alloc_id(
&mut fx.constants_cx,
fx.module,
ptr.alloc_id,
alloc.mutability,
);
let local_data_id =
fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
}
fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
}
Some(GlobalAlloc::Function(instance)) => {
let func_id =
crate::abi::import_function(fx.tcx, fx.cx.module, instance);
let func_id = crate::abi::import_function(fx.tcx, fx.module, instance);
let local_func_id =
fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
fx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
}
Some(GlobalAlloc::Static(def_id)) => {
assert!(fx.tcx.is_static(def_id));
let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
let local_data_id =
fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", def_id));
}
@ -243,10 +252,11 @@ fn pointer_for_allocation<'tcx>(
alloc: &'tcx Allocation,
) -> crate::pointer::Pointer {
let alloc_id = fx.tcx.create_memory_alloc(alloc);
fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
let data_id =
data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability);
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(local_data_id, format!("{:?}", alloc_id));
}
@ -255,18 +265,14 @@ fn pointer_for_allocation<'tcx>(
}
fn data_id_for_alloc_id(
cx: &mut ConstantCx,
module: &mut dyn Module,
alloc_id: AllocId,
mutability: rustc_hir::Mutability,
) -> DataId {
module
.declare_data(
&format!(".L__alloc_{:x}", alloc_id.0),
Linkage::Local,
mutability == rustc_hir::Mutability::Mut,
false,
)
.unwrap()
*cx.anon_allocs.entry(alloc_id).or_insert_with(|| {
module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap()
})
}
fn data_id_for_static(
@ -344,7 +350,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
GlobalAlloc::Memory(alloc) => alloc,
GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(),
};
let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability);
let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability);
(data_id, alloc, None)
}
TodoItem::Static(def_id) => {
@ -397,7 +403,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
}
GlobalAlloc::Memory(target_alloc) => {
cx.todo.push(TodoItem::Alloc(reloc));
data_id_for_alloc_id(module, reloc, target_alloc.mutability)
data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability)
}
GlobalAlloc::Static(def_id) => {
if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)
@ -419,8 +425,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64);
}
// FIXME don't duplicate definitions in lazy jit mode
let _ = module.define_data(data_id, &data_ctx);
module.define_data(data_id, &data_ctx).unwrap();
cx.done.insert(data_id);
}

66
compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
View file

@ -5,17 +5,19 @@ use crate::prelude::*;
use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa};
use gimli::write::{Address, CieId, EhFrame, FrameTable, Section};
use gimli::RunTimeEndian;
use crate::backend::WriteDebugInfo;
pub(crate) struct UnwindContext<'tcx> {
tcx: TyCtxt<'tcx>,
pub(crate) struct UnwindContext {
endian: RunTimeEndian,
frame_table: FrameTable,
cie_id: Option<CieId>,
}
impl<'tcx> UnwindContext<'tcx> {
pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
impl UnwindContext {
pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
let endian = super::target_endian(tcx);
let mut frame_table = FrameTable::default();
let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> {
None
};
UnwindContext { tcx, frame_table, cie_id }
UnwindContext { endian, frame_table, cie_id }
}
pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> {
}
pub(crate) fn emit<P: WriteDebugInfo>(self, product: &mut P) {
let mut eh_frame =
EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian));
self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
if !eh_frame.0.writer.slice().is_empty() {
@ -71,16 +72,12 @@ impl<'tcx> UnwindContext<'tcx> {
}
#[cfg(feature = "jit")]
pub(crate) unsafe fn register_jit(
self,
jit_module: &cranelift_jit::JITModule,
) -> Option<UnwindRegistry> {
let mut eh_frame =
EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx)));
pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) {
let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian));
self.frame_table.write_eh_frame(&mut eh_frame).unwrap();
if eh_frame.0.writer.slice().is_empty() {
return None;
return;
}
let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module);
@ -88,7 +85,10 @@ impl<'tcx> UnwindContext<'tcx> {
// GCC expects a terminating "empty" length, so write a 0 length at the end of the table.
eh_frame.extend(&[0, 0, 0, 0]);
let mut registrations = Vec::new();
// FIXME support unregistering unwind tables once cranelift-jit supports deallocating
// individual functions
#[allow(unused_variables)]
let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame);
// =======================================================================
// Everything after this line up to the end of the file is loosly based on
@ -96,8 +96,8 @@ impl<'tcx> UnwindContext<'tcx> {
#[cfg(target_os = "macos")]
{
// On macOS, `__register_frame` takes a pointer to a single FDE
let start = eh_frame.as_ptr();
let end = start.add(eh_frame.len());
let start = eh_frame;
let end = start.add(eh_frame_len);
let mut current = start;
// Walk all of the entries in the frame table and register them
@ -107,7 +107,6 @@ impl<'tcx> UnwindContext<'tcx> {
// Skip over the CIE
if current != start {
__register_frame(current);
registrations.push(current as usize);
}
// Move to the next table entry (+4 because the length itself is not inclusive)
@ -117,41 +116,12 @@ impl<'tcx> UnwindContext<'tcx> {
#[cfg(not(target_os = "macos"))]
{
// On other platforms, `__register_frame` will walk the FDEs until an entry of length 0
let ptr = eh_frame.as_ptr();
__register_frame(ptr);
registrations.push(ptr as usize);
__register_frame(eh_frame);
}
Some(UnwindRegistry { _frame_table: eh_frame, registrations })
}
}
/// Represents a registry of function unwind information for System V ABI.
pub(crate) struct UnwindRegistry {
_frame_table: Vec<u8>,
registrations: Vec<usize>,
}
extern "C" {
// libunwind import
fn __register_frame(fde: *const u8);
fn __deregister_frame(fde: *const u8);
}
impl Drop for UnwindRegistry {
fn drop(&mut self) {
unsafe {
// libgcc stores the frame entries as a linked list in decreasing sort order
// based on the PC value of the registered entry.
//
// As we store the registrations in increasing order, it would be O(N^2) to
// deregister in that order.
//
// To ensure that we just pop off the first element in the list upon every
// deregistration, walk our list of registrations backwards.
for fde in self.registrations.iter().rev() {
__deregister_frame(*fde as *const _);
}
}
}
}

69
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -16,12 +16,6 @@ use cranelift_object::ObjectModule;
use crate::{prelude::*, BackendConfig};
fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule {
let module = crate::backend::make_module(tcx.sess, name);
assert_eq!(pointer_ty(tcx), module.target_config().pointer_type());
module
}
struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
impl<HCX> HashStable<HCX> for ModuleCodegenResult {
@ -32,11 +26,12 @@ impl<HCX> HashStable<HCX> for ModuleCodegenResult {
fn emit_module(
tcx: TyCtxt<'_>,
backend_config: &BackendConfig,
name: String,
kind: ModuleKind,
module: ObjectModule,
debug: Option<DebugContext<'_>>,
unwind_context: UnwindContext<'_>,
unwind_context: UnwindContext,
) -> ModuleCodegenResult {
let mut product = module.finish();
@ -52,7 +47,7 @@ fn emit_module(
tcx.sess.fatal(&format!("error writing object file: {}", err));
}
let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() {
let work_product = if backend_config.disable_incr_cache {
None
} else {
rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
@ -110,23 +105,24 @@ fn module_codegen(
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let mut module = new_module(tcx, cgu_name.as_str().to_string());
let isa = crate::build_isa(tcx.sess, &backend_config);
let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string());
let mut cx = crate::CodegenCx::new(
tcx,
backend_config,
&mut module,
backend_config.clone(),
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
);
super::predefine_mono_items(&mut cx, &mono_items);
super::predefine_mono_items(tcx, &mut module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
}
MonoItem::Static(def_id) => {
crate::constant::codegen_static(&mut cx.constants_cx, def_id)
cx.tcx
.sess
.time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst));
}
MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id),
MonoItem::GlobalAsm(item_id) => {
let item = cx.tcx.hir().item(item_id);
if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind {
@ -138,25 +134,28 @@ fn module_codegen(
}
}
}
let (global_asm, debug, mut unwind_context) =
tcx.sess.time("finalize CodegenCx", || cx.finalize());
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context);
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false);
let codegen_result = emit_module(
tcx,
cgu.name().as_str().to_string(),
ModuleKind::Regular,
module,
debug,
unwind_context,
);
let debug_context = cx.debug_context;
let unwind_context = cx.unwind_context;
let codegen_result = tcx.sess.time("write object file", || {
emit_module(
tcx,
&backend_config,
cgu.name().as_str().to_string(),
ModuleKind::Regular,
module,
debug_context,
unwind_context,
)
});
codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm);
codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm);
codegen_result
}
pub(super) fn run_aot(
pub(crate) fn run_aot(
tcx: TyCtxt<'_>,
backend_config: BackendConfig,
metadata: EncodedMetadata,
@ -193,14 +192,14 @@ pub(super) fn run_aot(
}
}
let modules = super::time(tcx, "codegen mono items", || {
let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
cgus.iter()
.map(|cgu| {
let cgu_reuse = determine_cgu_reuse(tcx, cgu);
tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse);
match cgu_reuse {
_ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {}
_ if backend_config.disable_incr_cache => {}
CguReuse::No => {}
CguReuse::PreLto => {
return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products);
@ -212,7 +211,7 @@ pub(super) fn run_aot(
let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
dep_node,
tcx,
(backend_config, cgu.name()),
(backend_config.clone(), cgu.name()),
module_codegen,
rustc_middle::dep_graph::hash_result,
);
@ -228,7 +227,10 @@ pub(super) fn run_aot(
tcx.sess.abort_if_errors();
let mut allocator_module = new_module(tcx, "allocator_shim".to_string());
let isa = crate::build_isa(tcx.sess, &backend_config);
let mut allocator_module =
crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string());
assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true);
let created_alloc_shim =
crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
@ -236,6 +238,7 @@ pub(super) fn run_aot(
let allocator_module = if created_alloc_shim {
let ModuleCodegenResult(module, work_product) = emit_module(
tcx,
&backend_config,
"allocator_shim".to_string(),
ModuleKind::Allocator,
allocator_module,

155
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -1,4 +1,4 @@
//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object
//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object
//! files.
use std::cell::RefCell;
@ -8,32 +8,62 @@ use std::os::raw::{c_char, c_int};
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::config::EntryFnType;
use cranelift_jit::{JITBuilder, JITModule};
use crate::{prelude::*, BackendConfig};
use crate::{CodegenCx, CodegenMode};
thread_local! {
pub static BACKEND_CONFIG: RefCell<Option<BackendConfig>> = RefCell::new(None);
pub static CURRENT_MODULE: RefCell<Option<JITModule>> = RefCell::new(None);
struct JitState {
backend_config: BackendConfig,
jit_module: JITModule,
}
pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
if !tcx.sess.opts.output_types.should_codegen() {
tcx.sess.fatal("JIT mode doesn't work with `cargo check`.");
}
thread_local! {
static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
}
fn create_jit_module<'tcx>(
tcx: TyCtxt<'tcx>,
backend_config: &BackendConfig,
hotswap: bool,
) -> (JITModule, CodegenCx<'tcx>) {
let imported_symbols = load_imported_symbols_for_jit(tcx);
let mut jit_builder =
JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
let isa = crate::build_isa(tcx.sess, backend_config);
let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
jit_builder.hotswap(hotswap);
crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
jit_builder.symbols(imported_symbols);
let mut jit_module = JITModule::new(jit_builder);
assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false);
crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut jit_module,
&mut cx.unwind_context,
true,
);
(jit_module, cx)
}
pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
if !tcx.sess.opts.output_types.should_codegen() {
tcx.sess.fatal("JIT mode doesn't work with `cargo check`");
}
if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) {
tcx.sess.fatal("can't jit non-executable crate");
}
let (mut jit_module, mut cx) = create_jit_module(
tcx,
&backend_config,
matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
);
let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
let mono_items = cgus
@ -44,52 +74,45 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
.into_iter()
.collect::<Vec<(_, (_, _))>>();
let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false);
super::time(tcx, "codegen mono items", || {
super::predefine_mono_items(&mut cx, &mono_items);
super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
cx.tcx.sess.time("codegen fn", || {
crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
});
}
CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
},
MonoItem::Static(def_id) => {
crate::constant::codegen_static(&mut cx.constants_cx, def_id);
crate::constant::codegen_static(tcx, &mut jit_module, def_id);
}
MonoItem::GlobalAsm(item_id) => {
let item = cx.tcx.hir().item(item_id);
let item = tcx.hir().item(item_id);
tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode");
}
}
}
});
let (global_asm, _debug, mut unwind_context) =
tcx.sess.time("finalize CodegenCx", || cx.finalize());
jit_module.finalize_definitions();
if !global_asm.is_empty() {
if !cx.global_asm.is_empty() {
tcx.sess.fatal("Inline asm is not supported in JIT mode");
}
crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
tcx.sess.abort_if_errors();
jit_module.finalize_definitions();
let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
unsafe { cx.unwind_context.register_jit(&jit_module) };
println!(
"Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
);
let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
.chain(args.split(' '))
.chain(backend_config.jit_args.iter().map(|arg| &**arg))
.map(|arg| CString::new(arg).unwrap())
.collect::<Vec<_>>();
let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
@ -98,8 +121,21 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
// useful as some dynamic linkers use it as a marker to jump over.
argv.push(std::ptr::null());
BACKEND_CONFIG.with(|tls_backend_config| {
assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
let start_sig = Signature {
params: vec![
AbiParam::new(jit_module.target_config().pointer_type()),
AbiParam::new(jit_module.target_config().pointer_type()),
],
returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
};
let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
LAZY_JIT_STATE.with(|lazy_jit_state| {
let mut lazy_jit_state = lazy_jit_state.borrow_mut();
assert!(lazy_jit_state.is_none());
*lazy_jit_state = Some(JitState { backend_config, jit_module });
});
let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
@ -161,24 +197,23 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
// lift is used to ensure the correct lifetime for instance.
let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
CURRENT_MODULE.with(|jit_module| {
let mut jit_module = jit_module.borrow_mut();
let jit_module = jit_module.as_mut().unwrap();
let backend_config =
BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap());
LAZY_JIT_STATE.with(|lazy_jit_state| {
let mut lazy_jit_state = lazy_jit_state.borrow_mut();
let lazy_jit_state = lazy_jit_state.as_mut().unwrap();
let jit_module = &mut lazy_jit_state.jit_module;
let backend_config = lazy_jit_state.backend_config.clone();
let name = tcx.symbol_name(instance).name.to_string();
let name = tcx.symbol_name(instance).name;
let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap();
let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap();
jit_module.prepare_for_function_redefine(func_id).unwrap();
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
let (global_asm, _debug_context, unwind_context) = cx.finalize();
assert!(global_asm.is_empty());
assert!(cx.global_asm.is_empty());
jit_module.finalize_definitions();
std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) });
unsafe { cx.unwind_context.register_jit(&jit_module) };
jit_module.get_finalized_function(func_id)
})
})
@ -248,35 +283,37 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
imported_symbols
}
fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
let tcx = cx.tcx;
let pointer_type = cx.module.target_config().pointer_type();
let pointer_type = module.target_config().pointer_type();
let name = tcx.symbol_name(inst).name.to_string();
let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst);
let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap();
let name = tcx.symbol_name(inst).name;
let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst);
let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap();
let instance_ptr = Box::into_raw(Box::new(inst));
let jit_fn = cx
.module
let jit_fn = module
.declare_function(
"__clif_jit_fn",
Linkage::Import,
&Signature {
call_conv: cx.module.target_config().default_call_conv,
call_conv: module.target_config().default_call_conv,
params: vec![AbiParam::new(pointer_type)],
returns: vec![AbiParam::new(pointer_type)],
},
)
.unwrap();
let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone());
let mut builder_ctx = FunctionBuilderContext::new();
let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx);
cx.cached_context.clear();
let trampoline = &mut cx.cached_context.func;
trampoline.signature = sig.clone();
let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func);
let mut builder_ctx = FunctionBuilderContext::new();
let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx);
let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func);
let sig_ref = trampoline_builder.func.import_signature(sig);
let entry_block = trampoline_builder.create_block();
@ -291,10 +328,10 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
trampoline_builder.ins().return_(&ret_vals);
cx.module
module
.define_function(
func_id,
&mut Context::for_function(trampoline),
&mut cx.cached_context,
&mut NullTrapSink {},
&mut NullStackMapSink {},
)

58
compiler/rustc_codegen_cranelift/src/driver/mod.rs
View file

@ -1,63 +1,37 @@
//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions
//! like JIT executing or writing object files.
//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and
//! performing any further actions like JIT executing or writing object files.
//!
//! [`codegen_fn`]: crate::base::codegen_fn
//! [`codegen_static`]: crate::constant::codegen_static
use std::any::Any;
use rustc_middle::middle::cstore::EncodedMetadata;
use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility};
use crate::prelude::*;
use crate::CodegenMode;
mod aot;
pub(crate) mod aot;
#[cfg(feature = "jit")]
mod jit;
pub(crate) fn codegen_crate(
tcx: TyCtxt<'_>,
metadata: EncodedMetadata,
need_metadata_module: bool,
backend_config: crate::BackendConfig,
) -> Box<dyn Any> {
tcx.sess.abort_if_errors();
match backend_config.codegen_mode {
CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
let is_executable =
tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable);
if !is_executable {
tcx.sess.fatal("can't jit non-executable crate");
}
#[cfg(feature = "jit")]
let _: ! = jit::run_jit(tcx, backend_config);
#[cfg(not(feature = "jit"))]
tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
}
}
}
pub(crate) mod jit;
fn predefine_mono_items<'tcx>(
cx: &mut crate::CodegenCx<'_, 'tcx>,
tcx: TyCtxt<'tcx>,
module: &mut dyn Module,
mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
) {
cx.tcx.sess.time("predefine functions", || {
let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
tcx.sess.time("predefine functions", || {
let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE);
for &(mono_item, (linkage, visibility)) in mono_items {
match mono_item {
MonoItem::Fn(instance) => {
let name = cx.tcx.symbol_name(instance).name.to_string();
let name = tcx.symbol_name(instance).name;
let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
let sig = get_function_sig(tcx, module.isa().triple(), instance);
let linkage = crate::linkage::get_clif_linkage(
mono_item,
linkage,
visibility,
is_compiler_builtins,
);
cx.module.declare_function(&name, linkage, &sig).unwrap();
module.declare_function(name, linkage, &sig).unwrap();
}
MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
}
@ -65,8 +39,8 @@ fn predefine_mono_items<'tcx>(
});
}
fn time<R>(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R {
if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") {
fn time<R>(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R {
if display {
println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name);
let before = std::time::Instant::now();
let res = tcx.sess.time(name, f);

64
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -24,6 +24,64 @@ pub(crate) fn codegen_inline_asm<'tcx>(
let true_ = fx.bcx.ins().iconst(types::I32, 1);
fx.bcx.ins().trapnz(true_, TrapCode::User(1));
return;
} else if template[0] == InlineAsmTemplatePiece::String("mov rsi, rbx".to_string())
&& template[1] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[2] == InlineAsmTemplatePiece::String("cpuid".to_string())
&& template[3] == InlineAsmTemplatePiece::String("\n".to_string())
&& template[4] == InlineAsmTemplatePiece::String("xchg rsi, rbx".to_string())
{
assert_eq!(operands.len(), 4);
let (leaf, eax_place) = match operands[0] {
InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax));
(
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place.unwrap()),
)
}
_ => unreachable!(),
};
let ebx_place = match operands[1] {
InlineAsmOperand::Out { reg, late: true, place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::si));
crate::base::codegen_place(fx, place.unwrap())
}
_ => unreachable!(),
};
let (sub_leaf, ecx_place) = match operands[2] {
InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx));
(
crate::base::codegen_operand(fx, in_value).load_scalar(fx),
crate::base::codegen_place(fx, out_place.unwrap()),
)
}
_ => unreachable!(),
};
let edx_place = match operands[3] {
InlineAsmOperand::Out { reg, late: true, place } => {
let reg = expect_reg(reg);
assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx));
crate::base::codegen_place(fx, place.unwrap())
}
_ => unreachable!(),
};
let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf);
eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
return;
} else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
// ___chkstk, ___chkstk_ms and __alloca are only used on Windows
crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
} else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
crate::trap::trap_unimplemented(fx, "Alloca is not supported");
}
let mut slot_size = Size::from_bytes(0);
@ -92,8 +150,7 @@ pub(crate) fn codegen_inline_asm<'tcx>(
let inline_asm_index = fx.inline_asm_index;
fx.inline_asm_index += 1;
let asm_name =
format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index);
let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index);
let generated_asm = generate_asm_wrapper(
&asm_name,
@ -202,7 +259,6 @@ fn call_inline_asm<'tcx>(
}
let inline_asm_func = fx
.cx
.module
.declare_function(
asm_name,
@ -214,7 +270,7 @@ fn call_inline_asm<'tcx>(
},
)
.unwrap();
let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(inline_asm_func, asm_name);
}

2
compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
View file

@ -8,7 +8,7 @@ use crate::prelude::*;
pub(crate) fn codegen_cpuid_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
leaf: Value,
_subleaf: Value,
_sub_leaf: Value,
) -> (Value, Value, Value, Value) {
let leaf_0 = fx.bcx.create_block();
let leaf_1 = fx.bcx.create_block();

10
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View file

@ -22,7 +22,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
};
// Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) {
"llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) {
let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
let lane_ty = fx.clif_type(lane_ty).unwrap();
assert!(lane_count <= 32);
@ -51,7 +51,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
ret.write_cvalue(fx, res);
};
llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
"llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) {
let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
0 => FloatCC::Equal,
@ -81,7 +81,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
});
};
llvm.x86.sse2.psrli.d, (c a, o imm8) {
"llvm.x86.sse2.psrli.d", (c a, o imm8) {
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
@ -91,7 +91,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
CValue::by_val(res_lane, res_lane_layout)
});
};
llvm.x86.sse2.pslli.d, (c a, o imm8) {
"llvm.x86.sse2.pslli.d", (c a, o imm8) {
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
@ -101,7 +101,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
CValue::by_val(res_lane, res_lane_layout)
});
};
llvm.x86.sse2.storeu.dq, (v mem_addr, c a) {
"llvm.x86.sse2.storeu.dq", (v mem_addr, c a) {
// FIXME correctly handle the unalignment
let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
dest.write_cvalue(fx, a);

144
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -8,23 +8,25 @@ mod simd;
pub(crate) use cpuid::codegen_cpuid_call;
pub(crate) use llvm::codegen_llvm_intrinsic_call;
use rustc_span::symbol::{sym, kw};
use rustc_middle::ty::print::with_no_trimmed_paths;
use crate::prelude::*;
use cranelift_codegen::ir::AtomicRmwOp;
use rustc_middle::ty::print::with_no_trimmed_paths;
macro intrinsic_pat {
(_) => {
_
},
($name:ident) => {
stringify!($name)
sym::$name
},
(kw.$name:ident) => {
kw::$name
},
($name:literal) => {
stringify!($name)
$name
},
($x:ident . $($xs:tt).*) => {
concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
}
}
macro intrinsic_arg {
@ -87,7 +89,7 @@ macro call_intrinsic_match {
)*) => {
match $intrinsic {
$(
stringify!($name) => {
sym::$name => {
assert!($substs.is_noop());
if let [$(ref $arg),*] = *$args {
let ($($arg,)*) = (
@ -400,18 +402,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let def_id = instance.def_id();
let substs = instance.substs;
let intrinsic = fx.tcx.item_name(def_id).as_str();
let intrinsic = &intrinsic[..];
let intrinsic = fx.tcx.item_name(def_id);
let ret = match destination {
Some((place, _)) => place,
None => {
// Insert non returning intrinsics here
match intrinsic {
"abort" => {
sym::abort => {
trap_abort(fx, "Called intrinsic::abort.");
}
"transmute" => {
sym::transmute => {
crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
}
_ => unimplemented!("unsupported instrinsic {}", intrinsic),
@ -420,7 +421,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
}
};
if intrinsic.starts_with("simd_") {
if intrinsic.as_str().starts_with("simd_") {
self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
fx.bcx.ins().jump(ret_block, &[]);
@ -470,8 +471,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
sinf64(flt) -> f64 => sin,
cosf32(flt) -> f32 => cosf,
cosf64(flt) -> f64 => cos,
tanf32(flt) -> f32 => tanf,
tanf64(flt) -> f64 => tan,
}
intrinsic_match! {
@ -496,12 +495,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
count
};
if intrinsic.contains("nonoverlapping") {
if intrinsic == sym::copy_nonoverlapping {
// FIXME emit_small_memcpy
fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
} else {
// FIXME emit_small_memmove
fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
}
};
// NOTE: the volatile variants have src and dst swapped
@ -515,12 +514,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
};
// FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
if intrinsic.contains("nonoverlapping") {
if intrinsic == sym::volatile_copy_nonoverlapping_memory {
// FIXME emit_small_memcpy
fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount);
} else {
// FIXME emit_small_memmove
fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount);
}
};
size_of_val, <T> (c ptr) {
@ -552,27 +551,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
};
_ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem
| unchecked_shl | unchecked_shr, (c x, c y) {
// FIXME trap on overflow
let bin_op = match intrinsic {
"unchecked_add" => BinOp::Add,
"unchecked_sub" => BinOp::Sub,
"unchecked_div" | "exact_div" => BinOp::Div,
"unchecked_rem" => BinOp::Rem,
"unchecked_shl" => BinOp::Shl,
"unchecked_shr" => BinOp::Shr,
_ => unreachable!("intrinsic {}", intrinsic),
sym::unchecked_add => BinOp::Add,
sym::unchecked_sub => BinOp::Sub,
sym::unchecked_div | sym::exact_div => BinOp::Div,
sym::unchecked_rem => BinOp::Rem,
sym::unchecked_shl => BinOp::Shl,
sym::unchecked_shr => BinOp::Shr,
_ => unreachable!(),
};
let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
ret.write_cvalue(fx, res);
};
_ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) {
assert_eq!(x.layout().ty, y.layout().ty);
let bin_op = match intrinsic {
"add_with_overflow" => BinOp::Add,
"sub_with_overflow" => BinOp::Sub,
"mul_with_overflow" => BinOp::Mul,
_ => unreachable!("intrinsic {}", intrinsic),
sym::add_with_overflow => BinOp::Add,
sym::sub_with_overflow => BinOp::Sub,
sym::mul_with_overflow => BinOp::Mul,
_ => unreachable!(),
};
let res = crate::num::codegen_checked_int_binop(
@ -583,12 +583,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
);
ret.write_cvalue(fx, res);
};
_ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
saturating_add | saturating_sub, <T> (c lhs, c rhs) {
assert_eq!(lhs.layout().ty, rhs.layout().ty);
let bin_op = match intrinsic {
"saturating_add" => BinOp::Add,
"saturating_sub" => BinOp::Sub,
_ => unreachable!("intrinsic {}", intrinsic),
sym::saturating_add => BinOp::Add,
sym::saturating_sub => BinOp::Sub,
_ => unreachable!(),
};
let signed = type_sign(T);
@ -609,15 +609,15 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
let val = match (intrinsic, signed) {
("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
("saturating_add", true) => {
(sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val),
(sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
(sym::saturating_add, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
("saturating_sub", true) => {
(sym::saturating_sub, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
@ -632,11 +632,21 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
};
rotate_left, <T>(v x, v y) {
let layout = fx.layout_of(T);
let y = if fx.bcx.func.dfg.value_type(y) == types::I128 {
fx.bcx.ins().ireduce(types::I64, y)
} else {
y
};
let res = fx.bcx.ins().rotl(x, y);
ret.write_cvalue(fx, CValue::by_val(res, layout));
};
rotate_right, <T>(v x, v y) {
let layout = fx.layout_of(T);
let y = if fx.bcx.func.dfg.value_type(y) == types::I128 {
fx.bcx.ins().ireduce(types::I64, y)
} else {
y
};
let res = fx.bcx.ins().rotr(x, y);
ret.write_cvalue(fx, CValue::by_val(res, layout));
};
@ -670,7 +680,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let dst_ptr = dst.load_scalar(fx);
// FIXME make the memset actually volatile when switching to emit_small_memset
// FIXME use emit_small_memset
fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count);
fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count);
};
ctlz | ctlz_nonzero, <T> (v arg) {
// FIXME trap on `ctlz_nonzero` with zero arg.
@ -806,7 +816,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
return;
}
if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
with_no_trimmed_paths(|| crate::base::codegen_panic(
fx,
&format!("attempted to zero-initialize type `{}`, which is invalid", T),
@ -815,7 +825,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
return;
}
if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
with_no_trimmed_paths(|| crate::base::codegen_panic(
fx,
&format!("attempted to leave type `{}` uninitialized, which is invalid", T),
@ -827,7 +837,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
volatile_load | unaligned_volatile_load, (c ptr) {
// Cranelift treats loads as volatile by default
// FIXME ignore during stack2reg optimization
// FIXME correctly handle unaligned_volatile_load
let inner_layout =
fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
@ -836,7 +845,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
};
volatile_store | unaligned_volatile_store, (v ptr, c val) {
// Cranelift treats stores as volatile by default
// FIXME ignore during stack2reg optimization
// FIXME correctly handle unaligned_volatile_store
let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
dest.write_cvalue(fx, val);
@ -878,14 +886,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
ret.write_cvalue(fx, caller_location);
};
_ if intrinsic.starts_with("atomic_fence"), () {
_ if intrinsic.as_str().starts_with("atomic_fence"), () {
fx.bcx.ins().fence();
};
_ if intrinsic.starts_with("atomic_singlethreadfence"), () {
_ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () {
// FIXME use a compiler fence once Cranelift supports it
fx.bcx.ins().fence();
};
_ if intrinsic.starts_with("atomic_load"), <T> (v ptr) {
_ if intrinsic.as_str().starts_with("atomic_load"), <T> (v ptr) {
validate_atomic_type!(fx, intrinsic, span, T);
let ty = fx.clif_type(T).unwrap();
@ -894,14 +902,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(T));
ret.write_cvalue(fx, val);
};
_ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
_ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
let val = val.load_scalar(fx);
fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
};
_ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) {
_ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) {
let layout = new.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -913,7 +921,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
_ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
let layout = new.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
@ -927,7 +935,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
ret.write_cvalue(fx, ret_val)
};
_ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) {
_ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) {
let layout = amount.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -939,7 +947,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) {
_ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) {
let layout = amount.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -951,7 +959,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_and"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -963,7 +971,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_or"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -975,7 +983,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -989,7 +997,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
};
// FIXME https://github.com/bytecodealliance/wasmtime/issues/2647
_ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -1001,7 +1009,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_max"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -1013,7 +1021,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -1025,7 +1033,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_min"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -1037,7 +1045,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) {
_ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) {
let layout = src.layout();
validate_atomic_type!(fx, intrinsic, span, layout.ty);
let ty = fx.clif_type(layout.ty).unwrap();
@ -1071,7 +1079,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
ret.write_cvalue(fx, val);
};
try, (v f, v data, v _catch_fn) {
kw.Try, (v f, v data, v _catch_fn) {
// FIXME once unwinding is supported, change this to actually catch panics
let f_sig = fx.bcx.func.import_signature(Signature {
call_conv: CallConv::triple_default(fx.triple()),
@ -1088,11 +1096,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
let res = crate::num::codegen_float_binop(fx, match intrinsic {
"fadd_fast" => BinOp::Add,
"fsub_fast" => BinOp::Sub,
"fmul_fast" => BinOp::Mul,
"fdiv_fast" => BinOp::Div,
"frem_fast" => BinOp::Rem,
sym::fadd_fast => BinOp::Add,
sym::fsub_fast => BinOp::Sub,
sym::fmul_fast => BinOp::Mul,
sym::fdiv_fast => BinOp::Div,
sym::frem_fast => BinOp::Rem,
_ => unreachable!(),
}, x, y);
ret.write_cvalue(fx, res);

7
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -13,8 +13,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let def_id = instance.def_id();
let substs = instance.substs;
let intrinsic = fx.tcx.item_name(def_id).as_str();
let intrinsic = &intrinsic[..];
let intrinsic = fx.tcx.item_name(def_id);
intrinsic_match! {
fx, intrinsic, substs, args,
@ -65,10 +64,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
// simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
_ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
_ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap();
let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap();
assert_eq!(x.layout(), y.layout());
let layout = x.layout();

136
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -1,4 +1,4 @@
#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)]
#![warn(rust_2018_idioms)]
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
@ -23,7 +23,6 @@ extern crate rustc_target;
extern crate rustc_driver;
use std::any::Any;
use std::str::FromStr;
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
@ -34,9 +33,10 @@ use rustc_middle::ty::query::Providers;
use rustc_session::config::OutputFilenames;
use rustc_session::Session;
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::settings::{self, Configurable};
use crate::constant::ConstantCx;
pub use crate::config::*;
use crate::prelude::*;
mod abi;
@ -49,6 +49,7 @@ mod cast;
mod codegen_i128;
mod common;
mod compiler_builtins;
mod config;
mod constant;
mod debuginfo;
mod discriminant;
@ -87,7 +88,6 @@ mod prelude {
pub(crate) use rustc_index::vec::Idx;
pub(crate) use cranelift_codegen::entity::EntitySet;
pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC};
pub(crate) use cranelift_codegen::ir::function::Function;
pub(crate) use cranelift_codegen::ir::types;
@ -119,95 +119,36 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
}
}
struct CodegenCx<'m, 'tcx: 'm> {
/// The codegen context holds any information shared between the codegen of individual functions
/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
struct CodegenCx<'tcx> {
tcx: TyCtxt<'tcx>,
module: &'m mut dyn Module,
global_asm: String,
constants_cx: ConstantCx,
cached_context: Context,
vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), DataId>,
debug_context: Option<DebugContext<'tcx>>,
unwind_context: UnwindContext<'tcx>,
unwind_context: UnwindContext,
}
impl<'m, 'tcx> CodegenCx<'m, 'tcx> {
impl<'tcx> CodegenCx<'tcx> {
fn new(
tcx: TyCtxt<'tcx>,
backend_config: BackendConfig,
module: &'m mut dyn Module,
isa: &dyn TargetIsa,
debug_info: bool,
) -> Self {
let unwind_context = UnwindContext::new(
tcx,
module.isa(),
matches!(backend_config.codegen_mode, CodegenMode::Aot),
);
let debug_context =
if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None };
assert_eq!(pointer_ty(tcx), isa.pointer_type());
let unwind_context =
UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot));
let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None };
CodegenCx {
tcx,
module,
global_asm: String::new(),
constants_cx: ConstantCx::default(),
cached_context: Context::new(),
vtables: FxHashMap::default(),
debug_context,
unwind_context,
}
}
fn finalize(self) -> (String, Option<DebugContext<'tcx>>, UnwindContext<'tcx>) {
self.constants_cx.finalize(self.tcx, self.module);
(self.global_asm, self.debug_context, self.unwind_context)
}
}
#[derive(Copy, Clone, Debug)]
pub enum CodegenMode {
Aot,
Jit,
JitLazy,
}
impl Default for CodegenMode {
fn default() -> Self {
CodegenMode::Aot
}
}
impl FromStr for CodegenMode {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"aot" => Ok(CodegenMode::Aot),
"jit" => Ok(CodegenMode::Jit),
"jit-lazy" => Ok(CodegenMode::JitLazy),
_ => Err(format!("Unknown codegen mode `{}`", s)),
}
}
}
#[derive(Copy, Clone, Debug, Default)]
pub struct BackendConfig {
pub codegen_mode: CodegenMode,
}
impl BackendConfig {
fn from_opts(opts: &[String]) -> Result<Self, String> {
let mut config = BackendConfig::default();
for opt in opts {
if let Some((name, value)) = opt.split_once('=') {
match name {
"mode" => config.codegen_mode = value.parse()?,
_ => return Err(format!("Unknown option `{}`", name)),
}
} else {
return Err(format!("Invalid option `{}`", opt));
}
}
Ok(config)
}
}
pub struct CraneliftCodegenBackend {
@ -240,13 +181,23 @@ impl CodegenBackend for CraneliftCodegenBackend {
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<dyn Any> {
let config = if let Some(config) = self.config {
tcx.sess.abort_if_errors();
let config = if let Some(config) = self.config.clone() {
config
} else {
BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
.unwrap_or_else(|err| tcx.sess.fatal(&err))
};
driver::codegen_crate(tcx, metadata, need_metadata_module, config)
match config.codegen_mode {
CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
#[cfg(feature = "jit")]
let _: ! = driver::jit::run_jit(tcx, config);
#[cfg(not(feature = "jit"))]
tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift");
}
}
}
fn join_codegen(
@ -284,7 +235,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
sess.target.llvm_target.parse().unwrap()
}
fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::TargetIsa + 'static> {
use target_lexicon::BinaryFormat;
let target_triple = crate::target_triple(sess);
@ -292,9 +243,8 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
let mut flags_builder = settings::builder();
flags_builder.enable("is_pic").unwrap();
flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
let enable_verifier =
cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
flags_builder.set("enable_verifier", enable_verifier).unwrap();
let tls_model = match target_triple.binary_format {
BinaryFormat::Elf => "elf_gd",
@ -322,10 +272,28 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
let flags = settings::Flags::new(flags_builder);
let variant = cranelift_codegen::isa::BackendVariant::MachInst;
let mut isa_builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
// Don't use "haswell", as it implies `has_lzcnt`.macOS CI is still at Ivy Bridge EP, so `lzcnt`
// is interpreted as `bsr`.
isa_builder.enable("nehalem").unwrap();
let isa_builder = match sess.opts.cg.target_cpu.as_deref() {
Some("native") => {
let builder = cranelift_native::builder_with_options(variant, true).unwrap();
builder
}
Some(value) => {
let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
if let Err(_) = builder.enable(value) {
sess.fatal("The specified target cpu isn't currently supported by Cranelift.");
}
builder
}
None => {
let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
// Don't use "haswell" as the default, as it implies `has_lzcnt`.
// macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
builder.enable("nehalem").unwrap();
builder
}
};
isa_builder.finish(flags)
}

1
compiler/rustc_codegen_cranelift/src/linkage.rs
View file

@ -13,6 +13,7 @@ pub(crate) fn get_clif_linkage(
(RLinkage::External, Visibility::Default) => Linkage::Export,
(RLinkage::Internal, Visibility::Default) => Linkage::Local,
(RLinkage::External, Visibility::Hidden) => Linkage::Hidden,
(RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible,
_ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility),
}
}

70
compiler/rustc_codegen_cranelift/src/main_shim.rs
View file

@ -1,6 +1,9 @@
use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
use rustc_hir::LangItem;
use rustc_middle::ty::subst::GenericArg;
use rustc_middle::ty::AssocKind;
use rustc_session::config::EntryFnType;
use rustc_span::symbol::Ident;
use crate::prelude::*;
@ -9,9 +12,10 @@ use crate::prelude::*;
pub(crate) fn maybe_create_entry_wrapper(
tcx: TyCtxt<'_>,
module: &mut impl Module,
unwind_context: &mut UnwindContext<'_>,
unwind_context: &mut UnwindContext,
is_jit: bool,
) {
let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) {
let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) {
Some((def_id, entry_ty)) => (
def_id,
match entry_ty {
@ -23,18 +27,19 @@ pub(crate) fn maybe_create_entry_wrapper(
};
let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
if module.get_name(&*tcx.symbol_name(instance).name).is_none() {
if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() {
return;
}
create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item);
create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn);
fn create_entry_fn(
tcx: TyCtxt<'_>,
m: &mut impl Module,
unwind_context: &mut UnwindContext<'_>,
unwind_context: &mut UnwindContext,
rust_main_def_id: DefId,
use_start_lang_item: bool,
ignore_lang_start_wrapper: bool,
is_main_fn: bool,
) {
let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
// Given that `main()` has no arguments,
@ -57,9 +62,9 @@ pub(crate) fn maybe_create_entry_wrapper(
let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx);
let main_name = tcx.symbol_name(instance).name.to_string();
let main_name = tcx.symbol_name(instance).name;
let main_sig = get_function_sig(tcx, m.isa().triple(), instance);
let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap();
let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap();
let mut ctx = Context::new();
ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig);
@ -74,7 +79,47 @@ pub(crate) fn maybe_create_entry_wrapper(
let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
let call_inst = if use_start_lang_item {
let result = if is_main_fn && ignore_lang_start_wrapper {
// regular main fn, but ignoring #[lang = "start"] as we are running in the jit
// FIXME set program arguments somehow
let call_inst = bcx.ins().call(main_func_ref, &[]);
let call_results = bcx.func.dfg.inst_results(call_inst).to_owned();
let termination_trait = tcx.require_lang_item(LangItem::Termination, None);
let report = tcx
.associated_items(termination_trait)
.find_by_name_and_kind(
tcx,
Ident::from_str("report"),
AssocKind::Fn,
termination_trait,
)
.unwrap();
let report = Instance::resolve(
tcx,
ParamEnv::reveal_all(),
report.def_id,
tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()),
)
.unwrap()
.unwrap();
let report_name = tcx.symbol_name(report).name;
let report_sig = get_function_sig(tcx, m.isa().triple(), report);
let report_func_id =
m.declare_function(report_name, Linkage::Import, &report_sig).unwrap();
let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func);
// FIXME do proper abi handling instead of expecting the pass mode to be identical
// for returns and arguments.
let report_call_inst = bcx.ins().call(report_func_ref, &call_results);
let res = bcx.func.dfg.inst_results(report_call_inst)[0];
match m.target_config().pointer_type() {
types::I32 => res,
types::I64 => bcx.ins().sextend(types::I64, res),
_ => unimplemented!("16bit systems are not yet supported"),
}
} else if is_main_fn {
let start_def_id = tcx.require_lang_item(LangItem::Start, None);
let start_instance = Instance::resolve(
tcx,
@ -90,13 +135,14 @@ pub(crate) fn maybe_create_entry_wrapper(
let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv])
let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]);
bcx.inst_results(call_inst)[0]
} else {
// using user-defined start fn
bcx.ins().call(main_func_ref, &[arg_argc, arg_argv])
let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]);
bcx.inst_results(call_inst)[0]
};
let result = bcx.inst_results(call_inst)[0];
bcx.ins().return_(&[result]);
bcx.seal_all_blocks();
bcx.finalize();

57
compiler/rustc_codegen_cranelift/src/metadata.rs
View file

@ -8,13 +8,24 @@ use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::owning_ref::OwningRef;
use rustc_data_structures::rustc_erase_owner;
use rustc_data_structures::sync::MetadataRef;
use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
use rustc_middle::middle::cstore::MetadataLoader;
use rustc_middle::ty::TyCtxt;
use rustc_session::config;
use rustc_target::spec::Target;
use crate::backend::WriteMetadata;
/// The metadata loader used by cg_clif.
///
/// The metadata is stored in the same format as cg_llvm.
///
/// # Metadata location
///
/// <dl>
/// <dt>rlib</dt>
/// <dd>The metadata can be found in the `lib.rmeta` file inside of the ar archive.</dd>
/// <dt>dylib</dt>
/// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
/// </dl>
pub(crate) struct CraneliftMetadataLoader;
fn load_metadata_with(
@ -58,54 +69,16 @@ impl MetadataLoader for CraneliftMetadataLoader {
}
// Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112
pub(crate) fn write_metadata<P: WriteMetadata>(
tcx: TyCtxt<'_>,
product: &mut P,
) -> EncodedMetadata {
pub(crate) fn write_metadata<O: WriteMetadata>(tcx: TyCtxt<'_>, object: &mut O) {
use snap::write::FrameEncoder;
use std::io::Write;
#[derive(PartialEq, Eq, PartialOrd, Ord)]
enum MetadataKind {
None,
Uncompressed,
Compressed,
}
let kind = tcx
.sess
.crate_types()
.iter()
.map(|ty| match *ty {
config::CrateType::Executable
| config::CrateType::Staticlib
| config::CrateType::Cdylib => MetadataKind::None,
config::CrateType::Rlib => MetadataKind::Uncompressed,
config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed,
})
.max()
.unwrap_or(MetadataKind::None);
if kind == MetadataKind::None {
return EncodedMetadata::new();
}
let metadata = tcx.encode_metadata();
if kind == MetadataKind::Uncompressed {
return metadata;
}
assert!(kind == MetadataKind::Compressed);
let mut compressed = tcx.metadata_encoding_version();
FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap();
product.add_rustc_section(
object.add_rustc_section(
rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx),
compressed,
tcx.sess.target.is_like_osx,
);
metadata
}

17
compiler/rustc_codegen_cranelift/src/num.rs
View file

@ -271,14 +271,17 @@ pub(crate) fn codegen_checked_int_binop<'tcx>(
let val_hi = fx.bcx.ins().umulhi(lhs, rhs);
fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0)
} else {
// Based on LLVM's instruction sequence for compiling
// a.checked_mul(b).is_some() to riscv64gc:
// mulh a2, a0, a1
// mul a0, a0, a1
// srai a0, a0, 63
// xor a0, a0, a2
// snez a0, a0
let val_hi = fx.bcx.ins().smulhi(lhs, rhs);
let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0);
let not_all_ones = fx.bcx.ins().icmp_imm(
IntCC::NotEqual,
val_hi,
u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64,
);
fx.bcx.ins().band(not_all_zero, not_all_ones)
let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1));
let xor = fx.bcx.ins().bxor(val_hi, val_sign);
fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0)
};
(val, has_overflow)
}

34
compiler/rustc_codegen_cranelift/src/optimize/code_layout.rs
View file

@ -1,34 +0,0 @@
//! This optimization moves cold code to the end of the function.
//!
//! Some code is executed much less often than other code. For example panicking or the
//! landingpads for unwinding. By moving this cold code to the end of the function the average
//! amount of jumps is reduced and the code locality is improved.
//!
//! # Undefined behaviour
//!
//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself.
use crate::prelude::*;
pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet<Block>) {
// FIXME Move the block in place instead of remove and append once
// bytecodealliance/cranelift#1339 is implemented.
let mut block_insts = FxHashMap::default();
for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
let insts = ctx.func.layout.block_insts(block).collect::<Vec<_>>();
for &inst in &insts {
ctx.func.layout.remove_inst(inst);
}
block_insts.insert(block, insts);
ctx.func.layout.remove_block(block);
}
// And then append them at the back again.
for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) {
ctx.func.layout.append_block(block);
for inst in block_insts.remove(&block).unwrap() {
ctx.func.layout.append_inst(inst, block);
}
}
}

17
compiler/rustc_codegen_cranelift/src/optimize/mod.rs
View file

@ -2,29 +2,16 @@
use crate::prelude::*;
mod code_layout;
pub(crate) mod peephole;
mod stack2reg;
pub(crate) fn optimize_function<'tcx>(
tcx: TyCtxt<'tcx>,
instance: Instance<'tcx>,
ctx: &mut Context,
cold_blocks: &EntitySet<Block>,
clif_comments: &mut crate::pretty_clif::CommentWriter,
) {
// The code_layout optimization is very cheap.
self::code_layout::optimize_function(ctx, cold_blocks);
// FIXME classify optimizations over opt levels once we have more
if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No {
return; // FIXME classify optimizations over opt levels
}
// FIXME(#1142) stack2reg miscompiles lewton
if false {
self::stack2reg::optimize_function(ctx, clif_comments);
}
crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments);
crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments);
crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
}

486
compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
View file

@ -1,486 +0,0 @@
//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible.
//!
//! # Undefined behaviour
//!
//! This optimization is based on the assumption that stack slots which don't have their address
//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the
//! function which has the stack slots. This optimization also assumes that stack slot accesses
//! are never out of bounds. If these assumptions are not correct, then this optimization may remove
//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value
//! being loaded by a `stack_load`.
use std::collections::BTreeMap;
use std::fmt;
use std::ops::Not;
use rustc_data_structures::fx::FxHashSet;
use cranelift_codegen::cursor::{Cursor, FuncCursor};
use cranelift_codegen::ir::immediates::Offset32;
use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef};
use crate::prelude::*;
/// Workaround for `StackSlot` not implementing `Ord`.
#[derive(Copy, Clone, PartialEq, Eq)]
struct OrdStackSlot(StackSlot);
impl fmt::Debug for OrdStackSlot {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self.0)
}
}
impl PartialOrd for OrdStackSlot {
fn partial_cmp(&self, rhs: &Self) -> Option<std::cmp::Ordering> {
self.0.as_u32().partial_cmp(&rhs.0.as_u32())
}
}
impl Ord for OrdStackSlot {
fn cmp(&self, rhs: &Self) -> std::cmp::Ordering {
self.0.as_u32().cmp(&rhs.0.as_u32())
}
}
#[derive(Debug, Default)]
struct StackSlotUsage {
stack_addr: FxHashSet<Inst>,
stack_load: FxHashSet<Inst>,
stack_store: FxHashSet<Inst>,
}
impl StackSlotUsage {
fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec<Inst> {
self.stack_store
.iter()
.cloned()
.filter(|&store| {
match spatial_overlap(&ctx.func, store, load) {
SpatialOverlap::No => false, // Can never be the source of the loaded value.
SpatialOverlap::Partial | SpatialOverlap::Full => true,
}
})
.filter(|&store| {
match temporal_order(ctx, store, load) {
TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value.
TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true,
}
})
.collect::<Vec<Inst>>()
}
fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec<Inst> {
self.stack_load
.iter()
.cloned()
.filter(|&load| {
match spatial_overlap(&ctx.func, store, load) {
SpatialOverlap::No => false, // Can never be the source of the loaded value.
SpatialOverlap::Partial | SpatialOverlap::Full => true,
}
})
.filter(|&load| {
match temporal_order(ctx, store, load) {
TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value.
TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true,
}
})
.collect::<Vec<Inst>>()
}
fn remove_unused_stack_addr(func: &mut Function, inst: Inst) {
func.dfg.detach_results(inst);
func.dfg.replace(inst).nop();
}
fn remove_unused_load(func: &mut Function, load: Inst) {
func.dfg.detach_results(load);
func.dfg.replace(load).nop();
}
fn remove_dead_store(&mut self, func: &mut Function, store: Inst) {
func.dfg.replace(store).nop();
self.stack_store.remove(&store);
}
fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) {
let loaded_value = func.dfg.inst_results(load)[0];
let loaded_type = func.dfg.value_type(loaded_value);
if func.dfg.value_type(value) == loaded_type {
func.dfg.detach_results(load);
func.dfg.replace(load).nop();
func.dfg.change_to_alias(loaded_value, value);
} else {
func.dfg.replace(load).bitcast(loaded_type, value);
}
self.stack_load.remove(&load);
}
}
struct OptimizeContext<'a> {
ctx: &'a mut Context,
stack_slot_usage_map: BTreeMap<OrdStackSlot, StackSlotUsage>,
}
impl<'a> OptimizeContext<'a> {
fn for_context(ctx: &'a mut Context) -> Self {
ctx.flowgraph(); // Compute cfg and domtree.
// Record all stack_addr, stack_load and stack_store instructions.
let mut stack_slot_usage_map = BTreeMap::<OrdStackSlot, StackSlotUsage>::new();
let mut cursor = FuncCursor::new(&mut ctx.func);
while let Some(_block) = cursor.next_block() {
while let Some(inst) = cursor.next_inst() {
match cursor.func.dfg[inst] {
InstructionData::StackLoad {
opcode: Opcode::StackAddr,
stack_slot,
offset: _,
} => {
stack_slot_usage_map
.entry(OrdStackSlot(stack_slot))
.or_insert_with(StackSlotUsage::default)
.stack_addr
.insert(inst);
}
InstructionData::StackLoad {
opcode: Opcode::StackLoad,
stack_slot,
offset: _,
} => {
stack_slot_usage_map
.entry(OrdStackSlot(stack_slot))
.or_insert_with(StackSlotUsage::default)
.stack_load
.insert(inst);
}
InstructionData::StackStore {
opcode: Opcode::StackStore,
arg: _,
stack_slot,
offset: _,
} => {
stack_slot_usage_map
.entry(OrdStackSlot(stack_slot))
.or_insert_with(StackSlotUsage::default)
.stack_store
.insert(inst);
}
_ => {}
}
}
}
OptimizeContext { ctx, stack_slot_usage_map }
}
}
pub(super) fn optimize_function(
ctx: &mut Context,
clif_comments: &mut crate::pretty_clif::CommentWriter,
) {
combine_stack_addr_with_load_store(&mut ctx.func);
let mut opt_ctx = OptimizeContext::for_context(ctx);
// FIXME Repeat following instructions until fixpoint.
remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
if clif_comments.enabled() {
for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
}
}
for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() {
if users.stack_addr.is_empty().not() {
// Stack addr leaked; there may be unknown loads and stores.
// FIXME use stacked borrows to optimize
continue;
}
for load in users.stack_load.clone().into_iter() {
let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
if clif_comments.enabled() {
for &store in &potential_stores {
clif_comments.add_comment(
load,
format!(
"Potential store -> load forwarding {} -> {} ({:?}, {:?})",
opt_ctx.ctx.func.dfg.display_inst(store, None),
opt_ctx.ctx.func.dfg.display_inst(load, None),
spatial_overlap(&opt_ctx.ctx.func, store, load),
temporal_order(&opt_ctx.ctx, store, load),
),
);
}
}
match *potential_stores {
[] => {
if clif_comments.enabled() {
clif_comments
.add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
}
}
[store]
if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
&& temporal_order(&opt_ctx.ctx, store, load)
== TemporalOrder::DefinitivelyBefore =>
{
// Only one store could have been the origin of the value.
let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
if clif_comments.enabled() {
clif_comments.add_comment(
load,
format!("Store to load forward {} -> {}", store, load),
);
}
users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
}
_ => {} // FIXME implement this
}
}
for store in users.stack_store.clone().into_iter() {
let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
if clif_comments.enabled() {
for &load in &potential_loads {
clif_comments.add_comment(
store,
format!(
"Potential load from store {} <- {} ({:?}, {:?})",
opt_ctx.ctx.func.dfg.display_inst(load, None),
opt_ctx.ctx.func.dfg.display_inst(store, None),
spatial_overlap(&opt_ctx.ctx.func, store, load),
temporal_order(&opt_ctx.ctx, store, load),
),
);
}
}
if potential_loads.is_empty() {
// Never loaded; can safely remove all stores and the stack slot.
// FIXME also remove stores when there is always a next store before a load.
if clif_comments.enabled() {
clif_comments.add_comment(
store,
format!(
"Remove dead stack store {} of {}",
opt_ctx.ctx.func.dfg.display_inst(store, None),
stack_slot.0
),
);
}
users.remove_dead_store(&mut opt_ctx.ctx.func, store);
}
}
if users.stack_store.is_empty() && users.stack_load.is_empty() {
opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0;
}
}
}
fn combine_stack_addr_with_load_store(func: &mut Function) {
// Turn load and store into stack_load and stack_store when possible.
let mut cursor = FuncCursor::new(func);
while let Some(_block) = cursor.next_block() {
while let Some(inst) = cursor.next_inst() {
match cursor.func.dfg[inst] {
InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => {
if cursor.func.dfg.ctrl_typevar(inst) == types::I128
|| cursor.func.dfg.ctrl_typevar(inst).is_vector()
{
continue; // WORKAROUD: stack_load.i128 not yet implemented
}
if let Some((stack_slot, stack_addr_offset)) =
try_get_stack_slot_and_offset_for_addr(cursor.func, addr)
{
if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into())
{
let ty = cursor.func.dfg.ctrl_typevar(inst);
cursor.func.dfg.replace(inst).stack_load(
ty,
stack_slot,
combined_offset,
);
}
}
}
InstructionData::Store {
opcode: Opcode::Store,
args: [value, addr],
flags: _,
offset,
} => {
if cursor.func.dfg.ctrl_typevar(inst) == types::I128
|| cursor.func.dfg.ctrl_typevar(inst).is_vector()
{
continue; // WORKAROUND: stack_store.i128 not yet implemented
}
if let Some((stack_slot, stack_addr_offset)) =
try_get_stack_slot_and_offset_for_addr(cursor.func, addr)
{
if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into())
{
cursor.func.dfg.replace(inst).stack_store(
value,
stack_slot,
combined_offset,
);
}
}
}
_ => {}
}
}
}
}
fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) {
// FIXME incrementally rebuild on each call?
let mut stack_addr_load_insts_users = FxHashMap::<Inst, FxHashSet<Inst>>::default();
let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func);
while let Some(_block) = cursor.next_block() {
while let Some(inst) = cursor.next_inst() {
for &arg in cursor.func.dfg.inst_args(inst) {
if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) {
match cursor.func.dfg[arg_origin].opcode() {
Opcode::StackAddr | Opcode::StackLoad => {
stack_addr_load_insts_users
.entry(arg_origin)
.or_insert_with(FxHashSet::default)
.insert(inst);
}
_ => {}
}
}
}
}
}
#[cfg(debug_assertions)]
for inst in stack_addr_load_insts_users.keys() {
let mut is_recorded_stack_addr_or_stack_load = false;
for stack_slot_users in opt_ctx.stack_slot_usage_map.values() {
is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst)
|| stack_slot_users.stack_load.contains(inst);
}
assert!(is_recorded_stack_addr_or_stack_load);
}
// Replace all unused stack_addr and stack_load instructions with nop.
let mut func = &mut opt_ctx.ctx.func;
for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() {
stack_slot_users
.stack_addr
.drain_filter(|inst| {
stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
})
.for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst));
stack_slot_users
.stack_load
.drain_filter(|inst| {
stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true)
})
.for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst));
}
}
fn try_get_stack_slot_and_offset_for_addr(
func: &Function,
addr: Value,
) -> Option<(StackSlot, Offset32)> {
if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) {
if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } =
func.dfg[addr_inst]
{
return Some((stack_slot, offset));
}
}
None
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum SpatialOverlap {
No,
Partial,
Full,
}
fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap {
fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) {
match func.dfg[inst] {
InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset }
| InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset }
| InstructionData::StackStore {
opcode: Opcode::StackStore,
stack_slot,
offset,
arg: _,
} => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()),
_ => unreachable!("{:?}", func.dfg[inst]),
}
}
debug_assert_ne!(src, dest);
let (src_ss, src_offset, src_size) = inst_info(func, src);
let (dest_ss, dest_offset, dest_size) = inst_info(func, dest);
if src_ss != dest_ss {
return SpatialOverlap::No;
}
if src_offset == dest_offset && src_size == dest_size {
return SpatialOverlap::Full;
}
let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into();
let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into();
if src_end <= dest_offset.into() || dest_end <= src_offset.into() {
return SpatialOverlap::No;
}
SpatialOverlap::Partial
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
enum TemporalOrder {
/// `src` will never be executed before `dest`.
NeverBefore,
/// `src` may be executed before `dest`.
MaybeBefore,
/// `src` will always be executed before `dest`.
/// There may still be other instructions in between.
DefinitivelyBefore,
}
fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder {
debug_assert_ne!(src, dest);
if ctx.domtree.dominates(src, dest, &ctx.func.layout) {
TemporalOrder::DefinitivelyBefore
} else if ctx.domtree.dominates(src, dest, &ctx.func.layout) {
TemporalOrder::NeverBefore
} else {
TemporalOrder::MaybeBefore
}
}

49
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -207,7 +207,7 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
pub(crate) fn write_ir_file(
tcx: TyCtxt<'_>,
name: &str,
name: impl FnOnce() -> String,
write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
) {
if !should_write_ir(tcx) {
@ -222,7 +222,7 @@ pub(crate) fn write_ir_file(
res @ Err(_) => res.unwrap(),
}
let clif_file_name = clif_output_dir.join(name);
let clif_file_name = clif_output_dir.join(name());
let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
if let Err(err) = res {
@ -238,30 +238,31 @@ pub(crate) fn write_clif_file<'tcx>(
context: &cranelift_codegen::Context,
mut clif_comments: &CommentWriter,
) {
write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| {
let value_ranges =
isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
write_ir_file(
tcx,
|| format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
|file| {
let value_ranges = isa
.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
let mut clif = String::new();
cranelift_codegen::write::decorate_function(
&mut clif_comments,
&mut clif,
&context.func,
&DisplayFunctionAnnotations {
isa: Some(&*crate::build_isa(tcx.sess)),
value_ranges: value_ranges.as_ref(),
},
)
.unwrap();
let mut clif = String::new();
cranelift_codegen::write::decorate_function(
&mut clif_comments,
&mut clif,
&context.func,
&DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() },
)
.unwrap();
writeln!(file, "test compile")?;
writeln!(file, "set is_pic")?;
writeln!(file, "set enable_simd")?;
writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
writeln!(file)?;
file.write_all(clif.as_bytes())?;
Ok(())
});
writeln!(file, "test compile")?;
writeln!(file, "set is_pic")?;
writeln!(file, "set enable_simd")?;
writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
writeln!(file)?;
file.write_all(clif.as_bytes())?;
Ok(())
},
);
}
impl fmt::Debug for FunctionCx<'_, '_, '_> {

6
compiler/rustc_codegen_cranelift/src/trap.rs
View file

@ -4,7 +4,6 @@ use crate::prelude::*;
fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
let puts = fx
.cx
.module
.declare_function(
"puts",
@ -16,13 +15,12 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
},
)
.unwrap();
let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(puts, "puts");
}
let symbol_name = fx.tcx.symbol_name(fx.instance);
let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg);
let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg);
let msg_ptr = fx.anonymous_str("trap", &real_msg);
fx.bcx.ins().call(puts, &[msg_ptr]);
}

2
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -554,7 +554,7 @@ impl<'tcx> CPlace<'tcx> {
let src_align = src_layout.align.abi.bytes() as u8;
let dst_align = dst_layout.align.abi.bytes() as u8;
fx.bcx.emit_small_memory_copy(
fx.cx.module.target_config(),
fx.module.target_config(),
to_addr,
from_addr,
size,

34
compiler/rustc_codegen_cranelift/src/vtable.rs
View file

@ -72,15 +72,15 @@ pub(crate) fn get_vtable<'tcx>(
layout: TyAndLayout<'tcx>,
trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
) -> Value {
let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) {
let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) {
*data_id
} else {
let data_id = build_vtable(fx, layout, trait_ref);
fx.cx.vtables.insert((layout.ty, trait_ref), data_id);
fx.vtables.insert((layout.ty, trait_ref), data_id);
data_id
};
let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
}
@ -94,7 +94,7 @@ fn build_vtable<'tcx>(
let drop_in_place_fn = import_function(
tcx,
fx.cx.module,
fx.module,
Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx),
);
@ -111,7 +111,7 @@ fn build_vtable<'tcx>(
opt_mth.map(|(def_id, substs)| {
import_function(
tcx,
fx.cx.module,
fx.module,
Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs)
.unwrap()
.polymorphize(fx.tcx),
@ -132,34 +132,16 @@ fn build_vtable<'tcx>(
for (i, component) in components.into_iter().enumerate() {
if let Some(func_id) = component {
let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx);
let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx);
data_ctx.write_function_addr((i * usize_size) as u32, func_ref);
}
}
data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes());
let data_id = fx
.cx
.module
.declare_data(
&format!(
"__vtable.{}.for.{:?}.{}",
trait_ref
.as_ref()
.map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into())
.unwrap_or(std::borrow::Cow::Borrowed("???")),
layout.ty,
fx.cx.vtables.len(),
),
Linkage::Local,
false,
false,
)
.unwrap();
let data_id = fx.module.declare_anonymous_data(false, false).unwrap();
// FIXME don't duplicate definitions in lazy jit mode
let _ = fx.cx.module.define_data(data_id, &data_ctx);
fx.module.define_data(data_id, &data_ctx).unwrap();
data_id
}