bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Merge commit '57845a397e' into sync_cg_clif-2024-12-06

Browse source
This commit is contained in:
bjorn3 2024-12-06 12:10:30 +00:00
commit b3d837afe1
29 changed files with 396 additions and 428 deletions
Download patch file Download diff file Expand all files Collapse all files

46
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -125,8 +125,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
returns: Vec<AbiParam>,
args: &[Value],
) -> Cow<'_, [Value]> {
if self.tcx.sess.target.is_like_windows {
let (mut params, mut args): (Vec<_>, Vec<_>) = params
// Pass i128 arguments by-ref on Windows.
let (params, args): (Vec<_>, Cow<'_, [_]>) = if self.tcx.sess.target.is_like_windows {
let (params, args): (Vec<_>, Vec<_>) = params
.into_iter()
.zip(args)
.map(|(param, &arg)| {
@ -140,29 +141,42 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
})
.unzip();
let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128;
(params, args.into())
} else {
(params, args.into())
};
if indirect_ret_val {
params.insert(0, AbiParam::new(self.pointer_type));
let ret_ptr = self.create_stack_slot(16, 16);
args.insert(0, ret_ptr.get_addr(self));
self.lib_call_unadjusted(name, params, vec![], &args);
return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]);
// Return i128 using a return area pointer on Windows and s390x.
let adjust_ret_param =
if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == "s390x" {
returns.len() == 1 && returns[0].value_type == types::I128
} else {
return self.lib_call_unadjusted(name, params, returns, &args);
}
}
false
};
self.lib_call_unadjusted(name, params, returns, args)
if adjust_ret_param {
let mut params = params;
let mut args = args.to_vec();
params.insert(0, AbiParam::new(self.pointer_type));
let ret_ptr = self.create_stack_slot(16, 16);
args.insert(0, ret_ptr.get_addr(self));
self.lib_call_unadjusted(name, params, vec![], &args);
Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())])
} else {
Cow::Borrowed(self.lib_call_unadjusted(name, params, returns, &args))
}
}
pub(crate) fn lib_call_unadjusted(
fn lib_call_unadjusted(
&mut self,
name: &str,
params: Vec<AbiParam>,
returns: Vec<AbiParam>,
args: &[Value],
) -> Cow<'_, [Value]> {
) -> &[Value] {
let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
@ -175,7 +189,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
}
let results = self.bcx.inst_results(call_inst);
assert!(results.len() <= 2, "{}", results.len());
Cow::Borrowed(results)
results
}
}

19
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -6,6 +6,7 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_module::ModuleError;
use rustc_ast::InlineAsmOptions;
use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_index::IndexVec;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::InlineAsmMacro;
@ -16,6 +17,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
use crate::constant::ConstantCx;
use crate::debuginfo::{FunctionDebugContext, TypeDebugContext};
use crate::enable_verifier;
use crate::inline_asm::codegen_naked_asm;
use crate::prelude::*;
use crate::pretty_clif::CommentWriter;
@ -169,12 +171,13 @@ pub(crate) fn codegen_fn<'tcx>(
pub(crate) fn compile_fn(
cx: &mut crate::CodegenCx,
profiler: &SelfProfilerRef,
cached_context: &mut Context,
module: &mut dyn Module,
codegened_func: CodegenedFunction,
) {
let _timer =
cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name);
profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name);
let clif_comments = codegened_func.clif_comments;
@ -212,7 +215,7 @@ pub(crate) fn compile_fn(
};
// Define function
cx.profiler.generic_activity("define function").run(|| {
profiler.generic_activity("define function").run(|| {
context.want_disasm = cx.should_write_ir;
match module.define_function(codegened_func.func_id, context) {
Ok(()) => {}
@ -253,7 +256,7 @@ pub(crate) fn compile_fn(
// Define debuginfo for function
let debug_context = &mut cx.debug_context;
cx.profiler.generic_activity("generate debug info").run(|| {
profiler.generic_activity("generate debug info").run(|| {
if let Some(debug_context) = debug_context {
codegened_func.func_debug_cx.unwrap().finalize(
debug_context,
@ -264,11 +267,11 @@ pub(crate) fn compile_fn(
});
}
pub(crate) fn verify_func(
tcx: TyCtxt<'_>,
writer: &crate::pretty_clif::CommentWriter,
func: &Function,
) {
fn verify_func(tcx: TyCtxt<'_>, writer: &crate::pretty_clif::CommentWriter, func: &Function) {
if !enable_verifier(tcx.sess) {
return;
}
tcx.prof.generic_activity("verify clif ir").run(|| {
let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder());
match cranelift_codegen::verify_function(&func, &flags) {

27
compiler/rustc_codegen_cranelift/src/codegen_i128.rs
View file

@ -81,26 +81,6 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
match bin_op {
BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => unreachable!(),
BinOp::Add | BinOp::Sub => None,
BinOp::Mul if is_signed => {
let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]);
let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32));
let lhs = lhs.load_scalar(fx);
let rhs = rhs.load_scalar(fx);
let oflow_ptr = oflow.to_ptr().get_addr(fx);
let res = fx.lib_call_unadjusted(
"__muloti4",
vec![
AbiParam::new(types::I128),
AbiParam::new(types::I128),
AbiParam::new(fx.pointer_type),
],
vec![AbiParam::new(types::I128)],
&[lhs, rhs, oflow_ptr],
)[0];
let oflow = oflow.to_cvalue(fx).load_scalar(fx);
let oflow = fx.bcx.ins().ireduce(types::I8, oflow);
Some(CValue::by_val_pair(res, oflow, fx.layout_of(out_ty)))
}
BinOp::Mul => {
let out_ty = Ty::new_tup(fx.tcx, &[lhs.layout().ty, fx.tcx.types.bool]);
let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
@ -110,7 +90,12 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
AbiParam::new(types::I128),
];
let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
fx.lib_call("__rust_u128_mulo", param_types, vec![], &args);
fx.lib_call(
if is_signed { "__rust_i128_mulo" } else { "__rust_u128_mulo" },
param_types,
vec![],
&args,
);
Some(out_place.to_cvalue(fx))
}
BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(),

80
compiler/rustc_codegen_cranelift/src/config.rs
View file

@ -1,10 +1,3 @@
use std::env;
use std::str::FromStr;
fn bool_env_var(key: &str) -> bool {
env::var(key).as_deref() == Ok("1")
}
/// The mode to use for compilation.
#[derive(Copy, Clone, Debug)]
pub enum CodegenMode {
@ -16,19 +9,6 @@ pub enum CodegenMode {
JitLazy,
}
impl FromStr for CodegenMode {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"aot" => Ok(CodegenMode::Aot),
"jit" => Ok(CodegenMode::Jit),
"jit-lazy" => Ok(CodegenMode::JitLazy),
_ => Err(format!("Unknown codegen mode `{}`", s)),
}
}
}
/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars.
#[derive(Clone, Debug)]
pub struct BackendConfig {
@ -41,51 +21,22 @@ pub struct BackendConfig {
///
/// Defaults to the value of `CG_CLIF_JIT_ARGS`.
pub jit_args: Vec<String>,
/// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run
/// once before passing the clif ir to Cranelift for compilation.
///
/// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is
/// compiled with debug assertions enabled or false otherwise. Can be set using
/// `-Cllvm-args=enable_verifier=...`.
pub enable_verifier: bool,
/// Don't cache object files in the incremental cache. Useful during development of cg_clif
/// to make it possible to use incremental mode for all analyses performed by rustc without
/// caching object files when their content should have been changed by a change to cg_clif.
///
/// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false
/// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`.
pub disable_incr_cache: bool,
}
impl Default for BackendConfig {
fn default() -> Self {
BackendConfig {
codegen_mode: CodegenMode::Aot,
jit_args: {
match std::env::var("CG_CLIF_JIT_ARGS") {
Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(),
Err(std::env::VarError::NotPresent) => vec![],
Err(std::env::VarError::NotUnicode(s)) => {
panic!("CG_CLIF_JIT_ARGS not unicode: {:?}", s);
}
}
},
enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),
}
}
}
impl BackendConfig {
/// Parse the configuration passed in using `-Cllvm-args`.
pub fn from_opts(opts: &[String]) -> Result<Self, String> {
fn parse_bool(name: &str, value: &str) -> Result<bool, String> {
value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name))
}
let mut config = BackendConfig {
codegen_mode: CodegenMode::Aot,
jit_args: match std::env::var("CG_CLIF_JIT_ARGS") {
Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(),
Err(std::env::VarError::NotPresent) => vec![],
Err(std::env::VarError::NotUnicode(s)) => {
panic!("CG_CLIF_JIT_ARGS not unicode: {:?}", s);
}
},
};
let mut config = BackendConfig::default();
for opt in opts {
if opt.starts_with("-import-instr-limit") {
// Silently ignore -import-instr-limit. It is set by rust's build system even when
@ -94,9 +45,14 @@ impl BackendConfig {
}
if let Some((name, value)) = opt.split_once('=') {
match name {
"mode" => config.codegen_mode = value.parse()?,
"enable_verifier" => config.enable_verifier = parse_bool(name, value)?,
"disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?,
"mode" => {
config.codegen_mode = match value {
"aot" => CodegenMode::Aot,
"jit" => CodegenMode::Jit,
"jit-lazy" => CodegenMode::JitLazy,
_ => return Err(format!("Unknown codegen mode `{}`", value)),
};
}
_ => return Err(format!("Unknown option `{}`", name)),
}
} else {

276
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View file

@ -1,6 +1,7 @@
//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
//! standalone executable.
use std::env;
use std::fs::{self, File};
use std::io::BufWriter;
use std::path::{Path, PathBuf};
@ -25,13 +26,18 @@ use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
use rustc_session::Session;
use rustc_session::config::{DebugInfo, OutFileName, OutputFilenames, OutputType};
use crate::BackendConfig;
use crate::CodegenCx;
use crate::base::CodegenedFunction;
use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
use crate::debuginfo::TypeDebugContext;
use crate::global_asm::GlobalAsmConfig;
use crate::prelude::*;
use crate::unwind_module::UnwindModule;
fn disable_incr_cache() -> bool {
env::var("CG_CLIF_DISABLE_INCR_CACHE").as_deref() == Ok("1")
}
struct ModuleCodegenResult {
module_regular: CompiledModule,
module_global_asm: Option<CompiledModule>,
@ -63,10 +69,10 @@ impl OngoingCodegen {
self,
sess: &Session,
outputs: &OutputFilenames,
backend_config: &BackendConfig,
) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
let mut work_products = FxIndexMap::default();
let mut modules = vec![];
let disable_incr_cache = disable_incr_cache();
for module_codegen in self.modules {
let module_codegen_result = match module_codegen {
@ -87,7 +93,7 @@ impl OngoingCodegen {
if let Some((work_product_id, work_product)) = existing_work_product {
work_products.insert(work_product_id, work_product);
} else {
let work_product = if backend_config.disable_incr_cache {
let work_product = if disable_incr_cache {
None
} else if let Some(module_global_asm) = &module_global_asm {
rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
@ -322,12 +328,8 @@ fn produce_final_output_artifacts(
// These are used in linking steps and will be cleaned up afterward.
}
fn make_module(
sess: &Session,
backend_config: &BackendConfig,
name: String,
) -> UnwindModule<ObjectModule> {
let isa = crate::build_isa(sess, backend_config);
fn make_module(sess: &Session, name: String) -> UnwindModule<ObjectModule> {
let isa = crate::build_isa(sess);
let mut builder =
ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
@ -412,7 +414,13 @@ fn emit_module(
Err(err) => return Err(format!("error writing object file: {}", err)),
};
prof.artifact_size("object_file", &*name, file.metadata().unwrap().len());
if prof.enabled() {
prof.artifact_size(
"object_file",
tmp_file.file_name().unwrap().to_string_lossy(),
file.metadata().unwrap().len(),
);
}
Ok(CompiledModule {
name,
@ -486,91 +494,101 @@ fn reuse_workproduct_for_cgu(
})
}
fn codegen_cgu_content(
tcx: TyCtxt<'_>,
module: &mut dyn Module,
cgu_name: rustc_span::Symbol,
) -> (CodegenCx, Vec<CodegenedFunction>) {
let _timer = tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str());
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let mut cx = crate::CodegenCx::new(
tcx,
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
let mut type_dbg = TypeDebugContext::default();
super::predefine_mono_items(tcx, module, &mono_items);
let mut codegened_functions = vec![];
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
if let Some(codegened_function) = crate::base::codegen_fn(
tcx,
&mut cx,
&mut type_dbg,
Function::new(),
module,
inst,
) {
codegened_functions.push(codegened_function);
}
}
MonoItem::Static(def_id) => {
let data_id = crate::constant::codegen_static(tcx, module, def_id);
if let Some(debug_context) = &mut cx.debug_context {
debug_context.define_static(tcx, &mut type_dbg, def_id, data_id);
}
}
MonoItem::GlobalAsm(item_id) => {
crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
}
}
}
crate::main_shim::maybe_create_entry_wrapper(tcx, module, false, cgu.is_primary());
(cx, codegened_functions)
}
fn module_codegen(
tcx: TyCtxt<'_>,
(backend_config, global_asm_config, cgu_name, token): (
BackendConfig,
(global_asm_config, cgu_name, token): (
Arc<GlobalAsmConfig>,
rustc_span::Symbol,
ConcurrencyLimiterToken,
),
) -> OngoingModuleCodegen {
let (cgu_name, mut cx, mut module, codegened_functions) =
tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| {
let cgu = tcx.codegen_unit(cgu_name);
let mono_items = cgu.items_in_deterministic_order(tcx);
let mut module = make_module(tcx.sess, cgu_name.as_str().to_string());
let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
let (mut cx, codegened_functions) = codegen_cgu_content(tcx, &mut module, cgu_name);
let mut cx = crate::CodegenCx::new(
tcx,
module.isa(),
tcx.sess.opts.debuginfo != DebugInfo::None,
cgu_name,
);
let mut type_dbg = TypeDebugContext::default();
super::predefine_mono_items(tcx, &mut module, &mono_items);
let mut codegened_functions = vec![];
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
if let Some(codegened_function) = crate::base::codegen_fn(
tcx,
&mut cx,
&mut type_dbg,
Function::new(),
&mut module,
inst,
) {
codegened_functions.push(codegened_function);
}
}
MonoItem::Static(def_id) => {
let data_id = crate::constant::codegen_static(tcx, &mut module, def_id);
if let Some(debug_context) = &mut cx.debug_context {
debug_context.define_static(tcx, &mut type_dbg, def_id, data_id);
}
}
MonoItem::GlobalAsm(item_id) => {
crate::global_asm::codegen_global_asm_item(
tcx,
&mut cx.global_asm,
item_id,
);
}
}
}
crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, false, cgu.is_primary());
let cgu_name = cgu.name().as_str().to_owned();
(cgu_name, cx, module, codegened_functions)
});
let cgu_name = cgu_name.as_str().to_owned();
let producer = crate::debuginfo::producer(tcx.sess);
let profiler = tcx.prof.clone();
OngoingModuleCodegen::Async(std::thread::spawn(move || {
cx.profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| {
profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| {
cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler(
cx.profiler.clone(),
profiler.clone(),
)));
let mut cached_context = Context::new();
for codegened_func in codegened_functions {
crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
crate::base::compile_fn(
&mut cx,
&profiler,
&mut cached_context,
&mut module,
codegened_func,
);
}
});
let global_asm_object_file =
cx.profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| {
profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| {
crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
})?;
let codegen_result =
cx.profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| {
profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| {
emit_cgu(
&global_asm_config.output_filenames,
&cx.profiler,
&profiler,
cgu_name,
module,
cx.debug_context,
@ -583,9 +601,63 @@ fn module_codegen(
}))
}
fn emit_metadata_module(tcx: TyCtxt<'_>, metadata: &EncodedMetadata) -> CompiledModule {
use rustc_middle::mir::mono::CodegenUnitNameBuilder;
let _timer = tcx.sess.timer("write compressed metadata");
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
let metadata_cgu_name = cgu_name_builder
.build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata"))
.as_str()
.to_string();
let tmp_file =
tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
let obj = create_compressed_metadata_file(tcx.sess, metadata, &symbol_name);
if let Err(err) = std::fs::write(&tmp_file, obj) {
tcx.dcx().fatal(format!("error writing metadata object file: {}", err));
}
CompiledModule {
name: metadata_cgu_name,
kind: ModuleKind::Metadata,
object: Some(tmp_file),
dwarf_object: None,
bytecode: None,
assembly: None,
llvm_ir: None,
}
}
fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> {
let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string());
let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module);
if created_alloc_shim {
let product = allocator_module.finish();
match emit_module(
tcx.output_filenames(()),
&tcx.sess.prof,
product.object,
ModuleKind::Allocator,
"allocator_shim".to_owned(),
&crate::debuginfo::producer(tcx.sess),
) {
Ok(allocator_module) => Some(allocator_module),
Err(err) => tcx.dcx().fatal(err),
}
} else {
None
}
}
pub(crate) fn run_aot(
tcx: TyCtxt<'_>,
backend_config: BackendConfig,
metadata: EncodedMetadata,
need_metadata_module: bool,
) -> Box<OngoingCodegen> {
@ -631,9 +703,10 @@ pub(crate) fn run_aot(
let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));
let disable_incr_cache = disable_incr_cache();
let (todo_cgus, done_cgus) =
cgus.into_iter().enumerate().partition::<Vec<_>, _>(|&(i, _)| match cgu_reuse[i] {
_ if backend_config.disable_incr_cache => true,
_ if disable_incr_cache => true,
CguReuse::No => true,
CguReuse::PreLto | CguReuse::PostLto => false,
});
@ -647,12 +720,7 @@ pub(crate) fn run_aot(
.with_task(
dep_node,
tcx,
(
backend_config.clone(),
global_asm_config.clone(),
cgu.name(),
concurrency_limiter.acquire(tcx.dcx()),
),
(global_asm_config.clone(), cgu.name(), concurrency_limiter.acquire(tcx.dcx())),
module_codegen,
Some(rustc_middle::dep_graph::hash_result),
)
@ -666,62 +734,10 @@ pub(crate) fn run_aot(
modules
});
let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module);
let allocator_module = emit_allocator_module(tcx);
let allocator_module = if created_alloc_shim {
let product = allocator_module.finish();
match emit_module(
tcx.output_filenames(()),
&tcx.sess.prof,
product.object,
ModuleKind::Allocator,
"allocator_shim".to_owned(),
&crate::debuginfo::producer(tcx.sess),
) {
Ok(allocator_module) => Some(allocator_module),
Err(err) => tcx.dcx().fatal(err),
}
} else {
None
};
let metadata_module = if need_metadata_module {
let (metadata_cgu_name, tmp_file) = tcx.sess.time("write compressed metadata", || {
use rustc_middle::mir::mono::CodegenUnitNameBuilder;
let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
let metadata_cgu_name = cgu_name_builder
.build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata"))
.as_str()
.to_string();
let tmp_file =
tcx.output_filenames(()).temp_path(OutputType::Metadata, Some(&metadata_cgu_name));
let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name);
if let Err(err) = std::fs::write(&tmp_file, obj) {
tcx.dcx().fatal(format!("error writing metadata object file: {}", err));
}
(metadata_cgu_name, tmp_file)
});
Some(CompiledModule {
name: metadata_cgu_name,
kind: ModuleKind::Metadata,
object: Some(tmp_file),
dwarf_object: None,
bytecode: None,
assembly: None,
llvm_ir: None,
})
} else {
None
};
let metadata_module =
if need_metadata_module { Some(emit_metadata_module(tcx, &metadata)) } else { None };
Box::new(OngoingCodegen {
modules,

29
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -11,12 +11,12 @@ use cranelift_jit::{JITBuilder, JITModule};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::Session;
use rustc_span::Symbol;
use rustc_span::sym;
use crate::debuginfo::TypeDebugContext;
use crate::prelude::*;
use crate::unwind_module::UnwindModule;
use crate::{BackendConfig, CodegenCx, CodegenMode};
use crate::{CodegenCx, CodegenMode};
struct JitState {
jit_module: UnwindModule<JITModule>,
@ -59,14 +59,10 @@ impl UnsafeMessage {
}
}
fn create_jit_module(
tcx: TyCtxt<'_>,
backend_config: &BackendConfig,
hotswap: bool,
) -> (UnwindModule<JITModule>, CodegenCx) {
fn create_jit_module(tcx: TyCtxt<'_>, hotswap: bool) -> (UnwindModule<JITModule>, CodegenCx) {
let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
let isa = crate::build_isa(tcx.sess, backend_config);
let isa = crate::build_isa(tcx.sess);
let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
jit_builder.hotswap(hotswap);
crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
@ -81,7 +77,7 @@ fn create_jit_module(
(jit_module, cx)
}
pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec<String>) -> ! {
if !tcx.sess.opts.output_types.should_codegen() {
tcx.dcx().fatal("JIT mode doesn't work with `cargo check`");
}
@ -90,11 +86,8 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
tcx.dcx().fatal("can't jit non-executable crate");
}
let (mut jit_module, mut cx) = create_jit_module(
tcx,
&backend_config,
matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
);
let (mut jit_module, mut cx) =
create_jit_module(tcx, matches!(codegen_mode, CodegenMode::JitLazy));
let mut cached_context = Context::new();
let (_, cgus) = tcx.collect_and_partition_mono_items(());
@ -110,7 +103,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
MonoItem::Fn(inst) => match codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
codegen_and_compile_fn(
@ -151,7 +144,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
);
let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
.chain(backend_config.jit_args.iter().map(|arg| &**arg))
.chain(jit_args.iter().map(|arg| &**arg))
.map(|arg| CString::new(arg).unwrap())
.collect::<Vec<_>>();
@ -211,7 +204,7 @@ pub(crate) fn codegen_and_compile_fn<'tcx>(
instance: Instance<'tcx>,
) {
cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler(
cx.profiler.clone(),
tcx.prof.clone(),
)));
tcx.prof.generic_activity("codegen and compile fn").run(|| {
@ -227,7 +220,7 @@ pub(crate) fn codegen_and_compile_fn<'tcx>(
module,
instance,
) {
crate::base::compile_fn(cx, cached_context, module, codegened_func);
crate::base::compile_fn(cx, &tcx.prof, cached_context, module, codegened_func);
}
});
}

15
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View file

@ -102,13 +102,12 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
// Pass a wrapper rather than the function itself as the function itself may not
// be exported from the main codegen unit and may thus be unreachable from the
// object file created by an external assembler.
let inline_asm_index = fx.cx.inline_asm_index.get();
fx.cx.inline_asm_index.set(inline_asm_index + 1);
let wrapper_name = format!(
"__inline_asm_{}_wrapper_n{}",
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
inline_asm_index
fx.cx.inline_asm_index
);
fx.cx.inline_asm_index += 1;
let sig =
get_function_sig(fx.tcx, fx.target_config.default_call_conv, instance);
create_wrapper_function(fx.module, sig, &wrapper_name, symbol.name);
@ -167,13 +166,12 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>(
asm_gen.allocate_registers();
asm_gen.allocate_stack_slots();
let inline_asm_index = fx.cx.inline_asm_index.get();
fx.cx.inline_asm_index.set(inline_asm_index + 1);
let asm_name = format!(
"__inline_asm_{}_n{}",
fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
inline_asm_index
fx.cx.inline_asm_index
);
fx.cx.inline_asm_index += 1;
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
fx.cx.global_asm.push_str(&generated_asm);
@ -266,13 +264,12 @@ pub(crate) fn codegen_naked_asm<'tcx>(
// Pass a wrapper rather than the function itself as the function itself may not
// be exported from the main codegen unit and may thus be unreachable from the
// object file created by an external assembler.
let inline_asm_index = cx.inline_asm_index.get();
cx.inline_asm_index.set(inline_asm_index + 1);
let wrapper_name = format!(
"__inline_asm_{}_wrapper_n{}",
cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
inline_asm_index
cx.inline_asm_index
);
cx.inline_asm_index += 1;
let sig =
get_function_sig(tcx, module.target_config().default_call_conv, instance);
create_wrapper_function(module, sig, &wrapper_name, symbol.name);

3
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -1270,8 +1270,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
}
sym::cold_path => {
// This is a no-op. The intrinsic is just a hint to the optimizer.
// We still have an impl here to avoid it being turned into a call.
fx.bcx.set_cold_block(fx.bcx.current_block().unwrap());
}
// Unimplemented intrinsics must have a fallback body. The fallback body is obtained

59
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -34,7 +34,7 @@ extern crate rustc_target;
extern crate rustc_driver;
use std::any::Any;
use std::cell::{Cell, RefCell};
use std::env;
use std::sync::Arc;
use cranelift_codegen::isa::TargetIsa;
@ -42,7 +42,6 @@ use cranelift_codegen::settings::{self, Configurable};
use rustc_codegen_ssa::CodegenResults;
use rustc_codegen_ssa::back::versioned_llvm_target;
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_session::Session;
@ -123,11 +122,10 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
/// The codegen context holds any information shared between the codegen of individual functions
/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
struct CodegenCx {
profiler: SelfProfilerRef,
output_filenames: Arc<OutputFilenames>,
should_write_ir: bool,
global_asm: String,
inline_asm_index: Cell<usize>,
inline_asm_index: usize,
debug_context: Option<DebugContext>,
cgu_name: Symbol,
}
@ -142,11 +140,10 @@ impl CodegenCx {
None
};
CodegenCx {
profiler: tcx.prof.clone(),
output_filenames: tcx.output_filenames(()).clone(),
should_write_ir: crate::pretty_clif::should_write_ir(tcx),
global_asm: String::new(),
inline_asm_index: Cell::new(0),
inline_asm_index: 0,
debug_context,
cgu_name,
}
@ -154,7 +151,7 @@ impl CodegenCx {
}
pub struct CraneliftCodegenBackend {
pub config: RefCell<Option<BackendConfig>>,
pub config: Option<BackendConfig>,
}
impl CodegenBackend for CraneliftCodegenBackend {
@ -176,13 +173,6 @@ impl CodegenBackend for CraneliftCodegenBackend {
sess.dcx()
.fatal("`-Cinstrument-coverage` is LLVM specific and not supported by Cranelift");
}
let mut config = self.config.borrow_mut();
if config.is_none() {
let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args)
.unwrap_or_else(|err| sess.dcx().fatal(err));
*config = Some(new_config);
}
}
fn target_features(&self, sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> {
@ -215,12 +205,15 @@ impl CodegenBackend for CraneliftCodegenBackend {
need_metadata_module: bool,
) -> Box<dyn Any> {
tcx.dcx().abort_if_errors();
let config = self.config.borrow().clone().unwrap();
let config = self.config.clone().unwrap_or_else(|| {
BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
.unwrap_or_else(|err| tcx.sess.dcx().fatal(err))
});
match config.codegen_mode {
CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
CodegenMode::Aot => driver::aot::run_aot(tcx, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
#[cfg(feature = "jit")]
driver::jit::run_jit(tcx, config);
driver::jit::run_jit(tcx, config.codegen_mode, config.jit_args);
#[cfg(not(feature = "jit"))]
tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift");
@ -236,14 +229,20 @@ impl CodegenBackend for CraneliftCodegenBackend {
) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
let _timer = sess.timer("finish_ongoing_codegen");
ongoing_codegen.downcast::<driver::aot::OngoingCodegen>().unwrap().join(
sess,
outputs,
self.config.borrow().as_ref().unwrap(),
)
ongoing_codegen.downcast::<driver::aot::OngoingCodegen>().unwrap().join(sess, outputs)
}
}
/// Determine if the Cranelift ir verifier should run.
///
/// Returns true when `-Zverify-llvm-ir` is passed, the `CG_CLIF_ENABLE_VERIFIER` env var is set to
/// 1 or when cg_clif is compiled with debug assertions enabled or false otherwise.
fn enable_verifier(sess: &Session) -> bool {
sess.verify_llvm_ir()
|| cfg!(debug_assertions)
|| env::var("CG_CLIF_ENABLE_VERIFIER").as_deref() == Ok("1")
}
fn target_triple(sess: &Session) -> target_lexicon::Triple {
// FIXME(madsmtm): Use `sess.target.llvm_target` once target-lexicon supports unversioned macOS.
// See <https://github.com/bytecodealliance/target-lexicon/pull/113>
@ -253,14 +252,14 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple {
}
}
fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIsa + 'static> {
fn build_isa(sess: &Session) -> Arc<dyn TargetIsa + 'static> {
use target_lexicon::BinaryFormat;
let target_triple = crate::target_triple(sess);
let mut flags_builder = settings::builder();
flags_builder.enable("is_pic").unwrap();
let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" };
let enable_verifier = if enable_verifier(sess) { "true" } else { "false" };
flags_builder.set("enable_verifier", enable_verifier).unwrap();
flags_builder.set("regalloc_checker", enable_verifier).unwrap();
@ -295,6 +294,16 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIs
}
}
if let target_lexicon::OperatingSystem::Windows = target_triple.operating_system {
// FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510
flags_builder.enable("enable_multi_ret_implicit_sret").unwrap();
}
if let target_lexicon::Architecture::S390x = target_triple.architecture {
// FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510
flags_builder.enable("enable_multi_ret_implicit_sret").unwrap();
}
if let target_lexicon::Architecture::Aarch64(_)
| target_lexicon::Architecture::Riscv64(_)
| target_lexicon::Architecture::X86_64 = target_triple.architecture
@ -347,5 +356,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIs
/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
#[no_mangle]
pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
Box::new(CraneliftCodegenBackend { config: RefCell::new(None) })
Box::new(CraneliftCodegenBackend { config: None })
}