Auto merge of #121309 - Nilstrieb:inline-all-the-fallbacks, r=oli-obk

Make intrinsic fallback bodies cross-crate inlineable

This change was prompted by the stage1 compiler spending 4% of its time when compiling the polymorphic-recursion MIR opt test in `unlikely`.

Intrinsic fallback bodies like `unlikely` should always be inlined, it's very silly if they are not. To do this, we enable the fallback bodies to be cross-crate inlineable. Not that this matters for our workloads since the compiler never actually _uses_ the "fallback bodies", it just uses whatever was cfg(bootstrap)ped, so I've also added `#[inline]` to those.

See the comments for more information.

r? oli-obk

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -9,6 +9,7 @@ use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::InliningThreshold;
 use rustc_session::config::OptLevel;
+use rustc_span::sym;
 
 pub fn provide(providers: &mut Providers) {
     providers.cross_crate_inlinable = cross_crate_inlinable;
@@ -34,6 +35,14 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         return true;
     }
 
+    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
+        // Intrinsic fallback bodies are always cross-crate inlineable.
+        // To ensure that the MIR inliner doesn't cluelessly try to inline fallback
+        // bodies even when the backend would implement something better, we stop
+        // the MIR inliner from ever inlining an intrinsic.
+        return true;
+    }
+
     // Obey source annotations first; this is important because it means we can use
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {

compiler/rustc_mir_transform/src/inline.rs

@@ -13,6 +13,7 @@ use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Spanned;
+use rustc_span::sym;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -170,6 +171,13 @@ impl<'tcx> Inliner<'tcx> {
         let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id());
         self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;
 
+        // Intrinsic fallback bodies are automatically made cross-crate inlineable,
+        // but at this stage we don't know whether codegen knows the intrinsic,
+        // so just conservatively don't inline it.
+        if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) {
+            return Err("Callee is an intrinsic, do not inline fallback bodies");
+        }
+
         let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
         let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
         let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty;