Make intrinsic fallback bodies cross-crate inlineable

This change was prompted by the stage1 compiler spending 4% of its time
when compiling the polymorphic-recursion MIR opt test in `unlikely`.

Intrinsic fallback bodies like `unlikely` should always be inlined, it's
very silly if they are not. To do this, we enable the fallback bodies to
be cross-crate inlineable. Not that this matters for our workloads since
the compiler never actually _uses_ the "fallback bodies", it just uses
whatever was cfg(bootstrap)ped, so I've also added `#[inline]` to those.

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -9,6 +9,7 @@ use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::InliningThreshold;
 use rustc_session::config::OptLevel;
+use rustc_span::sym;
 
 pub fn provide(providers: &mut Providers) {
     providers.cross_crate_inlinable = cross_crate_inlinable;
@@ -34,6 +35,14 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         return true;
     }
 
+    if tcx.has_attr(def_id, sym::rustc_intrinsic) {
+        // Intrinsic fallback bodies are always cross-crate inlineable.
+        // To ensure that the MIR inliner doesn't cluelessly try to inline fallback
+        // bodies even when the backend would implement something better, we stop
+        // the MIR inliner from ever inlining an intrinsic.
+        return true;
+    }
+
     // Obey source annotations first; this is important because it means we can use
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {

compiler/rustc_mir_transform/src/inline.rs

@@ -13,6 +13,7 @@ use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Spanned;
+use rustc_span::sym;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -170,6 +171,13 @@ impl<'tcx> Inliner<'tcx> {
         let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id());
         self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;
 
+        // Intrinsic fallback bodies are automatically made cross-crate inlineable,
+        // but at this stage we don't know whether codegen knows the intrinsic,
+        // so just conservatively don't inline it.
+        if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) {
+            return Err("Callee is an intrinsic, do not inline fallback bodies");
+        }
+
         let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
         let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
         let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty;

library/core/src/intrinsics.rs

@@ -953,6 +953,7 @@ extern "rust-intrinsic" {
 #[rustc_nounwind]
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[cfg_attr(not(bootstrap), rustc_intrinsic)]
+#[cfg_attr(bootstrap, inline)]
 pub const unsafe fn assume(b: bool) {
     if !b {
         // SAFETY: the caller must guarantee the argument is never `false`
@@ -975,6 +976,7 @@ pub const unsafe fn assume(b: bool) {
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[cfg_attr(not(bootstrap), rustc_intrinsic)]
 #[rustc_nounwind]
+#[cfg_attr(bootstrap, inline)]
 pub const fn likely(b: bool) -> bool {
     b
 }
@@ -994,6 +996,7 @@ pub const fn likely(b: bool) -> bool {
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[cfg_attr(not(bootstrap), rustc_intrinsic)]
 #[rustc_nounwind]
+#[cfg_attr(bootstrap, inline)]
 pub const fn unlikely(b: bool) -> bool {
     b
 }
@@ -2556,6 +2559,7 @@ extern "rust-intrinsic" {
 #[rustc_nounwind]
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[cfg_attr(not(bootstrap), rustc_intrinsic)]
+#[cfg_attr(bootstrap, inline)]
 pub const fn is_val_statically_known<T: Copy>(_arg: T) -> bool {
     false
 }
@@ -2592,6 +2596,7 @@ pub(crate) const fn debug_assertions() -> bool {
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[rustc_nounwind]
 #[cfg_attr(not(bootstrap), rustc_intrinsic)]
+#[cfg_attr(bootstrap, inline)]
 pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 {
     // const eval overrides this function, but runtime code should always just return null pointers.
     crate::ptr::null_mut()
@@ -2611,6 +2616,7 @@ pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 {
 #[unstable(feature = "core_intrinsics", issue = "none")]
 #[rustc_nounwind]
 #[cfg_attr(not(bootstrap), rustc_intrinsic)]
+#[cfg_attr(bootstrap, inline)]
 pub const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {}
 
 // Some functions are defined here because they accidentally got made

src/doc/unstable-book/src/language-features/intrinsics.md

@@ -14,6 +14,8 @@ the intrinsic directly when you can.
 Many intrinsics can be written in pure rust, albeit inefficiently or without supporting
 some features that only exist on some backends. Backends can simply not implement those
 intrinsics without causing any code miscompilations or failures to compile.
+All intrinsic fallback bodies are automatically made cross-crate inlineable (like `#[inline]`)
+by the codegen backend, but not the MIR inliner.
 
 ```rust
 #![feature(rustc_attrs, effects)]