Add -Zcross-crate-inline-threshold=yes

compiler/rustc_interface/src/tests.rs

@@ -4,11 +4,11 @@ use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::{
     build_configuration, build_session_options, rustc_optgroups, BranchProtection, CFGuard, Cfg,
-    DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, Input,
+    DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs,
-    InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli,
+    InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained,
-    MirSpanview, OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet,
+    LinkerPluginLto, LocationDetail, LtoCli, MirSpanview, OomStrategy, Options, OutFileName,
-    Passes, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion,
+    OutputType, OutputTypes, PAuthKey, PacRet, Passes, Polonius, ProcMacroExecutionStrategy, Strip,
-    TraitSolver, WasiExecModel,
+    SwitchWithOptPath, SymbolManglingVersion, TraitSolver, WasiExecModel,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
@@ -748,7 +748,7 @@ fn test_unstable_options_tracking_hash() {
     );
     tracked!(codegen_backend, Some("abc".to_string()));
     tracked!(crate_attr, vec!["abc".to_string()]);
-    tracked!(cross_crate_inline_threshold, Some(200));
+    tracked!(cross_crate_inline_threshold, InliningThreshold::Always);
     tracked!(debug_info_for_profiling, true);
     tracked!(debug_macros, true);
     tracked!(dep_info_omit_d_target, true);

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -7,6 +7,7 @@ use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
+use rustc_session::config::InliningThreshold;
 use rustc_session::config::OptLevel;
 
 pub fn provide(providers: &mut Providers) {
@@ -54,6 +55,12 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         return false;
     }
 
+    let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold {
+        InliningThreshold::Always => return true,
+        InliningThreshold::Sometimes(threshold) => threshold,
+        InliningThreshold::Never => return false,
+    };
+
     let mir = tcx.optimized_mir(def_id);
     let mut checker =
         CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };
@@ -61,8 +68,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     checker.calls == 0
         && checker.resumes == 0
         && checker.landing_pads == 0
-        && checker.statements
+        && checker.statements <= threshold
-            <= tcx.sess.opts.unstable_opts.cross_crate_inline_threshold.unwrap_or(100)
 }
 
 struct CostChecker<'b, 'tcx> {

compiler/rustc_session/src/config.rs

@@ -3161,10 +3161,10 @@ impl PpMode {
 pub(crate) mod dep_tracking {
     use super::{
         BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression,
-        ErrorOutputType, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail,
+        ErrorOutputType, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto,
-        LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Polonius,
+        LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes,
-        RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind,
+        Polonius, RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm,
-        SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
+        SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
     };
     use crate::lint;
     use crate::options::WasiExecModel;
@@ -3270,6 +3270,7 @@ pub(crate) mod dep_tracking {
         LanguageIdentifier,
         TraitSolver,
         Polonius,
+        InliningThreshold,
     );
 
     impl<T1, T2> DepTrackingHash for (T1, T2)
@@ -3435,3 +3436,16 @@ impl Polonius {
         matches!(self, Polonius::Next)
     }
 }
+
+#[derive(Clone, Copy, PartialEq, Hash, Debug)]
+pub enum InliningThreshold {
+    Always,
+    Sometimes(usize),
+    Never,
+}
+
+impl Default for InliningThreshold {
+    fn default() -> Self {
+        Self::Sometimes(100)
+    }
+}

compiler/rustc_session/src/options.rs

@@ -428,6 +428,8 @@ mod desc {
         "one of supported execution strategies (`same-thread`, or `cross-thread`)";
     pub const parse_dump_solver_proof_tree: &str = "one of: `always`, `on-request`, `on-error`";
     pub const parse_remap_path_scope: &str = "comma separated list of scopes: `macro`, `diagnostics`, `unsplit-debuginfo`, `split-debuginfo`, `split-debuginfo-path`, `object`, `all`";
+    pub const parse_inlining_threshold: &str =
+        "either a boolean (`yes`, `no`, `on`, `off`, etc), or a non-negative number";
 }
 
 mod parse {
@@ -1310,6 +1312,26 @@ mod parse {
         };
         true
     }
+
+    pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool {
+        match v {
+            Some("always" | "yes") => {
+                *slot = InliningThreshold::Always;
+            }
+            Some("never") => {
+                *slot = InliningThreshold::Never;
+            }
+            Some(v) => {
+                if let Ok(threshold) = v.parse() {
+                    *slot = InliningThreshold::Sometimes(threshold);
+                } else {
+                    return false;
+                }
+            }
+            None => return false,
+        }
+        true
+    }
 }
 
 options! {
@@ -1479,7 +1501,7 @@ options! {
         "combine CGUs into a single one"),
     crate_attr: Vec<String> = (Vec::new(), parse_string_push, [TRACKED],
         "inject the given attribute in the crate"),
-    cross_crate_inline_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
+    cross_crate_inline_threshold: InliningThreshold = (InliningThreshold::Sometimes(100), parse_inlining_threshold, [TRACKED],
         "threshold to allow cross crate inlining of functions"),
     debug_info_for_profiling: bool = (false, parse_bool, [TRACKED],
         "emit discriminators and other data necessary for AutoFDO"),

tests/codegen/cross-crate-inlining/always-inline.rs

@@ -0,0 +1,13 @@
+// compile-flags: -O
+// aux-build:always.rs
+
+#![crate_type = "lib"]
+
+extern crate always;
+
+// Check that we inline a cross-crate call, even though it isn't a leaf
+#[no_mangle]
+pub fn outer() -> String {
+    // CHECK-NOT: call {{.*}}stem_fn
+    always::stem_fn()
+}

tests/codegen/cross-crate-inlining/auxiliary/always.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O -Zcross-crate-inline-threshold=always
+
+#![crate_type = "lib"]
+
+// This function *looks* like it contains a call, but that call will be optimized out by MIR
+// optimizations.
+pub fn leaf_fn() -> String {
+    String::new()
+}
+
+// This function contains a call, even after MIR optimizations. It is only eligible for
+// cross-crate-inlining with "always".
+pub fn stem_fn() -> String {
+    inner()
+}
+
+#[inline(never)]
+fn inner() -> String {
+    String::from("test")
+}

tests/codegen/cross-crate-inlining/auxiliary/leaf.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O
+
+#![crate_type = "lib"]
+
+// This function *looks* like it contains a call, but that call will be optimized out by MIR
+// optimizations.
+pub fn leaf_fn() -> String {
+    String::new()
+}
+
+// This function contains a call, even after MIR optimizations. It is only eligible for
+// cross-crate-inlining with "always".
+pub fn stem_fn() -> String {
+    inner()
+}
+
+#[inline(never)]
+fn inner() -> String {
+    String::from("test")
+}

tests/codegen/cross-crate-inlining/auxiliary/never.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O -Zcross-crate-inline-threshold=never
+
+#![crate_type = "lib"]
+
+// This function *looks* like it contains a call, but that call will be optimized out by MIR
+// optimizations.
+pub fn leaf_fn() -> String {
+    String::new()
+}
+
+// This function contains a call, even after MIR optimizations. It is only eligible for
+// cross-crate-inlining with "always".
+pub fn stem_fn() -> String {
+    inner()
+}
+
+#[inline(never)]
+fn inner() -> String {
+    String::from("test")
+}

tests/codegen/cross-crate-inlining/leaf-inlining.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O -Zcross-crate-inline-threshold=yes
+// aux-build:leaf.rs
+
+#![crate_type = "lib"]
+
+extern crate leaf;
+
+// Check that we inline a leaf cross-crate call
+#[no_mangle]
+pub fn leaf_outer() -> String {
+    // CHECK-NOT: call {{.*}}leaf_fn
+    leaf::leaf_fn()
+}
+
+// Check that we do not inline a non-leaf cross-crate call
+#[no_mangle]
+pub fn stem_outer() -> String {
+    // CHECK: call {{.*}}stem_fn
+    leaf::stem_fn()
+}

tests/codegen/cross-crate-inlining/never-inline.rs

@@ -0,0 +1,13 @@
+// compile-flags: -O
+// aux-build:never.rs
+
+#![crate_type = "lib"]
+
+extern crate never;
+
+// Check that we do not inline a cross-crate call, even though it is a leaf
+#[no_mangle]
+pub fn outer() -> String {
+    // CHECK: call {{.*}}leaf_fn
+    never::leaf_fn()
+}