Explicitly don't inline user-written rust-call fns

compiler/rustc_mir_transform/src/inline.rs

@@ -222,7 +222,14 @@ impl<'tcx> Inliner<'tcx> {
             trace!(?output_type, ?destination_ty);
             return Err("failed to normalize return type");
         }
-        if callsite.fn_sig.abi() == Abi::RustCall && callee_body.spread_arg.is_none() {
+        if callsite.fn_sig.abi() == Abi::RustCall {
+            // FIXME: Don't inline user-written `extern "rust-call"` functions,
+            // since this is generally perf-negative on rustc, and we hope that
+            // LLVM will inline these functions instead.
+            if callee_body.spread_arg.is_some() {
+                return Err("do not inline user-written rust-call functions");
+            }
+
             let (self_arg, arg_tuple) = match &args[..] {
                 [arg_tuple] => (None, arg_tuple),
                 [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),

tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff

@@ -5,13 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {main};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {main}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {main};
 +         let _4: ();
-+         scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+         }
 +     }
   
       bb0: {
@@ -22,9 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
++         _4 = <fn() {main} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind unreachable];
-+         _5 = const ();
-+         _4 = move (*_3)() -> [return: bb2, unwind unreachable];
       }
   
       bb1: {
@@ -36,7 +31,6 @@
 +     }
 + 
 +     bb2: {
-+         StorageDead(_5);
 +         StorageDead(_3);
 +         drop(_2) -> [return: bb1, unwind unreachable];
       }

tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff

@@ -5,13 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {main};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {main}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {main};
 +         let _4: ();
-+         scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+         }
 +     }
   
       bb0: {
@@ -22,9 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
++         _4 = <fn() {main} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind: bb3];
-+         _5 = const ();
-+         _4 = move (*_3)() -> [return: bb4, unwind: bb2];
       }
   
       bb1: {
@@ -35,18 +30,17 @@
           return;
 +     }
 + 
-+     bb2 (cleanup): {
++     bb2: {
-+         drop(_2) -> [return: bb3, unwind terminate];
++         StorageDead(_3);
++         drop(_2) -> [return: bb1, unwind continue];
 +     }
 + 
 +     bb3 (cleanup): {
-+         resume;
++         drop(_2) -> [return: bb4, unwind terminate];
 +     }
 + 
-+     bb4: {
++     bb4 (cleanup): {
-+         StorageDead(_5);
++         resume;
-+         StorageDead(_3);
-+         drop(_2) -> [return: bb1, unwind continue];
       }
   }
   

tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff

@@ -5,21 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {g};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {g}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {g};
 +         let _4: ();
-+         scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) {
-+             scope 3 (inlined g) {
-+                 scope 4 (inlined f::<fn() {main}>) {
-+                     debug g => main;
-+                     let _6: ();
-+                     scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -30,10 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
++         _4 = <fn() {g} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind unreachable];
-+         _5 = const ();
-+         StorageLive(_6);
-+         _6 = main() -> [return: bb2, unwind unreachable];
       }
   
       bb1: {
@@ -45,8 +31,6 @@
 +     }
 + 
 +     bb2: {
-+         StorageDead(_6);
-+         StorageDead(_5);
 +         StorageDead(_3);
 +         drop(_2) -> [return: bb1, unwind unreachable];
       }

tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff

@@ -5,21 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {g};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {g}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {g};
 +         let _4: ();
-+         scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) {
-+             scope 3 (inlined g) {
-+                 scope 4 (inlined f::<fn() {main}>) {
-+                     debug g => main;
-+                     let _6: ();
-+                     scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -30,10 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
++         _4 = <fn() {g} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind: bb3];
-+         _5 = const ();
-+         StorageLive(_6);
-+         _6 = main() -> [return: bb4, unwind: bb2];
       }
   
       bb1: {
@@ -44,19 +30,17 @@
           return;
 +     }
 + 
-+     bb2 (cleanup): {
++     bb2: {
-+         drop(_2) -> [return: bb3, unwind terminate];
++         StorageDead(_3);
++         drop(_2) -> [return: bb1, unwind continue];
 +     }
 + 
 +     bb3 (cleanup): {
-+         resume;
++         drop(_2) -> [return: bb4, unwind terminate];
 +     }
 + 
-+     bb4: {
++     bb4 (cleanup): {
-+         StorageDead(_6);
++         resume;
-+         StorageDead(_5);
-+         StorageDead(_3);
-+         drop(_2) -> [return: bb1, unwind continue];
       }
   }
   

tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.diff

@@ -7,40 +7,23 @@
       let mut _0: ();
       let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
       let mut _4: I;
-+     scope 1 (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut) {
-+         debug self => _3;
-+         debug args => _4;
-+         let mut _5: &mut dyn std::ops::FnMut<I, Output = ()>;
-+         let mut _6: std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
-+         let mut _7: *const dyn std::ops::FnMut<I, Output = ()>;
-+     }
   
       bb0: {
           StorageLive(_3);
           _3 = &mut _1;
           StorageLive(_4);
           _4 = move _2;
--         _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
+          _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
-+         StorageLive(_5);
-+         _6 = deref_copy (*_3);
-+         _7 = (((_6.0: std::ptr::Unique<dyn std::ops::FnMut<I, Output = ()>>).0: std::ptr::NonNull<dyn std::ops::FnMut<I, Output = ()>>).0: *const dyn std::ops::FnMut<I, Output = ()>);
-+         _5 = &mut (*_7);
-+         _0 = <dyn FnMut<I, Output = ()> as FnMut<I>>::call_mut(move _5, move _4) -> [return: bb2, unwind unreachable];
       }
   
       bb1: {
--         StorageDead(_4);
+          StorageDead(_4);
--         StorageDead(_3);
+          StorageDead(_3);
--         drop(_1) -> [return: bb2, unwind unreachable];
+          drop(_1) -> [return: bb2, unwind unreachable];
-+         return;
       }
   
       bb2: {
--         return;
+          return;
-+         StorageDead(_5);
-+         StorageDead(_4);
-+         StorageDead(_3);
-+         drop(_1) -> [return: bb1, unwind unreachable];
       }
   }
   

tests/mir-opt/inline/inline_box_fn.call.Inline.diff

@@ -7,13 +7,6 @@
       let _2: ();
       let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
       let mut _4: (i32,);
-+     scope 1 (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call) {
-+         debug self => _3;
-+         debug args => _4;
-+         let mut _5: &dyn std::ops::Fn(i32);
-+         let mut _6: std::boxed::Box<dyn std::ops::Fn(i32)>;
-+         let mut _7: *const dyn std::ops::Fn(i32);
-+     }
   
       bb0: {
           StorageLive(_2);
@@ -21,30 +14,19 @@
           _3 = &_1;
           StorageLive(_4);
           _4 = (const 1_i32,);
--         _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
+          _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
-+         StorageLive(_5);
-+         _6 = deref_copy (*_3);
-+         _7 = (((_6.0: std::ptr::Unique<dyn std::ops::Fn(i32)>).0: std::ptr::NonNull<dyn std::ops::Fn(i32)>).0: *const dyn std::ops::Fn(i32));
-+         _5 = &(*_7);
-+         _2 = <dyn Fn(i32) as Fn<(i32,)>>::call(move _5, move _4) -> [return: bb2, unwind unreachable];
       }
   
       bb1: {
-+         return;
-+     }
-+ 
-+     bb2: {
-+         StorageDead(_5);
           StorageDead(_4);
           StorageDead(_3);
           StorageDead(_2);
           _0 = const ();
--         drop(_1) -> [return: bb2, unwind unreachable];
+          drop(_1) -> [return: bb2, unwind unreachable];
--     }
+      }
-- 
+  
--     bb2: {
+      bb2: {
--         return;
+          return;
-+         drop(_1) -> [return: bb1, unwind unreachable];
       }
   }
   

tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff

@@ -5,20 +5,9 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {f};
-+     let mut _4: ();
 +     scope 1 (inlined call::<fn() {f}>) {
 +         debug f => _2;
 +         let _3: ();
-+         scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+             scope 3 (inlined f) {
-+                 scope 4 (inlined call::<fn() {f}>) {
-+                     debug f => f;
-+                     let _5: ();
-+                     scope 5 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -27,15 +16,10 @@
 +         StorageLive(_2);
 +         _2 = f;
 +         StorageLive(_3);
-+         StorageLive(_4);
++         _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind unreachable];
-+         _4 = const ();
-+         StorageLive(_5);
-+         _5 = f() -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
-+         StorageDead(_5);
-+         StorageDead(_4);
 +         StorageDead(_3);
 +         StorageDead(_2);
           StorageDead(_1);

tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff

@@ -5,20 +5,9 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {f};
-+     let mut _4: ();
 +     scope 1 (inlined call::<fn() {f}>) {
 +         debug f => _2;
 +         let _3: ();
-+         scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+             scope 3 (inlined f) {
-+                 scope 4 (inlined call::<fn() {f}>) {
-+                     debug f => f;
-+                     let _5: ();
-+                     scope 5 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -27,15 +16,10 @@
 +         StorageLive(_2);
 +         _2 = f;
 +         StorageLive(_3);
-+         StorageLive(_4);
++         _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind continue];
-+         _4 = const ();
-+         StorageLive(_5);
-+         _5 = f() -> [return: bb1, unwind continue];
       }
   
       bb1: {
-+         StorageDead(_5);
-+         StorageDead(_4);
 +         StorageDead(_3);
 +         StorageDead(_2);
           StorageDead(_1);

tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff

@@ -5,7 +5,6 @@
       let mut _0: ();
       let _1: (!, !);
 +     let mut _2: fn() -> ! {sleep};
-+     let mut _7: ();
 +     scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) {
 +         debug f => _2;
 +         let mut _3: &fn() -> ! {sleep};
@@ -18,10 +17,6 @@
 +                 debug b => _6;
 +             }
 +         }
-+         scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
-+             scope 5 (inlined sleep) {
-+             }
-+         }
 +     }
   
       bb0: {
@@ -33,13 +28,24 @@
 +         StorageLive(_6);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_7);
++         _4 = <fn() -> ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind unreachable];
-+         _7 = const ();
-+         goto -> bb1;
 +     }
 + 
 +     bb1: {
-+         goto -> bb1;
++         StorageDead(_3);
++         StorageLive(_5);
++         _5 = &_2;
++         _6 = <fn() -> ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind unreachable];
++     }
++ 
++     bb2: {
++         StorageDead(_5);
++         _1 = (move _4, move _6);
++         drop(_2) -> [return: bb3, unwind unreachable];
++     }
++ 
++     bb3: {
++         unreachable;
       }
   }
   

tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff

@@ -5,7 +5,6 @@
       let mut _0: ();
       let _1: (!, !);
 +     let mut _2: fn() -> ! {sleep};
-+     let mut _8: ();
 +     scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) {
 +         debug f => _2;
 +         let mut _3: &fn() -> ! {sleep};
@@ -19,10 +18,6 @@
 +                 debug b => _6;
 +             }
 +         }
-+         scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
-+             scope 5 (inlined sleep) {
-+             }
-+         }
 +     }
   
       bb0: {
@@ -34,13 +29,40 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_8);
++         _4 = <fn() -> ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind: bb5];
-+         _8 = const ();
-+         goto -> bb1;
 +     }
 + 
 +     bb1: {
-+         goto -> bb1;
++         StorageDead(_3);
++         StorageLive(_5);
++         _5 = &_2;
++         _6 = <fn() -> ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind: bb4];
++     }
++ 
++     bb2: {
++         StorageDead(_5);
++         StorageLive(_7);
++         _7 = move _4;
++         _1 = (move _7, move _6);
++         StorageDead(_7);
++         StorageDead(_4);
++         drop(_2) -> [return: bb3, unwind continue];
++     }
++ 
++     bb3: {
++         unreachable;
++     }
++ 
++     bb4 (cleanup): {
++         drop(_4) -> [return: bb5, unwind terminate];
++     }
++ 
++     bb5 (cleanup): {
++         drop(_2) -> [return: bb6, unwind terminate];
++     }
++ 
++     bb6 (cleanup): {
++         resume;
       }
   }
   

tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff

@@ -8,8 +8,6 @@
       let mut _3: &fn() {foo};
       let _4: fn() {foo};
       let mut _5: ();
-+     scope 1 (inlined <fn() {foo} as Fn<()>>::call - shim(fn() {foo})) {
-+     }
   
       bb0: {
           StorageLive(_2);
@@ -22,26 +20,20 @@
           _3 = &_4;
           StorageLive(_5);
           _5 = ();
--         _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind unreachable];
+          _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind unreachable];
-+         _2 = move (*_3)() -> [return: bb3, unwind unreachable];
       }
   
       bb2: {
-+         return;
-+     }
-+ 
-+     bb3: {
           StorageDead(_5);
           StorageDead(_3);
           StorageDead(_4);
           StorageDead(_2);
           _0 = const ();
--         drop(_1) -> [return: bb3, unwind unreachable];
+          drop(_1) -> [return: bb3, unwind unreachable];
--     }
+      }
-- 
+  
--     bb3: {
+      bb3: {
--         return;
+          return;
-+         drop(_1) -> [return: bb2, unwind unreachable];
       }
   }
   

tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff

@@ -8,55 +8,40 @@
       let mut _3: &fn() {foo};
       let _4: fn() {foo};
       let mut _5: ();
-+     scope 1 (inlined <fn() {foo} as Fn<()>>::call - shim(fn() {foo})) {
-+     }
   
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = hide_foo() -> [return: bb1, unwind: bb4];
+          _4 = hide_foo() -> [return: bb1, unwind: bb4];
-+         _4 = hide_foo() -> [return: bb1, unwind: bb3];
       }
   
       bb1: {
           _3 = &_4;
           StorageLive(_5);
           _5 = ();
--         _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind: bb4];
+          _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind: bb4];
-+         _2 = move (*_3)() -> [return: bb5, unwind: bb3];
       }
   
       bb2: {
--         StorageDead(_5);
+          StorageDead(_5);
--         StorageDead(_3);
+          StorageDead(_3);
--         StorageDead(_4);
+          StorageDead(_4);
--         StorageDead(_2);
+          StorageDead(_2);
--         _0 = const ();
+          _0 = const ();
--         drop(_1) -> [return: bb3, unwind: bb5];
+          drop(_1) -> [return: bb3, unwind: bb5];
-+         return;
       }
   
--     bb3: {
+      bb3: {
--         return;
+          return;
-+     bb3 (cleanup): {
-+         drop(_1) -> [return: bb4, unwind terminate];
       }
   
       bb4 (cleanup): {
--         drop(_1) -> [return: bb5, unwind terminate];
+          drop(_1) -> [return: bb5, unwind terminate];
-+         resume;
       }
   
--     bb5 (cleanup): {
+      bb5 (cleanup): {
--         resume;
+          resume;
-+     bb5: {
-+         StorageDead(_5);
-+         StorageDead(_3);
-+         StorageDead(_4);
-+         StorageDead(_2);
-+         _0 = const ();
-+         drop(_1) -> [return: bb2, unwind: bb4];
       }
   }