Auto merge of #110833 - compiler-errors:rustc-call-inliner-ice, r=cjgillot

Only unpack tupled args in inliner if we expect args to be unpacked

`"rust-call"` is a strange function abi. sometimes, it expects the arguments to be unpacked by the caller and passed as individual args (closure bodies), and sometimes it does not (user functions annotated with the `"rust-call"` abi).

make sure the mir inliner respects this difference when checking that arguments are compatible, and doesn't try to ICE when we call a `extern "rust-call"` function in a generic context.

fixes #110829

compiler/rustc_mir_transform/src/inline.rs

@@ -223,19 +223,29 @@ impl<'tcx> Inliner<'tcx> {
             return Err("failed to normalize return type");
         }
         if callsite.fn_sig.abi() == Abi::RustCall {
-            let (arg_tuple, skipped_args) = match &args[..] {
-                [arg_tuple] => (arg_tuple, 0),
-                [_, arg_tuple] => (arg_tuple, 1),
+            // FIXME: Don't inline user-written `extern "rust-call"` functions,
+            // since this is generally perf-negative on rustc, and we hope that
+            // LLVM will inline these functions instead.
+            if callee_body.spread_arg.is_some() {
+                return Err("do not inline user-written rust-call functions");
+            }
+
+            let (self_arg, arg_tuple) = match &args[..] {
+                [arg_tuple] => (None, arg_tuple),
+                [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
                 _ => bug!("Expected `rust-call` to have 1 or 2 args"),
             };
 
+            let self_arg_ty =
+                self_arg.map(|self_arg| self_arg.ty(&caller_body.local_decls, self.tcx));
+
             let arg_tuple_ty = arg_tuple.ty(&caller_body.local_decls, self.tcx);
-            let ty::Tuple(arg_tuple_tys) = arg_tuple_ty.kind() else {
+            let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
                 bug!("Closure arguments are not passed as a tuple");
             };
 
             for (arg_ty, input) in
-                arg_tuple_tys.iter().zip(callee_body.args_iter().skip(skipped_args))
+                self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter())
             {
                 let input_type = callee_body.local_decls[input].ty;
                 if !util::is_subtype(self.tcx, self.param_env, input_type, arg_ty) {

tests/mir-opt/inline/cycle.g.Inline.panic-abort.diff

@@ -5,13 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {main};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {main}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {main};
 +         let _4: ();
-+         scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+         }
 +     }
   
       bb0: {
@@ -22,9 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
-+         _5 = const ();
-+         _4 = move (*_3)() -> [return: bb2, unwind unreachable];
++         _4 = <fn() {main} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind unreachable];
       }
   
       bb1: {
@@ -36,7 +31,6 @@
 +     }
 + 
 +     bb2: {
-+         StorageDead(_5);
 +         StorageDead(_3);
 +         drop(_2) -> [return: bb1, unwind unreachable];
       }

tests/mir-opt/inline/cycle.g.Inline.panic-unwind.diff

@@ -5,13 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {main};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {main}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {main};
 +         let _4: ();
-+         scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+         }
 +     }
   
       bb0: {
@@ -22,9 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
-+         _5 = const ();
-+         _4 = move (*_3)() -> [return: bb4, unwind: bb2];
++         _4 = <fn() {main} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind: bb3];
       }
   
       bb1: {
@@ -35,18 +30,17 @@
           return;
 +     }
 + 
-+     bb2 (cleanup): {
-+         drop(_2) -> [return: bb3, unwind terminate];
++     bb2: {
++         StorageDead(_3);
++         drop(_2) -> [return: bb1, unwind continue];
 +     }
 + 
 +     bb3 (cleanup): {
-+         resume;
++         drop(_2) -> [return: bb4, unwind terminate];
 +     }
 + 
-+     bb4: {
-+         StorageDead(_5);
-+         StorageDead(_3);
-+         drop(_2) -> [return: bb1, unwind continue];
++     bb4 (cleanup): {
++         resume;
       }
   }
   

tests/mir-opt/inline/cycle.main.Inline.panic-abort.diff

@@ -5,21 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {g};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {g}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {g};
 +         let _4: ();
-+         scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) {
-+             scope 3 (inlined g) {
-+                 scope 4 (inlined f::<fn() {main}>) {
-+                     debug g => main;
-+                     let _6: ();
-+                     scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -30,10 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
-+         _5 = const ();
-+         StorageLive(_6);
-+         _6 = main() -> [return: bb2, unwind unreachable];
++         _4 = <fn() {g} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind unreachable];
       }
   
       bb1: {
@@ -45,8 +31,6 @@
 +     }
 + 
 +     bb2: {
-+         StorageDead(_6);
-+         StorageDead(_5);
 +         StorageDead(_3);
 +         drop(_2) -> [return: bb1, unwind unreachable];
       }

tests/mir-opt/inline/cycle.main.Inline.panic-unwind.diff

@@ -5,21 +5,10 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {g};
-+     let mut _5: ();
 +     scope 1 (inlined f::<fn() {g}>) {
 +         debug g => _2;
 +         let mut _3: &fn() {g};
 +         let _4: ();
-+         scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) {
-+             scope 3 (inlined g) {
-+                 scope 4 (inlined f::<fn() {main}>) {
-+                     debug g => main;
-+                     let _6: ();
-+                     scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -30,10 +19,7 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_5);
-+         _5 = const ();
-+         StorageLive(_6);
-+         _6 = main() -> [return: bb4, unwind: bb2];
++         _4 = <fn() {g} as Fn<()>>::call(move _3, const ()) -> [return: bb2, unwind: bb3];
       }
   
       bb1: {
@@ -44,19 +30,17 @@
           return;
 +     }
 + 
-+     bb2 (cleanup): {
-+         drop(_2) -> [return: bb3, unwind terminate];
++     bb2: {
++         StorageDead(_3);
++         drop(_2) -> [return: bb1, unwind continue];
 +     }
 + 
 +     bb3 (cleanup): {
-+         resume;
++         drop(_2) -> [return: bb4, unwind terminate];
 +     }
 + 
-+     bb4: {
-+         StorageDead(_6);
-+         StorageDead(_5);
-+         StorageDead(_3);
-+         drop(_2) -> [return: bb1, unwind continue];
++     bb4 (cleanup): {
++         resume;
       }
   }
   

tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-abort.diff

@@ -0,0 +1,29 @@
+- // MIR for `call` before Inline
++ // MIR for `call` after Inline
+  
+  fn call(_1: Box<dyn FnMut<I, Output = ()>>, _2: I) -> () {
+      debug mock => _1;
+      debug input => _2;
+      let mut _0: ();
+      let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
+      let mut _4: I;
+  
+      bb0: {
+          StorageLive(_3);
+          _3 = &mut _1;
+          StorageLive(_4);
+          _4 = move _2;
+          _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+          StorageDead(_3);
+          drop(_1) -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          return;
+      }
+  }
+  

tests/mir-opt/inline/dont_ice_on_generic_rust_call.call.Inline.panic-unwind.diff

@@ -0,0 +1,37 @@
+- // MIR for `call` before Inline
++ // MIR for `call` after Inline
+  
+  fn call(_1: Box<dyn FnMut<I, Output = ()>>, _2: I) -> () {
+      debug mock => _1;
+      debug input => _2;
+      let mut _0: ();
+      let mut _3: &mut std::boxed::Box<dyn std::ops::FnMut<I, Output = ()>>;
+      let mut _4: I;
+  
+      bb0: {
+          StorageLive(_3);
+          _3 = &mut _1;
+          StorageLive(_4);
+          _4 = move _2;
+          _0 = <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut(move _3, move _4) -> [return: bb1, unwind: bb3];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+          StorageDead(_3);
+          drop(_1) -> [return: bb2, unwind: bb4];
+      }
+  
+      bb2: {
+          return;
+      }
+  
+      bb3 (cleanup): {
+          drop(_1) -> [return: bb4, unwind terminate];
+      }
+  
+      bb4 (cleanup): {
+          resume;
+      }
+  }
+  

tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs

@@ -0,0 +1,11 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// compile-flags: -Zmir-enable-passes=+Inline --crate-type=lib
+
+#![feature(fn_traits, tuple_trait, unboxed_closures)]
+
+use std::marker::Tuple;
+
+// EMIT_MIR dont_ice_on_generic_rust_call.call.Inline.diff
+pub fn call<I: Tuple>(mut mock: Box<dyn FnMut<I, Output = ()>>, input: I) {
+    mock.call_mut(input)
+}

tests/mir-opt/inline/inline_box_fn.call.Inline.panic-abort.diff

@@ -0,0 +1,32 @@
+- // MIR for `call` before Inline
++ // MIR for `call` after Inline
+  
+  fn call(_1: Box<dyn Fn(i32)>) -> () {
+      debug x => _1;
+      let mut _0: ();
+      let _2: ();
+      let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
+      let mut _4: (i32,);
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = &_1;
+          StorageLive(_4);
+          _4 = (const 1_i32,);
+          _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+          StorageDead(_3);
+          StorageDead(_2);
+          _0 = const ();
+          drop(_1) -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          return;
+      }
+  }
+  

tests/mir-opt/inline/inline_box_fn.call.Inline.panic-unwind.diff

@@ -0,0 +1,40 @@
+- // MIR for `call` before Inline
++ // MIR for `call` after Inline
+  
+  fn call(_1: Box<dyn Fn(i32)>) -> () {
+      debug x => _1;
+      let mut _0: ();
+      let _2: ();
+      let mut _3: &std::boxed::Box<dyn std::ops::Fn(i32)>;
+      let mut _4: (i32,);
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = &_1;
+          StorageLive(_4);
+          _4 = (const 1_i32,);
+          _2 = <Box<dyn Fn(i32)> as Fn<(i32,)>>::call(move _3, move _4) -> [return: bb1, unwind: bb3];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+          StorageDead(_3);
+          StorageDead(_2);
+          _0 = const ();
+          drop(_1) -> [return: bb2, unwind: bb4];
+      }
+  
+      bb2: {
+          return;
+      }
+  
+      bb3 (cleanup): {
+          drop(_1) -> [return: bb4, unwind terminate];
+      }
+  
+      bb4 (cleanup): {
+          resume;
+      }
+  }
+  

tests/mir-opt/inline/inline_box_fn.rs

@@ -0,0 +1,8 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// unit-test: Inline
+// compile-flags: --crate-type=lib
+
+// EMIT_MIR inline_box_fn.call.Inline.diff
+fn call(x: Box<dyn Fn(i32)>) {
+    x(1);
+}

tests/mir-opt/inline/inline_cycle.two.Inline.panic-abort.diff

@@ -5,20 +5,9 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {f};
-+     let mut _4: ();
 +     scope 1 (inlined call::<fn() {f}>) {
 +         debug f => _2;
 +         let _3: ();
-+         scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+             scope 3 (inlined f) {
-+                 scope 4 (inlined call::<fn() {f}>) {
-+                     debug f => f;
-+                     let _5: ();
-+                     scope 5 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -27,15 +16,10 @@
 +         StorageLive(_2);
 +         _2 = f;
 +         StorageLive(_3);
-+         StorageLive(_4);
-+         _4 = const ();
-+         StorageLive(_5);
-+         _5 = f() -> [return: bb1, unwind unreachable];
++         _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
-+         StorageDead(_5);
-+         StorageDead(_4);
 +         StorageDead(_3);
 +         StorageDead(_2);
           StorageDead(_1);

tests/mir-opt/inline/inline_cycle.two.Inline.panic-unwind.diff

@@ -5,20 +5,9 @@
       let mut _0: ();
       let _1: ();
 +     let mut _2: fn() {f};
-+     let mut _4: ();
 +     scope 1 (inlined call::<fn() {f}>) {
 +         debug f => _2;
 +         let _3: ();
-+         scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+             scope 3 (inlined f) {
-+                 scope 4 (inlined call::<fn() {f}>) {
-+                     debug f => f;
-+                     let _5: ();
-+                     scope 5 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) {
-+                     }
-+                 }
-+             }
-+         }
 +     }
   
       bb0: {
@@ -27,15 +16,10 @@
 +         StorageLive(_2);
 +         _2 = f;
 +         StorageLive(_3);
-+         StorageLive(_4);
-+         _4 = const ();
-+         StorageLive(_5);
-+         _5 = f() -> [return: bb1, unwind continue];
++         _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, const ()) -> [return: bb1, unwind continue];
       }
   
       bb1: {
-+         StorageDead(_5);
-+         StorageDead(_4);
 +         StorageDead(_3);
 +         StorageDead(_2);
           StorageDead(_1);

tests/mir-opt/inline/inline_diverging.h.Inline.panic-abort.diff

@@ -5,7 +5,6 @@
       let mut _0: ();
       let _1: (!, !);
 +     let mut _2: fn() -> ! {sleep};
-+     let mut _7: ();
 +     scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) {
 +         debug f => _2;
 +         let mut _3: &fn() -> ! {sleep};
@@ -18,10 +17,6 @@
 +                 debug b => _6;
 +             }
 +         }
-+         scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
-+             scope 5 (inlined sleep) {
-+             }
-+         }
 +     }
   
       bb0: {
@@ -33,13 +28,24 @@
 +         StorageLive(_6);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_7);
-+         _7 = const ();
-+         goto -> bb1;
++         _4 = <fn() -> ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind unreachable];
 +     }
 + 
 +     bb1: {
-+         goto -> bb1;
++         StorageDead(_3);
++         StorageLive(_5);
++         _5 = &_2;
++         _6 = <fn() -> ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind unreachable];
++     }
++ 
++     bb2: {
++         StorageDead(_5);
++         _1 = (move _4, move _6);
++         drop(_2) -> [return: bb3, unwind unreachable];
++     }
++ 
++     bb3: {
++         unreachable;
       }
   }
   

tests/mir-opt/inline/inline_diverging.h.Inline.panic-unwind.diff

@@ -5,7 +5,6 @@
       let mut _0: ();
       let _1: (!, !);
 +     let mut _2: fn() -> ! {sleep};
-+     let mut _8: ();
 +     scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) {
 +         debug f => _2;
 +         let mut _3: &fn() -> ! {sleep};
@@ -19,10 +18,6 @@
 +                 debug b => _6;
 +             }
 +         }
-+         scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) {
-+             scope 5 (inlined sleep) {
-+             }
-+         }
 +     }
   
       bb0: {
@@ -34,13 +29,40 @@
 +         StorageLive(_4);
 +         StorageLive(_3);
 +         _3 = &_2;
-+         StorageLive(_8);
-+         _8 = const ();
-+         goto -> bb1;
++         _4 = <fn() -> ! {sleep} as Fn<()>>::call(move _3, const ()) -> [return: bb1, unwind: bb5];
 +     }
 + 
 +     bb1: {
-+         goto -> bb1;
++         StorageDead(_3);
++         StorageLive(_5);
++         _5 = &_2;
++         _6 = <fn() -> ! {sleep} as Fn<()>>::call(move _5, const ()) -> [return: bb2, unwind: bb4];
++     }
++ 
++     bb2: {
++         StorageDead(_5);
++         StorageLive(_7);
++         _7 = move _4;
++         _1 = (move _7, move _6);
++         StorageDead(_7);
++         StorageDead(_4);
++         drop(_2) -> [return: bb3, unwind continue];
++     }
++ 
++     bb3: {
++         unreachable;
++     }
++ 
++     bb4 (cleanup): {
++         drop(_4) -> [return: bb5, unwind terminate];
++     }
++ 
++     bb5 (cleanup): {
++         drop(_2) -> [return: bb6, unwind terminate];
++     }
++ 
++     bb6 (cleanup): {
++         resume;
       }
   }
   

tests/mir-opt/inline/issue_78442.bar.Inline.panic-abort.diff

@@ -8,8 +8,6 @@
       let mut _3: &fn() {foo};
       let _4: fn() {foo};
       let mut _5: ();
-+     scope 1 (inlined <fn() {foo} as Fn<()>>::call - shim(fn() {foo})) {
-+     }
   
       bb0: {
           StorageLive(_2);
@@ -22,26 +20,20 @@
           _3 = &_4;
           StorageLive(_5);
           _5 = ();
--         _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind unreachable];
-+         _2 = move (*_3)() -> [return: bb3, unwind unreachable];
+          _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
-+         return;
-+     }
-+ 
-+     bb3: {
           StorageDead(_5);
           StorageDead(_3);
           StorageDead(_4);
           StorageDead(_2);
           _0 = const ();
--         drop(_1) -> [return: bb3, unwind unreachable];
--     }
-- 
--     bb3: {
--         return;
-+         drop(_1) -> [return: bb2, unwind unreachable];
+          drop(_1) -> [return: bb3, unwind unreachable];
+      }
+  
+      bb3: {
+          return;
       }
   }
   

tests/mir-opt/inline/issue_78442.bar.Inline.panic-unwind.diff

@@ -8,55 +8,40 @@
       let mut _3: &fn() {foo};
       let _4: fn() {foo};
       let mut _5: ();
-+     scope 1 (inlined <fn() {foo} as Fn<()>>::call - shim(fn() {foo})) {
-+     }
   
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = hide_foo() -> [return: bb1, unwind: bb4];
-+         _4 = hide_foo() -> [return: bb1, unwind: bb3];
+          _4 = hide_foo() -> [return: bb1, unwind: bb4];
       }
   
       bb1: {
           _3 = &_4;
           StorageLive(_5);
           _5 = ();
--         _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind: bb4];
-+         _2 = move (*_3)() -> [return: bb5, unwind: bb3];
+          _2 = <fn() {foo} as Fn<()>>::call(move _3, move _5) -> [return: bb2, unwind: bb4];
       }
   
       bb2: {
--         StorageDead(_5);
--         StorageDead(_3);
--         StorageDead(_4);
--         StorageDead(_2);
--         _0 = const ();
--         drop(_1) -> [return: bb3, unwind: bb5];
-+         return;
+          StorageDead(_5);
+          StorageDead(_3);
+          StorageDead(_4);
+          StorageDead(_2);
+          _0 = const ();
+          drop(_1) -> [return: bb3, unwind: bb5];
       }
   
--     bb3: {
--         return;
-+     bb3 (cleanup): {
-+         drop(_1) -> [return: bb4, unwind terminate];
+      bb3: {
+          return;
       }
   
       bb4 (cleanup): {
--         drop(_1) -> [return: bb5, unwind terminate];
-+         resume;
+          drop(_1) -> [return: bb5, unwind terminate];
       }
   
--     bb5 (cleanup): {
--         resume;
-+     bb5: {
-+         StorageDead(_5);
-+         StorageDead(_3);
-+         StorageDead(_4);
-+         StorageDead(_2);
-+         _0 = const ();
-+         drop(_1) -> [return: bb2, unwind: bb4];
+      bb5 (cleanup): {
+          resume;
       }
   }