Auto merge of #113128 - WaffleLapkin:become_trully_unuwuable, r=oli-obk,RalfJung

Support tail calls in mir via `TerminatorKind::TailCall`

This is one of the interesting bits in tail call implementation — MIR support.

This adds a new `TerminatorKind` which represents a tail call:
```rust
    TailCall {
        func: Operand<'tcx>,
        args: Vec<Operand<'tcx>>,
        fn_span: Span,
    },
```

*Structurally* this is very similar to a normal `Call` but is missing a few fields:
- `destination` — tail calls don't write to destination, instead they pass caller's destination to the callee (such that eventual `return` will write to the caller of the function that used tail call)
- `target` — similarly to `destination` tail calls pass the caller's return address to the callee, so there is nothing to do
- `unwind` — I _think_ this is applicable too, although it's a bit confusing
- `call_source` — `become` forbids operators and is not created as a lowering of something else; tail calls always come from HIR (at least for now)

It might be helpful to read the interpreter implementation to understand what `TailCall` means exactly, although I've tried documenting it too.

-----

There are a few `FIXME`-questions still left, ideally we'd be able to answer them during review ':)

-----

r? `@oli-obk`
cc `@scottmcm` `@DrMeepster` `@JakobDegen`

compiler/rustc_mir_transform/src/coroutine.rs

@@ -1367,6 +1367,10 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
             | TerminatorKind::Call { .. }
             | TerminatorKind::InlineAsm { .. }
             | TerminatorKind::Assert { .. } => return true,
+
+            TerminatorKind::TailCall { .. } => {
+                unreachable!("tail calls can't be present in generators")
+            }
         }
     }
 
@@ -1916,6 +1920,7 @@ impl<'tcx> Visitor<'tcx> for EnsureCoroutineFieldAssignmentsNeverAlias<'_> {
             | TerminatorKind::UnwindResume
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
+            | TerminatorKind::TailCall { .. }
             | TerminatorKind::Unreachable
             | TerminatorKind::Drop { .. }
             | TerminatorKind::Assert { .. }

compiler/rustc_mir_transform/src/cost_checker.rs

@@ -40,7 +40,9 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> {
         fn is_call_like(bbd: &BasicBlockData<'_>) -> bool {
             use TerminatorKind::*;
             match bbd.terminator().kind {
-                Call { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => true,
+                Call { .. } | TailCall { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => {
+                    true
+                }
 
                 Goto { .. }
                 | SwitchInt { .. }
@@ -137,6 +139,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                     self.penalty += LANDINGPAD_PENALTY;
                 }
             }
+            TerminatorKind::TailCall { .. } => {
+                self.penalty += CALL_PENALTY;
+            }
             TerminatorKind::SwitchInt { discr, targets } => {
                 if discr.constant().is_some() {
                     // Not only will this become a `Goto`, but likely other

compiler/rustc_mir_transform/src/coverage/graph.rs

@@ -358,9 +358,12 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera
         }
 
         // These terminators have no coverage-relevant successors.
-        CoroutineDrop | Return | Unreachable | UnwindResume | UnwindTerminate(_) => {
-            CoverageSuccessors::NotChainable(&[])
-        }
+        CoroutineDrop
+        | Return
+        | TailCall { .. }
+        | Unreachable
+        | UnwindResume
+        | UnwindTerminate(_) => CoverageSuccessors::NotChainable(&[]),
     }
 }
 

compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs

@@ -193,7 +193,8 @@ fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> {
         | TerminatorKind::Goto { .. } => None,
 
         // Call `func` operand can have a more specific span when part of a chain of calls
-        | TerminatorKind::Call { ref func, .. } => {
+        TerminatorKind::Call { ref func, .. }
+        | TerminatorKind::TailCall { ref func, .. } => {
             let mut span = terminator.source_info.span;
             if let mir::Operand::Constant(box constant) = func {
                 if constant.span.lo() > span.lo() {

compiler/rustc_mir_transform/src/dest_prop.rs

@@ -628,6 +628,12 @@ impl WriteInfo {
                     self.add_operand(&arg.node);
                 }
             }
+            TerminatorKind::TailCall { func, args, .. } => {
+                self.add_operand(func);
+                for arg in args {
+                    self.add_operand(&arg.node);
+                }
+            }
             TerminatorKind::InlineAsm { operands, .. } => {
                 for asm_operand in operands {
                     match asm_operand {

compiler/rustc_mir_transform/src/inline.rs

@@ -383,6 +383,8 @@ impl<'tcx> Inliner<'tcx> {
     ) -> Option<CallSite<'tcx>> {
         // Only consider direct calls to functions
         let terminator = bb_data.terminator();
+
+        // FIXME(explicit_tail_calls): figure out if we can inline tail calls
         if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
             let func_ty = func.ty(caller_body, self.tcx);
             if let ty::FnDef(def_id, args) = *func_ty.kind() {
@@ -550,6 +552,9 @@ impl<'tcx> Inliner<'tcx> {
                 // inline-asm is detected. LLVM will still possibly do an inline later on
                 // if the no-attribute function ends up with the same instruction set anyway.
                 return Err("Cannot move inline-asm across instruction sets");
+            } else if let TerminatorKind::TailCall { .. } = term.kind {
+                // FIXME(explicit_tail_calls): figure out how exactly functions containing tail calls can be inlined (and if they even should)
+                return Err("can't inline functions with tail calls");
             } else {
                 work_list.extend(term.successors())
             }
@@ -1038,6 +1043,10 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
                 *target = self.map_block(*target);
                 *unwind = self.map_unwind(*unwind);
             }
+            TerminatorKind::TailCall { .. } => {
+                // check_mir_body forbids tail calls
+                unreachable!()
+            }
             TerminatorKind::Call { ref mut target, ref mut unwind, .. } => {
                 if let Some(ref mut tgt) = *target {
                     *tgt = self.map_block(*tgt);

compiler/rustc_mir_transform/src/jump_threading.rs

@@ -596,6 +596,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> {
             TerminatorKind::UnwindResume
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
+            | TerminatorKind::TailCall { .. }
             | TerminatorKind::Unreachable
             | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"),
             // Disallowed during optimizations.

compiler/rustc_mir_transform/src/known_panics_lint.rs

@@ -799,6 +799,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
             | TerminatorKind::UnwindResume
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
+            | TerminatorKind::TailCall { .. }
             | TerminatorKind::Unreachable
             | TerminatorKind::Drop { .. }
             | TerminatorKind::Yield { .. }

compiler/rustc_mir_transform/src/mentioned_items.rs

@@ -38,7 +38,7 @@ impl<'tcx> Visitor<'tcx> for MentionedItemsVisitor<'_, 'tcx> {
         self.super_terminator(terminator, location);
         let span = || self.body.source_info(location).span;
         match &terminator.kind {
-            mir::TerminatorKind::Call { func, .. } => {
+            mir::TerminatorKind::Call { func, .. } | mir::TerminatorKind::TailCall { func, .. } => {
                 let callee_ty = func.ty(self.body, self.tcx);
                 self.mentioned_items
                     .push(Spanned { node: MentionedItem::Fn(callee_ty), span: span() });

compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs

@@ -75,6 +75,7 @@ impl RemoveNoopLandingPads {
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Unreachable
             | TerminatorKind::Call { .. }
+            | TerminatorKind::TailCall { .. }
             | TerminatorKind::Assert { .. }
             | TerminatorKind::Drop { .. }
             | TerminatorKind::InlineAsm { .. } => false,

compiler/rustc_mir_transform/src/validate.rs

@@ -400,40 +400,44 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *target, EdgeKind::Normal);
                 self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::Call { args, destination, target, unwind, .. } => {
-                if let Some(target) = target {
-                    self.check_edge(location, *target, EdgeKind::Normal);
-                }
-                self.check_unwind_edge(location, *unwind);
+            TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => {
+                // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks
+                if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind {
+                    if let Some(target) = target {
+                        self.check_edge(location, target, EdgeKind::Normal);
+                    }
+                    self.check_unwind_edge(location, unwind);
 
-                // The code generation assumes that there are no critical call edges. The assumption
-                // is used to simplify inserting code that should be executed along the return edge
-                // from the call. FIXME(tmiasko): Since this is a strictly code generation concern,
-                // the code generation should be responsible for handling it.
-                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized)
-                    && self.is_critical_call_edge(*target, *unwind)
-                {
-                    self.fail(
-                        location,
-                        format!(
-                            "encountered critical edge in `Call` terminator {:?}",
-                            terminator.kind,
-                        ),
-                    );
+                    // The code generation assumes that there are no critical call edges. The assumption
+                    // is used to simplify inserting code that should be executed along the return edge
+                    // from the call. FIXME(tmiasko): Since this is a strictly code generation concern,
+                    // the code generation should be responsible for handling it.
+                    if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized)
+                        && self.is_critical_call_edge(target, unwind)
+                    {
+                        self.fail(
+                            location,
+                            format!(
+                                "encountered critical edge in `Call` terminator {:?}",
+                                terminator.kind,
+                            ),
+                        );
+                    }
+
+                    // The call destination place and Operand::Move place used as an argument might be
+                    // passed by a reference to the callee. Consequently they cannot be packed.
+                    if is_within_packed(self.tcx, &self.body.local_decls, destination).is_some() {
+                        // This is bad! The callee will expect the memory to be aligned.
+                        self.fail(
+                            location,
+                            format!(
+                                "encountered packed place in `Call` terminator destination: {:?}",
+                                terminator.kind,
+                            ),
+                        );
+                    }
                 }
 
-                // The call destination place and Operand::Move place used as an argument might be
-                // passed by a reference to the callee. Consequently they cannot be packed.
-                if is_within_packed(self.tcx, &self.body.local_decls, *destination).is_some() {
-                    // This is bad! The callee will expect the memory to be aligned.
-                    self.fail(
-                        location,
-                        format!(
-                            "encountered packed place in `Call` terminator destination: {:?}",
-                            terminator.kind,
-                        ),
-                    );
-                }
                 for arg in args {
                     if let Operand::Move(place) = &arg.node {
                         if is_within_packed(self.tcx, &self.body.local_decls, *place).is_some() {
@@ -1498,15 +1502,22 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     }
                 }
             }
-            TerminatorKind::Call { func, .. } => {
+            TerminatorKind::Call { func, .. } | TerminatorKind::TailCall { func, .. } => {
                 let func_ty = func.ty(&self.body.local_decls, self.tcx);
                 match func_ty.kind() {
                     ty::FnPtr(..) | ty::FnDef(..) => {}
                     _ => self.fail(
                         location,
-                        format!("encountered non-callable type {func_ty} in `Call` terminator"),
+                        format!(
+                            "encountered non-callable type {func_ty} in `{}` terminator",
+                            terminator.kind.name()
+                        ),
                     ),
                 }
+
+                if let TerminatorKind::TailCall { .. } = terminator.kind {
+                    // FIXME(explicit_tail_calls): implement tail-call specific checks here (such as signature matching, forbidding closures, etc)
+                }
             }
             TerminatorKind::Assert { cond, .. } => {
                 let cond_ty = cond.ty(&self.body.local_decls, self.tcx);