Change InlineAsm to allow multiple targets instead

compiler/rustc_borrowck/src/lib.rs

@@ -723,7 +723,7 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -161,7 +161,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1771,8 +1771,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
+            TerminatorKind::InlineAsm { ref targets, unwind, .. } => {
-                if let Some(target) = destination {
+                for &target in targets {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);

compiler/rustc_codegen_cranelift/src/base.rs

@@ -445,7 +445,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 template,
                 operands,
                 options,
-                destination,
+                targets,
                 line_spans: _,
                 unwind: _,
             } => {
@@ -456,13 +456,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     );
                 }
 
+                let have_labels = if options.contains(InlineAsmOptions::NORETURN) {
+                    !targets.is_empty()
+                } else {
+                    targets.len() > 1
+                };
+                if have_labels {
+                    fx.tcx.dcx().span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support labels in inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm_terminator(
                     fx,
                     source_info.span,
                     template,
                     operands,
                     *options,
-                    *destination,
+                    targets.get(0).copied(),
                 );
             }
             TerminatorKind::UnwindTerminate(reason) => {

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1285,7 +1285,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 ref operands,
                 options,
                 line_spans,
-                destination,
+                ref targets,
                 unwind,
             } => self.codegen_asm_terminator(
                 helper,
@@ -1295,7 +1295,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 operands,
                 options,
                 line_spans,
-                destination,
+                targets.get(0).copied(),
                 unwind,
                 self.instance,
                 mergeable_succ(),

compiler/rustc_const_eval/messages.ftl

@@ -232,9 +232,6 @@ const_eval_non_const_fn_call =
 const_eval_non_const_impl =
     impl defined here, but it is not `const`
 
-const_eval_noreturn_asm_returned =
-    returned from noreturn inline assembly
-
 const_eval_not_enough_caller_args =
     calling a function with fewer arguments than it requires
 

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -374,11 +374,17 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
         kind: Option<MemoryKind<Self::MemoryKind>>,
     ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>;
 
+    /// Evaluate the inline assembly.
+    ///
+    /// This should take care of jumping to the next block (one of `targets`) when asm goto
+    /// is triggered, `targets[0]` when the assembly falls through, or diverge in case of
+    /// `InlineAsmOptions::NORETURN` being set.
     fn eval_inline_asm(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
         _template: &'tcx [InlineAsmTemplatePiece],
         _operands: &[mir::InlineAsmOperand<'tcx>],
         _options: InlineAsmOptions,
+        _targets: &[mir::BasicBlock],
     ) -> InterpResult<'tcx> {
         throw_unsup_format!("inline assembly is not supported")
     }

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -1,6 +1,5 @@
 use std::borrow::Cow;
 
-use rustc_ast::ast::InlineAsmOptions;
 use rustc_middle::{
     mir,
     ty::{
@@ -224,15 +223,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 terminator.kind
             ),
 
-            InlineAsm { template, ref operands, options, destination, .. } => {
+            InlineAsm { template, ref operands, options, ref targets, .. } => {
-                M::eval_inline_asm(self, template, operands, options)?;
+                M::eval_inline_asm(self, template, operands, options, targets)?;
-                if options.contains(InlineAsmOptions::NORETURN) {
-                    throw_ub_custom!(fluent::const_eval_noreturn_asm_returned);
-                }
-                self.go_to_block(
-                    destination
-                        .expect("InlineAsm terminators without noreturn must have a destination"),
-                )
             }
         }
 

compiler/rustc_const_eval/src/transform/validate.rs

@@ -471,9 +471,9 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *real_target, EdgeKind::Normal);
                 self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
+            TerminatorKind::InlineAsm { targets, unwind, .. } => {
-                if let Some(destination) = destination {
+                for &target in targets {
-                    self.check_edge(location, *destination, EdgeKind::Normal);
+                    self.check_edge(location, target, EdgeKind::Normal);
                 }
                 self.check_unwind_edge(location, *unwind);
             }

compiler/rustc_middle/src/mir/mod.rs

@@ -1718,13 +1718,13 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 136);
+    static_assert_size!(BasicBlockData<'_>, 144);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 72);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(Terminator<'_>, 104);
+    static_assert_size!(Terminator<'_>, 112);
-    static_assert_size!(TerminatorKind<'_>, 88);
+    static_assert_size!(TerminatorKind<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/pretty.rs

@@ -7,7 +7,7 @@ use std::path::{Path, PathBuf};
 use crate::mir::interpret::ConstAllocation;
 
 use super::graphviz::write_mir_fn_graphviz;
-use rustc_ast::InlineAsmTemplatePiece;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir::interpret::{
     alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, GlobalAlloc, Pointer,
     Provenance,
@@ -868,16 +868,19 @@ impl<'tcx> TerminatorKind<'tcx> {
                 vec!["real".into(), "unwind".into()]
             }
             FalseUnwind { unwind: _, .. } => vec!["real".into()],
-            InlineAsm { destination: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
+            InlineAsm { options, ref targets, unwind, .. } => {
-                vec!["return".into(), "unwind".into()]
+                let mut vec = Vec::with_capacity(targets.len() + 1);
+                if !options.contains(InlineAsmOptions::NORETURN) {
+                    vec.push("return".into());
                 }
-            InlineAsm { destination: Some(_), unwind: _, .. } => {
+                vec.resize(targets.len(), "label".into());
-                vec!["return".into()]
+
+                if let UnwindAction::Cleanup(_) = unwind {
+                    vec.push("unwind".into());
                 }
-            InlineAsm { destination: None, unwind: UnwindAction::Cleanup(_), .. } => {
+
-                vec!["unwind".into()]
+                vec
             }
-            InlineAsm { destination: None, unwind: _, .. } => vec![],
         }
     }
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -793,9 +793,10 @@ pub enum TerminatorKind<'tcx> {
         /// used to map assembler errors back to the line in the source code.
         line_spans: &'tcx [Span],
 
-        /// Destination block after the inline assembly returns, unless it is
+        /// Valid targets for the inline assembly.
-        /// diverging (InlineAsmOptions::NORETURN).
+        /// The first element is the fallthrough destination, unless
-        destination: Option<BasicBlock>,
+        /// InlineAsmOptions::NORETURN is set.
+        targets: Vec<BasicBlock>,
 
         /// Action to be taken if the inline assembly unwinds. This is present
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.

compiler/rustc_middle/src/mir/terminator.rs

@@ -374,8 +374,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Yield { resume: ref t, drop: Some(u), .. }
             | Drop { target: ref t, unwind: UnwindAction::Cleanup(u), .. }
             | Assert { target: ref t, unwind: UnwindAction::Cleanup(u), .. }
-            | FalseUnwind { real_target: ref t, unwind: UnwindAction::Cleanup(u) }
+            | FalseUnwind { real_target: ref t, unwind: UnwindAction::Cleanup(u) } => {
-            | InlineAsm { destination: Some(ref t), unwind: UnwindAction::Cleanup(u), .. } => {
                 slice::from_ref(t).into_iter().copied().chain(Some(u))
             }
             Goto { target: ref t }
@@ -384,9 +383,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Yield { resume: ref t, drop: None, .. }
             | Drop { target: ref t, unwind: _, .. }
             | Assert { target: ref t, unwind: _, .. }
-            | FalseUnwind { real_target: ref t, unwind: _ }
+            | FalseUnwind { real_target: ref t, unwind: _ } => {
-            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref t), .. }
-            | InlineAsm { destination: Some(ref t), unwind: _, .. } => {
                 slice::from_ref(t).into_iter().copied().chain(None)
             }
             UnwindResume
@@ -394,10 +391,11 @@ impl<'tcx> TerminatorKind<'tcx> {
             | CoroutineDrop
             | Return
             | Unreachable
-            | Call { target: None, unwind: _, .. }
+            | Call { target: None, unwind: _, .. } => (&[]).into_iter().copied().chain(None),
-            | InlineAsm { destination: None, unwind: _, .. } => {
+            InlineAsm { ref targets, unwind: UnwindAction::Cleanup(u), .. } => {
-                (&[]).into_iter().copied().chain(None)
+                targets.iter().copied().chain(Some(u))
             }
+            InlineAsm { ref targets, unwind: _, .. } => targets.iter().copied().chain(None),
             SwitchInt { ref targets, .. } => targets.targets.iter().copied().chain(None),
             FalseEdge { ref real_target, imaginary_target } => {
                 slice::from_ref(real_target).into_iter().copied().chain(Some(imaginary_target))
@@ -413,21 +411,16 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Yield { resume: ref mut t, drop: Some(ref mut u), .. }
             | Drop { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
             | Assert { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
-            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) }
+            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) } => {
-            | InlineAsm {
+                slice::from_mut(t).into_iter().chain(Some(u))
-                destination: Some(ref mut t),
+            }
-                unwind: UnwindAction::Cleanup(ref mut u),
-                ..
-            } => slice::from_mut(t).into_iter().chain(Some(u)),
             Goto { target: ref mut t }
             | Call { target: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
             | Call { target: Some(ref mut t), unwind: _, .. }
             | Yield { resume: ref mut t, drop: None, .. }
             | Drop { target: ref mut t, unwind: _, .. }
             | Assert { target: ref mut t, unwind: _, .. }
-            | FalseUnwind { real_target: ref mut t, unwind: _ }
+            | FalseUnwind { real_target: ref mut t, unwind: _ } => {
-            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
-            | InlineAsm { destination: Some(ref mut t), unwind: _, .. } => {
                 slice::from_mut(t).into_iter().chain(None)
             }
             UnwindResume
@@ -435,8 +428,11 @@ impl<'tcx> TerminatorKind<'tcx> {
             | CoroutineDrop
             | Return
             | Unreachable
-            | Call { target: None, unwind: _, .. }
+            | Call { target: None, unwind: _, .. } => (&mut []).into_iter().chain(None),
-            | InlineAsm { destination: None, unwind: _, .. } => (&mut []).into_iter().chain(None),
+            InlineAsm { ref mut targets, unwind: UnwindAction::Cleanup(ref mut u), .. } => {
+                targets.iter_mut().chain(Some(u))
+            }
+            InlineAsm { ref mut targets, unwind: _, .. } => targets.iter_mut().chain(None),
             SwitchInt { ref mut targets, .. } => targets.targets.iter_mut().chain(None),
             FalseEdge { ref mut real_target, ref mut imaginary_target } => {
                 slice::from_mut(real_target).into_iter().chain(Some(imaginary_target))
@@ -511,7 +507,7 @@ pub enum TerminatorEdges<'mir, 'tcx> {
     Double(BasicBlock, BasicBlock),
     /// Special action for `Yield`, `Call` and `InlineAsm` terminators.
     AssignOnReturn {
-        return_: Option<BasicBlock>,
+        return_: &'mir [BasicBlock],
         /// The cleanup block, if it exists.
         cleanup: Option<BasicBlock>,
         place: CallReturnPlaces<'mir, 'tcx>,
@@ -575,31 +571,37 @@ impl<'tcx> TerminatorKind<'tcx> {
                 TerminatorEdges::Double(real_target, imaginary_target)
             }
 
-            Yield { resume: target, drop, resume_arg, value: _ } => {
+            Yield { resume: ref target, drop, resume_arg, value: _ } => {
                 TerminatorEdges::AssignOnReturn {
-                    return_: Some(target),
+                    return_: slice::from_ref(target),
                     cleanup: drop,
                     place: CallReturnPlaces::Yield(resume_arg),
                 }
             }
 
-            Call { unwind, destination, target, func: _, args: _, fn_span: _, call_source: _ } => {
+            Call {
-                TerminatorEdges::AssignOnReturn {
+                unwind,
-                    return_: target,
+                destination,
+                ref target,
+                func: _,
+                args: _,
+                fn_span: _,
+                call_source: _,
+            } => TerminatorEdges::AssignOnReturn {
+                return_: target.as_ref().map(slice::from_ref).unwrap_or_default(),
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::Call(destination),
-                }
+            },
-            }
 
             InlineAsm {
                 template: _,
                 ref operands,
                 options: _,
                 line_spans: _,
-                destination,
+                ref targets,
                 unwind,
             } => TerminatorEdges::AssignOnReturn {
-                return_: destination,
+                return_: targets,
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::InlineAsm(operands),
             },

compiler/rustc_middle/src/mir/visit.rs

@@ -565,7 +565,7 @@ macro_rules! make_mir_visitor {
                         operands,
                         options: _,
                         line_spans: _,
-                        destination: _,
+                        targets: _,
                         unwind: _,
                     } => {
                         for op in operands {

compiler/rustc_mir_build/src/build/expr/into.rs

@@ -474,10 +474,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         operands,
                         options,
                         line_spans,
-                        destination: if options.contains(InlineAsmOptions::NORETURN) {
+                        targets: if options.contains(InlineAsmOptions::NORETURN) {
-                            None
+                            Vec::new()
                         } else {
-                            Some(destination_block)
+                            vec![destination_block]
                         },
                         unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) {
                             UnwindAction::Continue

compiler/rustc_mir_build/src/lints.rs

@@ -199,9 +199,10 @@ impl<'mir, 'tcx, C: TerminatorClassifier<'tcx>> TriColorVisitor<BasicBlocks<'tcx
             | TerminatorKind::Unreachable
             | TerminatorKind::Yield { .. } => ControlFlow::Break(NonRecursive),
 
-            // A diverging InlineAsm is treated as non-recursing
+            // A InlineAsm without targets (diverging and contains no labels)
-            TerminatorKind::InlineAsm { destination, .. } => {
+            // is treated as non-recursing.
-                if destination.is_some() {
+            TerminatorKind::InlineAsm { ref targets, .. } => {
+                if !targets.is_empty() {
                     ControlFlow::Continue(())
                 } else {
                     ControlFlow::Break(NonRecursive)

compiler/rustc_mir_dataflow/src/framework/direction.rs

@@ -242,9 +242,9 @@ impl Direction for Backward {
                     propagate(pred, &tmp);
                 }
 
-                mir::TerminatorKind::InlineAsm {
+                mir::TerminatorKind::InlineAsm { ref targets, ref operands, .. }
-                    destination: Some(dest), ref operands, ..
+                    if targets.contains(&bb) =>
-                } if dest == bb => {
+                {
                     let mut tmp = exit_state.clone();
                     analysis.apply_call_return_effect(
                         &mut tmp,
@@ -491,9 +491,12 @@ impl Direction for Forward {
                 if let Some(cleanup) = cleanup {
                     propagate(cleanup, exit_state);
                 }
-                if let Some(return_) = return_ {
+
+                if !return_.is_empty() {
                     analysis.apply_call_return_effect(exit_state, bb, place);
-                    propagate(return_, exit_state);
+                    for &target in return_ {
+                        propagate(target, exit_state);
+                    }
                 }
             }
             TerminatorEdges::SwitchInt { targets, discr } => {

compiler/rustc_mir_dataflow/src/framework/graphviz.rs

@@ -299,7 +299,9 @@ where
                 })?;
             }
 
-            mir::TerminatorKind::InlineAsm { destination: Some(_), ref operands, .. } => {
+            mir::TerminatorKind::InlineAsm { ref targets, ref operands, .. }
+                if !targets.is_empty() =>
+            {
                 self.write_row(w, "", "(on successful return)", |this, w, fmt| {
                     let state_on_unwind = this.results.get().clone();
                     this.results.apply_custom_effect(|analysis, state| {

compiler/rustc_mir_dataflow/src/move_paths/builder.rs

@@ -491,7 +491,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                 ref operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {

compiler/rustc_mir_transform/src/coverage/graph.rs

@@ -349,12 +349,20 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera
         | FalseUnwind { real_target: target, .. }
         | Goto { target } => CoverageSuccessors::Chainable(target),
 
-        // These terminators can normally be chained, except when they have no
+        // A call terminator can normally be chained, except when they have no
         // successor because they are known to diverge.
-        Call { target: maybe_target, .. } | InlineAsm { destination: maybe_target, .. } => {
+        Call { target: maybe_target, .. } => match maybe_target {
-            match maybe_target {
             Some(target) => CoverageSuccessors::Chainable(target),
             None => CoverageSuccessors::NotChainable(&[]),
+        },
+
+        // An inline asm terminator can normally be chained, except when it diverges or uses asm
+        // goto.
+        InlineAsm { ref targets, .. } => {
+            if targets.len() == 1 {
+                CoverageSuccessors::Chainable(targets[0])
+            } else {
+                CoverageSuccessors::NotChainable(targets)
             }
         }
 

compiler/rustc_mir_transform/src/coverage/tests.rs

@@ -88,7 +88,6 @@ impl<'tcx> MockBlocks<'tcx> {
             | TerminatorKind::FalseEdge { real_target: ref mut target, .. }
             | TerminatorKind::FalseUnwind { real_target: ref mut target, .. }
             | TerminatorKind::Goto { ref mut target }
-            | TerminatorKind::InlineAsm { destination: Some(ref mut target), .. }
             | TerminatorKind::Yield { resume: ref mut target, .. } => *target = to_block,
             ref invalid => bug!("Invalid from_block: {:?}", invalid),
         }
@@ -185,10 +184,12 @@ fn debug_basic_blocks(mir_body: &Body<'_>) -> String {
                     | TerminatorKind::FalseEdge { real_target: target, .. }
                     | TerminatorKind::FalseUnwind { real_target: target, .. }
                     | TerminatorKind::Goto { target }
-                    | TerminatorKind::InlineAsm { destination: Some(target), .. }
                     | TerminatorKind::Yield { resume: target, .. } => {
                         format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), target)
                     }
+                    TerminatorKind::InlineAsm { targets, .. } => {
+                        format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets)
+                    }
                     TerminatorKind::SwitchInt { targets, .. } => {
                         format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets)
                     }

compiler/rustc_mir_transform/src/inline.rs

@@ -1032,8 +1032,8 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
             {
                 bug!("False unwinds should have been removed before inlining")
             }
-            TerminatorKind::InlineAsm { ref mut destination, ref mut unwind, .. } => {
+            TerminatorKind::InlineAsm { ref mut targets, ref mut unwind, .. } => {
-                if let Some(ref mut tgt) = *destination {
+                for tgt in targets.iter_mut() {
                     *tgt = self.map_block(*tgt);
                 }
                 *unwind = self.map_unwind(*unwind);

compiler/rustc_smir/src/rustc_smir/convert/mir.rs

@@ -632,14 +632,15 @@ impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> {
                 operands,
                 options,
                 line_spans,
-                destination,
+                targets,
                 unwind,
             } => TerminatorKind::InlineAsm {
                 template: format!("{template:?}"),
                 operands: operands.iter().map(|operand| operand.stable(tables)).collect(),
                 options: format!("{options:?}"),
                 line_spans: format!("{line_spans:?}"),
-                destination: destination.map(|d| d.as_usize()),
+                // FIXME: Figure out how to do labels in SMIR
+                destination: targets.first().map(|d| d.as_usize()),
                 unwind: unwind.stable(tables),
             },
             mir::TerminatorKind::Yield { .. }