LLVM codgen support for unwinding inline assembly

compiler/rustc_codegen_cranelift/src/base.rs

@@ -239,7 +239,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             fx.add_comment(inst, terminator_head);
         }
 
-        fx.set_debug_loc(bb_data.terminator().source_info);
+        let source_info = bb_data.terminator().source_info;
+        fx.set_debug_loc(source_info);
 
         match &bb_data.terminator().kind {
             TerminatorKind::Goto { target } => {
@@ -295,19 +296,19 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                         let len = codegen_operand(fx, len).load_scalar(fx);
                         let index = codegen_operand(fx, index).load_scalar(fx);
                         let location = fx
-                            .get_caller_location(bb_data.terminator().source_info.span)
+                            .get_caller_location(source_info.span)
                             .load_scalar(fx);
 
                         codegen_panic_inner(
                             fx,
                             rustc_hir::LangItem::PanicBoundsCheck,
                             &[index, len, location],
-                            bb_data.terminator().source_info.span,
+                            source_info.span,
                         );
                     }
                     _ => {
                         let msg_str = msg.description();
-                        codegen_panic(fx, msg_str, bb_data.terminator().source_info.span);
+                        codegen_panic(fx, msg_str, source_info.span);
                     }
                 }
             }
@@ -378,10 +379,18 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                 options,
                 destination,
                 line_spans: _,
+                cleanup,
             } => {
+                if cleanup.is_some() {
+                    fx.tcx.sess.span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support unwinding from inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm(
                     fx,
-                    bb_data.terminator().source_info.span,
+                    source_info.span,
                     template,
                     operands,
                     *options,
@@ -415,7 +424,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             }
             TerminatorKind::Drop { place, target, unwind: _ } => {
                 let drop_place = codegen_place(fx, *place);
-                crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place);
+                crate::abi::codegen_drop(fx, source_info.span, drop_place);
 
                 let target_block = fx.get_block(*target);
                 fx.bcx.ins().jump(target_block, &[]);

compiler/rustc_codegen_llvm/src/asm.rs

@@ -1,6 +1,8 @@
 use crate::builder::Builder;
+use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm;
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -98,6 +100,8 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             ia.alignstack,
             ia.dialect,
             &[span],
+            false,
+            None,
         );
         if r.is_none() {
             return false;
@@ -121,6 +125,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
+        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -355,6 +360,8 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             alignstack,
             dialect,
             line_spans,
+            options.contains(InlineAsmOptions::MAY_UNWIND),
+            dest_catch_funclet,
         )
         .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
 
@@ -447,10 +454,16 @@ pub(crate) fn inline_asm_call(
     alignstack: bool,
     dia: LlvmAsmDialect,
     line_spans: &[Span],
+    unwind: bool,
+    dest_catch_funclet: Option<(
+        &'ll llvm::BasicBlock,
+        &'ll llvm::BasicBlock,
+        Option<&Funclet<'ll>>,
+    )>,
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
     let alignstack = if alignstack { llvm::True } else { llvm::False };
-    let can_throw = llvm::False;
+    let can_throw = if unwind { llvm::True } else { llvm::False };
 
     let argtys = inputs
         .iter()
@@ -467,6 +480,13 @@ pub(crate) fn inline_asm_call(
         let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len());
         debug!("constraint verification result: {:?}", constraints_ok);
         if constraints_ok {
+            if unwind && llvm_util::get_version() < (13, 0, 0) {
+                bx.cx.sess().span_fatal(
+                    line_spans[0],
+                    "unwinding from inline assembly is only supported on llvm >= 13.",
+                );
+            }
+
             let v = llvm::LLVMRustInlineAsm(
                 fty,
                 asm.as_ptr().cast(),
@@ -478,7 +498,12 @@ pub(crate) fn inline_asm_call(
                 llvm::AsmDialect::from_generic(dia),
                 can_throw,
             );
-            let call = bx.call(fty, v, inputs, None);
+
+            let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
+                bx.invoke(fty, v, inputs, dest, catch, funclet)
+            } else {
+                bx.call(fty, v, inputs, None)
+            };
 
             // Store mark in a metadata node so we can map LLVM errors
             // back to source locations.  See #17552.

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -350,6 +350,8 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     false,
                     ast::LlvmAsmDialect::Att,
                     &[span],
+                    false,
+                    None,
                 )
                 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -276,9 +276,9 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
                 | TerminatorKind::SwitchInt { .. }
                 | TerminatorKind::Yield { .. }
                 | TerminatorKind::FalseEdge { .. }
-                | TerminatorKind::FalseUnwind { .. }
+                | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
-                | TerminatorKind::InlineAsm { .. } => { /* nothing to do */ }
                 TerminatorKind::Call { cleanup: unwind, .. }
+                | TerminatorKind::InlineAsm { cleanup: unwind, .. }
                 | TerminatorKind::Assert { cleanup: unwind, .. }
                 | TerminatorKind::DropAndReplace { unwind, .. }
                 | TerminatorKind::Drop { unwind, .. } => {

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -10,6 +10,7 @@ use crate::traits::*;
 use crate::MemFlags;
 
 use rustc_ast as ast;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
 use rustc_index::vec::Idx;
 use rustc_middle::mir::AssertKind;
@@ -174,6 +175,45 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             }
         }
     }
+
+    /// Generates inline assembly with optional `destination` and `cleanup`.
+    fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
+        &self,
+        fx: &mut FunctionCx<'a, 'tcx, Bx>,
+        bx: &mut Bx,
+        template: &[InlineAsmTemplatePiece],
+        operands: &[InlineAsmOperandRef<'tcx, Bx>],
+        options: InlineAsmOptions,
+        line_spans: &[Span],
+        destination: Option<mir::BasicBlock>,
+        cleanup: Option<mir::BasicBlock>,
+        instance: Instance<'_>,
+    ) {
+        if let Some(cleanup) = cleanup {
+            let ret_llbb = if let Some(target) = destination {
+                fx.llbb(target)
+            } else {
+                fx.unreachable_block()
+            };
+
+            bx.codegen_inline_asm(
+                template,
+                &operands,
+                options,
+                line_spans,
+                instance,
+                Some((ret_llbb, self.llblock(fx, cleanup), self.funclet(fx))),
+            );
+        } else {
+            bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None);
+
+            if let Some(target) = destination {
+                self.funclet_br(fx, bx, target);
+            } else {
+                bx.unreachable();
+            }
+        }
+    }
 }
 
 /// Codegen implementations for some terminator variants.
@@ -877,6 +917,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         options: ast::InlineAsmOptions,
         line_spans: &[Span],
         destination: Option<mir::BasicBlock>,
+        cleanup: Option<mir::BasicBlock>,
         instance: Instance<'_>,
     ) {
         let span = terminator.source_info.span;
@@ -931,13 +972,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             })
             .collect();
 
-        bx.codegen_inline_asm(template, &operands, options, line_spans, instance);
+        helper.do_inlineasm(
-
+            self,
-        if let Some(target) = destination {
+            &mut bx,
-            helper.funclet_br(self, &mut bx, target);
+            template,
-        } else {
+            &operands,
-            bx.unreachable();
+            options,
-        }
+            line_spans,
+            destination,
+            cleanup,
+            instance,
+        );
     }
 }
 
@@ -1041,7 +1086,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 options,
                 line_spans,
                 destination,
-                cleanup: _, // TODO
+                cleanup,
             } => {
                 self.codegen_asm_terminator(
                     helper,
@@ -1052,6 +1097,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     options,
                     line_spans,
                     destination,
+                    cleanup,
                     self.instance,
                 );
             }

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -59,6 +59,7 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
+        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
     );
 }
 

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -446,11 +446,19 @@ LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen,
                   char *Constraints, size_t ConstraintsLen,
                   LLVMBool HasSideEffects, LLVMBool IsAlignStack,
                   LLVMRustAsmDialect Dialect, LLVMBool CanThrow) {
+#if LLVM_VERSION_GE(13, 0)
   return wrap(InlineAsm::get(unwrap<FunctionType>(Ty),
                              StringRef(AsmString, AsmStringLen),
                              StringRef(Constraints, ConstraintsLen),
                              HasSideEffects, IsAlignStack,
                              fromRust(Dialect), CanThrow));
+#else
+  return wrap(InlineAsm::get(unwrap<FunctionType>(Ty),
+                             StringRef(AsmString, AsmStringLen),
+                             StringRef(Constraints, ConstraintsLen),
+                             HasSideEffects, IsAlignStack,
+                             fromRust(Dialect)));
+#endif
 }
 
 extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints,