diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index 78e8e32b972..07edd26e27a 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -107,7 +107,7 @@ enum ConstraintOrRegister { impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { - fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) { + fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) { if options.contains(InlineAsmOptions::MAY_UNWIND) { self.sess().dcx() .create_err(UnwindingInlineAsm { span: span[0] }) @@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // added to `outputs.len()` let mut inputs = vec![]; + // GCC index of a label equals its position in the array added to + // `outputs.len() + inputs.len()`. + let mut labels = vec![]; + // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _` let mut clobbers = vec![]; @@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // some targets to add a leading underscore (Mach-O). constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len(); } + + InlineAsmOperandRef::Label { label } => { + labels.push(label); + } } } @@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { InlineAsmOperandRef::Const { .. } => { // processed in the previous pass } + + InlineAsmOperandRef::Label { .. } => { + // processed in the previous pass + } } } @@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { InlineAsmOperandRef::Const { ref string } => { template_str.push_str(string); } + + InlineAsmOperandRef::Label { label } => { + let label_gcc_index = labels.iter() + .position(|&l| l == label) + .expect("wrong rust index"); + let gcc_index = label_gcc_index + outputs.len() + inputs.len(); + push_to_template(Some('l'), gcc_index); + } } } } @@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { // 4. Generate Extended Asm block let block = self.llbb(); - let extended_asm = block.add_extended_asm(None, &template_str); + let extended_asm = if let Some(dest) = dest { + assert!(!labels.is_empty()); + block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest)) + } else { + block.add_extended_asm(None, &template_str) + }; for op in &outputs { extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var); @@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { if !options.contains(InlineAsmOptions::NOSTACK) { // TODO(@Commeownist): figure out how to align stack } - if options.contains(InlineAsmOptions::NORETURN) { + if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) { let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable"); let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) }; self.call(self.type_void(), None, None, builtin_unreachable, &[], None); diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index a413466093b..f5d0b3c4657 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ) { let asm_arch = self.tcx.sess.asm_arch.unwrap(); @@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } // Build the template string + let mut labels = vec![]; let mut template_str = String::new(); for piece in template { match *piece { @@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // Only emit the raw symbol name template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); } + InlineAsmOperandRef::Label { label } => { + template_str.push_str(&format!("${{{}:l}}", constraints.len())); + constraints.push("!i".to_owned()); + labels.push(label); + } } } } @@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { &constraints.join(","), &inputs, output_type, + &labels, volatile, alignstack, dialect, line_spans, options.contains(InlineAsmOptions::MAY_UNWIND), - dest_catch_funclet, + dest, + catch_funclet, ) .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); @@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` - if let Some((dest, _, _)) = dest_catch_funclet { + if let Some(dest) = dest { self.switch_to_block(dest); } @@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>( cons: &str, inputs: &[&'ll Value], output: &'ll llvm::Type, + labels: &[&'ll llvm::BasicBlock], volatile: bool, alignstack: bool, dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, - dest_catch_funclet: Option<( - &'ll llvm::BasicBlock, - &'ll llvm::BasicBlock, - Option<&Funclet<'ll>>, - )>, + dest: Option<&'ll llvm::BasicBlock>, + catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>, ) -> Option<&'ll Value> { let volatile = if volatile { llvm::True } else { llvm::False }; let alignstack = if alignstack { llvm::True } else { llvm::False }; @@ -457,8 +464,10 @@ pub(crate) fn inline_asm_call<'ll>( can_throw, ); - let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest, catch, funclet) + let call = if !labels.is_empty() { + bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None) + } else if let Some((catch, funclet)) = catch_funclet { + bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet) } else { bx.call(fty, None, None, v, inputs, None) }; diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 23e6f054a7c..ee76038800f 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -377,12 +377,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { constraint, inputs, self.type_void(), + &[], true, false, llvm::AsmDialect::Att, &[span], false, None, + None, ) .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index c9a1e7789c2..2eba489cbfb 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -264,7 +264,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { mir::UnwindAction::Unreachable => None, }; - if let Some(cleanup) = unwind_target { + if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) { let ret_llbb = if let Some(target) = destination { fx.llbb(target) } else { @@ -277,11 +277,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { options, line_spans, instance, - Some((ret_llbb, cleanup, self.funclet(fx))), + Some(ret_llbb), + None, + ); + MergingSucc::False + } else if let Some(cleanup) = unwind_target { + let ret_llbb = if let Some(target) = destination { + fx.llbb(target) + } else { + fx.unreachable_block() + }; + + bx.codegen_inline_asm( + template, + operands, + options, + line_spans, + instance, + Some(ret_llbb), + Some((cleanup, self.funclet(fx))), ); MergingSucc::False } else { - bx.codegen_inline_asm(template, operands, options, line_spans, instance, None); + bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None); if let Some(target) = destination { self.funclet_br(fx, bx, target, mergeable_succ) @@ -1067,7 +1085,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operands: &[mir::InlineAsmOperand<'tcx>], options: ast::InlineAsmOptions, line_spans: &[Span], - destination: Option, + targets: &[mir::BasicBlock], unwind: mir::UnwindAction, instance: Instance<'_>, mergeable_succ: bool, @@ -1119,8 +1137,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { mir::InlineAsmOperand::SymStatic { def_id } => { InlineAsmOperandRef::SymStatic { def_id } } - mir::InlineAsmOperand::Label { target_index: _ } => { - todo!(); + mir::InlineAsmOperand::Label { target_index } => { + InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) } } }) .collect(); @@ -1132,7 +1150,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { &operands, options, line_spans, - destination, + if options.contains(InlineAsmOptions::NORETURN) { + None + } else { + targets.get(0).copied() + }, unwind, instance, mergeable_succ, @@ -1298,7 +1320,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { operands, options, line_spans, - targets.get(0).copied(), + targets, unwind, self.instance, mergeable_succ(), diff --git a/compiler/rustc_codegen_ssa/src/traits/asm.rs b/compiler/rustc_codegen_ssa/src/traits/asm.rs index c2ae74b18d8..8d67b626bbd 100644 --- a/compiler/rustc_codegen_ssa/src/traits/asm.rs +++ b/compiler/rustc_codegen_ssa/src/traits/asm.rs @@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> { SymStatic { def_id: DefId, }, + Label { + label: B::BasicBlock, + }, } #[derive(Debug)] @@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes { options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, - dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + dest: Option, + catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, ); }