Auto merge of #132173 - veluca93:abi_checks, r=RalfJung,compiler-errors

Emit warning when calling/declaring functions with unavailable vectors.

On some architectures, vector types may have a different ABI depending on whether the relevant target features are enabled. (The ABI when the feature is disabled is often not specified, but LLVM implements some de-facto ABI.)

As discussed in rust-lang/lang-team#235, this turns out to very easily lead to unsound code.

This commit makes it a post-monomorphization future-incompat warning to declare or call functions using those vector types in a context in which the corresponding target features are disabled, if using an ABI for which the difference is relevant. This ensures that these functions are always called with a consistent ABI.

See the [nomination comment](https://github.com/rust-lang/rust/pull/127731#issuecomment-2288558187) for more discussion.

Part of #116558

r? RalfJung

compiler/rustc_monomorphize/src/collector.rs

@@ -205,6 +205,7 @@
 //! this is not implemented however: a mono item will be produced
 //! regardless of whether it is actually needed or not.
 
+mod abi_check;
 mod move_check;
 
 use std::path::PathBuf;
@@ -1207,6 +1208,8 @@ fn collect_items_of_instance<'tcx>(
     mentioned_items: &mut MonoItems<'tcx>,
     mode: CollectionMode,
 ) {
+    tcx.ensure().check_feature_dependent_abi(instance);
+
     let body = tcx.instance_mir(instance.def);
     // Naively, in "used" collection mode, all functions get added to *both* `used_items` and
     // `mentioned_items`. Mentioned items processing will then notice that they have already been
@@ -1623,4 +1626,5 @@ pub(crate) fn collect_crate_mono_items<'tcx>(
 
 pub(crate) fn provide(providers: &mut Providers) {
     providers.hooks.should_codegen_locally = should_codegen_locally;
+    abi_check::provide(providers);
 }

compiler/rustc_monomorphize/src/collector/abi_check.rs

@@ -0,0 +1,162 @@
+//! This module ensures that if a function's ABI requires a particular target feature,
+//! that target feature is enabled both on the callee and all callers.
+use rustc_hir::CRATE_HIR_ID;
+use rustc_middle::mir::visit::Visitor as MirVisitor;
+use rustc_middle::mir::{self, Location, traversal};
+use rustc_middle::query::Providers;
+use rustc_middle::ty::inherent::*;
+use rustc_middle::ty::{self, Instance, InstanceKind, ParamEnv, Ty, TyCtxt};
+use rustc_session::lint::builtin::ABI_UNSUPPORTED_VECTOR_TYPES;
+use rustc_span::def_id::DefId;
+use rustc_span::{DUMMY_SP, Span, Symbol};
+use rustc_target::abi::call::{FnAbi, PassMode};
+use rustc_target::abi::{BackendRepr, RegKind};
+
+use crate::errors::{AbiErrorDisabledVectorTypeCall, AbiErrorDisabledVectorTypeDef};
+
+fn uses_vector_registers(mode: &PassMode, repr: &BackendRepr) -> bool {
+    match mode {
+        PassMode::Ignore | PassMode::Indirect { .. } => false,
+        PassMode::Cast { pad_i32: _, cast } => {
+            cast.prefix.iter().any(|r| r.is_some_and(|x| x.kind == RegKind::Vector))
+                || cast.rest.unit.kind == RegKind::Vector
+        }
+        PassMode::Direct(..) | PassMode::Pair(..) => matches!(repr, BackendRepr::Vector { .. }),
+    }
+}
+
+fn do_check_abi<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    abi: &FnAbi<'tcx, Ty<'tcx>>,
+    target_feature_def: DefId,
+    mut emit_err: impl FnMut(&'static str),
+) {
+    let Some(feature_def) = tcx.sess.target.features_for_correct_vector_abi() else {
+        return;
+    };
+    let codegen_attrs = tcx.codegen_fn_attrs(target_feature_def);
+    for arg_abi in abi.args.iter().chain(std::iter::once(&abi.ret)) {
+        let size = arg_abi.layout.size;
+        if uses_vector_registers(&arg_abi.mode, &arg_abi.layout.backend_repr) {
+            // Find the first feature that provides at least this vector size.
+            let feature = match feature_def.iter().find(|(bits, _)| size.bits() <= *bits) {
+                Some((_, feature)) => feature,
+                None => {
+                    emit_err("<no available feature for this size>");
+                    continue;
+                }
+            };
+            let feature_sym = Symbol::intern(feature);
+            if !tcx.sess.unstable_target_features.contains(&feature_sym)
+                && !codegen_attrs.target_features.iter().any(|x| x.name == feature_sym)
+            {
+                emit_err(feature);
+            }
+        }
+    }
+}
+
+/// Checks that the ABI of a given instance of a function does not contain vector-passed arguments
+/// or return values for which the corresponding target feature is not enabled.
+fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
+    let param_env = ParamEnv::reveal_all();
+    let Ok(abi) = tcx.fn_abi_of_instance(param_env.and((instance, ty::List::empty()))) else {
+        // An error will be reported during codegen if we cannot determine the ABI of this
+        // function.
+        return;
+    };
+    do_check_abi(tcx, abi, instance.def_id(), |required_feature| {
+        let span = tcx.def_span(instance.def_id());
+        tcx.emit_node_span_lint(
+            ABI_UNSUPPORTED_VECTOR_TYPES,
+            CRATE_HIR_ID,
+            span,
+            AbiErrorDisabledVectorTypeDef { span, required_feature },
+        );
+    })
+}
+
+/// Checks that a call expression does not try to pass a vector-passed argument which requires a
+/// target feature that the caller does not have, as doing so causes UB because of ABI mismatch.
+fn check_call_site_abi<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    callee: Ty<'tcx>,
+    span: Span,
+    caller: InstanceKind<'tcx>,
+) {
+    if callee.fn_sig(tcx).abi().is_rust() {
+        // "Rust" ABI never passes arguments in vector registers.
+        return;
+    }
+    let param_env = ParamEnv::reveal_all();
+    let callee_abi = match *callee.kind() {
+        ty::FnPtr(..) => {
+            tcx.fn_abi_of_fn_ptr(param_env.and((callee.fn_sig(tcx), ty::List::empty())))
+        }
+        ty::FnDef(def_id, args) => {
+            // Intrinsics are handled separately by the compiler.
+            if tcx.intrinsic(def_id).is_some() {
+                return;
+            }
+            let instance = ty::Instance::expect_resolve(tcx, param_env, def_id, args, DUMMY_SP);
+            tcx.fn_abi_of_instance(param_env.and((instance, ty::List::empty())))
+        }
+        _ => {
+            panic!("Invalid function call");
+        }
+    };
+
+    let Ok(callee_abi) = callee_abi else {
+        // ABI failed to compute; this will not get through codegen.
+        return;
+    };
+    do_check_abi(tcx, callee_abi, caller.def_id(), |required_feature| {
+        tcx.emit_node_span_lint(
+            ABI_UNSUPPORTED_VECTOR_TYPES,
+            CRATE_HIR_ID,
+            span,
+            AbiErrorDisabledVectorTypeCall { span, required_feature },
+        );
+    });
+}
+
+struct MirCallesAbiCheck<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    body: &'a mir::Body<'tcx>,
+    instance: Instance<'tcx>,
+}
+
+impl<'a, 'tcx> MirVisitor<'tcx> for MirCallesAbiCheck<'a, 'tcx> {
+    fn visit_terminator(&mut self, terminator: &mir::Terminator<'tcx>, _: Location) {
+        match terminator.kind {
+            mir::TerminatorKind::Call { ref func, ref fn_span, .. }
+            | mir::TerminatorKind::TailCall { ref func, ref fn_span, .. } => {
+                let callee_ty = func.ty(self.body, self.tcx);
+                let callee_ty = self.instance.instantiate_mir_and_normalize_erasing_regions(
+                    self.tcx,
+                    ty::ParamEnv::reveal_all(),
+                    ty::EarlyBinder::bind(callee_ty),
+                );
+                check_call_site_abi(self.tcx, callee_ty, *fn_span, self.body.source.instance);
+            }
+            _ => {}
+        }
+    }
+}
+
+fn check_callees_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
+    let body = tcx.instance_mir(instance.def);
+    let mut visitor = MirCallesAbiCheck { tcx, body, instance };
+    for (bb, data) in traversal::mono_reachable(body, tcx, instance) {
+        visitor.visit_basic_block_data(bb, data)
+    }
+}
+
+fn check_feature_dependent_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
+    check_instance_abi(tcx, instance);
+    check_callees_abi(tcx, instance);
+}
+
+pub(super) fn provide(providers: &mut Providers) {
+    *providers = Providers { check_feature_dependent_abi, ..*providers }
+}

compiler/rustc_monomorphize/src/errors.rs

@@ -92,3 +92,21 @@ pub(crate) struct StartNotFound;
 pub(crate) struct UnknownCguCollectionMode<'a> {
     pub mode: &'a str,
 }
+
+#[derive(LintDiagnostic)]
+#[diag(monomorphize_abi_error_disabled_vector_type_def)]
+#[help]
+pub(crate) struct AbiErrorDisabledVectorTypeDef<'a> {
+    #[label]
+    pub span: Span,
+    pub required_feature: &'a str,
+}
+
+#[derive(LintDiagnostic)]
+#[diag(monomorphize_abi_error_disabled_vector_type_call)]
+#[help]
+pub(crate) struct AbiErrorDisabledVectorTypeCall<'a> {
+    #[label]
+    pub span: Span,
+    pub required_feature: &'a str,
+}