CFI: Fix SIGILL reached via trait objects

Fix #106547 by transforming the concrete self into a reference to a
trait object before emitting type metadata identifiers for trait
methods.

compiler/rustc_symbol_mangling/src/typeid.rs

@@ -4,7 +4,7 @@
 /// For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler,
 /// see design document in the tracking issue #89653.
 use bitflags::bitflags;
-use rustc_middle::ty::{FnSig, Ty, TyCtxt};
+use rustc_middle::ty::{FnSig, Instance, Ty, TyCtxt};
 use rustc_target::abi::call::FnAbi;
 use std::hash::Hasher;
 use twox_hash::XxHash64;
@@ -38,6 +38,15 @@ pub fn typeid_for_fnsig<'tcx>(
     typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, options)
 }
 
+/// Returns a type metadata identifier for the specified Instance.
+pub fn typeid_for_instance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: &Instance<'tcx>,
+    options: TypeIdOptions,
+) -> String {
+    typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options)
+}
+
 /// Returns a KCFI type metadata identifier for the specified FnAbi.
 pub fn kcfi_typeid_for_fnabi<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -63,3 +72,16 @@ pub fn kcfi_typeid_for_fnsig<'tcx>(
     hash.write(typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, options).as_bytes());
     hash.finish() as u32
 }
+
+/// Returns a KCFI type metadata identifier for the specified Instance.
+pub fn kcfi_typeid_for_instance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: &Instance<'tcx>,
+    options: TypeIdOptions,
+) -> u32 {
+    // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the
+    // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.)
+    let mut hash: XxHash64 = Default::default();
+    hash.write(typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options).as_bytes());
+    hash.finish() as u32
+}

compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs

@@ -14,8 +14,8 @@ use rustc_errors::DiagnosticMessage;
 use rustc_hir as hir;
 use rustc_middle::ty::subst::{GenericArg, GenericArgKind, SubstsRef};
 use rustc_middle::ty::{
-    self, Const, ExistentialPredicate, FloatTy, FnSig, IntTy, List, Region, RegionKind, TermKind,
-    Ty, TyCtxt, UintTy,
+    self, Const, ExistentialPredicate, FloatTy, FnSig, Instance, IntTy, List, Region, RegionKind,
+    TermKind, Ty, TyCtxt, UintTy,
 };
 use rustc_span::def_id::DefId;
 use rustc_span::sym;
@@ -1010,3 +1010,56 @@ pub fn typeid_for_fnsig<'tcx>(
 
     typeid
 }
+
+/// Returns a type metadata identifier for the specified Instance using the Itanium C++ ABI with
+/// vendor extended type qualifiers and types for Rust types that are not used at the FFI boundary.
+pub fn typeid_for_instance<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    instance: &Instance<'tcx>,
+    options: TypeIdOptions,
+) -> String {
+    let fn_abi = tcx
+        .fn_abi_of_instance(tcx.param_env(instance.def_id()).and((*instance, ty::List::empty())))
+        .unwrap_or_else(|instance| {
+            bug!("typeid_for_instance: couldn't get fn_abi of instance {:?}", instance)
+        });
+
+    // If this instance is a method and self is a reference, get the impl it belongs to
+    let impl_def_id = tcx.impl_of_method(instance.def_id());
+    if impl_def_id.is_some() && !fn_abi.args.is_empty() && fn_abi.args[0].layout.ty.is_ref() {
+        // If this impl is not an inherent impl, get the trait it implements
+        if let Some(trait_ref) = tcx.impl_trait_ref(impl_def_id.unwrap()) {
+            // Transform the concrete self into a reference to a trait object
+            let existential_predicate = trait_ref.map_bound(|trait_ref| {
+                ty::ExistentialPredicate::Trait(ty::ExistentialTraitRef::erase_self_ty(
+                    tcx, trait_ref,
+                ))
+            });
+            let existential_predicates = tcx.mk_poly_existential_predicates(&[ty::Binder::dummy(
+                existential_predicate.skip_binder(),
+            )]);
+            // Is the concrete self mutable?
+            let self_ty = if fn_abi.args[0].layout.ty.is_mutable_ptr() {
+                tcx.mk_mut_ref(
+                    tcx.lifetimes.re_erased,
+                    tcx.mk_dynamic(existential_predicates, tcx.lifetimes.re_erased, ty::Dyn),
+                )
+            } else {
+                tcx.mk_imm_ref(
+                    tcx.lifetimes.re_erased,
+                    tcx.mk_dynamic(existential_predicates, tcx.lifetimes.re_erased, ty::Dyn),
+                )
+            };
+
+            // Replace the concrete self in an fn_abi clone by the reference to a trait object
+            let mut fn_abi = fn_abi.clone();
+            // HACK(rcvalle): It is okay to not replace or update the entire ArgAbi here because the
+            //   other fields are never used.
+            fn_abi.args[0].layout.ty = self_ty;
+
+            return typeid_for_fnabi(tcx, &fn_abi, options);
+        }
+    }
+
+    typeid_for_fnabi(tcx, &fn_abi, options)
+}