Add cross-language LLVM CFI support to the Rust compiler

This commit adds cross-language LLVM Control Flow Integrity (CFI)
support to the Rust compiler by adding the
`-Zsanitizer-cfi-normalize-integers` option to be used with Clang
`-fsanitize-cfi-icall-normalize-integers` for normalizing integer types
(see https://reviews.llvm.org/D139395).

It provides forward-edge control flow protection for C or C++ and Rust
-compiled code "mixed binaries" (i.e., for when C or C++ and Rust
-compiled code share the same virtual address space). For more
information about LLVM CFI and cross-language LLVM CFI support for the
Rust compiler, see design document in the tracking issue #89653.

Cross-language LLVM CFI can be enabled with -Zsanitizer=cfi and
-Zsanitizer-cfi-normalize-integers, and requires proper (i.e.,
non-rustc) LTO (i.e., -Clinker-plugin-lto).
This commit is contained in:
Ramon de C Valle 2022-12-12 22:42:44 -08:00
parent fec9adcdbc
commit 004aa15b47
70 changed files with 1384 additions and 387 deletions

View file

@ -442,9 +442,9 @@ pub(crate) fn inline_asm_call<'ll>(
);
let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
bx.invoke(fty, None, v, inputs, dest, catch, funclet)
bx.invoke(fty, None, None, v, inputs, dest, catch, funclet)
} else {
bx.call(fty, None, v, inputs, None)
bx.call(fty, None, None, v, inputs, None)
};
// Store mark in a metadata node so we can map LLVM errors

View file

@ -15,14 +15,15 @@ use rustc_codegen_ssa::traits::*;
use rustc_codegen_ssa::MemFlags;
use rustc_data_structures::small_c_str::SmallCStr;
use rustc_hir::def_id::DefId;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout,
};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::Span;
use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi;
use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions};
use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
use rustc_target::spec::{HasTargetSpec, Target};
use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
use std::borrow::Cow;
use std::ffi::CStr;
use std::iter;
@ -216,6 +217,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
fn invoke(
&mut self,
llty: &'ll Type,
fn_attrs: Option<&CodegenFnAttrs>,
fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
llfn: &'ll Value,
args: &[&'ll Value],
@ -230,19 +232,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
let mut bundles = vec![funclet_bundle];
// Set KCFI operand bundle
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
let kcfi_bundle =
if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call {
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
} else {
None
};
if kcfi_bundle.is_some() {
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
bundles.push(kcfi_bundle);
}
// Emit CFI pointer type membership test
self.cfi_type_test(fn_attrs, fn_abi, llfn);
// Emit KCFI operand bundle
let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn);
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
bundles.push(kcfi_bundle);
bundles.retain(|bundle| bundle.is_some());
let invoke = unsafe {
@ -1183,6 +1179,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
fn call(
&mut self,
llty: &'ll Type,
fn_attrs: Option<&CodegenFnAttrs>,
fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
llfn: &'ll Value,
args: &[&'ll Value],
@ -1195,19 +1192,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
let mut bundles = vec![funclet_bundle];
// Set KCFI operand bundle
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
let kcfi_bundle =
if let Some(fn_abi) = fn_abi && self.tcx.sess.is_sanitizer_kcfi_enabled() && is_indirect_call {
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
} else {
None
};
if kcfi_bundle.is_some() {
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
bundles.push(kcfi_bundle);
}
// Emit CFI pointer type membership test
self.cfi_type_test(fn_attrs, fn_abi, llfn);
// Emit KCFI operand bundle
let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn);
let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
bundles.push(kcfi_bundle);
bundles.retain(|bundle| bundle.is_some());
let call = unsafe {
@ -1456,7 +1447,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
let (ty, f) = self.cx.get_intrinsic(intrinsic);
self.call(ty, None, f, args, None)
self.call(ty, None, None, f, args, None)
}
fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
@ -1518,7 +1509,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width)
};
let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None)
self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None)
}
pub(crate) fn landing_pad(
@ -1535,4 +1526,71 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
}
}
// Emits CFI pointer type membership tests.
fn cfi_type_test(
&mut self,
fn_attrs: Option<&CodegenFnAttrs>,
fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
llfn: &'ll Value,
) {
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
if is_indirect_call && fn_abi.is_some() && self.tcx.sess.is_sanitizer_cfi_enabled() {
if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::CFI) {
return;
}
let mut options = TypeIdOptions::empty();
if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
options.insert(TypeIdOptions::GENERALIZE_POINTERS);
}
if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
}
let typeid = typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options);
let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
// Test whether the function pointer is associated with the type identifier.
let cond = self.type_test(llfn, typeid_metadata);
let bb_pass = self.append_sibling_block("type_test.pass");
let bb_fail = self.append_sibling_block("type_test.fail");
self.cond_br(cond, bb_pass, bb_fail);
self.switch_to_block(bb_fail);
self.abort();
self.unreachable();
self.switch_to_block(bb_pass);
}
}
// Emits KCFI operand bundles.
fn kcfi_operand_bundle(
&mut self,
fn_attrs: Option<&CodegenFnAttrs>,
fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
llfn: &'ll Value,
) -> Option<llvm::OperandBundleDef<'ll>> {
let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() };
let kcfi_bundle = if is_indirect_call && self.tcx.sess.is_sanitizer_kcfi_enabled() {
if fn_attrs.is_some() && fn_attrs.unwrap().no_sanitize.contains(SanitizerSet::KCFI) {
return None;
}
let mut options = TypeIdOptions::empty();
if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
options.insert(TypeIdOptions::GENERALIZE_POINTERS);
}
if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
}
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap(), options);
Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)]))
} else {
None
};
kcfi_bundle
}
}

View file

@ -228,18 +228,29 @@ pub unsafe fn create_module<'ll>(
llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1);
}
if sess.is_sanitizer_cfi_enabled() {
// FIXME(rcvalle): Add support for non canonical jump tables.
// Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.)
if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() {
let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast();
// FIXME(rcvalle): Add it with Override behavior flag.
llvm::LLVMRustAddModuleFlag(
llmod,
llvm::LLVMModFlagBehavior::Warning,
llvm::LLVMModFlagBehavior::Override,
canonical_jump_tables,
1,
);
}
// Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.)
if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() {
let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast();
llvm::LLVMRustAddModuleFlag(
llmod,
llvm::LLVMModFlagBehavior::Override,
enable_split_lto_unit,
1,
);
}
// Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.)
if sess.is_sanitizer_kcfi_enabled() {
let kcfi = "kcfi\0".as_ptr().cast();
llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1);

View file

@ -20,7 +20,7 @@ use crate::type_::Type;
use crate::value::Value;
use rustc_codegen_ssa::traits::TypeMembershipMethods;
use rustc_middle::ty::Ty;
use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi};
use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions};
use smallvec::SmallVec;
/// Declare a function.
@ -132,12 +132,31 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
fn_abi.apply_attrs_llfn(self, llfn);
if self.tcx.sess.is_sanitizer_cfi_enabled() {
let typeid = typeid_for_fnabi(self.tcx, fn_abi);
let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty());
self.set_type_metadata(llfn, typeid);
let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS);
self.add_type_metadata(llfn, typeid);
let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS);
self.add_type_metadata(llfn, typeid);
let typeid = typeid_for_fnabi(
self.tcx,
fn_abi,
TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS,
);
self.add_type_metadata(llfn, typeid);
}
if self.tcx.sess.is_sanitizer_kcfi_enabled() {
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi);
// LLVM KCFI does not support multiple !kcfi_type attachments
let mut options = TypeIdOptions::empty();
if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
options.insert(TypeIdOptions::GENERALIZE_POINTERS);
}
if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
options.insert(TypeIdOptions::NORMALIZE_INTEGERS);
}
let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options);
self.set_kcfi_type_metadata(llfn, kcfi_typeid);
}

View file

@ -110,6 +110,7 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
self.call(
simple_ty,
None,
None,
simple_fn,
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
None,
@ -444,7 +445,7 @@ fn try_intrinsic<'ll>(
) {
if bx.sess().panic_strategy() == PanicStrategy::Abort {
let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void());
bx.call(try_func_ty, None, try_func, &[data], None);
bx.call(try_func_ty, None, None, try_func, &[data], None);
// Return 0 unconditionally from the intrinsic call;
// we can never unwind.
let ret_align = bx.tcx().data_layout.i32_align.abi;
@ -543,7 +544,7 @@ fn codegen_msvc_try<'ll>(
let ptr_align = bx.tcx().data_layout.pointer_align.abi;
let slot = bx.alloca(bx.type_i8p(), ptr_align);
let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void());
bx.invoke(try_func_ty, None, try_func, &[data], normal, catchswitch, None);
bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None);
bx.switch_to_block(normal);
bx.ret(bx.const_i32(0));
@ -587,7 +588,7 @@ fn codegen_msvc_try<'ll>(
let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]);
let ptr = bx.load(bx.type_i8p(), slot, ptr_align);
let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void());
bx.call(catch_ty, None, catch_func, &[data, ptr], Some(&funclet));
bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet));
bx.catch_ret(&funclet, caught);
// The flag value of 64 indicates a "catch-all".
@ -595,7 +596,7 @@ fn codegen_msvc_try<'ll>(
let flags = bx.const_i32(64);
let null = bx.const_null(bx.type_i8p());
let funclet = bx.catch_pad(cs, &[null, flags, null]);
bx.call(catch_ty, None, catch_func, &[data, null], Some(&funclet));
bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet));
bx.catch_ret(&funclet, caught);
bx.switch_to_block(caught);
@ -604,7 +605,7 @@ fn codegen_msvc_try<'ll>(
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None);
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
}
@ -647,7 +648,7 @@ fn codegen_gnu_try<'ll>(
let data = llvm::get_param(bx.llfn(), 1);
let catch_func = llvm::get_param(bx.llfn(), 2);
let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void());
bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None);
bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None);
bx.switch_to_block(then);
bx.ret(bx.const_i32(0));
@ -665,13 +666,13 @@ fn codegen_gnu_try<'ll>(
bx.add_clause(vals, tydesc);
let ptr = bx.extract_value(vals, 0);
let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void());
bx.call(catch_ty, None, catch_func, &[data, ptr], None);
bx.call(catch_ty, None, None, catch_func, &[data, ptr], None);
bx.ret(bx.const_i32(1));
});
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None);
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
}
@ -711,7 +712,7 @@ fn codegen_emcc_try<'ll>(
let data = llvm::get_param(bx.llfn(), 1);
let catch_func = llvm::get_param(bx.llfn(), 2);
let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void());
bx.invoke(try_func_ty, None, try_func, &[data], then, catch, None);
bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None);
bx.switch_to_block(then);
bx.ret(bx.const_i32(0));
@ -750,13 +751,13 @@ fn codegen_emcc_try<'ll>(
let catch_data = bx.bitcast(catch_data, bx.type_i8p());
let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void());
bx.call(catch_ty, None, catch_func, &[data, catch_data], None);
bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None);
bx.ret(bx.const_i32(1));
});
// Note that no invoke is used here because by definition this function
// can't panic (that's what it's catching).
let ret = bx.call(llty, None, llfn, &[try_func, data, catch_func], None);
let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None);
let i32_align = bx.tcx().data_layout.i32_align.abi;
bx.store(ret, dest, i32_align);
}
@ -1205,6 +1206,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
let c = bx.call(
fn_ty,
None,
None,
f,
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
None,
@ -1423,6 +1425,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
let v = bx.call(
fn_ty,
None,
None,
f,
&[args[1].immediate(), alignment, mask, args[0].immediate()],
None,
@ -1564,6 +1567,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
let v = bx.call(
fn_ty,
None,
None,
f,
&[args[0].immediate(), args[1].immediate(), alignment, mask],
None,
@ -2037,7 +2041,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty);
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(fn_ty, None, f, &[lhs, rhs], None);
let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None);
return Ok(v);
}

View file

@ -291,8 +291,24 @@ impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
}
impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> {
fn add_type_metadata(&self, function: &'ll Value, typeid: String) {
let typeid_metadata = self.typeid_metadata(typeid).unwrap();
let v = [self.const_usize(0), typeid_metadata];
unsafe {
llvm::LLVMRustGlobalAddMetadata(
function,
llvm::MD_type as c_uint,
llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext(
self.llcx,
v.as_ptr(),
v.len() as c_uint,
)),
)
}
}
fn set_type_metadata(&self, function: &'ll Value, typeid: String) {
let typeid_metadata = self.typeid_metadata(typeid);
let typeid_metadata = self.typeid_metadata(typeid).unwrap();
let v = [self.const_usize(0), typeid_metadata];
unsafe {
llvm::LLVMGlobalSetMetadata(
@ -307,13 +323,28 @@ impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> {
}
}
fn typeid_metadata(&self, typeid: String) -> &'ll Value {
unsafe {
fn typeid_metadata(&self, typeid: String) -> Option<&'ll Value> {
Some(unsafe {
llvm::LLVMMDStringInContext(
self.llcx,
typeid.as_ptr() as *const c_char,
typeid.len() as c_uint,
)
})
}
fn add_kcfi_type_metadata(&self, function: &'ll Value, kcfi_typeid: u32) {
let kcfi_type_metadata = self.const_u32(kcfi_typeid);
unsafe {
llvm::LLVMRustGlobalAddMetadata(
function,
llvm::MD_kcfi_type as c_uint,
llvm::LLVMMDNodeInContext2(
self.llcx,
&llvm::LLVMValueAsMetadata(kcfi_type_metadata),
1,
),
)
}
}