bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #128371 - andjo403:rangeAttribute, r=nikic

Browse source 
Add range attribute to scalar function results and arguments

as LLVM 19 adds the range attribute this starts to use it for better optimization.
hade been interesting to see a perf run with the https://github.com/rust-lang/rust/pull/127513

closes https://github.com/rust-lang/rust/issues/50156
cc https://github.com/rust-lang/rust/issues/49572 shall be fixed but not possible to see as there is asserts that already trigger the optimization.
This commit is contained in:
bors 2024-08-12 10:20:00 +00:00
commit e08b80c0fb
12 changed files with 159 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

61
compiler/rustc_codegen_llvm/src/abi.rs
View file

@ -430,9 +430,32 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
i += 1;
i - 1
};
let apply_range_attr = |idx: AttributePlace, scalar: rustc_target::abi::Scalar| {
if cx.sess().opts.optimize != config::OptLevel::No
&& llvm_util::get_version() >= (19, 0, 0)
&& matches!(scalar.primitive(), Int(..))
// If the value is a boolean, the range is 0..2 and that ultimately
// become 0..0 when the type becomes i1, which would be rejected
// by the LLVM verifier.
&& !scalar.is_bool()
// LLVM also rejects full range.
&& !scalar.is_always_valid(cx)
{
attributes::apply_to_llfn(
llfn,
idx,
&[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))],
);
}
};
match &self.ret.mode {
PassMode::Direct(attrs) => {
attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
if let abi::Abi::Scalar(scalar) = self.ret.layout.abi {
apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
}
}
PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
assert!(!on_stack);
@ -471,8 +494,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
);
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
}
PassMode::Direct(attrs)
| PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
PassMode::Direct(attrs) => {
let i = apply(attrs);
if let abi::Abi::Scalar(scalar) = arg.layout.abi {
apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
}
}
PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
apply(attrs);
}
PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
@ -481,8 +509,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
apply(meta_attrs);
}
PassMode::Pair(a, b) => {
apply(a);
apply(b);
let i = apply(a);
let ii = apply(b);
if let abi::Abi::ScalarPair(scalar_a, scalar_b) = arg.layout.abi {
apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a);
apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b);
}
}
PassMode::Cast { cast, pad_i32 } => {
if *pad_i32 {
@ -537,15 +569,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
}
_ => {}
}
if let abi::Abi::Scalar(scalar) = self.ret.layout.abi {
// If the value is a boolean, the range is 0..2 and that ultimately
// become 0..0 when the type becomes i1, which would be rejected
// by the LLVM verifier.
if let Int(..) = scalar.primitive() {
if !scalar.is_bool() && !scalar.is_always_valid(bx) {
bx.range_metadata(callsite, scalar.valid_range(bx));
}
}
if bx.cx.sess().opts.optimize != config::OptLevel::No
&& llvm_util::get_version() < (19, 0, 0)
&& let abi::Abi::Scalar(scalar) = self.ret.layout.abi
&& matches!(scalar.primitive(), Int(..))
// If the value is a boolean, the range is 0..2 and that ultimately
// become 0..0 when the type becomes i1, which would be rejected
// by the LLVM verifier.
&& !scalar.is_bool()
// LLVM also rejects full range.
&& !scalar.is_always_valid(bx)
{
bx.range_metadata(callsite, scalar.valid_range(bx));
}
for arg in self.args.iter() {
match &arg.mode {

6
compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file

@ -1575,6 +1575,12 @@ extern "C" {
pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute;
pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute;
pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute;
pub fn LLVMRustCreateRangeAttribute(
C: &Context,
num_bits: c_uint,
lower_words: *const u64,
upper_words: *const u64,
) -> &Attribute;
// Operations on functions
pub fn LLVMRustGetOrInsertFunction<'a>(

17
compiler/rustc_codegen_llvm/src/llvm/mod.rs
View file

@ -8,7 +8,7 @@ use std::string::FromUtf8Error;
use libc::c_uint;
use rustc_data_structures::small_c_str::SmallCStr;
use rustc_llvm::RustString;
use rustc_target::abi::Align;
use rustc_target::abi::{Align, Size, WrappingRange};
pub use self::AtomicRmwBinOp::*;
pub use self::CallConv::*;
@ -105,6 +105,21 @@ pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribu
unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) }
}
pub fn CreateRangeAttr(llcx: &Context, size: Size, range: WrappingRange) -> &Attribute {
let lower = range.start;
let upper = range.end.wrapping_add(1);
let lower_words = [lower as u64, (lower >> 64) as u64];
let upper_words = [upper as u64, (upper >> 64) as u64];
unsafe {
LLVMRustCreateRangeAttribute(
llcx,
size.bits().try_into().unwrap(),
lower_words.as_ptr(),
upper_words.as_ptr(),
)
}
}
#[derive(Copy, Clone)]
pub enum AttributePlace {
ReturnValue,