bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Lower BinOp::Cmp to llvm.{s,u}cmp.* intrinsics

Browse source 
Lowers `mir::BinOp::Cmp` (`three_way_compare` intrinsic) to the corresponding
LLVM `llvm.{s,u}cmp.i8.*` intrinsics, added in LLVM 19.
This commit is contained in:
DaniPopes 2024-12-06 23:16:06 +01:00
parent 002da76821
commit 58c10c66c1
No known key found for this signature in database
GPG key ID: 0F09640DDB7AC692
8 changed files with 124 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

30
compiler/rustc_codegen_llvm/src/builder.rs
View file

@ -14,6 +14,7 @@ use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::*;
use rustc_data_structures::small_c_str::SmallCStr;
use rustc_hir::def_id::DefId;
use rustc_middle::bug;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
@ -1119,6 +1120,35 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
}
fn three_way_compare(
&mut self,
ty: Ty<'tcx>,
lhs: Self::Value,
rhs: Self::Value,
) -> Option<Self::Value> {
// FIXME: See comment on the definition of `three_way_compare`.
if crate::llvm_util::get_version() < (20, 0, 0) {
return None;
}
let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) {
(true, 8) => "llvm.scmp.i8.i8",
(true, 16) => "llvm.scmp.i8.i16",
(true, 32) => "llvm.scmp.i8.i32",
(true, 64) => "llvm.scmp.i8.i64",
(true, 128) => "llvm.scmp.i8.i128",
(false, 8) => "llvm.ucmp.i8.i8",
(false, 16) => "llvm.ucmp.i8.i16",
(false, 32) => "llvm.ucmp.i8.i32",
(false, 64) => "llvm.ucmp.i8.i64",
(false, 128) => "llvm.ucmp.i8.i128",
_ => bug!("three-way compare unsupported for type {ty:?}"),
};
Some(self.call_intrinsic(name, &[lhs, rhs]))
}
/* Miscellaneous instructions */
fn memcpy(
&mut self,

12
compiler/rustc_codegen_llvm/src/context.rs
View file

@ -1108,6 +1108,18 @@ impl<'ll> CodegenCx<'ll, '_> {
ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64);
ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128);
ifn!("llvm.scmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.scmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
ifn!("llvm.scmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
ifn!("llvm.scmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
ifn!("llvm.scmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
ifn!("llvm.ucmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.ucmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
ifn!("llvm.ucmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
ifn!("llvm.ucmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
ifn!("llvm.ucmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void);
ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void);

3
compiler/rustc_codegen_ssa/src/mir/rvalue.rs
View file

@ -1005,6 +1005,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
mir::BinOp::Cmp => {
use std::cmp::Ordering;
assert!(!is_float);
if let Some(value) = bx.three_way_compare(lhs_ty, lhs, rhs) {
return value;
}
let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed);
if bx.cx().tcx().sess.opts.optimize == OptLevel::No {
// FIXME: This actually generates tighter assembly, and is a classic trick

12
compiler/rustc_codegen_ssa/src/traits/builder.rs
View file

@ -397,6 +397,18 @@ pub trait BuilderMethods<'a, 'tcx>:
fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
fn fcmp(&mut self, op: RealPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;
/// Returns `-1` if `lhs < rhs`, `0` if `lhs == rhs`, and `1` if `lhs > rhs`.
// FIXME: Move the default implementation from `codegen_scalar_binop` into this method and
// remove the `Option` return once LLVM 20 is the minimum version.
fn three_way_compare(
&mut self,
_ty: Ty<'tcx>,
_lhs: Self::Value,
_rhs: Self::Value,
) -> Option<Self::Value> {
None
}
fn memcpy(
&mut self,
dst: Self::Value,

46
tests/assembly/x86_64-cmp.rs
View file

@ -1,5 +1,8 @@
//@ revisions: DEBUG LLVM-PRE-20-OPTIM LLVM-20-OPTIM
//@ [DEBUG] compile-flags: -C opt-level=0
//@ revisions: LLVM-PRE-20-DEBUG LLVM-20-DEBUG LLVM-PRE-20-OPTIM LLVM-20-OPTIM
//@ [LLVM-PRE-20-DEBUG] compile-flags: -C opt-level=0
//@ [LLVM-PRE-20-DEBUG] max-llvm-major-version: 19
//@ [LLVM-20-DEBUG] compile-flags: -C opt-level=0
//@ [LLVM-20-DEBUG] min-llvm-version: 20
//@ [LLVM-PRE-20-OPTIM] compile-flags: -C opt-level=3
//@ [LLVM-PRE-20-OPTIM] max-llvm-major-version: 19
//@ [LLVM-20-OPTIM] compile-flags: -C opt-level=3
@ -16,13 +19,19 @@ use std::intrinsics::three_way_compare;
#[no_mangle]
// CHECK-LABEL: signed_cmp:
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
// DEBUG: cmp
// DEBUG: setg
// DEBUG: and
// DEBUG: cmp
// DEBUG: setl
// DEBUG: and
// DEBUG: sub
// LLVM-PRE-20-DEBUG: cmp
// LLVM-PRE-20-DEBUG: setg
// LLVM-PRE-20-DEBUG: and
// LLVM-PRE-20-DEBUG: cmp
// LLVM-PRE-20-DEBUG: setl
// LLVM-PRE-20-DEBUG: and
// LLVM-PRE-20-DEBUG: sub
//
// LLVM-20-DEBUG: sub
// LLVM-20-DEBUG: setl
// LLVM-20-DEBUG: setg
// LLVM-20-DEBUG: sub
// LLVM-20-DEBUG: ret
// LLVM-PRE-20-OPTIM: xor
// LLVM-PRE-20-OPTIM: cmp
@ -42,13 +51,18 @@ pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
#[no_mangle]
// CHECK-LABEL: unsigned_cmp:
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
// DEBUG: cmp
// DEBUG: seta
// DEBUG: and
// DEBUG: cmp
// DEBUG: setb
// DEBUG: and
// DEBUG: sub
// LLVM-PRE-20-DEBUG: cmp
// LLVM-PRE-20-DEBUG: seta
// LLVM-PRE-20-DEBUG: and
// LLVM-PRE-20-DEBUG: cmp
// LLVM-PRE-20-DEBUG: setb
// LLVM-PRE-20-DEBUG: and
// LLVM-PRE-20-DEBUG: sub
//
// LLVM-20-DEBUG: sub
// LLVM-20-DEBUG: seta
// LLVM-20-DEBUG: sbb
// LLVM-20-DEBUG: ret
// LLVM-PRE-20-OPTIM: xor
// LLVM-PRE-20-OPTIM: cmp

1
tests/codegen/comparison-operators-2-tuple.rs
View file

@ -1,5 +1,4 @@
//@ compile-flags: -C opt-level=1 -Z merge-functions=disabled
//@ only-x86_64
//@ min-llvm-version: 20
#![crate_type = "lib"]

35
tests/codegen/integer-cmp.rs
View file

@ -4,7 +4,7 @@
//@ revisions: llvm-pre-20 llvm-20
//@ [llvm-20] min-llvm-version: 20
//@ [llvm-pre-20] max-llvm-major-version: 19
//@ compile-flags: -C opt-level=3
//@ compile-flags: -C opt-level=3 -Zmerge-functions=disabled
#![crate_type = "lib"]
@ -13,7 +13,7 @@ use std::cmp::Ordering;
// CHECK-LABEL: @cmp_signed
#[no_mangle]
pub fn cmp_signed(a: i64, b: i64) -> Ordering {
// llvm-20: @llvm.scmp.i8.i64
// llvm-20: call{{.*}} i8 @llvm.scmp.i8.i64
// llvm-pre-20: icmp slt
// llvm-pre-20: icmp ne
// llvm-pre-20: zext i1
@ -24,10 +24,39 @@ pub fn cmp_signed(a: i64, b: i64) -> Ordering {
// CHECK-LABEL: @cmp_unsigned
#[no_mangle]
pub fn cmp_unsigned(a: u32, b: u32) -> Ordering {
// llvm-20: @llvm.ucmp.i8.i32
// llvm-20: call{{.*}} i8 @llvm.ucmp.i8.i32
// llvm-pre-20: icmp ult
// llvm-pre-20: icmp ne
// llvm-pre-20: zext i1
// llvm-pre-20: select i1
a.cmp(&b)
}
// CHECK-LABEL: @cmp_char
#[no_mangle]
pub fn cmp_char(a: char, b: char) -> Ordering {
// llvm-20: call{{.*}} i8 @llvm.ucmp.i8.i32
// llvm-pre-20: icmp ult
// llvm-pre-20: icmp ne
// llvm-pre-20: zext i1
// llvm-pre-20: select i1
a.cmp(&b)
}
// CHECK-LABEL: @cmp_tuple
#[no_mangle]
pub fn cmp_tuple(a: (i16, u16), b: (i16, u16)) -> Ordering {
// llvm-20-DAG: call{{.*}} i8 @llvm.ucmp.i8.i16
// llvm-20-DAG: call{{.*}} i8 @llvm.scmp.i8.i16
// llvm-20: ret i8
// llvm-pre-20: icmp slt
// llvm-pre-20: icmp ne
// llvm-pre-20: zext i1
// llvm-pre-20: select i1
// llvm-pre-20: icmp ult
// llvm-pre-20: icmp ne
// llvm-pre-20: zext i1
// llvm-pre-20: select i1
// llvm-pre-20: select i1
a.cmp(&b)
}

27
tests/codegen/intrinsics/three_way_compare.rs
View file

@ -2,6 +2,7 @@
//@ [DEBUG] compile-flags: -C opt-level=0
//@ [OPTIM] compile-flags: -C opt-level=3
//@ compile-flags: -C no-prepopulate-passes
//@ min-llvm-version: 20
#![crate_type = "lib"]
#![feature(core_intrinsics)]
@ -12,17 +13,8 @@ use std::intrinsics::three_way_compare;
// CHECK-LABEL: @signed_cmp
// CHECK-SAME: (i16{{.*}} %a, i16{{.*}} %b)
pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
// DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
// DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
// OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
// OPTIM: ret i8 %[[CGEL]]
// CHECK: %[[CMP:.+]] = call i8 @llvm.scmp.i8.i16(i16 %a, i16 %b)
// CHECK-NEXT: ret i8 %[[CMP]]
three_way_compare(a, b)
}
@ -30,16 +22,7 @@ pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
// CHECK-LABEL: @unsigned_cmp
// CHECK-SAME: (i16{{.*}} %a, i16{{.*}} %b)
pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
// DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b
// DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
// DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b
// DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
// DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
// OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b
// OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
// OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
// OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
// OPTIM: ret i8 %[[CGEL]]
// CHECK: %[[CMP:.+]] = call i8 @llvm.ucmp.i8.i16(i16 %a, i16 %b)
// CHECK-NEXT: ret i8 %[[CMP]]
three_way_compare(a, b)
}