2021-04-03 13:05:11 +02:00
|
|
|
//
|
2018-05-04 20:07:35 +02:00
|
|
|
|
|
|
|
//@ compile-flags: -C no-prepopulate-passes
|
|
|
|
|
|
|
|
#![crate_type = "lib"]
|
2025-02-24 17:26:56 +01:00
|
|
|
#![feature(repr_simd, core_intrinsics)]
|
2018-05-04 20:07:35 +02:00
|
|
|
#![allow(non_camel_case_types)]
|
|
|
|
|
2025-02-24 17:26:56 +01:00
|
|
|
use std::intrinsics::simd::simd_scatter;
|
|
|
|
|
2018-05-04 20:07:35 +02:00
|
|
|
#[repr(simd)]
|
|
|
|
#[derive(Copy, Clone, PartialEq, Debug)]
|
2024-08-22 01:28:20 -07:00
|
|
|
pub struct Vec2<T>(pub [T; 2]);
|
2018-05-04 20:07:35 +02:00
|
|
|
|
|
|
|
#[repr(simd)]
|
|
|
|
#[derive(Copy, Clone, PartialEq, Debug)]
|
2024-08-22 01:28:20 -07:00
|
|
|
pub struct Vec4<T>(pub [T; 4]);
|
2018-05-04 20:07:35 +02:00
|
|
|
|
|
|
|
// CHECK-LABEL: @scatter_f32x2
|
|
|
|
#[no_mangle]
|
2025-02-24 17:26:56 +01:00
|
|
|
pub unsafe fn scatter_f32x2(pointers: Vec2<*mut f32>, mask: Vec2<i32>, values: Vec2<f32>) {
|
2025-01-27 15:10:42 +01:00
|
|
|
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
|
Consistently use the most significant bit of vector masks
This improves the codegen for vector `select`, `gather`, `scatter` and
boolean reduction intrinsics and fixes rust-lang/portable-simd#316.
The current behavior of most mask operations during llvm codegen is to
truncate the mask vector to <N x i1>, telling llvm to use the least
significat bit. The exception is the `simd_bitmask` intrinsics, which
already used the most signifiant bit.
Since sse/avx instructions are defined to use the most significant bit,
truncating means that llvm has to insert a left shift to move the bit
into the most significant position, before the mask can actually be
used.
Similarly on aarch64, mask operations like blend work bit by bit,
repeating the least significant bit across the whole lane involves
shifting it into the sign position and then comparing against zero.
By shifting before truncating to <N x i1>, we tell llvm that we only
consider the most significant bit, removing the need for additional
shift instructions in the assembly.
2023-01-04 23:55:40 +01:00
|
|
|
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
|
|
|
|
// CHECK: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
|
2018-05-04 20:07:35 +02:00
|
|
|
simd_scatter(values, pointers, mask)
|
|
|
|
}
|
|
|
|
|
2025-03-03 18:18:33 +01:00
|
|
|
// CHECK-LABEL: @scatter_f32x2_unsigned
|
|
|
|
#[no_mangle]
|
|
|
|
pub unsafe fn scatter_f32x2_unsigned(pointers: Vec2<*mut f32>, mask: Vec2<u32>, values: Vec2<f32>) {
|
|
|
|
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
|
|
|
|
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
|
|
|
|
// CHECK: call void @llvm.masked.scatter.v2f32.v2p0(<2 x float> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
|
|
|
|
simd_scatter(values, pointers, mask)
|
|
|
|
}
|
|
|
|
|
2018-05-04 20:07:35 +02:00
|
|
|
// CHECK-LABEL: @scatter_pf32x2
|
|
|
|
#[no_mangle]
|
2025-02-24 17:26:56 +01:00
|
|
|
pub unsafe fn scatter_pf32x2(
|
|
|
|
pointers: Vec2<*mut *const f32>,
|
|
|
|
mask: Vec2<i32>,
|
|
|
|
values: Vec2<*const f32>,
|
|
|
|
) {
|
2025-01-27 15:10:42 +01:00
|
|
|
// CHECK: [[A:%[0-9]+]] = lshr <2 x i32> {{.*}}, {{<i32 31, i32 31>|splat \(i32 31\)}}
|
Consistently use the most significant bit of vector masks
This improves the codegen for vector `select`, `gather`, `scatter` and
boolean reduction intrinsics and fixes rust-lang/portable-simd#316.
The current behavior of most mask operations during llvm codegen is to
truncate the mask vector to <N x i1>, telling llvm to use the least
significat bit. The exception is the `simd_bitmask` intrinsics, which
already used the most signifiant bit.
Since sse/avx instructions are defined to use the most significant bit,
truncating means that llvm has to insert a left shift to move the bit
into the most significant position, before the mask can actually be
used.
Similarly on aarch64, mask operations like blend work bit by bit,
repeating the least significant bit across the whole lane involves
shifting it into the sign position and then comparing against zero.
By shifting before truncating to <N x i1>, we tell llvm that we only
consider the most significant bit, removing the need for additional
shift instructions in the assembly.
2023-01-04 23:55:40 +01:00
|
|
|
// CHECK: [[B:%[0-9]+]] = trunc <2 x i32> [[A]] to <2 x i1>
|
|
|
|
// CHECK: call void @llvm.masked.scatter.v2p0.v2p0(<2 x ptr> {{.*}}, <2 x ptr> {{.*}}, i32 {{.*}}, <2 x i1> [[B]]
|
2018-05-04 20:07:35 +02:00
|
|
|
simd_scatter(values, pointers, mask)
|
|
|
|
}
|