bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Sync core::simd up to rust-lang/portable-simd@2e081db92a

Browse source
This commit is contained in:
Jubilee Young 2022-07-20 17:57:56 -07:00
commit a14404a028
31 changed files with 1605 additions and 880 deletions
Download patch file Download diff file Expand all files Collapse all files

3
library/portable-simd/crates/core_simd/Cargo.toml
View file

@ -9,7 +9,8 @@ categories = ["hardware-support", "no-std"]
license = "MIT OR Apache-2.0"
[features]
default = []
default = ["as_crate"]
as_crate = []
std = []
generic_const_exprs = []

120
library/portable-simd/crates/core_simd/src/comparisons.rs
View file

@ -1,120 +0,0 @@
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Mask, Simd, SimdElement, SupportedLaneCount};
impl<T, const LANES: usize> Simd<T, LANES>
where
T: SimdElement + PartialEq,
LaneCount<LANES>: SupportedLaneCount,
{
/// Test if each lane is equal to the corresponding lane in `other`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn lanes_eq(self, other: Self) -> Mask<T::Mask, LANES> {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) }
}
/// Test if each lane is not equal to the corresponding lane in `other`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn lanes_ne(self, other: Self) -> Mask<T::Mask, LANES> {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) }
}
}
impl<T, const LANES: usize> Simd<T, LANES>
where
T: SimdElement + PartialOrd,
LaneCount<LANES>: SupportedLaneCount,
{
/// Test if each lane is less than the corresponding lane in `other`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn lanes_lt(self, other: Self) -> Mask<T::Mask, LANES> {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) }
}
/// Test if each lane is greater than the corresponding lane in `other`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn lanes_gt(self, other: Self) -> Mask<T::Mask, LANES> {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) }
}
/// Test if each lane is less than or equal to the corresponding lane in `other`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn lanes_le(self, other: Self) -> Mask<T::Mask, LANES> {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) }
}
/// Test if each lane is greater than or equal to the corresponding lane in `other`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn lanes_ge(self, other: Self) -> Mask<T::Mask, LANES> {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) }
}
}
macro_rules! impl_ord_methods_vector {
{ $type:ty } => {
impl<const LANES: usize> Simd<$type, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
/// Returns the lane-wise minimum with `other`.
#[must_use = "method returns a new vector and does not mutate the original value"]
#[inline]
pub fn min(self, other: Self) -> Self {
self.lanes_gt(other).select(other, self)
}
/// Returns the lane-wise maximum with `other`.
#[must_use = "method returns a new vector and does not mutate the original value"]
#[inline]
pub fn max(self, other: Self) -> Self {
self.lanes_lt(other).select(other, self)
}
/// Restrict each lane to a certain interval.
///
/// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is
/// less than `min`. Otherwise returns `self`.
///
/// # Panics
///
/// Panics if `min > max` on any lane.
#[must_use = "method returns a new vector and does not mutate the original value"]
#[inline]
pub fn clamp(self, min: Self, max: Self) -> Self {
assert!(
min.lanes_le(max).all(),
"each lane in `min` must be less than or equal to the corresponding lane in `max`",
);
self.max(min).min(max)
}
}
}
}
impl_ord_methods_vector!(i8);
impl_ord_methods_vector!(i16);
impl_ord_methods_vector!(i32);
impl_ord_methods_vector!(i64);
impl_ord_methods_vector!(isize);
impl_ord_methods_vector!(u8);
impl_ord_methods_vector!(u16);
impl_ord_methods_vector!(u32);
impl_ord_methods_vector!(u64);
impl_ord_methods_vector!(usize);

11
library/portable-simd/crates/core_simd/src/elements.rs Normal file
View file

@ -0,0 +1,11 @@
mod float;
mod int;
mod uint;
mod sealed {
pub trait Sealed {}
}
pub use float::*;
pub use int::*;
pub use uint::*;

357
library/portable-simd/crates/core_simd/src/elements/float.rs Normal file
View file

@ -0,0 +1,357 @@
use super::sealed::Sealed;
use crate::simd::{
intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialEq, SimdPartialOrd,
SupportedLaneCount,
};
/// Operations on SIMD vectors of floats.
pub trait SimdFloat: Copy + Sealed {
/// Mask type used for manipulating this SIMD vector type.
type Mask;
/// Scalar type contained by this SIMD vector type.
type Scalar;
/// Bit representation of this SIMD vector type.
type Bits;
/// Raw transmutation to an unsigned integer vector type with the
/// same size and number of lanes.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn to_bits(self) -> Self::Bits;
/// Raw transmutation from an unsigned integer vector type with the
/// same size and number of lanes.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn from_bits(bits: Self::Bits) -> Self;
/// Produces a vector where every lane has the absolute value of the
/// equivalently-indexed lane in `self`.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn abs(self) -> Self;
/// Takes the reciprocal (inverse) of each lane, `1/x`.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn recip(self) -> Self;
/// Converts each lane from radians to degrees.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn to_degrees(self) -> Self;
/// Converts each lane from degrees to radians.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn to_radians(self) -> Self;
/// Returns true for each lane if it has a positive sign, including
/// `+0.0`, `NaN`s with positive sign bit and positive infinity.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_sign_positive(self) -> Self::Mask;
/// Returns true for each lane if it has a negative sign, including
/// `-0.0`, `NaN`s with negative sign bit and negative infinity.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_sign_negative(self) -> Self::Mask;
/// Returns true for each lane if its value is `NaN`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_nan(self) -> Self::Mask;
/// Returns true for each lane if its value is positive infinity or negative infinity.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_infinite(self) -> Self::Mask;
/// Returns true for each lane if its value is neither infinite nor `NaN`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_finite(self) -> Self::Mask;
/// Returns true for each lane if its value is subnormal.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_subnormal(self) -> Self::Mask;
/// Returns true for each lane if its value is neither zero, infinite,
/// subnormal, nor `NaN`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_normal(self) -> Self::Mask;
/// Replaces each lane with a number that represents its sign.
///
/// * `1.0` if the number is positive, `+0.0`, or `INFINITY`
/// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY`
/// * `NAN` if the number is `NAN`
#[must_use = "method returns a new vector and does not mutate the original value"]
fn signum(self) -> Self;
/// Returns each lane with the magnitude of `self` and the sign of `sign`.
///
/// For any lane containing a `NAN`, a `NAN` with the sign of `sign` is returned.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn copysign(self, sign: Self) -> Self;
/// Returns the minimum of each lane.
///
/// If one of the values is `NAN`, then the other value is returned.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn simd_min(self, other: Self) -> Self;
/// Returns the maximum of each lane.
///
/// If one of the values is `NAN`, then the other value is returned.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn simd_max(self, other: Self) -> Self;
/// Restrict each lane to a certain interval unless it is NaN.
///
/// For each lane in `self`, returns the corresponding lane in `max` if the lane is
/// greater than `max`, and the corresponding lane in `min` if the lane is less
/// than `min`. Otherwise returns the lane in `self`.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn simd_clamp(self, min: Self, max: Self) -> Self;
/// Returns the sum of the lanes of the vector.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{f32x2, SimdFloat};
/// let v = f32x2::from_array([1., 2.]);
/// assert_eq!(v.reduce_sum(), 3.);
/// ```
fn reduce_sum(self) -> Self::Scalar;
/// Reducing multiply. Returns the product of the lanes of the vector.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{f32x2, SimdFloat};
/// let v = f32x2::from_array([3., 4.]);
/// assert_eq!(v.reduce_product(), 12.);
/// ```
fn reduce_product(self) -> Self::Scalar;
/// Returns the maximum lane in the vector.
///
/// Returns values based on equality, so a vector containing both `0.` and `-0.` may
/// return either.
///
/// This function will not return `NaN` unless all lanes are `NaN`.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{f32x2, SimdFloat};
/// let v = f32x2::from_array([1., 2.]);
/// assert_eq!(v.reduce_max(), 2.);
///
/// // NaN values are skipped...
/// let v = f32x2::from_array([1., f32::NAN]);
/// assert_eq!(v.reduce_max(), 1.);
///
/// // ...unless all values are NaN
/// let v = f32x2::from_array([f32::NAN, f32::NAN]);
/// assert!(v.reduce_max().is_nan());
/// ```
fn reduce_max(self) -> Self::Scalar;
/// Returns the minimum lane in the vector.
///
/// Returns values based on equality, so a vector containing both `0.` and `-0.` may
/// return either.
///
/// This function will not return `NaN` unless all lanes are `NaN`.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{f32x2, SimdFloat};
/// let v = f32x2::from_array([3., 7.]);
/// assert_eq!(v.reduce_min(), 3.);
///
/// // NaN values are skipped...
/// let v = f32x2::from_array([1., f32::NAN]);
/// assert_eq!(v.reduce_min(), 1.);
///
/// // ...unless all values are NaN
/// let v = f32x2::from_array([f32::NAN, f32::NAN]);
/// assert!(v.reduce_min().is_nan());
/// ```
fn reduce_min(self) -> Self::Scalar;
}
macro_rules! impl_trait {
{ $($ty:ty { bits: $bits_ty:ty, mask: $mask_ty:ty }),* } => {
$(
impl<const LANES: usize> Sealed for Simd<$ty, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
}
impl<const LANES: usize> SimdFloat for Simd<$ty, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
type Mask = Mask<<$mask_ty as SimdElement>::Mask, LANES>;
type Scalar = $ty;
type Bits = Simd<$bits_ty, LANES>;
#[inline]
fn to_bits(self) -> Simd<$bits_ty, LANES> {
assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<Self::Bits>());
// Safety: transmuting between vector types is safe
unsafe { core::mem::transmute_copy(&self) }
}
#[inline]
fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self {
assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<Self::Bits>());
// Safety: transmuting between vector types is safe
unsafe { core::mem::transmute_copy(&bits) }
}
#[inline]
fn abs(self) -> Self {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_fabs(self) }
}
#[inline]
fn recip(self) -> Self {
Self::splat(1.0) / self
}
#[inline]
fn to_degrees(self) -> Self {
// to_degrees uses a special constant for better precision, so extract that constant
self * Self::splat(Self::Scalar::to_degrees(1.))
}
#[inline]
fn to_radians(self) -> Self {
self * Self::splat(Self::Scalar::to_radians(1.))
}
#[inline]
fn is_sign_positive(self) -> Self::Mask {
!self.is_sign_negative()
}
#[inline]
fn is_sign_negative(self) -> Self::Mask {
let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1);
sign_bits.simd_gt(Simd::splat(0))
}
#[inline]
fn is_nan(self) -> Self::Mask {
self.simd_ne(self)
}
#[inline]
fn is_infinite(self) -> Self::Mask {
self.abs().simd_eq(Self::splat(Self::Scalar::INFINITY))
}
#[inline]
fn is_finite(self) -> Self::Mask {
self.abs().simd_lt(Self::splat(Self::Scalar::INFINITY))
}
#[inline]
fn is_subnormal(self) -> Self::Mask {
self.abs().simd_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(Self::Scalar::INFINITY).to_bits()).simd_eq(Simd::splat(0))
}
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
fn is_normal(self) -> Self::Mask {
!(self.abs().simd_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite())
}
#[inline]
fn signum(self) -> Self {
self.is_nan().select(Self::splat(Self::Scalar::NAN), Self::splat(1.0).copysign(self))
}
#[inline]
fn copysign(self, sign: Self) -> Self {
let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits();
let magnitude = self.to_bits() & !Self::splat(-0.).to_bits();
Self::from_bits(sign_bit | magnitude)
}
#[inline]
fn simd_min(self, other: Self) -> Self {
// Safety: `self` and `other` are float vectors
unsafe { intrinsics::simd_fmin(self, other) }
}
#[inline]
fn simd_max(self, other: Self) -> Self {
// Safety: `self` and `other` are floating point vectors
unsafe { intrinsics::simd_fmax(self, other) }
}
#[inline]
fn simd_clamp(self, min: Self, max: Self) -> Self {
assert!(
min.simd_le(max).all(),
"each lane in `min` must be less than or equal to the corresponding lane in `max`",
);
let mut x = self;
x = x.simd_lt(min).select(min, x);
x = x.simd_gt(max).select(max, x);
x
}
#[inline]
fn reduce_sum(self) -> Self::Scalar {
// LLVM sum is inaccurate on i586
if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) {
self.as_array().iter().sum()
} else {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_add_ordered(self, 0.) }
}
}
#[inline]
fn reduce_product(self) -> Self::Scalar {
// LLVM product is inaccurate on i586
if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) {
self.as_array().iter().product()
} else {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_mul_ordered(self, 1.) }
}
}
#[inline]
fn reduce_max(self) -> Self::Scalar {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_max(self) }
}
#[inline]
fn reduce_min(self) -> Self::Scalar {
// Safety: `self` is a float vector
unsafe { intrinsics::simd_reduce_min(self) }
}
}
)*
}
}
impl_trait! { f32 { bits: u32, mask: i32 }, f64 { bits: u64, mask: i64 } }

298
library/portable-simd/crates/core_simd/src/elements/int.rs Normal file
View file

@ -0,0 +1,298 @@
use super::sealed::Sealed;
use crate::simd::{
intrinsics, LaneCount, Mask, Simd, SimdElement, SimdPartialOrd, SupportedLaneCount,
};
/// Operations on SIMD vectors of signed integers.
pub trait SimdInt: Copy + Sealed {
/// Mask type used for manipulating this SIMD vector type.
type Mask;
/// Scalar type contained by this SIMD vector type.
type Scalar;
/// Lanewise saturating add.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdInt};
/// use core::i32::{MIN, MAX};
/// let x = Simd::from_array([MIN, 0, 1, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x + max;
/// let sat = x.saturating_add(max);
/// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2]));
/// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX]));
/// ```
fn saturating_add(self, second: Self) -> Self;
/// Lanewise saturating subtract.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdInt};
/// use core::i32::{MIN, MAX};
/// let x = Simd::from_array([MIN, -2, -1, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x - max;
/// let sat = x.saturating_sub(max);
/// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0]));
/// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0]));
fn saturating_sub(self, second: Self) -> Self;
/// Lanewise absolute value, implemented in Rust.
/// Every lane becomes its absolute value.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdInt};
/// use core::i32::{MIN, MAX};
/// let xs = Simd::from_array([MIN, MIN +1, -5, 0]);
/// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0]));
/// ```
fn abs(self) -> Self;
/// Lanewise saturating absolute value, implemented in Rust.
/// As abs(), except the MIN value becomes MAX instead of itself.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdInt};
/// use core::i32::{MIN, MAX};
/// let xs = Simd::from_array([MIN, -2, 0, 3]);
/// let unsat = xs.abs();
/// let sat = xs.saturating_abs();
/// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3]));
/// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3]));
/// ```
fn saturating_abs(self) -> Self;
/// Lanewise saturating negation, implemented in Rust.
/// As neg(), except the MIN value becomes MAX instead of itself.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdInt};
/// use core::i32::{MIN, MAX};
/// let x = Simd::from_array([MIN, -2, 3, MAX]);
/// let unsat = -x;
/// let sat = x.saturating_neg();
/// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1]));
/// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1]));
/// ```
fn saturating_neg(self) -> Self;
/// Returns true for each positive lane and false if it is zero or negative.
fn is_positive(self) -> Self::Mask;
/// Returns true for each negative lane and false if it is zero or positive.
fn is_negative(self) -> Self::Mask;
/// Returns numbers representing the sign of each lane.
/// * `0` if the number is zero
/// * `1` if the number is positive
/// * `-1` if the number is negative
fn signum(self) -> Self;
/// Returns the sum of the lanes of the vector, with wrapping addition.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{i32x4, SimdInt};
/// let v = i32x4::from_array([1, 2, 3, 4]);
/// assert_eq!(v.reduce_sum(), 10);
///
/// // SIMD integer addition is always wrapping
/// let v = i32x4::from_array([i32::MAX, 1, 0, 0]);
/// assert_eq!(v.reduce_sum(), i32::MIN);
/// ```
fn reduce_sum(self) -> Self::Scalar;
/// Returns the product of the lanes of the vector, with wrapping multiplication.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{i32x4, SimdInt};
/// let v = i32x4::from_array([1, 2, 3, 4]);
/// assert_eq!(v.reduce_product(), 24);
///
/// // SIMD integer multiplication is always wrapping
/// let v = i32x4::from_array([i32::MAX, 2, 1, 1]);
/// assert!(v.reduce_product() < i32::MAX);
/// ```
fn reduce_product(self) -> Self::Scalar;
/// Returns the maximum lane in the vector.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{i32x4, SimdInt};
/// let v = i32x4::from_array([1, 2, 3, 4]);
/// assert_eq!(v.reduce_max(), 4);
/// ```
fn reduce_max(self) -> Self::Scalar;
/// Returns the minimum lane in the vector.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{i32x4, SimdInt};
/// let v = i32x4::from_array([1, 2, 3, 4]);
/// assert_eq!(v.reduce_min(), 1);
/// ```
fn reduce_min(self) -> Self::Scalar;
/// Returns the cumulative bitwise "and" across the lanes of the vector.
fn reduce_and(self) -> Self::Scalar;
/// Returns the cumulative bitwise "or" across the lanes of the vector.
fn reduce_or(self) -> Self::Scalar;
/// Returns the cumulative bitwise "xor" across the lanes of the vector.
fn reduce_xor(self) -> Self::Scalar;
}
macro_rules! impl_trait {
{ $($ty:ty),* } => {
$(
impl<const LANES: usize> Sealed for Simd<$ty, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
}
impl<const LANES: usize> SimdInt for Simd<$ty, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
type Mask = Mask<<$ty as SimdElement>::Mask, LANES>;
type Scalar = $ty;
#[inline]
fn saturating_add(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_add(self, second) }
}
#[inline]
fn saturating_sub(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_sub(self, second) }
}
#[inline]
fn abs(self) -> Self {
const SHR: $ty = <$ty>::BITS as $ty - 1;
let m = self >> Simd::splat(SHR);
(self^m) - m
}
#[inline]
fn saturating_abs(self) -> Self {
// arith shift for -1 or 0 mask based on sign bit, giving 2s complement
const SHR: $ty = <$ty>::BITS as $ty - 1;
let m = self >> Simd::splat(SHR);
(self^m).saturating_sub(m)
}
#[inline]
fn saturating_neg(self) -> Self {
Self::splat(0).saturating_sub(self)
}
#[inline]
fn is_positive(self) -> Self::Mask {
self.simd_gt(Self::splat(0))
}
#[inline]
fn is_negative(self) -> Self::Mask {
self.simd_lt(Self::splat(0))
}
#[inline]
fn signum(self) -> Self {
self.is_positive().select(
Self::splat(1),
self.is_negative().select(Self::splat(-1), Self::splat(0))
)
}
#[inline]
fn reduce_sum(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_add_ordered(self, 0) }
}
#[inline]
fn reduce_product(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) }
}
#[inline]
fn reduce_max(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_max(self) }
}
#[inline]
fn reduce_min(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_min(self) }
}
#[inline]
fn reduce_and(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_and(self) }
}
#[inline]
fn reduce_or(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_or(self) }
}
#[inline]
fn reduce_xor(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_xor(self) }
}
}
)*
}
}
impl_trait! { i8, i16, i32, i64, isize }

139
library/portable-simd/crates/core_simd/src/elements/uint.rs Normal file
View file

@ -0,0 +1,139 @@
use super::sealed::Sealed;
use crate::simd::{intrinsics, LaneCount, Simd, SupportedLaneCount};
/// Operations on SIMD vectors of unsigned integers.
pub trait SimdUint: Copy + Sealed {
/// Scalar type contained by this SIMD vector type.
type Scalar;
/// Lanewise saturating add.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdUint};
/// use core::u32::MAX;
/// let x = Simd::from_array([2, 1, 0, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x + max;
/// let sat = x.saturating_add(max);
/// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1]));
/// assert_eq!(sat, max);
/// ```
fn saturating_add(self, second: Self) -> Self;
/// Lanewise saturating subtract.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdUint};
/// use core::u32::MAX;
/// let x = Simd::from_array([2, 1, 0, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x - max;
/// let sat = x.saturating_sub(max);
/// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0]));
/// assert_eq!(sat, Simd::splat(0));
fn saturating_sub(self, second: Self) -> Self;
/// Returns the sum of the lanes of the vector, with wrapping addition.
fn reduce_sum(self) -> Self::Scalar;
/// Returns the product of the lanes of the vector, with wrapping multiplication.
fn reduce_product(self) -> Self::Scalar;
/// Returns the maximum lane in the vector.
fn reduce_max(self) -> Self::Scalar;
/// Returns the minimum lane in the vector.
fn reduce_min(self) -> Self::Scalar;
/// Returns the cumulative bitwise "and" across the lanes of the vector.
fn reduce_and(self) -> Self::Scalar;
/// Returns the cumulative bitwise "or" across the lanes of the vector.
fn reduce_or(self) -> Self::Scalar;
/// Returns the cumulative bitwise "xor" across the lanes of the vector.
fn reduce_xor(self) -> Self::Scalar;
}
macro_rules! impl_trait {
{ $($ty:ty),* } => {
$(
impl<const LANES: usize> Sealed for Simd<$ty, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
}
impl<const LANES: usize> SimdUint for Simd<$ty, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
type Scalar = $ty;
#[inline]
fn saturating_add(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_add(self, second) }
}
#[inline]
fn saturating_sub(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { intrinsics::simd_saturating_sub(self, second) }
}
#[inline]
fn reduce_sum(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_add_ordered(self, 0) }
}
#[inline]
fn reduce_product(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_mul_ordered(self, 1) }
}
#[inline]
fn reduce_max(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_max(self) }
}
#[inline]
fn reduce_min(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_min(self) }
}
#[inline]
fn reduce_and(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_and(self) }
}
#[inline]
fn reduce_or(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_or(self) }
}
#[inline]
fn reduce_xor(self) -> Self::Scalar {
// Safety: `self` is an integer vector
unsafe { intrinsics::simd_reduce_xor(self) }
}
}
)*
}
}
impl_trait! { u8, u16, u32, u64, usize }

73
library/portable-simd/crates/core_simd/src/eq.rs Normal file
View file

@ -0,0 +1,73 @@
use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdElement, SupportedLaneCount};
/// Parallel `PartialEq`.
pub trait SimdPartialEq {
/// The mask type returned by each comparison.
type Mask;
/// Test if each lane is equal to the corresponding lane in `other`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn simd_eq(self, other: Self) -> Self::Mask;
/// Test if each lane is equal to the corresponding lane in `other`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn simd_ne(self, other: Self) -> Self::Mask;
}
macro_rules! impl_number {
{ $($number:ty),* } => {
$(
impl<const LANES: usize> SimdPartialEq for Simd<$number, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
type Mask = Mask<<$number as SimdElement>::Mask, LANES>;
#[inline]
fn simd_eq(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) }
}
#[inline]
fn simd_ne(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) }
}
}
)*
}
}
impl_number! { f32, f64, u8, u16, u32, u64, usize, i8, i16, i32, i64, isize }
macro_rules! impl_mask {
{ $($integer:ty),* } => {
$(
impl<const LANES: usize> SimdPartialEq for Mask<$integer, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
type Mask = Self;
#[inline]
fn simd_eq(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_eq(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_ne(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_ne(self.to_int(), other.to_int())) }
}
}
)*
}
}
impl_mask! { i8, i16, i32, i64, isize }

8
library/portable-simd/crates/core_simd/src/lane_count.rs
View file

@ -3,7 +3,7 @@ mod sealed {
}
use sealed::Sealed;
/// A type representing a vector lane count.
/// Specifies the number of lanes in a SIMD vector as a type.
pub struct LaneCount<const LANES: usize>;
impl<const LANES: usize> LaneCount<LANES> {
@ -11,7 +11,11 @@ impl<const LANES: usize> LaneCount<LANES> {
pub const BITMASK_LEN: usize = (LANES + 7) / 8;
}
/// Helper trait for vector lane counts.
/// Statically guarantees that a lane count is marked as supported.
///
/// This trait is *sealed*: the list of implementors below is total.
/// Users do not have the ability to mark additional `LaneCount<N>` values as supported.
/// Only SIMD vectors with supported lane counts are constructable.
pub trait SupportedLaneCount: Sealed {
#[doc(hidden)]
type BitMask: Copy + Default + AsRef<[u8]> + AsMut<[u8]>;

2
library/portable-simd/crates/core_simd/src/lib.rs
View file

@ -12,7 +12,7 @@
#![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))]
#![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))]
#![warn(missing_docs)]
#![deny(unsafe_op_in_unsafe_fn)]
#![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)]
#![unstable(feature = "portable_simd", issue = "86656")]
//! Portable SIMD module.

55
library/portable-simd/crates/core_simd/src/masks.rs
View file

@ -15,7 +15,10 @@ mod mask_impl;
mod to_bitmask;
pub use to_bitmask::ToBitMask;
use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SupportedLaneCount};
#[cfg(feature = "generic_const_exprs")]
pub use to_bitmask::{bitmask_len, ToBitMaskArray};
use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount};
use core::cmp::Ordering;
use core::{fmt, mem};
@ -56,7 +59,7 @@ macro_rules! impl_element {
where
LaneCount<LANES>: SupportedLaneCount,
{
(value.lanes_eq(Simd::splat(0)) | value.lanes_eq(Simd::splat(-1))).all()
(value.simd_eq(Simd::splat(0 as _)) | value.simd_eq(Simd::splat(-1 as _))).all()
}
fn eq(self, other: Self) -> bool { self == other }
@ -65,6 +68,7 @@ macro_rules! impl_element {
const FALSE: Self = 0;
}
// Safety: this is a valid mask element type
unsafe impl MaskElement for $ty {}
}
}
@ -77,6 +81,8 @@ impl_element! { isize }
/// A SIMD vector mask for `LANES` elements of width specified by `Element`.
///
/// Masks represent boolean inclusion/exclusion on a per-lane basis.
///
/// The layout of this type is unspecified.
#[repr(transparent)]
pub struct Mask<T, const LANES: usize>(mask_impl::Mask<T, LANES>)
@ -179,6 +185,13 @@ where
self.0.to_int()
}
/// Converts the mask to a mask of any other lane size.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn cast<U: MaskElement>(self) -> Mask<U, LANES> {
Mask(self.0.convert())
}
/// Tests the value of the specified lane.
///
/// # Safety
@ -507,58 +520,58 @@ where
}
}
/// Vector of eight 8-bit masks
/// A mask for SIMD vectors with eight elements of 8 bits.
pub type mask8x8 = Mask<i8, 8>;
/// Vector of 16 8-bit masks
/// A mask for SIMD vectors with 16 elements of 8 bits.
pub type mask8x16 = Mask<i8, 16>;
/// Vector of 32 8-bit masks
/// A mask for SIMD vectors with 32 elements of 8 bits.
pub type mask8x32 = Mask<i8, 32>;
/// Vector of 16 8-bit masks
/// A mask for SIMD vectors with 64 elements of 8 bits.
pub type mask8x64 = Mask<i8, 64>;
/// Vector of four 16-bit masks
/// A mask for SIMD vectors with four elements of 16 bits.
pub type mask16x4 = Mask<i16, 4>;
/// Vector of eight 16-bit masks
/// A mask for SIMD vectors with eight elements of 16 bits.
pub type mask16x8 = Mask<i16, 8>;
/// Vector of 16 16-bit masks
/// A mask for SIMD vectors with 16 elements of 16 bits.
pub type mask16x16 = Mask<i16, 16>;
/// Vector of 32 16-bit masks
/// A mask for SIMD vectors with 32 elements of 16 bits.
pub type mask16x32 = Mask<i16, 32>;
/// Vector of two 32-bit masks
/// A mask for SIMD vectors with two elements of 32 bits.
pub type mask32x2 = Mask<i32, 2>;
/// Vector of four 32-bit masks
/// A mask for SIMD vectors with four elements of 32 bits.
pub type mask32x4 = Mask<i32, 4>;
/// Vector of eight 32-bit masks
/// A mask for SIMD vectors with eight elements of 32 bits.
pub type mask32x8 = Mask<i32, 8>;
/// Vector of 16 32-bit masks
/// A mask for SIMD vectors with 16 elements of 32 bits.
pub type mask32x16 = Mask<i32, 16>;
/// Vector of two 64-bit masks
/// A mask for SIMD vectors with two elements of 64 bits.
pub type mask64x2 = Mask<i64, 2>;
/// Vector of four 64-bit masks
/// A mask for SIMD vectors with four elements of 64 bits.
pub type mask64x4 = Mask<i64, 4>;
/// Vector of eight 64-bit masks
/// A mask for SIMD vectors with eight elements of 64 bits.
pub type mask64x8 = Mask<i64, 8>;
/// Vector of two pointer-width masks
/// A mask for SIMD vectors with two elements of pointer width.
pub type masksizex2 = Mask<isize, 2>;
/// Vector of four pointer-width masks
/// A mask for SIMD vectors with four elements of pointer width.
pub type masksizex4 = Mask<isize, 4>;
/// Vector of eight pointer-width masks
/// A mask for SIMD vectors with eight elements of pointer width.
pub type masksizex8 = Mask<isize, 8>;
macro_rules! impl_from {
@ -569,7 +582,7 @@ macro_rules! impl_from {
LaneCount<LANES>: SupportedLaneCount,
{
fn from(value: Mask<$from, LANES>) -> Self {
Self(value.0.convert())
value.cast()
}
}
)*

20
library/portable-simd/crates/core_simd/src/masks/bitmask.rs
View file

@ -115,6 +115,26 @@ where
unsafe { Self(intrinsics::simd_bitmask(value), PhantomData) }
}
#[cfg(feature = "generic_const_exprs")]
#[inline]
#[must_use = "method returns a new array and does not mutate the original value"]
pub fn to_bitmask_array<const N: usize>(self) -> [u8; N] {
assert!(core::mem::size_of::<Self>() == N);
// Safety: converting an integer to an array of bytes of the same size is safe
unsafe { core::mem::transmute_copy(&self.0) }
}
#[cfg(feature = "generic_const_exprs")]
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn from_bitmask_array<const N: usize>(bitmask: [u8; N]) -> Self {
assert!(core::mem::size_of::<Self>() == N);
// Safety: converting an array of bytes to an integer of the same size is safe
Self(unsafe { core::mem::transmute_copy(&bitmask) }, PhantomData)
}
#[inline]
pub fn to_bitmask_integer<U>(self) -> U
where

85
library/portable-simd/crates/core_simd/src/masks/full_masks.rs
View file

@ -4,6 +4,9 @@ use super::MaskElement;
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Simd, SupportedLaneCount, ToBitMask};
#[cfg(feature = "generic_const_exprs")]
use crate::simd::ToBitMaskArray;
#[repr(transparent)]
pub struct Mask<T, const LANES: usize>(Simd<T, LANES>)
where
@ -68,14 +71,26 @@ where
// Used for bitmask bit order workaround
pub(crate) trait ReverseBits {
fn reverse_bits(self) -> Self;
// Reverse the least significant `n` bits of `self`.
// (Remaining bits must be 0.)
fn reverse_bits(self, n: usize) -> Self;
}
macro_rules! impl_reverse_bits {
{ $($int:ty),* } => {
$(
impl ReverseBits for $int {
fn reverse_bits(self) -> Self { <$int>::reverse_bits(self) }
#[inline(always)]
fn reverse_bits(self, n: usize) -> Self {
let rev = <$int>::reverse_bits(self);
let bitsize = core::mem::size_of::<$int>() * 8;
if n < bitsize {
// Shift things back to the right
rev >> (bitsize - n)
} else {
rev
}
}
}
)*
}
@ -127,6 +142,68 @@ where
unsafe { Mask(intrinsics::simd_cast(self.0)) }
}
#[cfg(feature = "generic_const_exprs")]
#[inline]
#[must_use = "method returns a new array and does not mutate the original value"]
pub fn to_bitmask_array<const N: usize>(self) -> [u8; N]
where
super::Mask<T, LANES>: ToBitMaskArray,
[(); <super::Mask<T, LANES> as ToBitMaskArray>::BYTES]: Sized,
{
assert_eq!(<super::Mask<T, LANES> as ToBitMaskArray>::BYTES, N);
// Safety: N is the correct bitmask size
unsafe {
// Compute the bitmask
let bitmask: [u8; <super::Mask<T, LANES> as ToBitMaskArray>::BYTES] =
intrinsics::simd_bitmask(self.0);
// Transmute to the return type, previously asserted to be the same size
let mut bitmask: [u8; N] = core::mem::transmute_copy(&bitmask);
// LLVM assumes bit order should match endianness
if cfg!(target_endian = "big") {
for x in bitmask.as_mut() {
*x = x.reverse_bits();
}
};
bitmask
}
}
#[cfg(feature = "generic_const_exprs")]
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn from_bitmask_array<const N: usize>(mut bitmask: [u8; N]) -> Self
where
super::Mask<T, LANES>: ToBitMaskArray,
[(); <super::Mask<T, LANES> as ToBitMaskArray>::BYTES]: Sized,
{
assert_eq!(<super::Mask<T, LANES> as ToBitMaskArray>::BYTES, N);
// Safety: N is the correct bitmask size
unsafe {
// LLVM assumes bit order should match endianness
if cfg!(target_endian = "big") {
for x in bitmask.as_mut() {
*x = x.reverse_bits();
}
}
// Transmute to the bitmask type, previously asserted to be the same size
let bitmask: [u8; <super::Mask<T, LANES> as ToBitMaskArray>::BYTES] =
core::mem::transmute_copy(&bitmask);
// Compute the regular mask
Self::from_int_unchecked(intrinsics::simd_select_bitmask(
bitmask,
Self::splat(true).to_int(),
Self::splat(false).to_int(),
))
}
}
#[inline]
pub(crate) fn to_bitmask_integer<U: ReverseBits>(self) -> U
where
@ -137,7 +214,7 @@ where
// LLVM assumes bit order should match endianness
if cfg!(target_endian = "big") {
bitmask.reverse_bits()
bitmask.reverse_bits(LANES)
} else {
bitmask
}
@ -150,7 +227,7 @@ where
{
// LLVM assumes bit order should match endianness
let bitmask = if cfg!(target_endian = "big") {
bitmask.reverse_bits()
bitmask.reverse_bits(LANES)
} else {
bitmask
};

61
library/portable-simd/crates/core_simd/src/masks/to_bitmask.rs
View file

@ -16,11 +16,7 @@ where
/// Converts masks to and from integer bitmasks.
///
/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB.
///
/// # Safety
/// This trait is `unsafe` and sealed, since the `BitMask` type must match the number of lanes in
/// the mask.
pub unsafe trait ToBitMask: Sealed {
pub trait ToBitMask: Sealed {
/// The integer bitmask type.
type BitMask;
@ -31,10 +27,25 @@ pub unsafe trait ToBitMask: Sealed {
fn from_bitmask(bitmask: Self::BitMask) -> Self;
}
/// Converts masks to and from byte array bitmasks.
///
/// Each bit of the bitmask corresponds to a mask lane, starting with the LSB of the first byte.
#[cfg(feature = "generic_const_exprs")]
pub trait ToBitMaskArray: Sealed {
/// The length of the bitmask array.
const BYTES: usize;
/// Converts a mask to a bitmask.
fn to_bitmask_array(self) -> [u8; Self::BYTES];
/// Converts a bitmask to a mask.
fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self;
}
macro_rules! impl_integer_intrinsic {
{ $(unsafe impl ToBitMask<BitMask=$int:ty> for Mask<_, $lanes:literal>)* } => {
{ $(impl ToBitMask<BitMask=$int:ty> for Mask<_, $lanes:literal>)* } => {
$(
unsafe impl<T: MaskElement> ToBitMask for Mask<T, $lanes> {
impl<T: MaskElement> ToBitMask for Mask<T, $lanes> {
type BitMask = $int;
fn to_bitmask(self) -> $int {
@ -50,11 +61,33 @@ macro_rules! impl_integer_intrinsic {
}
impl_integer_intrinsic! {
unsafe impl ToBitMask<BitMask=u8> for Mask<_, 1>
unsafe impl ToBitMask<BitMask=u8> for Mask<_, 2>
unsafe impl ToBitMask<BitMask=u8> for Mask<_, 4>
unsafe impl ToBitMask<BitMask=u8> for Mask<_, 8>
unsafe impl ToBitMask<BitMask=u16> for Mask<_, 16>
unsafe impl ToBitMask<BitMask=u32> for Mask<_, 32>
unsafe impl ToBitMask<BitMask=u64> for Mask<_, 64>
impl ToBitMask<BitMask=u8> for Mask<_, 1>
impl ToBitMask<BitMask=u8> for Mask<_, 2>
impl ToBitMask<BitMask=u8> for Mask<_, 4>
impl ToBitMask<BitMask=u8> for Mask<_, 8>
impl ToBitMask<BitMask=u16> for Mask<_, 16>
impl ToBitMask<BitMask=u32> for Mask<_, 32>
impl ToBitMask<BitMask=u64> for Mask<_, 64>
}
/// Returns the minimum numnber of bytes in a bitmask with `lanes` lanes.
#[cfg(feature = "generic_const_exprs")]
pub const fn bitmask_len(lanes: usize) -> usize {
(lanes + 7) / 8
}
#[cfg(feature = "generic_const_exprs")]
impl<T: MaskElement, const LANES: usize> ToBitMaskArray for Mask<T, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
const BYTES: usize = bitmask_len(LANES);
fn to_bitmask_array(self) -> [u8; Self::BYTES] {
self.0.to_bitmask_array()
}
fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self {
Mask(mask_impl::Mask::from_bitmask_array(bitmask))
}
}

156
library/portable-simd/crates/core_simd/src/math.rs
View file

@ -1,156 +0,0 @@
use crate::simd::intrinsics::{simd_saturating_add, simd_saturating_sub};
use crate::simd::{LaneCount, Simd, SupportedLaneCount};
macro_rules! impl_uint_arith {
($($ty:ty),+) => {
$( impl<const LANES: usize> Simd<$ty, LANES> where LaneCount<LANES>: SupportedLaneCount {
/// Lanewise saturating add.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::MAX;")]
/// let x = Simd::from_array([2, 1, 0, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x + max;
/// let sat = x.saturating_add(max);
/// assert_eq!(unsat, Simd::from_array([1, 0, MAX, MAX - 1]));
/// assert_eq!(sat, max);
/// ```
#[inline]
pub fn saturating_add(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { simd_saturating_add(self, second) }
}
/// Lanewise saturating subtract.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::MAX;")]
/// let x = Simd::from_array([2, 1, 0, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x - max;
/// let sat = x.saturating_sub(max);
/// assert_eq!(unsat, Simd::from_array([3, 2, 1, 0]));
/// assert_eq!(sat, Simd::splat(0));
#[inline]
pub fn saturating_sub(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { simd_saturating_sub(self, second) }
}
})+
}
}
macro_rules! impl_int_arith {
($($ty:ty),+) => {
$( impl<const LANES: usize> Simd<$ty, LANES> where LaneCount<LANES>: SupportedLaneCount {
/// Lanewise saturating add.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")]
/// let x = Simd::from_array([MIN, 0, 1, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x + max;
/// let sat = x.saturating_add(max);
/// assert_eq!(unsat, Simd::from_array([-1, MAX, MIN, -2]));
/// assert_eq!(sat, Simd::from_array([-1, MAX, MAX, MAX]));
/// ```
#[inline]
pub fn saturating_add(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { simd_saturating_add(self, second) }
}
/// Lanewise saturating subtract.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")]
/// let x = Simd::from_array([MIN, -2, -1, MAX]);
/// let max = Simd::splat(MAX);
/// let unsat = x - max;
/// let sat = x.saturating_sub(max);
/// assert_eq!(unsat, Simd::from_array([1, MAX, MIN, 0]));
/// assert_eq!(sat, Simd::from_array([MIN, MIN, MIN, 0]));
#[inline]
pub fn saturating_sub(self, second: Self) -> Self {
// Safety: `self` is a vector
unsafe { simd_saturating_sub(self, second) }
}
/// Lanewise absolute value, implemented in Rust.
/// Every lane becomes its absolute value.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")]
/// let xs = Simd::from_array([MIN, MIN +1, -5, 0]);
/// assert_eq!(xs.abs(), Simd::from_array([MIN, MAX, 5, 0]));
/// ```
#[inline]
pub fn abs(self) -> Self {
const SHR: $ty = <$ty>::BITS as $ty - 1;
let m = self >> Simd::splat(SHR);
(self^m) - m
}
/// Lanewise saturating absolute value, implemented in Rust.
/// As abs(), except the MIN value becomes MAX instead of itself.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")]
/// let xs = Simd::from_array([MIN, -2, 0, 3]);
/// let unsat = xs.abs();
/// let sat = xs.saturating_abs();
/// assert_eq!(unsat, Simd::from_array([MIN, 2, 0, 3]));
/// assert_eq!(sat, Simd::from_array([MAX, 2, 0, 3]));
/// ```
#[inline]
pub fn saturating_abs(self) -> Self {
// arith shift for -1 or 0 mask based on sign bit, giving 2s complement
const SHR: $ty = <$ty>::BITS as $ty - 1;
let m = self >> Simd::splat(SHR);
(self^m).saturating_sub(m)
}
/// Lanewise saturating negation, implemented in Rust.
/// As neg(), except the MIN value becomes MAX instead of itself.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::Simd;
#[doc = concat!("# use core::", stringify!($ty), "::{MIN, MAX};")]
/// let x = Simd::from_array([MIN, -2, 3, MAX]);
/// let unsat = -x;
/// let sat = x.saturating_neg();
/// assert_eq!(unsat, Simd::from_array([MIN, 2, -3, MIN + 1]));
/// assert_eq!(sat, Simd::from_array([MAX, 2, -3, MIN + 1]));
/// ```
#[inline]
pub fn saturating_neg(self) -> Self {
Self::splat(0).saturating_sub(self)
}
})+
}
}
impl_uint_arith! { u8, u16, u32, u64, usize }
impl_int_arith! { i8, i16, i32, i64, isize }

12
library/portable-simd/crates/core_simd/src/mod.rs
View file

@ -1,6 +1,3 @@
#[macro_use]
mod reduction;
#[macro_use]
mod swizzle;
@ -9,14 +6,14 @@ pub(crate) mod intrinsics;
#[cfg(feature = "generic_const_exprs")]
mod to_bytes;
mod comparisons;
mod elements;
mod eq;
mod fmt;
mod iter;
mod lane_count;
mod masks;
mod math;
mod ops;
mod round;
mod ord;
mod select;
mod vector;
mod vendor;
@ -25,8 +22,11 @@ mod vendor;
pub mod simd {
pub(crate) use crate::core_simd::intrinsics;
pub use crate::core_simd::elements::*;
pub use crate::core_simd::eq::*;
pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount};
pub use crate::core_simd::masks::*;
pub use crate::core_simd::ord::*;
pub use crate::core_simd::swizzle::*;
pub use crate::core_simd::vector::*;
}

14
library/portable-simd/crates/core_simd/src/ops.rs
View file

@ -1,4 +1,4 @@
use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
use crate::simd::{LaneCount, Simd, SimdElement, SimdPartialEq, SupportedLaneCount};
use core::ops::{Add, Mul};
use core::ops::{BitAnd, BitOr, BitXor};
use core::ops::{Div, Rem, Sub};
@ -33,6 +33,7 @@ where
macro_rules! unsafe_base {
($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => {
// Safety: $lhs and $rhs are vectors
unsafe { $crate::simd::intrinsics::$simd_call($lhs, $rhs) }
};
}
@ -48,6 +49,8 @@ macro_rules! unsafe_base {
// cg_clif defaults to this, and scalar MIR shifts also default to wrapping
macro_rules! wrap_bitshift {
($lhs:ident, $rhs:ident, {$simd_call:ident}, $int:ident) => {
#[allow(clippy::suspicious_arithmetic_impl)]
// Safety: $lhs and the bitand result are vectors
unsafe {
$crate::simd::intrinsics::$simd_call(
$lhs,
@ -74,7 +77,7 @@ macro_rules! int_divrem_guard {
$simd_call:ident
},
$int:ident ) => {
if $rhs.lanes_eq(Simd::splat(0)).any() {
if $rhs.simd_eq(Simd::splat(0 as _)).any() {
panic!($zero);
} else {
// Prevent otherwise-UB overflow on the MIN / -1 case.
@ -82,14 +85,15 @@ macro_rules! int_divrem_guard {
// This should, at worst, optimize to a few branchless logical ops
// Ideally, this entire conditional should evaporate
// Fire LLVM and implement those manually if it doesn't get the hint
($lhs.lanes_eq(Simd::splat(<$int>::MIN))
($lhs.simd_eq(Simd::splat(<$int>::MIN))
// type inference can break here, so cut an SInt to size
& $rhs.lanes_eq(Simd::splat(-1i64 as _)))
.select(Simd::splat(1), $rhs)
& $rhs.simd_eq(Simd::splat(-1i64 as _)))
.select(Simd::splat(1 as _), $rhs)
} else {
// Nice base case to make it easy to const-fold away the other branch.
$rhs
};
// Safety: $lhs and rhs are vectors
unsafe { $crate::simd::intrinsics::$simd_call($lhs, rhs) }
}
};

1
library/portable-simd/crates/core_simd/src/ops/unary.rs
View file

@ -14,6 +14,7 @@ macro_rules! neg {
#[inline]
#[must_use = "operator returns a new vector without mutating the input"]
fn neg(self) -> Self::Output {
// Safety: `self` is a signed vector
unsafe { intrinsics::simd_neg(self) }
}
})*

213
library/portable-simd/crates/core_simd/src/ord.rs Normal file
View file

@ -0,0 +1,213 @@
use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount};
/// Parallel `PartialOrd`.
pub trait SimdPartialOrd: SimdPartialEq {
/// Test if each lane is less than the corresponding lane in `other`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn simd_lt(self, other: Self) -> Self::Mask;
/// Test if each lane is less than or equal to the corresponding lane in `other`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn simd_le(self, other: Self) -> Self::Mask;
/// Test if each lane is greater than the corresponding lane in `other`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn simd_gt(self, other: Self) -> Self::Mask;
/// Test if each lane is greater than or equal to the corresponding lane in `other`.
#[must_use = "method returns a new mask and does not mutate the original value"]
fn simd_ge(self, other: Self) -> Self::Mask;
}
/// Parallel `Ord`.
pub trait SimdOrd: SimdPartialOrd {
/// Returns the lane-wise maximum with `other`.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn simd_max(self, other: Self) -> Self;
/// Returns the lane-wise minimum with `other`.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn simd_min(self, other: Self) -> Self;
/// Restrict each lane to a certain interval.
///
/// For each lane, returns `max` if `self` is greater than `max`, and `min` if `self` is
/// less than `min`. Otherwise returns `self`.
///
/// # Panics
///
/// Panics if `min > max` on any lane.
#[must_use = "method returns a new vector and does not mutate the original value"]
fn simd_clamp(self, min: Self, max: Self) -> Self;
}
macro_rules! impl_integer {
{ $($integer:ty),* } => {
$(
impl<const LANES: usize> SimdPartialOrd for Simd<$integer, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
#[inline]
fn simd_lt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) }
}
#[inline]
fn simd_le(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) }
}
#[inline]
fn simd_gt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) }
}
#[inline]
fn simd_ge(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) }
}
}
impl<const LANES: usize> SimdOrd for Simd<$integer, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
#[inline]
fn simd_max(self, other: Self) -> Self {
self.simd_lt(other).select(other, self)
}
#[inline]
fn simd_min(self, other: Self) -> Self {
self.simd_gt(other).select(other, self)
}
#[inline]
fn simd_clamp(self, min: Self, max: Self) -> Self {
assert!(
min.simd_le(max).all(),
"each lane in `min` must be less than or equal to the corresponding lane in `max`",
);
self.simd_max(min).simd_min(max)
}
}
)*
}
}
impl_integer! { u8, u16, u32, u64, usize, i8, i16, i32, i64, isize }
macro_rules! impl_float {
{ $($float:ty),* } => {
$(
impl<const LANES: usize> SimdPartialOrd for Simd<$float, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
#[inline]
fn simd_lt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) }
}
#[inline]
fn simd_le(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) }
}
#[inline]
fn simd_gt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) }
}
#[inline]
fn simd_ge(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) }
}
}
)*
}
}
impl_float! { f32, f64 }
macro_rules! impl_mask {
{ $($integer:ty),* } => {
$(
impl<const LANES: usize> SimdPartialOrd for Mask<$integer, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
#[inline]
fn simd_lt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_lt(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_le(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_le(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_gt(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_gt(self.to_int(), other.to_int())) }
}
#[inline]
fn simd_ge(self, other: Self) -> Self::Mask {
// Safety: `self` is a vector, and the result of the comparison
// is always a valid mask.
unsafe { Self::from_int_unchecked(intrinsics::simd_ge(self.to_int(), other.to_int())) }
}
}
impl<const LANES: usize> SimdOrd for Mask<$integer, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
#[inline]
fn simd_max(self, other: Self) -> Self {
self.simd_gt(other).select_mask(other, self)
}
#[inline]
fn simd_min(self, other: Self) -> Self {
self.simd_lt(other).select_mask(other, self)
}
#[inline]
fn simd_clamp(self, min: Self, max: Self) -> Self {
assert!(
min.simd_le(max).all(),
"each lane in `min` must be less than or equal to the corresponding lane in `max`",
);
self.simd_max(min).simd_min(max)
}
}
)*
}
}
impl_mask! { i8, i16, i32, i64, isize }

153
library/portable-simd/crates/core_simd/src/reduction.rs
View file

@ -1,153 +0,0 @@
use crate::simd::intrinsics::{
simd_reduce_add_ordered, simd_reduce_and, simd_reduce_max, simd_reduce_min,
simd_reduce_mul_ordered, simd_reduce_or, simd_reduce_xor,
};
use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
use core::ops::{BitAnd, BitOr, BitXor};
macro_rules! impl_integer_reductions {
{ $scalar:ty } => {
impl<const LANES: usize> Simd<$scalar, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
/// Reducing wrapping add. Returns the sum of the lanes of the vector, with wrapping addition.
#[inline]
pub fn reduce_sum(self) -> $scalar {
// Safety: `self` is an integer vector
unsafe { simd_reduce_add_ordered(self, 0) }
}
/// Reducing wrapping multiply. Returns the product of the lanes of the vector, with wrapping multiplication.
#[inline]
pub fn reduce_product(self) -> $scalar {
// Safety: `self` is an integer vector
unsafe { simd_reduce_mul_ordered(self, 1) }
}
/// Reducing maximum. Returns the maximum lane in the vector.
#[inline]
pub fn reduce_max(self) -> $scalar {
// Safety: `self` is an integer vector
unsafe { simd_reduce_max(self) }
}
/// Reducing minimum. Returns the minimum lane in the vector.
#[inline]
pub fn reduce_min(self) -> $scalar {
// Safety: `self` is an integer vector
unsafe { simd_reduce_min(self) }
}
}
}
}
impl_integer_reductions! { i8 }
impl_integer_reductions! { i16 }
impl_integer_reductions! { i32 }
impl_integer_reductions! { i64 }
impl_integer_reductions! { isize }
impl_integer_reductions! { u8 }
impl_integer_reductions! { u16 }
impl_integer_reductions! { u32 }
impl_integer_reductions! { u64 }
impl_integer_reductions! { usize }
macro_rules! impl_float_reductions {
{ $scalar:ty } => {
impl<const LANES: usize> Simd<$scalar, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
/// Reducing add. Returns the sum of the lanes of the vector.
#[inline]
pub fn reduce_sum(self) -> $scalar {
// LLVM sum is inaccurate on i586
if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) {
self.as_array().iter().sum()
} else {
// Safety: `self` is a float vector
unsafe { simd_reduce_add_ordered(self, 0.) }
}
}
/// Reducing multiply. Returns the product of the lanes of the vector.
#[inline]
pub fn reduce_product(self) -> $scalar {
// LLVM product is inaccurate on i586
if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) {
self.as_array().iter().product()
} else {
// Safety: `self` is a float vector
unsafe { simd_reduce_mul_ordered(self, 1.) }
}
}
/// Reducing maximum. Returns the maximum lane in the vector.
///
/// Returns values based on equality, so a vector containing both `0.` and `-0.` may
/// return either. This function will not return `NaN` unless all lanes are `NaN`.
#[inline]
pub fn reduce_max(self) -> $scalar {
// Safety: `self` is a float vector
unsafe { simd_reduce_max(self) }
}
/// Reducing minimum. Returns the minimum lane in the vector.
///
/// Returns values based on equality, so a vector containing both `0.` and `-0.` may
/// return either. This function will not return `NaN` unless all lanes are `NaN`.
#[inline]
pub fn reduce_min(self) -> $scalar {
// Safety: `self` is a float vector
unsafe { simd_reduce_min(self) }
}
}
}
}
impl_float_reductions! { f32 }
impl_float_reductions! { f64 }
impl<T, const LANES: usize> Simd<T, LANES>
where
Self: BitAnd<Self, Output = Self>,
T: SimdElement + BitAnd<T, Output = T>,
LaneCount<LANES>: SupportedLaneCount,
{
/// Reducing bitwise "and". Returns the cumulative bitwise "and" across the lanes of
/// the vector.
#[inline]
pub fn reduce_and(self) -> T {
unsafe { simd_reduce_and(self) }
}
}
impl<T, const LANES: usize> Simd<T, LANES>
where
Self: BitOr<Self, Output = Self>,
T: SimdElement + BitOr<T, Output = T>,
LaneCount<LANES>: SupportedLaneCount,
{
/// Reducing bitwise "or". Returns the cumulative bitwise "or" across the lanes of
/// the vector.
#[inline]
pub fn reduce_or(self) -> T {
unsafe { simd_reduce_or(self) }
}
}
impl<T, const LANES: usize> Simd<T, LANES>
where
Self: BitXor<Self, Output = Self>,
T: SimdElement + BitXor<T, Output = T>,
LaneCount<LANES>: SupportedLaneCount,
{
/// Reducing bitwise "xor". Returns the cumulative bitwise "xor" across the lanes of
/// the vector.
#[inline]
pub fn reduce_xor(self) -> T {
unsafe { simd_reduce_xor(self) }
}
}

40
library/portable-simd/crates/core_simd/src/round.rs
View file

@ -1,40 +0,0 @@
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
use core::convert::FloatToInt;
macro_rules! implement {
{
$type:ty
} => {
impl<const LANES: usize> Simd<$type, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
/// Rounds toward zero and converts to the same-width integer type, assuming that
/// the value is finite and fits in that type.
///
/// # Safety
/// The value must:
///
/// * Not be NaN
/// * Not be infinite
/// * Be representable in the return type, after truncating off its fractional part
///
/// If these requirements are infeasible or costly, consider using the safe function [cast],
/// which saturates on conversion.
///
/// [cast]: Simd::cast
#[inline]
pub unsafe fn to_int_unchecked<I>(self) -> Simd<I, LANES>
where
$type: FloatToInt<I>,
I: SimdElement,
{
unsafe { intrinsics::simd_cast(self) }
}
}
}
}
implement! { f32 }
implement! { f64 }

52
library/portable-simd/crates/core_simd/src/swizzle.rs
View file

@ -1,44 +1,46 @@
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
/// Constructs a new vector by selecting values from the lanes of the source vector or vectors to use.
/// Constructs a new SIMD vector by copying elements from selected lanes in other vectors.
///
/// When swizzling one vector, the indices of the result vector are indicated by a `const` array
/// of `usize`, like [`Swizzle`].
/// When swizzling two vectors, the indices are indicated by a `const` array of [`Which`], like
/// [`Swizzle2`].
/// When swizzling one vector, lanes are selected by a `const` array of `usize`,
/// like [`Swizzle`].
///
/// When swizzling two vectors, lanes are selected by a `const` array of [`Which`],
/// like [`Swizzle2`].
///
/// # Examples
/// ## One source vector
///
/// With a single SIMD vector, the const array specifies lane indices in that vector:
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::{Simd, simd_swizzle};
/// let v = Simd::<f32, 4>::from_array([0., 1., 2., 3.]);
/// # use core::simd::{u32x2, u32x4, simd_swizzle};
/// let v = u32x4::from_array([10, 11, 12, 13]);
///
/// // Keeping the same size
/// let r = simd_swizzle!(v, [3, 0, 1, 2]);
/// assert_eq!(r.to_array(), [3., 0., 1., 2.]);
/// let r: u32x4 = simd_swizzle!(v, [3, 0, 1, 2]);
/// assert_eq!(r.to_array(), [13, 10, 11, 12]);
///
/// // Changing the number of lanes
/// let r = simd_swizzle!(v, [3, 1]);
/// assert_eq!(r.to_array(), [3., 1.]);
/// let r: u32x2 = simd_swizzle!(v, [3, 1]);
/// assert_eq!(r.to_array(), [13, 11]);
/// ```
///
/// ## Two source vectors
/// With two input SIMD vectors, the const array uses `Which` to specify the source of each index:
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::{Simd, simd_swizzle, Which};
/// use Which::*;
/// let a = Simd::<f32, 4>::from_array([0., 1., 2., 3.]);
/// let b = Simd::<f32, 4>::from_array([4., 5., 6., 7.]);
/// # use core::simd::{u32x2, u32x4, simd_swizzle, Which};
/// use Which::{First, Second};
/// let a = u32x4::from_array([0, 1, 2, 3]);
/// let b = u32x4::from_array([4, 5, 6, 7]);
///
/// // Keeping the same size
/// let r = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]);
/// assert_eq!(r.to_array(), [0., 1., 6., 7.]);
/// let r: u32x4 = simd_swizzle!(a, b, [First(0), First(1), Second(2), Second(3)]);
/// assert_eq!(r.to_array(), [0, 1, 6, 7]);
///
/// // Changing the number of lanes
/// let r = simd_swizzle!(a, b, [First(0), Second(0)]);
/// assert_eq!(r.to_array(), [0., 4.]);
/// let r: u32x2 = simd_swizzle!(a, b, [First(0), Second(0)]);
/// assert_eq!(r.to_array(), [0, 4]);
/// ```
#[allow(unused_macros)]
pub macro simd_swizzle {
@ -68,12 +70,14 @@ pub macro simd_swizzle {
}
}
/// An index into one of two vectors.
/// Specifies a lane index into one of two SIMD vectors.
///
/// This is an input type for [Swizzle2] and helper macros like [simd_swizzle].
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Which {
/// Indexes the first vector.
/// Index of a lane in the first input SIMD vector.
First(usize),
/// Indexes the second vector.
/// Index of a lane in the second input SIMD vector.
Second(usize),
}

157
library/portable-simd/crates/core_simd/src/vector.rs
View file

@ -9,8 +9,9 @@ pub use uint::*;
// Vectors of pointers are not for public use at the current time.
pub(crate) mod ptr;
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Mask, MaskElement, SupportedLaneCount};
use crate::simd::{
intrinsics, LaneCount, Mask, MaskElement, SimdPartialOrd, SupportedLaneCount, Swizzle,
};
/// A SIMD vector of `LANES` elements of type `T`. `Simd<T, N>` has the same shape as [`[T; N]`](array), but operates like `T`.
///
@ -99,17 +100,50 @@ where
/// Number of lanes in this vector.
pub const LANES: usize = LANES;
/// Get the number of lanes in this vector.
/// Returns the number of lanes in this SIMD vector.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::u32x4;
/// let v = u32x4::splat(0);
/// assert_eq!(v.lanes(), 4);
/// ```
pub const fn lanes(&self) -> usize {
LANES
}
/// Construct a SIMD vector by setting all lanes to the given value.
pub const fn splat(value: T) -> Self {
Self([value; LANES])
/// Constructs a new SIMD vector with all lanes set to the given value.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::u32x4;
/// let v = u32x4::splat(8);
/// assert_eq!(v.as_array(), &[8, 8, 8, 8]);
/// ```
pub fn splat(value: T) -> Self {
// This is preferred over `[value; LANES]`, since it's explicitly a splat:
// https://github.com/rust-lang/rust/issues/97804
struct Splat;
impl<const LANES: usize> Swizzle<1, LANES> for Splat {
const INDEX: [usize; LANES] = [0; LANES];
}
Splat::swizzle(Simd::<T, 1>::from([value]))
}
/// Returns an array reference containing the entire SIMD vector.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::{Simd, u64x4};
/// let v: u64x4 = Simd::from_array([0, 1, 2, 3]);
/// assert_eq!(v.as_array(), &[0, 1, 2, 3]);
/// ```
pub const fn as_array(&self) -> &[T; LANES] {
&self.0
}
@ -129,9 +163,21 @@ where
self.0
}
/// Converts a slice to a SIMD vector containing `slice[..LANES]`
/// Converts a slice to a SIMD vector containing `slice[..LANES]`.
///
/// # Panics
/// `from_slice` will panic if the slice's `len` is less than the vector's `Simd::LANES`.
///
/// Panics if the slice's length is less than the vector's `Simd::LANES`.
///
/// # Examples
///
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::u32x4;
/// let source = vec![1, 2, 3, 4, 5, 6];
/// let v = u32x4::from_slice(&source);
/// assert_eq!(v.as_array(), &[1, 2, 3, 4]);
/// ```
#[must_use]
pub const fn from_slice(slice: &[T]) -> Self {
assert!(slice.len() >= LANES, "slice length must be at least the number of lanes");
@ -145,6 +191,7 @@ where
}
/// Performs lanewise conversion of a SIMD vector's elements to another SIMD-valid type.
///
/// This follows the semantics of Rust's `as` conversion for casting
/// integers to unsigned integers (interpreting as the other type, so `-1` to `MAX`),
/// and from floats to integers (truncating, or saturating at the limits) for each lane,
@ -169,10 +216,35 @@ where
#[must_use]
#[inline]
pub fn cast<U: SimdElement>(self) -> Simd<U, LANES> {
// Safety: The input argument is a vector of a known SIMD type.
// Safety: The input argument is a vector of a valid SIMD element type.
unsafe { intrinsics::simd_as(self) }
}
/// Rounds toward zero and converts to the same-width integer type, assuming that
/// the value is finite and fits in that type.
///
/// # Safety
/// The value must:
///
/// * Not be NaN
/// * Not be infinite
/// * Be representable in the return type, after truncating off its fractional part
///
/// If these requirements are infeasible or costly, consider using the safe function [cast],
/// which saturates on conversion.
///
/// [cast]: Simd::cast
#[inline]
pub unsafe fn to_int_unchecked<I>(self) -> Simd<I, LANES>
where
T: core::convert::FloatToInt<I>,
I: SimdElement,
{
// Safety: `self` is a vector, and `FloatToInt` ensures the type can be casted to
// an integer.
unsafe { intrinsics::simd_cast(self) }
}
/// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
/// If an index is out-of-bounds, the lane is instead selected from the `or` vector.
///
@ -239,7 +311,7 @@ where
idxs: Simd<usize, LANES>,
or: Self,
) -> Self {
let enable: Mask<isize, LANES> = enable & idxs.lanes_lt(Simd::splat(slice.len()));
let enable: Mask<isize, LANES> = enable & idxs.simd_lt(Simd::splat(slice.len()));
// Safety: We have masked-off out-of-bounds lanes.
unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) }
}
@ -256,13 +328,15 @@ where
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::{Simd, Mask};
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdPartialOrd, Mask};
/// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
/// let idxs = Simd::from_array([9, 3, 0, 5]);
/// let alt = Simd::from_array([-5, -4, -3, -2]);
/// let enable = Mask::from_array([true, true, true, false]); // Note the final mask lane.
/// // If this mask was used to gather, it would be unsound. Let's fix that.
/// let enable = enable & idxs.lanes_lt(Simd::splat(vec.len()));
/// let enable = enable & idxs.simd_lt(Simd::splat(vec.len()));
///
/// // We have masked the OOB lane, so it's safe to gather now.
/// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) };
@ -313,7 +387,9 @@ where
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::{Simd, Mask};
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, Mask};
/// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
/// let idxs = Simd::from_array([9, 3, 0, 0]);
/// let vals = Simd::from_array([-27, 82, -41, 124]);
@ -329,7 +405,7 @@ where
enable: Mask<isize, LANES>,
idxs: Simd<usize, LANES>,
) {
let enable: Mask<isize, LANES> = enable & idxs.lanes_lt(Simd::splat(slice.len()));
let enable: Mask<isize, LANES> = enable & idxs.simd_lt(Simd::splat(slice.len()));
// Safety: We have masked-off out-of-bounds lanes.
unsafe { self.scatter_select_unchecked(slice, enable, idxs) }
}
@ -347,13 +423,15 @@ where
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # use core::simd::{Simd, Mask};
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, SimdPartialOrd, Mask};
/// let mut vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
/// let idxs = Simd::from_array([9, 3, 0, 0]);
/// let vals = Simd::from_array([-27, 82, -41, 124]);
/// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane.
/// // If this mask was used to scatter, it would be unsound. Let's fix that.
/// let enable = enable & idxs.lanes_lt(Simd::splat(vec.len()));
/// let enable = enable & idxs.simd_lt(Simd::splat(vec.len()));
///
/// // We have masked the OOB lane, so it's safe to scatter now.
/// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); }
@ -425,8 +503,27 @@ where
{
#[inline]
fn eq(&self, other: &Self) -> bool {
// TODO use SIMD equality
self.to_array() == other.to_array()
// Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
let mask = unsafe {
let tfvec: Simd<<T as SimdElement>::Mask, LANES> = intrinsics::simd_eq(*self, *other);
Mask::from_int_unchecked(tfvec)
};
// Two vectors are equal if all lanes tested true for vertical equality.
mask.all()
}
#[allow(clippy::partialeq_ne_impl)]
#[inline]
fn ne(&self, other: &Self) -> bool {
// Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask.
let mask = unsafe {
let tfvec: Simd<<T as SimdElement>::Mask, LANES> = intrinsics::simd_ne(*self, *other);
Mask::from_int_unchecked(tfvec)
};
// Two vectors are non-equal if any lane tested true for vertical non-equality.
mask.any()
}
}
@ -561,61 +658,85 @@ pub unsafe trait SimdElement: Sealed + Copy {
}
impl Sealed for u8 {}
// Safety: u8 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for u8 {
type Mask = i8;
}
impl Sealed for u16 {}
// Safety: u16 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for u16 {
type Mask = i16;
}
impl Sealed for u32 {}
// Safety: u32 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for u32 {
type Mask = i32;
}
impl Sealed for u64 {}
// Safety: u64 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for u64 {
type Mask = i64;
}
impl Sealed for usize {}
// Safety: usize is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for usize {
type Mask = isize;
}
impl Sealed for i8 {}
// Safety: i8 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for i8 {
type Mask = i8;
}
impl Sealed for i16 {}
// Safety: i16 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for i16 {
type Mask = i16;
}
impl Sealed for i32 {}
// Safety: i32 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for i32 {
type Mask = i32;
}
impl Sealed for i64 {}
// Safety: i64 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for i64 {
type Mask = i64;
}
impl Sealed for isize {}
// Safety: isize is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for isize {
type Mask = isize;
}
impl Sealed for f32 {}
// Safety: f32 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for f32 {
type Mask = i32;
}
impl Sealed for f64 {}
// Safety: f64 is a valid SIMD element type, and is supported by this API
unsafe impl SimdElement for f64 {
type Mask = i64;
}

191
library/portable-simd/crates/core_simd/src/vector/float.rs
View file

@ -1,199 +1,24 @@
#![allow(non_camel_case_types)]
use crate::simd::intrinsics;
use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount};
use crate::simd::Simd;
/// Implements inherent methods for a float vector containing multiple
/// `$lanes` of float `$type`, which uses `$bits_ty` as its binary
/// representation.
macro_rules! impl_float_vector {
{ $type:ty, $bits_ty:ty, $mask_ty:ty } => {
impl<const LANES: usize> Simd<$type, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
/// Raw transmutation to an unsigned integer vector type with the
/// same size and number of lanes.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn to_bits(self) -> Simd<$bits_ty, LANES> {
assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<Simd<$bits_ty, LANES>>());
unsafe { core::mem::transmute_copy(&self) }
}
/// Raw transmutation from an unsigned integer vector type with the
/// same size and number of lanes.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn from_bits(bits: Simd<$bits_ty, LANES>) -> Self {
assert_eq!(core::mem::size_of::<Self>(), core::mem::size_of::<Simd<$bits_ty, LANES>>());
unsafe { core::mem::transmute_copy(&bits) }
}
/// Produces a vector where every lane has the absolute value of the
/// equivalently-indexed lane in `self`.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn abs(self) -> Self {
unsafe { intrinsics::simd_fabs(self) }
}
/// Takes the reciprocal (inverse) of each lane, `1/x`.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn recip(self) -> Self {
Self::splat(1.0) / self
}
/// Converts each lane from radians to degrees.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn to_degrees(self) -> Self {
// to_degrees uses a special constant for better precision, so extract that constant
self * Self::splat(<$type>::to_degrees(1.))
}
/// Converts each lane from degrees to radians.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn to_radians(self) -> Self {
self * Self::splat(<$type>::to_radians(1.))
}
/// Returns true for each lane if it has a positive sign, including
/// `+0.0`, `NaN`s with positive sign bit and positive infinity.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_sign_positive(self) -> Mask<$mask_ty, LANES> {
!self.is_sign_negative()
}
/// Returns true for each lane if it has a negative sign, including
/// `-0.0`, `NaN`s with negative sign bit and negative infinity.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_sign_negative(self) -> Mask<$mask_ty, LANES> {
let sign_bits = self.to_bits() & Simd::splat((!0 >> 1) + 1);
sign_bits.lanes_gt(Simd::splat(0))
}
/// Returns true for each lane if its value is `NaN`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_nan(self) -> Mask<$mask_ty, LANES> {
self.lanes_ne(self)
}
/// Returns true for each lane if its value is positive infinity or negative infinity.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_infinite(self) -> Mask<$mask_ty, LANES> {
self.abs().lanes_eq(Self::splat(<$type>::INFINITY))
}
/// Returns true for each lane if its value is neither infinite nor `NaN`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_finite(self) -> Mask<$mask_ty, LANES> {
self.abs().lanes_lt(Self::splat(<$type>::INFINITY))
}
/// Returns true for each lane if its value is subnormal.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_subnormal(self) -> Mask<$mask_ty, LANES> {
self.abs().lanes_ne(Self::splat(0.0)) & (self.to_bits() & Self::splat(<$type>::INFINITY).to_bits()).lanes_eq(Simd::splat(0))
}
/// Returns true for each lane if its value is neither zero, infinite,
/// subnormal, nor `NaN`.
#[inline]
#[must_use = "method returns a new mask and does not mutate the original value"]
pub fn is_normal(self) -> Mask<$mask_ty, LANES> {
!(self.abs().lanes_eq(Self::splat(0.0)) | self.is_nan() | self.is_subnormal() | self.is_infinite())
}
/// Replaces each lane with a number that represents its sign.
///
/// * `1.0` if the number is positive, `+0.0`, or `INFINITY`
/// * `-1.0` if the number is negative, `-0.0`, or `NEG_INFINITY`
/// * `NAN` if the number is `NAN`
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn signum(self) -> Self {
self.is_nan().select(Self::splat(<$type>::NAN), Self::splat(1.0).copysign(self))
}
/// Returns each lane with the magnitude of `self` and the sign of `sign`.
///
/// If any lane is a `NAN`, then a `NAN` with the sign of `sign` is returned.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn copysign(self, sign: Self) -> Self {
let sign_bit = sign.to_bits() & Self::splat(-0.).to_bits();
let magnitude = self.to_bits() & !Self::splat(-0.).to_bits();
Self::from_bits(sign_bit | magnitude)
}
/// Returns the minimum of each lane.
///
/// If one of the values is `NAN`, then the other value is returned.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn min(self, other: Self) -> Self {
unsafe { intrinsics::simd_fmin(self, other) }
}
/// Returns the maximum of each lane.
///
/// If one of the values is `NAN`, then the other value is returned.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn max(self, other: Self) -> Self {
unsafe { intrinsics::simd_fmax(self, other) }
}
/// Restrict each lane to a certain interval unless it is NaN.
///
/// For each lane in `self`, returns the corresponding lane in `max` if the lane is
/// greater than `max`, and the corresponding lane in `min` if the lane is less
/// than `min`. Otherwise returns the lane in `self`.
#[inline]
#[must_use = "method returns a new vector and does not mutate the original value"]
pub fn clamp(self, min: Self, max: Self) -> Self {
assert!(
min.lanes_le(max).all(),
"each lane in `min` must be less than or equal to the corresponding lane in `max`",
);
let mut x = self;
x = x.lanes_lt(min).select(min, x);
x = x.lanes_gt(max).select(max, x);
x
}
}
};
}
impl_float_vector! { f32, u32, i32 }
impl_float_vector! { f64, u64, i64 }
/// Vector of two `f32` values
/// A 64-bit SIMD vector with two elements of type `f32`.
pub type f32x2 = Simd<f32, 2>;
/// Vector of four `f32` values
/// A 128-bit SIMD vector with four elements of type `f32`.
pub type f32x4 = Simd<f32, 4>;
/// Vector of eight `f32` values
/// A 256-bit SIMD vector with eight elements of type `f32`.
pub type f32x8 = Simd<f32, 8>;
/// Vector of 16 `f32` values
/// A 512-bit SIMD vector with 16 elements of type `f32`.
pub type f32x16 = Simd<f32, 16>;
/// Vector of two `f64` values
/// A 128-bit SIMD vector with two elements of type `f64`.
pub type f64x2 = Simd<f64, 2>;
/// Vector of four `f64` values
/// A 256-bit SIMD vector with four elements of type `f64`.
pub type f64x4 = Simd<f64, 4>;
/// Vector of eight `f64` values
/// A 512-bit SIMD vector with eight elements of type `f64`.
pub type f64x8 = Simd<f64, 8>;

82
library/portable-simd/crates/core_simd/src/vector/int.rs
View file

@ -1,103 +1,63 @@
#![allow(non_camel_case_types)]
use crate::simd::{LaneCount, Mask, Simd, SupportedLaneCount};
use crate::simd::Simd;
/// Implements additional integer traits (Eq, Ord, Hash) on the specified vector `$name`, holding multiple `$lanes` of `$type`.
macro_rules! impl_integer_vector {
{ $type:ty } => {
impl<const LANES: usize> Simd<$type, LANES>
where
LaneCount<LANES>: SupportedLaneCount,
{
/// Returns true for each positive lane and false if it is zero or negative.
#[inline]
pub fn is_positive(self) -> Mask<$type, LANES> {
self.lanes_gt(Self::splat(0))
}
/// Returns true for each negative lane and false if it is zero or positive.
#[inline]
pub fn is_negative(self) -> Mask<$type, LANES> {
self.lanes_lt(Self::splat(0))
}
/// Returns numbers representing the sign of each lane.
/// * `0` if the number is zero
/// * `1` if the number is positive
/// * `-1` if the number is negative
#[inline]
pub fn signum(self) -> Self {
self.is_positive().select(
Self::splat(1),
self.is_negative().select(Self::splat(-1), Self::splat(0))
)
}
}
}
}
impl_integer_vector! { isize }
impl_integer_vector! { i16 }
impl_integer_vector! { i32 }
impl_integer_vector! { i64 }
impl_integer_vector! { i8 }
/// Vector of two `isize` values
/// A SIMD vector with two elements of type `isize`.
pub type isizex2 = Simd<isize, 2>;
/// Vector of four `isize` values
/// A SIMD vector with four elements of type `isize`.
pub type isizex4 = Simd<isize, 4>;
/// Vector of eight `isize` values
/// A SIMD vector with eight elements of type `isize`.
pub type isizex8 = Simd<isize, 8>;
/// Vector of two `i16` values
/// A 32-bit SIMD vector with two elements of type `i16`.
pub type i16x2 = Simd<i16, 2>;
/// Vector of four `i16` values
/// A 64-bit SIMD vector with four elements of type `i16`.
pub type i16x4 = Simd<i16, 4>;
/// Vector of eight `i16` values
/// A 128-bit SIMD vector with eight elements of type `i16`.
pub type i16x8 = Simd<i16, 8>;
/// Vector of 16 `i16` values
/// A 256-bit SIMD vector with 16 elements of type `i16`.
pub type i16x16 = Simd<i16, 16>;
/// Vector of 32 `i16` values
/// A 512-bit SIMD vector with 32 elements of type `i16`.
pub type i16x32 = Simd<i16, 32>;
/// Vector of two `i32` values
/// A 64-bit SIMD vector with two elements of type `i32`.
pub type i32x2 = Simd<i32, 2>;
/// Vector of four `i32` values
/// A 128-bit SIMD vector with four elements of type `i32`.
pub type i32x4 = Simd<i32, 4>;
/// Vector of eight `i32` values
/// A 256-bit SIMD vector with eight elements of type `i32`.
pub type i32x8 = Simd<i32, 8>;
/// Vector of 16 `i32` values
/// A 512-bit SIMD vector with 16 elements of type `i32`.
pub type i32x16 = Simd<i32, 16>;
/// Vector of two `i64` values
/// A 128-bit SIMD vector with two elements of type `i64`.
pub type i64x2 = Simd<i64, 2>;
/// Vector of four `i64` values
/// A 256-bit SIMD vector with four elements of type `i64`.
pub type i64x4 = Simd<i64, 4>;
/// Vector of eight `i64` values
/// A 512-bit SIMD vector with eight elements of type `i64`.
pub type i64x8 = Simd<i64, 8>;
/// Vector of four `i8` values
/// A 32-bit SIMD vector with four elements of type `i8`.
pub type i8x4 = Simd<i8, 4>;
/// Vector of eight `i8` values
/// A 64-bit SIMD vector with eight elements of type `i8`.
pub type i8x8 = Simd<i8, 8>;
/// Vector of 16 `i8` values
/// A 128-bit SIMD vector with 16 elements of type `i8`.
pub type i8x16 = Simd<i8, 16>;
/// Vector of 32 `i8` values
/// A 256-bit SIMD vector with 32 elements of type `i8`.
pub type i8x32 = Simd<i8, 32>;
/// Vector of 64 `i8` values
/// A 512-bit SIMD vector with 64 elements of type `i8`.
pub type i8x64 = Simd<i8, 64>;

40
library/portable-simd/crates/core_simd/src/vector/uint.rs
View file

@ -2,62 +2,62 @@
use crate::simd::Simd;
/// Vector of two `usize` values
/// A SIMD vector with two elements of type `usize`.
pub type usizex2 = Simd<usize, 2>;
/// Vector of four `usize` values
/// A SIMD vector with four elements of type `usize`.
pub type usizex4 = Simd<usize, 4>;
/// Vector of eight `usize` values
/// A SIMD vector with eight elements of type `usize`.
pub type usizex8 = Simd<usize, 8>;
/// Vector of two `u16` values
/// A 32-bit SIMD vector with two elements of type `u16`.
pub type u16x2 = Simd<u16, 2>;
/// Vector of four `u16` values
/// A 64-bit SIMD vector with four elements of type `u16`.
pub type u16x4 = Simd<u16, 4>;
/// Vector of eight `u16` values
/// A 128-bit SIMD vector with eight elements of type `u16`.
pub type u16x8 = Simd<u16, 8>;
/// Vector of 16 `u16` values
/// A 256-bit SIMD vector with 16 elements of type `u16`.
pub type u16x16 = Simd<u16, 16>;
/// Vector of 32 `u16` values
/// A 512-bit SIMD vector with 32 elements of type `u16`.
pub type u16x32 = Simd<u16, 32>;
/// Vector of two `u32` values
/// A 64-bit SIMD vector with two elements of type `u32`.
pub type u32x2 = Simd<u32, 2>;
/// Vector of four `u32` values
/// A 128-bit SIMD vector with four elements of type `u32`.
pub type u32x4 = Simd<u32, 4>;
/// Vector of eight `u32` values
/// A 256-bit SIMD vector with eight elements of type `u32`.
pub type u32x8 = Simd<u32, 8>;
/// Vector of 16 `u32` values
/// A 512-bit SIMD vector with 16 elements of type `u32`.
pub type u32x16 = Simd<u32, 16>;
/// Vector of two `u64` values
/// A 128-bit SIMD vector with two elements of type `u64`.
pub type u64x2 = Simd<u64, 2>;
/// Vector of four `u64` values
/// A 256-bit SIMD vector with four elements of type `u64`.
pub type u64x4 = Simd<u64, 4>;
/// Vector of eight `u64` values
/// A 512-bit SIMD vector with eight elements of type `u64`.
pub type u64x8 = Simd<u64, 8>;
/// Vector of four `u8` values
/// A 32-bit SIMD vector with four elements of type `u8`.
pub type u8x4 = Simd<u8, 4>;
/// Vector of eight `u8` values
/// A 64-bit SIMD vector with eight elements of type `u8`.
pub type u8x8 = Simd<u8, 8>;
/// Vector of 16 `u8` values
/// A 128-bit SIMD vector with 16 elements of type `u8`.
pub type u8x16 = Simd<u8, 16>;
/// Vector of 32 `u8` values
/// A 256-bit SIMD vector with 32 elements of type `u8`.
pub type u8x32 = Simd<u8, 32>;
/// Vector of 64 `u8` values
/// A 512-bit SIMD vector with 64 elements of type `u8`.
pub type u8x64 = Simd<u8, 64>;

27
library/portable-simd/crates/core_simd/tests/i16_ops.rs
View file

@ -1,32 +1,5 @@
#![feature(portable_simd)]
use core_simd::i16x2;
#[macro_use]
mod ops_macros;
impl_signed_tests! { i16 }
#[test]
fn max_is_not_lexicographic() {
let a = i16x2::splat(10);
let b = i16x2::from_array([-4, 12]);
assert_eq!(a.max(b), i16x2::from_array([10, 12]));
}
#[test]
fn min_is_not_lexicographic() {
let a = i16x2::splat(10);
let b = i16x2::from_array([12, -4]);
assert_eq!(a.min(b), i16x2::from_array([10, -4]));
}
#[test]
fn clamp_is_not_lexicographic() {
let a = i16x2::splat(10);
let lo = i16x2::from_array([-12, -4]);
let up = i16x2::from_array([-4, 12]);
assert_eq!(a.clamp(lo, up), i16x2::from_array([-4, 10]));
let x = i16x2::from_array([1, 10]);
let y = x.clamp(i16x2::splat(0), i16x2::splat(9));
assert_eq!(y, i16x2::from_array([1, 9]));
}

56
library/portable-simd/crates/core_simd/tests/masks.rs
View file

@ -80,6 +80,62 @@ macro_rules! test_mask_api {
assert_eq!(bitmask, 0b1000001101001001);
assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask(bitmask), mask);
}
#[test]
fn roundtrip_bitmask_conversion_short() {
use core_simd::ToBitMask;
let values = [
false, false, false, true,
];
let mask = core_simd::Mask::<$type, 4>::from_array(values);
let bitmask = mask.to_bitmask();
assert_eq!(bitmask, 0b1000);
assert_eq!(core_simd::Mask::<$type, 4>::from_bitmask(bitmask), mask);
let values = [true, false];
let mask = core_simd::Mask::<$type, 2>::from_array(values);
let bitmask = mask.to_bitmask();
assert_eq!(bitmask, 0b01);
assert_eq!(core_simd::Mask::<$type, 2>::from_bitmask(bitmask), mask);
}
#[test]
fn cast() {
fn cast_impl<T: core_simd::MaskElement>()
where
core_simd::Mask<$type, 8>: Into<core_simd::Mask<T, 8>>,
{
let values = [true, false, false, true, false, false, true, false];
let mask = core_simd::Mask::<$type, 8>::from_array(values);
let cast_mask = mask.cast::<T>();
assert_eq!(values, cast_mask.to_array());
let into_mask: core_simd::Mask<T, 8> = mask.into();
assert_eq!(values, into_mask.to_array());
}
cast_impl::<i8>();
cast_impl::<i16>();
cast_impl::<i32>();
cast_impl::<i64>();
cast_impl::<isize>();
}
#[cfg(feature = "generic_const_exprs")]
#[test]
fn roundtrip_bitmask_array_conversion() {
use core_simd::ToBitMaskArray;
let values = [
true, false, false, true, false, false, true, false,
true, true, false, false, false, false, false, true,
];
let mask = core_simd::Mask::<$type, 16>::from_array(values);
let bitmask = mask.to_bitmask_array();
assert_eq!(bitmask, [0b01001001, 0b10000011]);
assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask_array(bitmask), mask);
}
}
}
}

48
library/portable-simd/crates/core_simd/tests/ops_macros.rs
View file

@ -172,6 +172,7 @@ macro_rules! impl_common_integer_tests {
macro_rules! impl_signed_tests {
{ $scalar:tt } => {
mod $scalar {
use core_simd::simd::SimdInt;
type Vector<const LANES: usize> = core_simd::Simd<Scalar, LANES>;
type Scalar = $scalar;
@ -222,34 +223,37 @@ macro_rules! impl_signed_tests {
assert_eq!(a % b, Vector::<LANES>::splat(0));
}
fn min<const LANES: usize>() {
fn simd_min<const LANES: usize>() {
use core_simd::simd::SimdOrd;
let a = Vector::<LANES>::splat(Scalar::MIN);
let b = Vector::<LANES>::splat(0);
assert_eq!(a.min(b), a);
assert_eq!(a.simd_min(b), a);
let a = Vector::<LANES>::splat(Scalar::MAX);
let b = Vector::<LANES>::splat(0);
assert_eq!(a.min(b), b);
assert_eq!(a.simd_min(b), b);
}
fn max<const LANES: usize>() {
fn simd_max<const LANES: usize>() {
use core_simd::simd::SimdOrd;
let a = Vector::<LANES>::splat(Scalar::MIN);
let b = Vector::<LANES>::splat(0);
assert_eq!(a.max(b), b);
assert_eq!(a.simd_max(b), b);
let a = Vector::<LANES>::splat(Scalar::MAX);
let b = Vector::<LANES>::splat(0);
assert_eq!(a.max(b), a);
assert_eq!(a.simd_max(b), a);
}
fn clamp<const LANES: usize>() {
fn simd_clamp<const LANES: usize>() {
use core_simd::simd::SimdOrd;
let min = Vector::<LANES>::splat(Scalar::MIN);
let max = Vector::<LANES>::splat(Scalar::MAX);
let zero = Vector::<LANES>::splat(0);
let one = Vector::<LANES>::splat(1);
let negone = Vector::<LANES>::splat(-1);
assert_eq!(zero.clamp(min, max), zero);
assert_eq!(zero.clamp(min, one), zero);
assert_eq!(zero.clamp(one, max), one);
assert_eq!(zero.clamp(min, negone), negone);
assert_eq!(zero.simd_clamp(min, max), zero);
assert_eq!(zero.simd_clamp(min, one), zero);
assert_eq!(zero.simd_clamp(one, max), one);
assert_eq!(zero.simd_clamp(min, negone), negone);
}
}
@ -309,6 +313,7 @@ macro_rules! impl_signed_tests {
macro_rules! impl_unsigned_tests {
{ $scalar:tt } => {
mod $scalar {
use core_simd::simd::SimdUint;
type Vector<const LANES: usize> = core_simd::Simd<Scalar, LANES>;
type Scalar = $scalar;
@ -343,6 +348,7 @@ macro_rules! impl_unsigned_tests {
macro_rules! impl_float_tests {
{ $scalar:tt, $int_scalar:tt } => {
mod $scalar {
use core_simd::SimdFloat;
type Vector<const LANES: usize> = core_simd::Simd<Scalar, LANES>;
type Scalar = $scalar;
@ -458,10 +464,10 @@ macro_rules! impl_float_tests {
)
}
fn min<const LANES: usize>() {
fn simd_min<const LANES: usize>() {
// Regular conditions (both values aren't zero)
test_helpers::test_binary_elementwise(
&Vector::<LANES>::min,
&Vector::<LANES>::simd_min,
&Scalar::min,
// Reject the case where both values are zero with different signs
&|a, b| {
@ -477,14 +483,14 @@ macro_rules! impl_float_tests {
// Special case where both values are zero
let p_zero = Vector::<LANES>::splat(0.);
let n_zero = Vector::<LANES>::splat(-0.);
assert!(p_zero.min(n_zero).to_array().iter().all(|x| *x == 0.));
assert!(n_zero.min(p_zero).to_array().iter().all(|x| *x == 0.));
assert!(p_zero.simd_min(n_zero).to_array().iter().all(|x| *x == 0.));
assert!(n_zero.simd_min(p_zero).to_array().iter().all(|x| *x == 0.));
}
fn max<const LANES: usize>() {
fn simd_max<const LANES: usize>() {
// Regular conditions (both values aren't zero)
test_helpers::test_binary_elementwise(
&Vector::<LANES>::max,
&Vector::<LANES>::simd_max,
&Scalar::max,
// Reject the case where both values are zero with different signs
&|a, b| {
@ -500,11 +506,11 @@ macro_rules! impl_float_tests {
// Special case where both values are zero
let p_zero = Vector::<LANES>::splat(0.);
let n_zero = Vector::<LANES>::splat(-0.);
assert!(p_zero.max(n_zero).to_array().iter().all(|x| *x == 0.));
assert!(n_zero.max(p_zero).to_array().iter().all(|x| *x == 0.));
assert!(p_zero.simd_max(n_zero).to_array().iter().all(|x| *x == 0.));
assert!(n_zero.simd_max(p_zero).to_array().iter().all(|x| *x == 0.));
}
fn clamp<const LANES: usize>() {
fn simd_clamp<const LANES: usize>() {
test_helpers::test_3(&|value: [Scalar; LANES], mut min: [Scalar; LANES], mut max: [Scalar; LANES]| {
for (min, max) in min.iter_mut().zip(max.iter_mut()) {
if max < min {
@ -522,7 +528,7 @@ macro_rules! impl_float_tests {
for i in 0..LANES {
result_scalar[i] = value[i].clamp(min[i], max[i]);
}
let result_vector = Vector::from_array(value).clamp(min.into(), max.into()).to_array();
let result_vector = Vector::from_array(value).simd_clamp(min.into(), max.into()).to_array();
test_helpers::prop_assert_biteq!(result_scalar, result_vector);
Ok(())
})

2
library/portable-simd/crates/core_simd/tests/round.rs
View file

@ -59,7 +59,7 @@ macro_rules! float_rounding_test {
const MAX_REPRESENTABLE_VALUE: Scalar =
(ALL_MANTISSA_BITS << (core::mem::size_of::<Scalar>() * 8 - <Scalar>::MANTISSA_DIGITS as usize - 1)) as Scalar;
let mut runner = proptest::test_runner::TestRunner::default();
let mut runner = test_helpers::make_runner();
runner.run(
&test_helpers::array::UniformArrayStrategy::new(-MAX_REPRESENTABLE_VALUE..MAX_REPRESENTABLE_VALUE),
|x| {