From b5f9d43ff1139fb5dbd1a919dbf63e48c2c56012 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Thu, 21 Jul 2022 14:53:07 -0700 Subject: [PATCH 001/130] rust-lang/portable-simd#289: Strengthen warnings about relying on Mask layout This makes it more clear that you can't rely on the layout of these, which seems worth doing given that the names vaguely suggest that you can (and the docs only clarify that you can't on Mask but not the maskNxM aliases). --- crates/core_simd/src/masks.rs | 76 ++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index c36c336d8a2..99535021735 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -83,7 +83,9 @@ impl_element! { isize } /// /// Masks represent boolean inclusion/exclusion on a per-lane basis. /// -/// The layout of this type is unspecified. +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[T; LANES]`. #[repr(transparent)] pub struct Mask(mask_impl::Mask) where @@ -521,57 +523,129 @@ where } /// A mask for SIMD vectors with eight elements of 8 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i8; 8]`. pub type mask8x8 = Mask; /// A mask for SIMD vectors with 16 elements of 8 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i8; 16]`. pub type mask8x16 = Mask; /// A mask for SIMD vectors with 32 elements of 8 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i8; 32]`. pub type mask8x32 = Mask; /// A mask for SIMD vectors with 64 elements of 8 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i8; 64]`. pub type mask8x64 = Mask; /// A mask for SIMD vectors with four elements of 16 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i16; 4]`. pub type mask16x4 = Mask; /// A mask for SIMD vectors with eight elements of 16 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i16; 8]`. pub type mask16x8 = Mask; /// A mask for SIMD vectors with 16 elements of 16 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i16; 16]`. pub type mask16x16 = Mask; /// A mask for SIMD vectors with 32 elements of 16 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i16; 32]`. pub type mask16x32 = Mask; /// A mask for SIMD vectors with two elements of 32 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i32; 2]`. pub type mask32x2 = Mask; /// A mask for SIMD vectors with four elements of 32 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i32; 4]`. pub type mask32x4 = Mask; /// A mask for SIMD vectors with eight elements of 32 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i32; 8]`. pub type mask32x8 = Mask; /// A mask for SIMD vectors with 16 elements of 32 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i32; 16]`. pub type mask32x16 = Mask; /// A mask for SIMD vectors with two elements of 64 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i64; 2]`. pub type mask64x2 = Mask; /// A mask for SIMD vectors with four elements of 64 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i64; 4]`. pub type mask64x4 = Mask; /// A mask for SIMD vectors with eight elements of 64 bits. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[i64; 8]`. pub type mask64x8 = Mask; /// A mask for SIMD vectors with two elements of pointer width. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[isize; 2]`. pub type masksizex2 = Mask; /// A mask for SIMD vectors with four elements of pointer width. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[isize; 4]`. pub type masksizex4 = Mask; /// A mask for SIMD vectors with eight elements of pointer width. +/// +/// The layout of this type is unspecified, and may change between platforms +/// and/or Rust versions, and code should not assume that it is equivalent to +/// `[isize; 8]`. pub type masksizex8 = Mask; macro_rules! impl_from { From ddede9fb9b5bd3a7cce71775ac8ce7bd30fdf87a Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Fri, 22 Jul 2022 09:39:23 -0400 Subject: [PATCH 002/130] make some Miri backtraces more pretty --- crates/core_simd/src/vector.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 8661be938d5..e8e8f6899d3 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -239,6 +239,7 @@ where /// /// [cast]: Simd::cast #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn to_int_unchecked(self) -> Simd where T: core::convert::FloatToInt, @@ -349,6 +350,7 @@ where /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[must_use] #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn gather_select_unchecked( slice: &[T], enable: Mask, @@ -444,6 +446,7 @@ where /// ``` /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn scatter_select_unchecked( self, slice: &mut [T], From 3183afb6b5fcbf688bb90cf1db3f635406f868dc Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 29 Jul 2022 11:57:05 -0400 Subject: [PATCH 003/130] Fix interleave/deinterleave for vectors with only one lane --- crates/core_simd/src/swizzle.rs | 12 ++++++++++-- crates/core_simd/tests/swizzle.rs | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 22999d24950..02567252a63 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -325,7 +325,11 @@ where const INDEX: [Which; LANES] = hi::(); } - (Lo::swizzle2(self, other), Hi::swizzle2(self, other)) + if LANES == 1 { + (self, other) + } else { + (Lo::swizzle2(self, other), Hi::swizzle2(self, other)) + } } /// Deinterleave two vectors. @@ -380,6 +384,10 @@ where const INDEX: [Which; LANES] = odd::(); } - (Even::swizzle2(self, other), Odd::swizzle2(self, other)) + if LANES == 1 { + (self, other) + } else { + (Even::swizzle2(self, other), Odd::swizzle2(self, other)) + } } } diff --git a/crates/core_simd/tests/swizzle.rs b/crates/core_simd/tests/swizzle.rs index 51c63611aba..33a7becb421 100644 --- a/crates/core_simd/tests/swizzle.rs +++ b/crates/core_simd/tests/swizzle.rs @@ -60,3 +60,17 @@ fn interleave() { assert_eq!(even, a); assert_eq!(odd, b); } + +// portable-simd#298 +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn interleave_one() { + let a = Simd::from_array([0]); + let b = Simd::from_array([1]); + let (lo, hi) = a.interleave(b); + assert_eq!(lo.to_array(), [0]); + assert_eq!(hi.to_array(), [1]); + let (even, odd) = lo.deinterleave(hi); + assert_eq!(even, a); + assert_eq!(odd, b); +} From 8742a86b1da28c1bb7f0e7f663becde9b0c5a73e Mon Sep 17 00:00:00 2001 From: Jacob Lifshay Date: Fri, 29 Jul 2022 16:12:24 -0700 Subject: [PATCH 004/130] add all_lane_counts feature to enable non-power-of-2 lane counts <= 64 --- .github/workflows/ci.yml | 4 + crates/core_simd/Cargo.toml | 1 + crates/core_simd/src/lane_count.rs | 36 ++-- crates/test_helpers/Cargo.toml | 3 + crates/test_helpers/src/lib.rs | 271 ++++++++++++++++++++--------- 5 files changed, 217 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d50dfa1be4c..acd47a3da72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,6 +241,10 @@ jobs: - "--features std" - "--features generic_const_exprs" - "--features std --features generic_const_exprs" + - "--features all_lane_counts" + - "--features all_lane_counts --features std" + - "--features all_lane_counts --features generic_const_exprs" + - "--features all_lane_counts --features std --features generic_const_exprs" steps: - uses: actions/checkout@v2 diff --git a/crates/core_simd/Cargo.toml b/crates/core_simd/Cargo.toml index 8a29cf15696..7435e24edd3 100644 --- a/crates/core_simd/Cargo.toml +++ b/crates/core_simd/Cargo.toml @@ -13,6 +13,7 @@ default = ["as_crate"] as_crate = [] std = [] generic_const_exprs = [] +all_lane_counts = [] [target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen] version = "0.2" diff --git a/crates/core_simd/src/lane_count.rs b/crates/core_simd/src/lane_count.rs index 63723e2ec13..2b91eb9e800 100644 --- a/crates/core_simd/src/lane_count.rs +++ b/crates/core_simd/src/lane_count.rs @@ -23,24 +23,20 @@ pub trait SupportedLaneCount: Sealed { impl Sealed for LaneCount {} -impl SupportedLaneCount for LaneCount<1> { - type BitMask = [u8; 1]; -} -impl SupportedLaneCount for LaneCount<2> { - type BitMask = [u8; 1]; -} -impl SupportedLaneCount for LaneCount<4> { - type BitMask = [u8; 1]; -} -impl SupportedLaneCount for LaneCount<8> { - type BitMask = [u8; 1]; -} -impl SupportedLaneCount for LaneCount<16> { - type BitMask = [u8; 2]; -} -impl SupportedLaneCount for LaneCount<32> { - type BitMask = [u8; 4]; -} -impl SupportedLaneCount for LaneCount<64> { - type BitMask = [u8; 8]; +macro_rules! supported_lane_count { + ($($lanes:literal),+) => { + $( + impl SupportedLaneCount for LaneCount<$lanes> { + type BitMask = [u8; ($lanes + 7) / 8]; + } + )+ + }; } + +supported_lane_count!(1, 2, 4, 8, 16, 32, 64); +#[cfg(feature = "all_lane_counts")] +supported_lane_count!( + 3, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, + 31, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63 +); diff --git a/crates/test_helpers/Cargo.toml b/crates/test_helpers/Cargo.toml index a04b0961d7f..1d2bc8b519a 100644 --- a/crates/test_helpers/Cargo.toml +++ b/crates/test_helpers/Cargo.toml @@ -8,3 +8,6 @@ publish = false version = "0.10" default-features = false features = ["alloc"] + +[features] +all_lane_counts = [] diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 141bee18a9a..650eadd12bf 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -333,6 +333,39 @@ pub fn test_ternary_elementwise< ); } +#[doc(hidden)] +#[macro_export] +macro_rules! test_lanes_helper { + ($($(#[$meta:meta])* $fn_name:ident $lanes:literal;)+) => { + $( + #[test] + $(#[$meta])* + fn $fn_name() { + implementation::<$lanes>(); + } + )+ + }; + ( + $(#[$meta:meta])+; + $($(#[$meta_before:meta])+ $fn_name_before:ident $lanes_before:literal;)* + $fn_name:ident $lanes:literal; + $($fn_name_rest:ident $lanes_rest:literal;)* + ) => { + $crate::test_lanes_helper!( + $(#[$meta])+; + $($(#[$meta_before])+ $fn_name_before $lanes_before;)* + $(#[$meta])+ $fn_name $lanes; + $($fn_name_rest $lanes_rest;)* + ); + }; + ( + $(#[$meta_ignored:meta])+; + $($(#[$meta:meta])+ $fn_name:ident $lanes:literal;)+ + ) => { + $crate::test_lanes_helper!($($(#[$meta])+ $fn_name $lanes;)+); + }; +} + /// Expand a const-generic test into separate tests for each possible lane count. #[macro_export] macro_rules! test_lanes { @@ -351,51 +384,90 @@ macro_rules! test_lanes { #[cfg(target_arch = "wasm32")] wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] - fn lanes_1() { - implementation::<1>(); - } + $crate::test_lanes_helper!( + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; + lanes_1 1; + lanes_2 2; + lanes_4 4; + ); - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] - fn lanes_2() { - implementation::<2>(); - } - - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] - fn lanes_4() { - implementation::<4>(); - } - - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow - fn lanes_8() { - implementation::<8>(); - } + $crate::test_lanes_helper!( + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; + lanes_8 8; + lanes_16 16; + lanes_32 32; + lanes_64 64; + ); - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] - #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow - fn lanes_16() { - implementation::<16>(); - } + #[cfg(feature = "all_lane_counts")] + $crate::test_lanes_helper!( + // test some odd and even non-power-of-2 lengths on miri + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; + lanes_3 3; + lanes_5 5; + lanes_6 6; + ); - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] + #[cfg(feature = "all_lane_counts")] #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow - fn lanes_32() { - implementation::<32>(); - } - - #[test] - #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] - #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow - fn lanes_64() { - implementation::<64>(); - } + $crate::test_lanes_helper!( + #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]; + lanes_7 7; + lanes_9 9; + lanes_10 10; + lanes_11 11; + lanes_12 12; + lanes_13 13; + lanes_14 14; + lanes_15 15; + lanes_17 17; + lanes_18 18; + lanes_19 19; + lanes_20 20; + lanes_21 21; + lanes_22 22; + lanes_23 23; + lanes_24 24; + lanes_25 25; + lanes_26 26; + lanes_27 27; + lanes_28 28; + lanes_29 29; + lanes_30 30; + lanes_31 31; + lanes_33 33; + lanes_34 34; + lanes_35 35; + lanes_36 36; + lanes_37 37; + lanes_38 38; + lanes_39 39; + lanes_40 40; + lanes_41 41; + lanes_42 42; + lanes_43 43; + lanes_44 44; + lanes_45 45; + lanes_46 46; + lanes_47 47; + lanes_48 48; + lanes_49 49; + lanes_50 50; + lanes_51 51; + lanes_52 52; + lanes_53 53; + lanes_54 54; + lanes_55 55; + lanes_56 56; + lanes_57 57; + lanes_58 58; + lanes_59 59; + lanes_60 60; + lanes_61 61; + lanes_62 62; + lanes_63 63; + ); } )* } @@ -416,47 +488,90 @@ macro_rules! test_lanes_panic { core_simd::LaneCount<$lanes>: core_simd::SupportedLaneCount, $body - #[test] - #[should_panic] - fn lanes_1() { - implementation::<1>(); - } + $crate::test_lanes_helper!( + #[should_panic]; + lanes_1 1; + lanes_2 2; + lanes_4 4; + ); - #[test] - #[should_panic] - fn lanes_2() { - implementation::<2>(); - } + #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow + $crate::test_lanes_helper!( + #[should_panic]; + lanes_8 8; + lanes_16 16; + lanes_32 32; + lanes_64 64; + ); - #[test] - #[should_panic] - fn lanes_4() { - implementation::<4>(); - } + #[cfg(feature = "all_lane_counts")] + $crate::test_lanes_helper!( + // test some odd and even non-power-of-2 lengths on miri + #[should_panic]; + lanes_3 3; + lanes_5 5; + lanes_6 6; + ); - #[test] - #[should_panic] - fn lanes_8() { - implementation::<8>(); - } - - #[test] - #[should_panic] - fn lanes_16() { - implementation::<16>(); - } - - #[test] - #[should_panic] - fn lanes_32() { - implementation::<32>(); - } - - #[test] - #[should_panic] - fn lanes_64() { - implementation::<64>(); - } + #[cfg(feature = "all_lane_counts")] + #[cfg(not(miri))] // Miri intrinsic implementations are uniform and larger tests are sloooow + $crate::test_lanes_helper!( + #[should_panic]; + lanes_7 7; + lanes_9 9; + lanes_10 10; + lanes_11 11; + lanes_12 12; + lanes_13 13; + lanes_14 14; + lanes_15 15; + lanes_17 17; + lanes_18 18; + lanes_19 19; + lanes_20 20; + lanes_21 21; + lanes_22 22; + lanes_23 23; + lanes_24 24; + lanes_25 25; + lanes_26 26; + lanes_27 27; + lanes_28 28; + lanes_29 29; + lanes_30 30; + lanes_31 31; + lanes_33 33; + lanes_34 34; + lanes_35 35; + lanes_36 36; + lanes_37 37; + lanes_38 38; + lanes_39 39; + lanes_40 40; + lanes_41 41; + lanes_42 42; + lanes_43 43; + lanes_44 44; + lanes_45 45; + lanes_46 46; + lanes_47 47; + lanes_48 48; + lanes_49 49; + lanes_50 50; + lanes_51 51; + lanes_52 52; + lanes_53 53; + lanes_54 54; + lanes_55 55; + lanes_56 56; + lanes_57 57; + lanes_58 58; + lanes_59 59; + lanes_60 60; + lanes_61 61; + lanes_62 62; + lanes_63 63; + ); } )* } From 6bf512823548b4fdbb7127489e883bff8a98b33f Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 1 Aug 2022 00:34:58 -0400 Subject: [PATCH 005/130] Simplify interleave/deinterleave and fix for odd-length vectors. --- crates/core_simd/src/swizzle.rs | 74 ++++++++++++--------------------- 1 file changed, 26 insertions(+), 48 deletions(-) diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 02567252a63..0b66b8a0ae0 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -265,13 +265,10 @@ where /// Interleave two vectors. /// - /// Produces two vectors with lanes taken alternately from `self` and `other`. + /// The resulting vectors contain lanes taken alternatively from `self` and `other`, first + /// filling the first result, and then the second. /// - /// The first result contains the first `LANES / 2` lanes from `self` and `other`, - /// alternating, starting with the first lane of `self`. - /// - /// The second result contains the last `LANES / 2` lanes from `self` and `other`, - /// alternating, starting with the lane `LANES / 2` from the start of `self`. + /// The reverse of this operation is [`Simd::deinterleave`]. /// /// ``` /// #![feature(portable_simd)] @@ -285,29 +282,17 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn interleave(self, other: Self) -> (Self, Self) { - const fn lo() -> [Which; LANES] { + const fn interleave(high: bool) -> [Which; LANES] { let mut idx = [Which::First(0); LANES]; let mut i = 0; while i < LANES { - let offset = i / 2; - idx[i] = if i % 2 == 0 { - Which::First(offset) + // Treat the source as a concatenated vector + let dst_index = if high { i + LANES } else { i }; + let src_index = dst_index / 2 + (dst_index % 2) * LANES; + idx[i] = if src_index < LANES { + Which::First(src_index) } else { - Which::Second(offset) - }; - i += 1; - } - idx - } - const fn hi() -> [Which; LANES] { - let mut idx = [Which::First(0); LANES]; - let mut i = 0; - while i < LANES { - let offset = (LANES + i) / 2; - idx[i] = if i % 2 == 0 { - Which::First(offset) - } else { - Which::Second(offset) + Which::Second(src_index % LANES) }; i += 1; } @@ -318,18 +303,14 @@ where struct Hi; impl Swizzle2 for Lo { - const INDEX: [Which; LANES] = lo::(); + const INDEX: [Which; LANES] = interleave::(false); } impl Swizzle2 for Hi { - const INDEX: [Which; LANES] = hi::(); + const INDEX: [Which; LANES] = interleave::(true); } - if LANES == 1 { - (self, other) - } else { - (Lo::swizzle2(self, other), Hi::swizzle2(self, other)) - } + (Lo::swizzle2(self, other), Hi::swizzle2(self, other)) } /// Deinterleave two vectors. @@ -340,6 +321,8 @@ where /// The second result takes every other lane of `self` and then `other`, starting with /// the second lane. /// + /// The reverse of this operation is [`Simd::interleave`]. + /// /// ``` /// #![feature(portable_simd)] /// # use core::simd::Simd; @@ -352,22 +335,17 @@ where #[inline] #[must_use = "method returns a new vector and does not mutate the original inputs"] pub fn deinterleave(self, other: Self) -> (Self, Self) { - const fn even() -> [Which; LANES] { + const fn deinterleave(second: bool) -> [Which; LANES] { let mut idx = [Which::First(0); LANES]; let mut i = 0; - while i < LANES / 2 { - idx[i] = Which::First(2 * i); - idx[i + LANES / 2] = Which::Second(2 * i); - i += 1; - } - idx - } - const fn odd() -> [Which; LANES] { - let mut idx = [Which::First(0); LANES]; - let mut i = 0; - while i < LANES / 2 { - idx[i] = Which::First(2 * i + 1); - idx[i + LANES / 2] = Which::Second(2 * i + 1); + while i < LANES { + // Treat the source as a concatenated vector + let src_index = i * 2 + if second { 1 } else { 0 }; + idx[i] = if src_index < LANES { + Which::First(src_index) + } else { + Which::Second(src_index % LANES) + }; i += 1; } idx @@ -377,11 +355,11 @@ where struct Odd; impl Swizzle2 for Even { - const INDEX: [Which; LANES] = even::(); + const INDEX: [Which; LANES] = deinterleave::(false); } impl Swizzle2 for Odd { - const INDEX: [Which; LANES] = odd::(); + const INDEX: [Which; LANES] = deinterleave::(true); } if LANES == 1 { From c739af3908613ba3f611dce115525e2f2f91bfca Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 1 Aug 2022 00:38:29 -0400 Subject: [PATCH 006/130] Hide rustc unstable feature from docs --- crates/core_simd/src/swizzle.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 0b66b8a0ae0..72cce7aeb04 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -271,7 +271,7 @@ where /// The reverse of this operation is [`Simd::deinterleave`]. /// /// ``` - /// #![feature(portable_simd)] + /// # #![feature(portable_simd)] /// # use core::simd::Simd; /// let a = Simd::from_array([0, 1, 2, 3]); /// let b = Simd::from_array([4, 5, 6, 7]); @@ -324,7 +324,7 @@ where /// The reverse of this operation is [`Simd::interleave`]. /// /// ``` - /// #![feature(portable_simd)] + /// # #![feature(portable_simd)] /// # use core::simd::Simd; /// let a = Simd::from_array([0, 4, 1, 5]); /// let b = Simd::from_array([2, 6, 3, 7]); From d030301161a372b545e5d8c1784cba113e5a8ebd Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 1 Aug 2022 19:52:35 -0400 Subject: [PATCH 007/130] Remove special case for length-1 vectors --- crates/core_simd/src/swizzle.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 72cce7aeb04..61cc604e4cd 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -362,10 +362,6 @@ where const INDEX: [Which; LANES] = deinterleave::(true); } - if LANES == 1 { - (self, other) - } else { - (Even::swizzle2(self, other), Odd::swizzle2(self, other)) - } + (Even::swizzle2(self, other), Odd::swizzle2(self, other)) } } From 5f7066430b9239cfe8243ddba4c29416f002ae6b Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Mon, 1 Aug 2022 19:57:41 -0400 Subject: [PATCH 008/130] Simplify expression --- crates/core_simd/src/swizzle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/swizzle.rs b/crates/core_simd/src/swizzle.rs index 61cc604e4cd..68f20516cf5 100644 --- a/crates/core_simd/src/swizzle.rs +++ b/crates/core_simd/src/swizzle.rs @@ -340,7 +340,7 @@ where let mut i = 0; while i < LANES { // Treat the source as a concatenated vector - let src_index = i * 2 + if second { 1 } else { 0 }; + let src_index = i * 2 + second as usize; idx[i] = if src_index < LANES { Which::First(src_index) } else { From 2c5ebfb6a26d384bc21db6796095890c1f13f19c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Fri, 30 Sep 2022 20:25:34 -0500 Subject: [PATCH 009/130] add feature flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit couldn't run the `hellosimd` without it 🤷🏾 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index db0af2da606..791051f69ae 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ core_simd = { git = "https://github.com/rust-lang/portable-simd" } and finally write this in `src/main.rs`: ```rust +#![feature(portable_simd)] use core_simd::*; fn main() { let a = f32x4::splat(10.0); From 4491309cb01cc917ef455c41b0dcf9cf5900aa35 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 16 Oct 2022 13:31:42 -0400 Subject: [PATCH 010/130] Mark more mask functions inline --- crates/core_simd/src/masks.rs | 9 +++++++++ crates/core_simd/src/masks/bitmask.rs | 4 ++++ crates/core_simd/src/masks/full_masks.rs | 4 ++++ crates/core_simd/src/masks/to_bitmask.rs | 4 ++++ 4 files changed, 21 insertions(+) diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 99535021735..7fd50fed447 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -55,6 +55,7 @@ pub unsafe trait MaskElement: SimdElement + Sealed {} macro_rules! impl_element { { $ty:ty } => { impl Sealed for $ty { + #[inline] fn valid(value: Simd) -> bool where LaneCount: SupportedLaneCount, @@ -62,6 +63,7 @@ macro_rules! impl_element { (value.simd_eq(Simd::splat(0 as _)) | value.simd_eq(Simd::splat(-1 as _))).all() } + #[inline] fn eq(self, other: Self) -> bool { self == other } const TRUE: Self = -1; @@ -104,6 +106,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn clone(&self) -> Self { *self } @@ -115,11 +118,13 @@ where LaneCount: SupportedLaneCount, { /// Construct a mask by setting all lanes to the given value. + #[inline] pub fn splat(value: bool) -> Self { Self(mask_impl::Mask::splat(value)) } /// Converts an array of bools to a SIMD mask. + #[inline] pub fn from_array(array: [bool; LANES]) -> Self { // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of // true: 0b_0000_0001 @@ -136,6 +141,7 @@ where } /// Converts a SIMD mask to an array of bools. + #[inline] pub fn to_array(self) -> [bool; LANES] { // This follows mostly the same logic as from_array. // SAFETY: Rust's bool has a layout of 1 byte (u8) with a value of @@ -263,6 +269,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn from(array: [bool; LANES]) -> Self { Self::from_array(array) } @@ -273,6 +280,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn from(vector: Mask) -> Self { vector.to_array() } @@ -655,6 +663,7 @@ macro_rules! impl_from { where LaneCount: SupportedLaneCount, { + #[inline] fn from(value: Mask<$from, LANES>) -> Self { value.cast() } diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 365ecc0a325..20465ba9b07 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -26,6 +26,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn clone(&self) -> Self { *self } @@ -36,6 +37,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn eq(&self, other: &Self) -> bool { self.0.as_ref() == other.0.as_ref() } @@ -46,6 +48,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn partial_cmp(&self, other: &Self) -> Option { self.0.as_ref().partial_cmp(other.0.as_ref()) } @@ -63,6 +66,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.0.as_ref().cmp(other.0.as_ref()) } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index adf0fcbeae2..bcedd2df225 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -37,6 +37,7 @@ where T: MaskElement + PartialEq, LaneCount: SupportedLaneCount, { + #[inline] fn eq(&self, other: &Self) -> bool { self.0.eq(&other.0) } @@ -47,6 +48,7 @@ where T: MaskElement + PartialOrd, LaneCount: SupportedLaneCount, { + #[inline] fn partial_cmp(&self, other: &Self) -> Option { self.0.partial_cmp(&other.0) } @@ -64,6 +66,7 @@ where T: MaskElement + Ord, LaneCount: SupportedLaneCount, { + #[inline] fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.0.cmp(&other.0) } @@ -262,6 +265,7 @@ where T: MaskElement, LaneCount: SupportedLaneCount, { + #[inline] fn from(value: Mask) -> Self { value.0 } diff --git a/crates/core_simd/src/masks/to_bitmask.rs b/crates/core_simd/src/masks/to_bitmask.rs index 65d3ce9be65..46914dfe0d9 100644 --- a/crates/core_simd/src/masks/to_bitmask.rs +++ b/crates/core_simd/src/masks/to_bitmask.rs @@ -48,10 +48,12 @@ macro_rules! impl_integer_intrinsic { impl ToBitMask for Mask { type BitMask = $int; + #[inline] fn to_bitmask(self) -> $int { self.0.to_bitmask_integer() } + #[inline] fn from_bitmask(bitmask: $int) -> Self { Self(mask_impl::Mask::from_bitmask_integer(bitmask)) } @@ -83,10 +85,12 @@ where { const BYTES: usize = bitmask_len(LANES); + #[inline] fn to_bitmask_array(self) -> [u8; Self::BYTES] { self.0.to_bitmask_array() } + #[inline] fn from_bitmask_array(bitmask: [u8; Self::BYTES]) -> Self { Mask(mask_impl::Mask::from_bitmask_array(bitmask)) } From ee9a23facb7871218f5f0bf596f77e27586187a9 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 16 Oct 2022 13:52:08 -0400 Subject: [PATCH 011/130] Update readme --- README.md | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 791051f69ae..4c1b4062100 100644 --- a/README.md +++ b/README.md @@ -24,20 +24,10 @@ or by setting up `rustup default nightly` or else with `cargo +nightly {build,te ```bash cargo new hellosimd ``` -to create a new crate. Edit `hellosimd/Cargo.toml` to be -```toml -[package] -name = "hellosimd" -version = "0.1.0" -edition = "2018" -[dependencies] -core_simd = { git = "https://github.com/rust-lang/portable-simd" } -``` - -and finally write this in `src/main.rs`: +to create a new crate. Finally write this in `src/main.rs`: ```rust #![feature(portable_simd)] -use core_simd::*; +use std::simd::f32x4; fn main() { let a = f32x4::splat(10.0); let b = f32x4::from_array([1.0, 2.0, 3.0, 4.0]); @@ -45,24 +35,22 @@ fn main() { } ``` -Explanation: We import all the bindings from the crate with the first line. Then, we construct our SIMD vectors with methods like `splat` or `from_array`. Finally, we can use operators on them like `+` and the appropriate SIMD instructions will be carried out. When we run `cargo run` you should get `[11.0, 12.0, 13.0, 14.0]`. +Explanation: We construct our SIMD vectors with methods like `splat` or `from_array`. Next, we can use operators like `+` on them, and the appropriate SIMD instructions will be carried out. When we run `cargo run` you should get `[11.0, 12.0, 13.0, 14.0]`. -## Code Organization +## Supported vectors -Currently the crate is organized so that each element type is a file, and then the 64-bit, 128-bit, 256-bit, and 512-bit vectors using those types are contained in said file. - -All types are then exported as a single, flat module. +Currently, vectors may have up to 64 elements, but aliases are provided only up to 512-bit vectors. Depending on the size of the primitive type, the number of lanes the vector will have varies. For example, 128-bit vectors have four `f32` lanes and two `f64` lanes. The supported element types are as follows: * **Floating Point:** `f32`, `f64` -* **Signed Integers:** `i8`, `i16`, `i32`, `i64`, `i128`, `isize` -* **Unsigned Integers:** `u8`, `u16`, `u32`, `u64`, `u128`, `usize` -* **Masks:** `mask8`, `mask16`, `mask32`, `mask64`, `mask128`, `masksize` +* **Signed Integers:** `i8`, `i16`, `i32`, `i64`, `isize` (`i128` excluded) +* **Unsigned Integers:** `u8`, `u16`, `u32`, `u64`, `usize` (`u128` excluded) +* **Masks:** 8-bit, 16-bit, 32-bit, 64-bit, and `usize`-sized masks Floating point, signed integers, and unsigned integers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to. -The `mask` types are "truthy" values, but they use the number of bits in their name instead of just 1 bit like a normal `bool` uses. +The mask types are "truthy" values, like `bool`, but have an unspecified layout in the vector type and cannot be constructed outside of a vector. [simd-guide]: ./beginners-guide.md [zulip-project-portable-simd]: https://rust-lang.zulipchat.com/#narrow/stream/257879-project-portable-simd From f236f5745a0058bd85e044fe3252b87676843018 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 16 Oct 2022 18:08:17 -0400 Subject: [PATCH 012/130] Update README.md Co-authored-by: Jacob Lifshay --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c1b4062100..80313157ea2 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The supported element types are as follows: * **Masks:** 8-bit, 16-bit, 32-bit, 64-bit, and `usize`-sized masks Floating point, signed integers, and unsigned integers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to. -The mask types are "truthy" values, like `bool`, but have an unspecified layout in the vector type and cannot be constructed outside of a vector. +The mask types have elements that are "truthy" values, like `bool`, but have an unspecified layout because different architectures prefer different layouts for mask types. [simd-guide]: ./beginners-guide.md [zulip-project-portable-simd]: https://rust-lang.zulipchat.com/#narrow/stream/257879-project-portable-simd From 61a6f1854f453bb1003b08358b9478eba7fd6ad8 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 16 Oct 2022 21:38:13 -0400 Subject: [PATCH 013/130] Specify aliases in one place, and make it more uniform which are defined --- crates/core_simd/src/alias.rs | 227 +++++++++++++++++++++++++++ crates/core_simd/src/masks.rs | 126 --------------- crates/core_simd/src/mod.rs | 2 + crates/core_simd/src/vector.rs | 8 - crates/core_simd/src/vector/float.rs | 24 --- crates/core_simd/src/vector/int.rs | 63 -------- crates/core_simd/src/vector/uint.rs | 63 -------- 7 files changed, 229 insertions(+), 284 deletions(-) create mode 100644 crates/core_simd/src/alias.rs delete mode 100644 crates/core_simd/src/vector/float.rs delete mode 100644 crates/core_simd/src/vector/int.rs delete mode 100644 crates/core_simd/src/vector/uint.rs diff --git a/crates/core_simd/src/alias.rs b/crates/core_simd/src/alias.rs new file mode 100644 index 00000000000..b4d5f45208a --- /dev/null +++ b/crates/core_simd/src/alias.rs @@ -0,0 +1,227 @@ +macro_rules! number { + { 1 } => { "one" }; + { 2 } => { "two" }; + { 4 } => { "four" }; + { 8 } => { "eight" }; + { $x:literal } => { stringify!($x) }; +} + +macro_rules! plural { + { 1 } => { "" }; + { $x:literal } => { "s" }; +} + +macro_rules! alias { + { + $( + $element:ty = { + $($alias:ident $elements:tt)* + } + )* + } => { + $( + $( + #[doc = concat!("A SIMD vector with ", number!($elements), " element", plural!($elements), " of type [`", stringify!($element), "`].")] + #[allow(non_camel_case_types)] + pub type $alias = $crate::simd::Simd<$element, $elements>; + )* + )* + } +} + +macro_rules! mask_alias { + { + $( + $element:ty : $size:literal = { + $($alias:ident $elements:tt)* + } + )* + } => { + $( + $( + #[doc = concat!("A SIMD mask with ", number!($elements), " element", plural!($elements), " for vectors with ", $size, " element types.")] + /// + #[doc = concat!( + "The layout of this type is unspecified, and may change between platforms and/or Rust versions, and code should not assume that it is equivalent to `[", + stringify!($element), "; ", $elements, "]`." + )] + #[allow(non_camel_case_types)] + pub type $alias = $crate::simd::Mask<$element, $elements>; + )* + )* + } +} + +alias! { + i8 = { + i8x1 1 + i8x2 2 + i8x4 4 + i8x8 8 + i8x16 16 + i8x32 32 + i8x64 64 + } + + i16 = { + i16x1 1 + i16x2 2 + i16x4 4 + i16x8 8 + i16x16 16 + i16x32 32 + i16x64 64 + } + + i32 = { + i32x1 1 + i32x2 2 + i32x4 4 + i32x8 8 + i32x16 16 + i32x32 32 + i32x64 64 + } + + i64 = { + i64x1 1 + i64x2 2 + i64x4 4 + i64x8 8 + i64x16 16 + i64x32 32 + i64x64 64 + } + + isize = { + isizex1 1 + isizex2 2 + isizex4 4 + isizex8 8 + isizex16 16 + isizex32 32 + isizex64 64 + } + + u8 = { + u8x1 1 + u8x2 2 + u8x4 4 + u8x8 8 + u8x16 16 + u8x32 32 + u8x64 64 + } + + u16 = { + u16x1 1 + u16x2 2 + u16x4 4 + u16x8 8 + u16x16 16 + u16x32 32 + u16x64 64 + } + + u32 = { + u32x1 1 + u32x2 2 + u32x4 4 + u32x8 8 + u32x16 16 + u32x32 32 + u32x64 64 + } + + u64 = { + u64x1 1 + u64x2 2 + u64x4 4 + u64x8 8 + u64x16 16 + u64x32 32 + u64x64 64 + } + + usize = { + usizex1 1 + usizex2 2 + usizex4 4 + usizex8 8 + usizex16 16 + usizex32 32 + usizex64 64 + } + + f32 = { + f32x1 1 + f32x2 2 + f32x4 4 + f32x8 8 + f32x16 16 + f32x32 32 + f32x64 64 + } + + f64 = { + f64x1 1 + f64x2 2 + f64x4 4 + f64x8 8 + f64x16 16 + f64x32 32 + f64x64 64 + } +} + +mask_alias! { + i8 : "8-bit" = { + mask8x1 1 + mask8x2 2 + mask8x4 4 + mask8x8 8 + mask8x16 16 + mask8x32 32 + mask8x64 64 + } + + i16 : "16-bit" = { + mask16x1 1 + mask16x2 2 + mask16x4 4 + mask16x8 8 + mask16x16 16 + mask16x32 32 + mask16x64 64 + } + + i32 : "32-bit" = { + mask32x1 1 + mask32x2 2 + mask32x4 4 + mask32x8 8 + mask32x16 16 + mask32x32 32 + mask32x64 64 + } + + i64 : "64-bit" = { + mask64x1 1 + mask64x2 2 + mask64x4 4 + mask64x8 8 + mask64x16 16 + mask64x32 32 + mask64x64 64 + } + + isize : "pointer-sized" = { + masksizex1 1 + masksizex2 2 + masksizex4 4 + masksizex8 8 + masksizex16 16 + masksizex32 32 + masksizex64 64 + } +} diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 7fd50fed447..e58df80fca8 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -530,132 +530,6 @@ where } } -/// A mask for SIMD vectors with eight elements of 8 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i8; 8]`. -pub type mask8x8 = Mask; - -/// A mask for SIMD vectors with 16 elements of 8 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i8; 16]`. -pub type mask8x16 = Mask; - -/// A mask for SIMD vectors with 32 elements of 8 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i8; 32]`. -pub type mask8x32 = Mask; - -/// A mask for SIMD vectors with 64 elements of 8 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i8; 64]`. -pub type mask8x64 = Mask; - -/// A mask for SIMD vectors with four elements of 16 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i16; 4]`. -pub type mask16x4 = Mask; - -/// A mask for SIMD vectors with eight elements of 16 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i16; 8]`. -pub type mask16x8 = Mask; - -/// A mask for SIMD vectors with 16 elements of 16 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i16; 16]`. -pub type mask16x16 = Mask; - -/// A mask for SIMD vectors with 32 elements of 16 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i16; 32]`. -pub type mask16x32 = Mask; - -/// A mask for SIMD vectors with two elements of 32 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i32; 2]`. -pub type mask32x2 = Mask; - -/// A mask for SIMD vectors with four elements of 32 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i32; 4]`. -pub type mask32x4 = Mask; - -/// A mask for SIMD vectors with eight elements of 32 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i32; 8]`. -pub type mask32x8 = Mask; - -/// A mask for SIMD vectors with 16 elements of 32 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i32; 16]`. -pub type mask32x16 = Mask; - -/// A mask for SIMD vectors with two elements of 64 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i64; 2]`. -pub type mask64x2 = Mask; - -/// A mask for SIMD vectors with four elements of 64 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i64; 4]`. -pub type mask64x4 = Mask; - -/// A mask for SIMD vectors with eight elements of 64 bits. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[i64; 8]`. -pub type mask64x8 = Mask; - -/// A mask for SIMD vectors with two elements of pointer width. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[isize; 2]`. -pub type masksizex2 = Mask; - -/// A mask for SIMD vectors with four elements of pointer width. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[isize; 4]`. -pub type masksizex4 = Mask; - -/// A mask for SIMD vectors with eight elements of pointer width. -/// -/// The layout of this type is unspecified, and may change between platforms -/// and/or Rust versions, and code should not assume that it is equivalent to -/// `[isize; 8]`. -pub type masksizex8 = Mask; - macro_rules! impl_from { { $from:ty => $($to:ty),* } => { $( diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index b472aa3abe2..9909d639874 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod intrinsics; #[cfg(feature = "generic_const_exprs")] mod to_bytes; +mod alias; mod elements; mod eq; mod fmt; @@ -22,6 +23,7 @@ mod vendor; pub mod simd { pub(crate) use crate::core_simd::intrinsics; + pub use crate::core_simd::alias::*; pub use crate::core_simd::elements::*; pub use crate::core_simd::eq::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index e8e8f6899d3..7f0e8350cf8 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -1,11 +1,3 @@ -mod float; -mod int; -mod uint; - -pub use float::*; -pub use int::*; -pub use uint::*; - // Vectors of pointers are not for public use at the current time. pub(crate) mod ptr; diff --git a/crates/core_simd/src/vector/float.rs b/crates/core_simd/src/vector/float.rs deleted file mode 100644 index f836c99b1e2..00000000000 --- a/crates/core_simd/src/vector/float.rs +++ /dev/null @@ -1,24 +0,0 @@ -#![allow(non_camel_case_types)] - -use crate::simd::Simd; - -/// A 64-bit SIMD vector with two elements of type `f32`. -pub type f32x2 = Simd; - -/// A 128-bit SIMD vector with four elements of type `f32`. -pub type f32x4 = Simd; - -/// A 256-bit SIMD vector with eight elements of type `f32`. -pub type f32x8 = Simd; - -/// A 512-bit SIMD vector with 16 elements of type `f32`. -pub type f32x16 = Simd; - -/// A 128-bit SIMD vector with two elements of type `f64`. -pub type f64x2 = Simd; - -/// A 256-bit SIMD vector with four elements of type `f64`. -pub type f64x4 = Simd; - -/// A 512-bit SIMD vector with eight elements of type `f64`. -pub type f64x8 = Simd; diff --git a/crates/core_simd/src/vector/int.rs b/crates/core_simd/src/vector/int.rs deleted file mode 100644 index 20e56c7dc64..00000000000 --- a/crates/core_simd/src/vector/int.rs +++ /dev/null @@ -1,63 +0,0 @@ -#![allow(non_camel_case_types)] - -use crate::simd::Simd; - -/// A SIMD vector with two elements of type `isize`. -pub type isizex2 = Simd; - -/// A SIMD vector with four elements of type `isize`. -pub type isizex4 = Simd; - -/// A SIMD vector with eight elements of type `isize`. -pub type isizex8 = Simd; - -/// A 32-bit SIMD vector with two elements of type `i16`. -pub type i16x2 = Simd; - -/// A 64-bit SIMD vector with four elements of type `i16`. -pub type i16x4 = Simd; - -/// A 128-bit SIMD vector with eight elements of type `i16`. -pub type i16x8 = Simd; - -/// A 256-bit SIMD vector with 16 elements of type `i16`. -pub type i16x16 = Simd; - -/// A 512-bit SIMD vector with 32 elements of type `i16`. -pub type i16x32 = Simd; - -/// A 64-bit SIMD vector with two elements of type `i32`. -pub type i32x2 = Simd; - -/// A 128-bit SIMD vector with four elements of type `i32`. -pub type i32x4 = Simd; - -/// A 256-bit SIMD vector with eight elements of type `i32`. -pub type i32x8 = Simd; - -/// A 512-bit SIMD vector with 16 elements of type `i32`. -pub type i32x16 = Simd; - -/// A 128-bit SIMD vector with two elements of type `i64`. -pub type i64x2 = Simd; - -/// A 256-bit SIMD vector with four elements of type `i64`. -pub type i64x4 = Simd; - -/// A 512-bit SIMD vector with eight elements of type `i64`. -pub type i64x8 = Simd; - -/// A 32-bit SIMD vector with four elements of type `i8`. -pub type i8x4 = Simd; - -/// A 64-bit SIMD vector with eight elements of type `i8`. -pub type i8x8 = Simd; - -/// A 128-bit SIMD vector with 16 elements of type `i8`. -pub type i8x16 = Simd; - -/// A 256-bit SIMD vector with 32 elements of type `i8`. -pub type i8x32 = Simd; - -/// A 512-bit SIMD vector with 64 elements of type `i8`. -pub type i8x64 = Simd; diff --git a/crates/core_simd/src/vector/uint.rs b/crates/core_simd/src/vector/uint.rs deleted file mode 100644 index b4a69c44363..00000000000 --- a/crates/core_simd/src/vector/uint.rs +++ /dev/null @@ -1,63 +0,0 @@ -#![allow(non_camel_case_types)] - -use crate::simd::Simd; - -/// A SIMD vector with two elements of type `usize`. -pub type usizex2 = Simd; - -/// A SIMD vector with four elements of type `usize`. -pub type usizex4 = Simd; - -/// A SIMD vector with eight elements of type `usize`. -pub type usizex8 = Simd; - -/// A 32-bit SIMD vector with two elements of type `u16`. -pub type u16x2 = Simd; - -/// A 64-bit SIMD vector with four elements of type `u16`. -pub type u16x4 = Simd; - -/// A 128-bit SIMD vector with eight elements of type `u16`. -pub type u16x8 = Simd; - -/// A 256-bit SIMD vector with 16 elements of type `u16`. -pub type u16x16 = Simd; - -/// A 512-bit SIMD vector with 32 elements of type `u16`. -pub type u16x32 = Simd; - -/// A 64-bit SIMD vector with two elements of type `u32`. -pub type u32x2 = Simd; - -/// A 128-bit SIMD vector with four elements of type `u32`. -pub type u32x4 = Simd; - -/// A 256-bit SIMD vector with eight elements of type `u32`. -pub type u32x8 = Simd; - -/// A 512-bit SIMD vector with 16 elements of type `u32`. -pub type u32x16 = Simd; - -/// A 128-bit SIMD vector with two elements of type `u64`. -pub type u64x2 = Simd; - -/// A 256-bit SIMD vector with four elements of type `u64`. -pub type u64x4 = Simd; - -/// A 512-bit SIMD vector with eight elements of type `u64`. -pub type u64x8 = Simd; - -/// A 32-bit SIMD vector with four elements of type `u8`. -pub type u8x4 = Simd; - -/// A 64-bit SIMD vector with eight elements of type `u8`. -pub type u8x8 = Simd; - -/// A 128-bit SIMD vector with 16 elements of type `u8`. -pub type u8x16 = Simd; - -/// A 256-bit SIMD vector with 32 elements of type `u8`. -pub type u8x32 = Simd; - -/// A 512-bit SIMD vector with 64 elements of type `u8`. -pub type u8x64 = Simd; From 402b50a2728ec4dd9a6da2e57b25cce3ffb48f06 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 16 Oct 2022 23:46:18 -0400 Subject: [PATCH 014/130] Improve variable names --- crates/core_simd/src/alias.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/core_simd/src/alias.rs b/crates/core_simd/src/alias.rs index b4d5f45208a..23f121c4619 100644 --- a/crates/core_simd/src/alias.rs +++ b/crates/core_simd/src/alias.rs @@ -14,16 +14,16 @@ macro_rules! plural { macro_rules! alias { { $( - $element:ty = { - $($alias:ident $elements:tt)* + $element_ty:ty = { + $($alias:ident $num_elements:tt)* } )* } => { $( $( - #[doc = concat!("A SIMD vector with ", number!($elements), " element", plural!($elements), " of type [`", stringify!($element), "`].")] + #[doc = concat!("A SIMD vector with ", number!($num_elements), " element", plural!($num_elements), " of type [`", stringify!($element_ty), "`].")] #[allow(non_camel_case_types)] - pub type $alias = $crate::simd::Simd<$element, $elements>; + pub type $alias = $crate::simd::Simd<$element_ty, $num_elements>; )* )* } @@ -32,21 +32,21 @@ macro_rules! alias { macro_rules! mask_alias { { $( - $element:ty : $size:literal = { - $($alias:ident $elements:tt)* + $element_ty:ty : $size:literal = { + $($alias:ident $num_elements:tt)* } )* } => { $( $( - #[doc = concat!("A SIMD mask with ", number!($elements), " element", plural!($elements), " for vectors with ", $size, " element types.")] + #[doc = concat!("A SIMD mask with ", number!($num_elements), " element", plural!($num_elements), " for vectors with ", $size, " element types.")] /// #[doc = concat!( "The layout of this type is unspecified, and may change between platforms and/or Rust versions, and code should not assume that it is equivalent to `[", - stringify!($element), "; ", $elements, "]`." + stringify!($element_ty), "; ", $num_elements, "]`." )] #[allow(non_camel_case_types)] - pub type $alias = $crate::simd::Mask<$element, $elements>; + pub type $alias = $crate::simd::Mask<$element_ty, $num_elements>; )* )* } From d3cfd7c5c9dba01a8f31b10cef4a1985ae1dc53f Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Tue, 21 Jun 2022 23:17:13 -0400 Subject: [PATCH 015/130] Add vectors of pointers --- crates/core_simd/src/cast.rs | 45 ++++++++ crates/core_simd/src/elements.rs | 4 + crates/core_simd/src/elements/const_ptr.rs | 59 +++++++++++ crates/core_simd/src/elements/mut_ptr.rs | 57 +++++++++++ crates/core_simd/src/eq.rs | 42 ++++++++ crates/core_simd/src/mod.rs | 2 + crates/core_simd/src/ord.rs | 114 +++++++++++++++++++++ crates/core_simd/src/vector.rs | 19 +++- 8 files changed, 339 insertions(+), 3 deletions(-) create mode 100644 crates/core_simd/src/cast.rs create mode 100644 crates/core_simd/src/elements/const_ptr.rs create mode 100644 crates/core_simd/src/elements/mut_ptr.rs diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs new file mode 100644 index 00000000000..e04a9042b1b --- /dev/null +++ b/crates/core_simd/src/cast.rs @@ -0,0 +1,45 @@ +use crate::simd::SimdElement; + +/// Supporting trait for `Simd::cast`. Typically doesn't need to be used directly. +pub trait SimdCast: SimdElement {} + +macro_rules! into_number { + { $($type:ty),* } => { + $( + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + impl SimdCast for $type {} + + impl SimdCast for $type {} + impl SimdCast for $type {} + )* + } +} + +into_number! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, f32, f64 } + +macro_rules! into_pointer { + { $($type:ty),* } => { + $( + impl SimdCast<$type> for *const T {} + impl SimdCast<$type> for *mut T {} + impl SimdCast<*const T> for $type {} + impl SimdCast<*mut T> for $type {} + )* + } +} + +into_pointer! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } + +impl SimdCast<*const T> for *const U {} +impl SimdCast<*const T> for *mut U {} +impl SimdCast<*mut T> for *const U {} +impl SimdCast<*mut T> for *mut U {} diff --git a/crates/core_simd/src/elements.rs b/crates/core_simd/src/elements.rs index 701eb66b248..dc7f52a4d57 100644 --- a/crates/core_simd/src/elements.rs +++ b/crates/core_simd/src/elements.rs @@ -1,11 +1,15 @@ +mod const_ptr; mod float; mod int; +mod mut_ptr; mod uint; mod sealed { pub trait Sealed {} } +pub use const_ptr::*; pub use float::*; pub use int::*; +pub use mut_ptr::*; pub use uint::*; diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs new file mode 100644 index 00000000000..ab6b5b8b5f4 --- /dev/null +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -0,0 +1,59 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; + +/// Operations on SIMD vectors of constant pointers. +pub trait SimdConstPtr: Copy + Sealed { + /// Vector type representing the pointers as bits. + type Bits; + + /// Vector of mutable pointers to the same type. + type MutPtr; + + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Returns `true` for each lane that is null. + fn is_null(self) -> Self::Mask; + + /// Changes constness without changing the type. + fn as_mut(self) -> Self::MutPtr; + + /// Cast pointers to raw bits. + fn to_bits(self) -> Self::Bits; + + /// Cast raw bits to pointers. + fn from_bits(bits: Self::Bits) -> Self; +} + +impl Sealed for Simd<*const T, LANES> where + LaneCount: SupportedLaneCount +{ +} + +impl SimdConstPtr for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Bits = Simd; + type MutPtr = Simd<*mut T, LANES>; + type Mask = Mask; + + fn is_null(self) -> Self::Mask { + Simd::splat(core::ptr::null()).simd_eq(self) + } + + fn as_mut(self) -> Self::MutPtr { + // Converting between pointers is safe + unsafe { intrinsics::simd_as(self) } + } + + fn to_bits(self) -> Self::Bits { + // Casting pointers to usize is safe + unsafe { intrinsics::simd_as(self) } + } + + fn from_bits(bits: Self::Bits) -> Self { + // Casting usize to pointers is safe + unsafe { intrinsics::simd_as(bits) } + } +} diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs new file mode 100644 index 00000000000..b49f9fda7e4 --- /dev/null +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -0,0 +1,57 @@ +use super::sealed::Sealed; +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; + +/// Operations on SIMD vectors of mutable pointers. +pub trait SimdMutPtr: Copy + Sealed { + /// Vector type representing the pointers as bits. + type Bits; + + /// Vector of constant pointers to the same type. + type ConstPtr; + + /// Mask type used for manipulating this SIMD vector type. + type Mask; + + /// Returns `true` for each lane that is null. + fn is_null(self) -> Self::Mask; + + /// Changes constness without changing the type. + fn as_const(self) -> Self::ConstPtr; + + /// Cast pointers to raw bits. + fn to_bits(self) -> Self::Bits; + + /// Cast raw bits to pointers. + fn from_bits(bits: Self::Bits) -> Self; +} + +impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount +{} + +impl SimdMutPtr for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Bits = Simd; + type ConstPtr = Simd<*const T, LANES>; + type Mask = Mask; + + fn is_null(self) -> Self::Mask { + Simd::splat(core::ptr::null_mut()).simd_eq(self) + } + + fn as_const(self) -> Self::ConstPtr { + // Converting between pointers is safe + unsafe { intrinsics::simd_as(self) } + } + + fn to_bits(self) -> Self::Bits { + // Casting pointers to usize is safe + unsafe { intrinsics::simd_as(self) } + } + + fn from_bits(bits: Self::Bits) -> Self { + // Casting usize to pointers is safe + unsafe { intrinsics::simd_as(bits) } + } +} diff --git a/crates/core_simd/src/eq.rs b/crates/core_simd/src/eq.rs index c7111f720a8..149380746e7 100644 --- a/crates/core_simd/src/eq.rs +++ b/crates/core_simd/src/eq.rs @@ -71,3 +71,45 @@ macro_rules! impl_mask { } impl_mask! { i8, i16, i32, i64, isize } + +impl SimdPartialEq for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Mask = Mask; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } +} + +impl SimdPartialEq for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + type Mask = Mask; + + #[inline] + fn simd_eq(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + } + + #[inline] + fn simd_ne(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + } +} diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index 9909d639874..ece026a448b 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod intrinsics; mod to_bytes; mod alias; +mod cast; mod elements; mod eq; mod fmt; @@ -24,6 +25,7 @@ pub mod simd { pub(crate) use crate::core_simd::intrinsics; pub use crate::core_simd::alias::*; + pub use crate::core_simd::cast::*; pub use crate::core_simd::elements::*; pub use crate::core_simd::eq::*; pub use crate::core_simd::lane_count::{LaneCount, SupportedLaneCount}; diff --git a/crates/core_simd/src/ord.rs b/crates/core_simd/src/ord.rs index 9a87bc2e344..95a1ecaeeda 100644 --- a/crates/core_simd/src/ord.rs +++ b/crates/core_simd/src/ord.rs @@ -211,3 +211,117 @@ macro_rules! impl_mask { } impl_mask! { i8, i16, i32, i64, isize } + +impl SimdPartialOrd for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } +} + +impl SimdOrd for Simd<*const T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } +} + +impl SimdPartialOrd for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_lt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + } + + #[inline] + fn simd_le(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + } + + #[inline] + fn simd_gt(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + } + + #[inline] + fn simd_ge(self, other: Self) -> Self::Mask { + // Safety: `self` is a vector, and the result of the comparison + // is always a valid mask. + unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + } +} + +impl SimdOrd for Simd<*mut T, LANES> +where + LaneCount: SupportedLaneCount, +{ + #[inline] + fn simd_max(self, other: Self) -> Self { + self.simd_lt(other).select(other, self) + } + + #[inline] + fn simd_min(self, other: Self) -> Self { + self.simd_gt(other).select(other, self) + } + + #[inline] + fn simd_clamp(self, min: Self, max: Self) -> Self { + assert!( + min.simd_le(max).all(), + "each lane in `min` must be less than or equal to the corresponding lane in `max`", + ); + self.simd_max(min).simd_min(max) + } +} diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 7f0e8350cf8..cbc8ced5a84 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -2,7 +2,7 @@ pub(crate) mod ptr; use crate::simd::{ - intrinsics, LaneCount, Mask, MaskElement, SimdPartialOrd, SupportedLaneCount, Swizzle, + intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdPartialOrd, SupportedLaneCount, Swizzle, }; /// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. @@ -211,7 +211,10 @@ where #[must_use] #[inline] #[cfg(not(bootstrap))] - pub fn cast(self) -> Simd { + pub fn cast(self) -> Simd + where + T: SimdCast, + { // Safety: The input argument is a vector of a valid SIMD element type. unsafe { intrinsics::simd_as(self) } } @@ -234,7 +237,7 @@ where #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn to_int_unchecked(self) -> Simd where - T: core::convert::FloatToInt, + T: core::convert::FloatToInt + SimdCast, I: SimdElement, { // Safety: `self` is a vector, and `FloatToInt` ensures the type can be casted to @@ -739,3 +742,13 @@ impl Sealed for f64 {} unsafe impl SimdElement for f64 { type Mask = i64; } + +impl Sealed for *const T {} +unsafe impl SimdElement for *const T { + type Mask = isize; +} + +impl Sealed for *mut T {} +unsafe impl SimdElement for *mut T { + type Mask = isize; +} From 7e96f5dbea3fd2291f0e835a21ed0c41f6ef086e Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Tue, 21 Jun 2022 23:20:06 -0400 Subject: [PATCH 016/130] Use safe casts --- crates/core_simd/src/elements/const_ptr.rs | 11 ++++------- crates/core_simd/src/elements/mut_ptr.rs | 11 ++++------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index ab6b5b8b5f4..62365eace89 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; /// Operations on SIMD vectors of constant pointers. pub trait SimdConstPtr: Copy + Sealed { @@ -43,17 +43,14 @@ where } fn as_mut(self) -> Self::MutPtr { - // Converting between pointers is safe - unsafe { intrinsics::simd_as(self) } + self.cast() } fn to_bits(self) -> Self::Bits { - // Casting pointers to usize is safe - unsafe { intrinsics::simd_as(self) } + self.cast() } fn from_bits(bits: Self::Bits) -> Self { - // Casting usize to pointers is safe - unsafe { intrinsics::simd_as(bits) } + bits.cast() } } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index b49f9fda7e4..8c68d42628f 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -1,5 +1,5 @@ use super::sealed::Sealed; -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; /// Operations on SIMD vectors of mutable pointers. pub trait SimdMutPtr: Copy + Sealed { @@ -41,17 +41,14 @@ where } fn as_const(self) -> Self::ConstPtr { - // Converting between pointers is safe - unsafe { intrinsics::simd_as(self) } + self.cast() } fn to_bits(self) -> Self::Bits { - // Casting pointers to usize is safe - unsafe { intrinsics::simd_as(self) } + self.cast() } fn from_bits(bits: Self::Bits) -> Self { - // Casting usize to pointers is safe - unsafe { intrinsics::simd_as(bits) } + bits.cast() } } From 4076ba8a77326c70645f6c4a4351b0d84c5c898f Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 23 Jun 2022 01:21:58 -0400 Subject: [PATCH 017/130] Implement scatter/gather with new pointer vector and add tests --- crates/core_simd/src/cast.rs | 128 +++++++++++++++++---- crates/core_simd/src/elements/const_ptr.rs | 30 +++-- crates/core_simd/src/elements/mut_ptr.rs | 30 +++-- crates/core_simd/src/eq.rs | 20 ++-- crates/core_simd/src/ord.rs | 36 ++---- crates/core_simd/src/vector.rs | 13 +-- crates/core_simd/src/vector/ptr.rs | 51 -------- crates/core_simd/tests/pointers.rs | 43 +++++++ crates/test_helpers/src/biteq.rs | 20 ++++ crates/test_helpers/src/lib.rs | 63 ++++++---- 10 files changed, 275 insertions(+), 159 deletions(-) delete mode 100644 crates/core_simd/src/vector/ptr.rs create mode 100644 crates/core_simd/tests/pointers.rs diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs index e04a9042b1b..d62d3f6635d 100644 --- a/crates/core_simd/src/cast.rs +++ b/crates/core_simd/src/cast.rs @@ -1,25 +1,41 @@ -use crate::simd::SimdElement; +use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SupportedLaneCount}; /// Supporting trait for `Simd::cast`. Typically doesn't need to be used directly. -pub trait SimdCast: SimdElement {} +pub trait SimdCast: SimdElement { + #[doc(hidden)] + fn cast(x: Simd) -> Simd + where + LaneCount: SupportedLaneCount; +} macro_rules! into_number { + { $from:ty, $to:ty } => { + impl SimdCast<$to> for $from { + fn cast(x: Simd) -> Simd<$to, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: simd_as can handle numeric conversions + unsafe { intrinsics::simd_as(x) } + } + } + }; { $($type:ty),* } => { $( - impl SimdCast for $type {} - impl SimdCast for $type {} - impl SimdCast for $type {} - impl SimdCast for $type {} - impl SimdCast for $type {} + into_number! { $type, i8 } + into_number! { $type, i16 } + into_number! { $type, i32 } + into_number! { $type, i64 } + into_number! { $type, isize } - impl SimdCast for $type {} - impl SimdCast for $type {} - impl SimdCast for $type {} - impl SimdCast for $type {} - impl SimdCast for $type {} + into_number! { $type, u8 } + into_number! { $type, u16 } + into_number! { $type, u32 } + into_number! { $type, u64 } + into_number! { $type, usize } - impl SimdCast for $type {} - impl SimdCast for $type {} + into_number! { $type, f32 } + into_number! { $type, f64 } )* } } @@ -29,17 +45,85 @@ into_number! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, f32, f64 } macro_rules! into_pointer { { $($type:ty),* } => { $( - impl SimdCast<$type> for *const T {} - impl SimdCast<$type> for *mut T {} - impl SimdCast<*const T> for $type {} - impl SimdCast<*mut T> for $type {} + impl SimdCast<$type> for *const T { + fn cast(x: Simd) -> Simd<$type, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: transmuting isize to pointers is safe + let x: Simd = unsafe { core::mem::transmute_copy(&x) }; + x.cast() + } + } + impl SimdCast<$type> for *mut T { + fn cast(x: Simd) -> Simd<$type, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: transmuting isize to pointers is safe + let x: Simd = unsafe { core::mem::transmute_copy(&x) }; + x.cast() + } + } + impl SimdCast<*const T> for $type { + fn cast(x: Simd<$type, LANES>) -> Simd<*const T, LANES> + where + LaneCount: SupportedLaneCount, + { + let x: Simd = x.cast(); + // Safety: transmuting isize to pointers is safe + unsafe { core::mem::transmute_copy(&x) } + } + } + impl SimdCast<*mut T> for $type { + fn cast(x: Simd<$type, LANES>) -> Simd<*mut T, LANES> + where + LaneCount: SupportedLaneCount, + { + let x: Simd = x.cast(); + // Safety: transmuting isize to pointers is safe + unsafe { core::mem::transmute_copy(&x) } + } + } )* } } into_pointer! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } -impl SimdCast<*const T> for *const U {} -impl SimdCast<*const T> for *mut U {} -impl SimdCast<*mut T> for *const U {} -impl SimdCast<*mut T> for *mut U {} +impl SimdCast<*const T> for *const U { + fn cast(x: Simd<*const U, LANES>) -> Simd<*const T, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: transmuting pointers is safe + unsafe { core::mem::transmute_copy(&x) } + } +} +impl SimdCast<*const T> for *mut U { + fn cast(x: Simd<*mut U, LANES>) -> Simd<*const T, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: transmuting pointers is safe + unsafe { core::mem::transmute_copy(&x) } + } +} +impl SimdCast<*mut T> for *const U { + fn cast(x: Simd<*const U, LANES>) -> Simd<*mut T, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: transmuting pointers is safe + unsafe { core::mem::transmute_copy(&x) } + } +} +impl SimdCast<*mut T> for *mut U { + fn cast(x: Simd<*mut U, LANES>) -> Simd<*mut T, LANES> + where + LaneCount: SupportedLaneCount, + { + // Safety: transmuting pointers is safe + unsafe { core::mem::transmute_copy(&x) } + } +} diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index 62365eace89..c4a254f5ab1 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -3,8 +3,8 @@ use crate::simd::{LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; /// Operations on SIMD vectors of constant pointers. pub trait SimdConstPtr: Copy + Sealed { - /// Vector type representing the pointers as bits. - type Bits; + /// Vector of usize with the same number of lanes. + type Usize; /// Vector of mutable pointers to the same type. type MutPtr; @@ -18,11 +18,15 @@ pub trait SimdConstPtr: Copy + Sealed { /// Changes constness without changing the type. fn as_mut(self) -> Self::MutPtr; - /// Cast pointers to raw bits. - fn to_bits(self) -> Self::Bits; + /// Gets the "address" portion of the pointer. + /// + /// Equivalent to calling [`pointer::addr`] on each lane. + fn addr(self) -> Self::Usize; - /// Cast raw bits to pointers. - fn from_bits(bits: Self::Bits) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + fn wrapping_add(self, count: Self::Usize) -> Self; } impl Sealed for Simd<*const T, LANES> where @@ -34,23 +38,29 @@ impl SimdConstPtr for Simd<*const T, LANES> where LaneCount: SupportedLaneCount, { - type Bits = Simd; + type Usize = Simd; type MutPtr = Simd<*mut T, LANES>; type Mask = Mask; + #[inline] fn is_null(self) -> Self::Mask { Simd::splat(core::ptr::null()).simd_eq(self) } + #[inline] fn as_mut(self) -> Self::MutPtr { self.cast() } - fn to_bits(self) -> Self::Bits { + #[inline] + fn addr(self) -> Self::Usize { self.cast() } - fn from_bits(bits: Self::Bits) -> Self { - bits.cast() + #[inline] + fn wrapping_add(self, count: Self::Usize) -> Self { + let addr = self.addr() + (count * Simd::splat(core::mem::size_of::())); + // Safety: transmuting usize to pointers is safe, even if accessing those pointers isn't. + unsafe { core::mem::transmute_copy(&addr) } } } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index 8c68d42628f..5920960c49c 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -3,8 +3,8 @@ use crate::simd::{LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; /// Operations on SIMD vectors of mutable pointers. pub trait SimdMutPtr: Copy + Sealed { - /// Vector type representing the pointers as bits. - type Bits; + /// Vector of usize with the same number of lanes. + type Usize; /// Vector of constant pointers to the same type. type ConstPtr; @@ -18,11 +18,15 @@ pub trait SimdMutPtr: Copy + Sealed { /// Changes constness without changing the type. fn as_const(self) -> Self::ConstPtr; - /// Cast pointers to raw bits. - fn to_bits(self) -> Self::Bits; + /// Gets the "address" portion of the pointer. + /// + /// Equivalent to calling [`pointer::addr`] on each lane. + fn addr(self) -> Self::Usize; - /// Cast raw bits to pointers. - fn from_bits(bits: Self::Bits) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + fn wrapping_add(self, count: Self::Usize) -> Self; } impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount @@ -32,23 +36,29 @@ impl SimdMutPtr for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount, { - type Bits = Simd; + type Usize = Simd; type ConstPtr = Simd<*const T, LANES>; type Mask = Mask; + #[inline] fn is_null(self) -> Self::Mask { Simd::splat(core::ptr::null_mut()).simd_eq(self) } + #[inline] fn as_const(self) -> Self::ConstPtr { self.cast() } - fn to_bits(self) -> Self::Bits { + #[inline] + fn addr(self) -> Self::Usize { self.cast() } - fn from_bits(bits: Self::Bits) -> Self { - bits.cast() + #[inline] + fn wrapping_add(self, count: Self::Usize) -> Self { + let addr = self.addr() + (count * Simd::splat(core::mem::size_of::())); + // Safety: transmuting usize to pointers is safe, even if accessing those pointers isn't. + unsafe { core::mem::transmute_copy(&addr) } } } diff --git a/crates/core_simd/src/eq.rs b/crates/core_simd/src/eq.rs index 149380746e7..80763c07272 100644 --- a/crates/core_simd/src/eq.rs +++ b/crates/core_simd/src/eq.rs @@ -1,4 +1,6 @@ -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::{ + intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdElement, SimdMutPtr, SupportedLaneCount, +}; /// Parallel `PartialEq`. pub trait SimdPartialEq { @@ -80,16 +82,12 @@ where #[inline] fn simd_eq(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + self.addr().simd_eq(other.addr()) } #[inline] fn simd_ne(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + self.addr().simd_ne(other.addr()) } } @@ -101,15 +99,11 @@ where #[inline] fn simd_eq(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_eq(self, other)) } + self.addr().simd_eq(other.addr()) } #[inline] fn simd_ne(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ne(self, other)) } + self.addr().simd_ne(other.addr()) } } diff --git a/crates/core_simd/src/ord.rs b/crates/core_simd/src/ord.rs index 95a1ecaeeda..1ae9cd061fb 100644 --- a/crates/core_simd/src/ord.rs +++ b/crates/core_simd/src/ord.rs @@ -1,4 +1,6 @@ -use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{ + intrinsics, LaneCount, Mask, Simd, SimdConstPtr, SimdMutPtr, SimdPartialEq, SupportedLaneCount, +}; /// Parallel `PartialOrd`. pub trait SimdPartialOrd: SimdPartialEq { @@ -218,30 +220,22 @@ where { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + self.addr().simd_lt(other.addr()) } #[inline] fn simd_le(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + self.addr().simd_le(other.addr()) } #[inline] fn simd_gt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + self.addr().simd_gt(other.addr()) } #[inline] fn simd_ge(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + self.addr().simd_ge(other.addr()) } } @@ -275,30 +269,22 @@ where { #[inline] fn simd_lt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_lt(self, other)) } + self.addr().simd_lt(other.addr()) } #[inline] fn simd_le(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_le(self, other)) } + self.addr().simd_le(other.addr()) } #[inline] fn simd_gt(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_gt(self, other)) } + self.addr().simd_gt(other.addr()) } #[inline] fn simd_ge(self, other: Self) -> Self::Mask { - // Safety: `self` is a vector, and the result of the comparison - // is always a valid mask. - unsafe { Mask::from_int_unchecked(intrinsics::simd_ge(self, other)) } + self.addr().simd_ge(other.addr()) } } diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index cbc8ced5a84..145394a519d 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -1,8 +1,6 @@ -// Vectors of pointers are not for public use at the current time. -pub(crate) mod ptr; - use crate::simd::{ - intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdPartialOrd, SupportedLaneCount, Swizzle, + intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdConstPtr, SimdMutPtr, SimdPartialOrd, + SupportedLaneCount, Swizzle, }; /// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. @@ -215,8 +213,7 @@ where where T: SimdCast, { - // Safety: The input argument is a vector of a valid SIMD element type. - unsafe { intrinsics::simd_as(self) } + SimdCast::cast(self) } /// Rounds toward zero and converts to the same-width integer type, assuming that @@ -352,7 +349,7 @@ where idxs: Simd, or: Self, ) -> Self { - let base_ptr = crate::simd::ptr::SimdConstPtr::splat(slice.as_ptr()); + let base_ptr = Simd::<*const T, LANES>::splat(slice.as_ptr()); // Ferris forgive me, I have done pointer arithmetic here. let ptrs = base_ptr.wrapping_add(idxs); // Safety: The ptrs have been bounds-masked to prevent memory-unsafe reads insha'allah @@ -460,7 +457,7 @@ where // 3. &mut [T] which will become our base ptr. unsafe { // Now Entering ☢️ *mut T Zone - let base_ptr = crate::simd::ptr::SimdMutPtr::splat(slice.as_mut_ptr()); + let base_ptr = Simd::<*mut T, LANES>::splat(slice.as_mut_ptr()); // Ferris forgive me, I have done pointer arithmetic here. let ptrs = base_ptr.wrapping_add(idxs); // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah diff --git a/crates/core_simd/src/vector/ptr.rs b/crates/core_simd/src/vector/ptr.rs deleted file mode 100644 index fa756344db9..00000000000 --- a/crates/core_simd/src/vector/ptr.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Private implementation details of public gather/scatter APIs. -use crate::simd::intrinsics; -use crate::simd::{LaneCount, Simd, SupportedLaneCount}; - -/// A vector of *const T. -#[derive(Debug, Copy, Clone)] -#[repr(simd)] -pub(crate) struct SimdConstPtr([*const T; LANES]); - -impl SimdConstPtr -where - LaneCount: SupportedLaneCount, - T: Sized, -{ - #[inline] - #[must_use] - pub fn splat(ptr: *const T) -> Self { - Self([ptr; LANES]) - } - - #[inline] - #[must_use] - pub fn wrapping_add(self, addend: Simd) -> Self { - // Safety: this intrinsic doesn't have a precondition - unsafe { intrinsics::simd_arith_offset(self, addend) } - } -} - -/// A vector of *mut T. Be very careful around potential aliasing. -#[derive(Debug, Copy, Clone)] -#[repr(simd)] -pub(crate) struct SimdMutPtr([*mut T; LANES]); - -impl SimdMutPtr -where - LaneCount: SupportedLaneCount, - T: Sized, -{ - #[inline] - #[must_use] - pub fn splat(ptr: *mut T) -> Self { - Self([ptr; LANES]) - } - - #[inline] - #[must_use] - pub fn wrapping_add(self, addend: Simd) -> Self { - // Safety: this intrinsic doesn't have a precondition - unsafe { intrinsics::simd_arith_offset(self, addend) } - } -} diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs new file mode 100644 index 00000000000..df26c462f93 --- /dev/null +++ b/crates/core_simd/tests/pointers.rs @@ -0,0 +1,43 @@ +#![feature(portable_simd, strict_provenance)] + +use core_simd::{Simd, SimdConstPtr, SimdMutPtr}; + +macro_rules! common_tests { + { $constness:ident } => { + test_helpers::test_lanes! { + fn is_null() { + test_helpers::test_unary_mask_elementwise( + &Simd::<*$constness (), LANES>::is_null, + &<*$constness ()>::is_null, + &|_| true, + ); + } + + fn addr() { + test_helpers::test_unary_elementwise( + &Simd::<*$constness (), LANES>::addr, + &<*$constness ()>::addr, + &|_| true, + ); + } + + fn wrapping_add() { + test_helpers::test_binary_elementwise( + &Simd::<*$constness (), LANES>::wrapping_add, + &<*$constness ()>::wrapping_add, + &|_, _| true, + ); + } + } + } +} + +mod const_ptr { + use super::*; + common_tests! { const } +} + +mod mut_ptr { + use super::*; + common_tests! { mut } +} diff --git a/crates/test_helpers/src/biteq.rs b/crates/test_helpers/src/biteq.rs index 00350e22418..7d91260d838 100644 --- a/crates/test_helpers/src/biteq.rs +++ b/crates/test_helpers/src/biteq.rs @@ -55,6 +55,26 @@ macro_rules! impl_float_biteq { impl_float_biteq! { f32, f64 } +impl BitEq for *const T { + fn biteq(&self, other: &Self) -> bool { + self == other + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} + +impl BitEq for *mut T { + fn biteq(&self, other: &Self) -> bool { + self == other + } + + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + write!(f, "{:?}", self) + } +} + impl BitEq for [T; N] { fn biteq(&self, other: &Self) -> bool { self.iter() diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 650eadd12bf..5f2a928b5e4 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -38,6 +38,28 @@ impl_num! { usize } impl_num! { f32 } impl_num! { f64 } +impl DefaultStrategy for *const T { + type Strategy = proptest::strategy::Map *const T>; + fn default_strategy() -> Self::Strategy { + fn map(x: isize) -> *const T { + x as _ + } + use proptest::strategy::Strategy; + proptest::num::isize::ANY.prop_map(map) + } +} + +impl DefaultStrategy for *mut T { + type Strategy = proptest::strategy::Map *mut T>; + fn default_strategy() -> Self::Strategy { + fn map(x: isize) -> *mut T { + x as _ + } + use proptest::strategy::Strategy; + proptest::num::isize::ANY.prop_map(map) + } +} + #[cfg(not(target_arch = "wasm32"))] impl DefaultStrategy for u128 { type Strategy = proptest::num::u128::Any; @@ -135,21 +157,21 @@ pub fn test_unary_elementwise ScalarResult, check: &dyn Fn([Scalar; LANES]) -> bool, ) where - Scalar: Copy + Default + core::fmt::Debug + DefaultStrategy, - ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Scalar: Copy + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + biteq::BitEq + core::fmt::Debug + DefaultStrategy, Vector: Into<[Scalar; LANES]> + From<[Scalar; LANES]> + Copy, VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, { test_1(&|x: [Scalar; LANES]| { proptest::prop_assume!(check(x)); let result_1: [ScalarResult; LANES] = fv(x.into()).into(); - let result_2: [ScalarResult; LANES] = { - let mut result = [ScalarResult::default(); LANES]; - for (i, o) in x.iter().zip(result.iter_mut()) { - *o = fs(*i); - } - result - }; + let result_2: [ScalarResult; LANES] = x + .iter() + .copied() + .map(fs) + .collect::>() + .try_into() + .unwrap(); crate::prop_assert_biteq!(result_1, result_2); Ok(()) }); @@ -162,7 +184,7 @@ pub fn test_unary_mask_elementwise( fs: &dyn Fn(Scalar) -> bool, check: &dyn Fn([Scalar; LANES]) -> bool, ) where - Scalar: Copy + Default + core::fmt::Debug + DefaultStrategy, + Scalar: Copy + core::fmt::Debug + DefaultStrategy, Vector: Into<[Scalar; LANES]> + From<[Scalar; LANES]> + Copy, Mask: Into<[bool; LANES]> + From<[bool; LANES]> + Copy, { @@ -196,9 +218,9 @@ pub fn test_binary_elementwise< fs: &dyn Fn(Scalar1, Scalar2) -> ScalarResult, check: &dyn Fn([Scalar1; LANES], [Scalar2; LANES]) -> bool, ) where - Scalar1: Copy + Default + core::fmt::Debug + DefaultStrategy, - Scalar2: Copy + Default + core::fmt::Debug + DefaultStrategy, - ScalarResult: Copy + Default + biteq::BitEq + core::fmt::Debug + DefaultStrategy, + Scalar1: Copy + core::fmt::Debug + DefaultStrategy, + Scalar2: Copy + core::fmt::Debug + DefaultStrategy, + ScalarResult: Copy + biteq::BitEq + core::fmt::Debug + DefaultStrategy, Vector1: Into<[Scalar1; LANES]> + From<[Scalar1; LANES]> + Copy, Vector2: Into<[Scalar2; LANES]> + From<[Scalar2; LANES]> + Copy, VectorResult: Into<[ScalarResult; LANES]> + From<[ScalarResult; LANES]> + Copy, @@ -206,13 +228,14 @@ pub fn test_binary_elementwise< test_2(&|x: [Scalar1; LANES], y: [Scalar2; LANES]| { proptest::prop_assume!(check(x, y)); let result_1: [ScalarResult; LANES] = fv(x.into(), y.into()).into(); - let result_2: [ScalarResult; LANES] = { - let mut result = [ScalarResult::default(); LANES]; - for ((i1, i2), o) in x.iter().zip(y.iter()).zip(result.iter_mut()) { - *o = fs(*i1, *i2); - } - result - }; + let result_2: [ScalarResult; LANES] = x + .iter() + .copied() + .zip(y.iter().copied()) + .map(|(x, y)| fs(x, y)) + .collect::>() + .try_into() + .unwrap(); crate::prop_assert_biteq!(result_1, result_2); Ok(()) }); From 6b3c599ba29e46fd7011cf1f01ec6c4cfda395cf Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 23 Jun 2022 01:40:51 -0400 Subject: [PATCH 018/130] Add missing safety comment --- crates/core_simd/src/vector.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 145394a519d..2fc090254d7 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -741,11 +741,15 @@ unsafe impl SimdElement for f64 { } impl Sealed for *const T {} + +// Safety: const pointers are valid SIMD element types, and are supported by this API unsafe impl SimdElement for *const T { type Mask = isize; } impl Sealed for *mut T {} + +// Safety: mut pointers are valid SIMD element types, and are supported by this API unsafe impl SimdElement for *mut T { type Mask = isize; } From f10e591de1d321b57af68502a78eef6f8f80c05c Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 24 Jun 2022 00:13:36 -0400 Subject: [PATCH 019/130] Fix wrapping pointer arithmetic --- crates/core_simd/src/elements/const_ptr.rs | 33 ++++++++++++++++++---- crates/core_simd/src/elements/mut_ptr.rs | 33 ++++++++++++++++++---- crates/core_simd/src/intrinsics.rs | 3 ++ crates/core_simd/tests/pointers.rs | 16 +++++++++++ 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index c4a254f5ab1..d10bd1481d0 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -1,11 +1,14 @@ use super::sealed::Sealed; -use crate::simd::{LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; /// Operations on SIMD vectors of constant pointers. pub trait SimdConstPtr: Copy + Sealed { - /// Vector of usize with the same number of lanes. + /// Vector of `usize` with the same number of lanes. type Usize; + /// Vector of `isize` with the same number of lanes. + type Isize; + /// Vector of mutable pointers to the same type. type MutPtr; @@ -23,10 +26,20 @@ pub trait SimdConstPtr: Copy + Sealed { /// Equivalent to calling [`pointer::addr`] on each lane. fn addr(self) -> Self::Usize; + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. + fn wrapping_offset(self, offset: Self::Isize) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. /// /// Equivalent to calling [`pointer::wrapping_add`] on each lane. fn wrapping_add(self, count: Self::Usize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + fn wrapping_sub(self, count: Self::Usize) -> Self; } impl Sealed for Simd<*const T, LANES> where @@ -39,6 +52,7 @@ where LaneCount: SupportedLaneCount, { type Usize = Simd; + type Isize = Simd; type MutPtr = Simd<*mut T, LANES>; type Mask = Mask; @@ -57,10 +71,19 @@ where self.cast() } + #[inline] + fn wrapping_offset(self, count: Self::Isize) -> Self { + // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets + unsafe { intrinsics::simd_arith_offset(self, count) } + } + #[inline] fn wrapping_add(self, count: Self::Usize) -> Self { - let addr = self.addr() + (count * Simd::splat(core::mem::size_of::())); - // Safety: transmuting usize to pointers is safe, even if accessing those pointers isn't. - unsafe { core::mem::transmute_copy(&addr) } + self.wrapping_offset(count.cast()) + } + + #[inline] + fn wrapping_sub(self, count: Self::Usize) -> Self { + self.wrapping_offset(-count.cast::()) } } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index 5920960c49c..4fc6202e14e 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -1,11 +1,14 @@ use super::sealed::Sealed; -use crate::simd::{LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; +use crate::simd::{intrinsics, LaneCount, Mask, Simd, SimdPartialEq, SupportedLaneCount}; /// Operations on SIMD vectors of mutable pointers. pub trait SimdMutPtr: Copy + Sealed { - /// Vector of usize with the same number of lanes. + /// Vector of `usize` with the same number of lanes. type Usize; + /// Vector of `isize` with the same number of lanes. + type Isize; + /// Vector of constant pointers to the same type. type ConstPtr; @@ -23,10 +26,20 @@ pub trait SimdMutPtr: Copy + Sealed { /// Equivalent to calling [`pointer::addr`] on each lane. fn addr(self) -> Self::Usize; + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. + fn wrapping_offset(self, offset: Self::Isize) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. /// /// Equivalent to calling [`pointer::wrapping_add`] on each lane. fn wrapping_add(self, count: Self::Usize) -> Self; + + /// Calculates the offset from a pointer using wrapping arithmetic. + /// + /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + fn wrapping_sub(self, count: Self::Usize) -> Self; } impl Sealed for Simd<*mut T, LANES> where LaneCount: SupportedLaneCount @@ -37,6 +50,7 @@ where LaneCount: SupportedLaneCount, { type Usize = Simd; + type Isize = Simd; type ConstPtr = Simd<*const T, LANES>; type Mask = Mask; @@ -55,10 +69,19 @@ where self.cast() } + #[inline] + fn wrapping_offset(self, count: Self::Isize) -> Self { + // Safety: simd_arith_offset takes a vector of pointers and a vector of offsets + unsafe { intrinsics::simd_arith_offset(self, count) } + } + #[inline] fn wrapping_add(self, count: Self::Usize) -> Self { - let addr = self.addr() + (count * Simd::splat(core::mem::size_of::())); - // Safety: transmuting usize to pointers is safe, even if accessing those pointers isn't. - unsafe { core::mem::transmute_copy(&addr) } + self.wrapping_offset(count.cast()) + } + + #[inline] + fn wrapping_sub(self, count: Self::Usize) -> Self { + self.wrapping_offset(-count.cast::()) } } diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index 6047890a093..41128cd1481 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -151,4 +151,7 @@ extern "platform-intrinsic" { pub(crate) fn simd_select(m: M, yes: T, no: T) -> T; #[allow(unused)] pub(crate) fn simd_select_bitmask(m: M, yes: T, no: T) -> T; + + // equivalent to wrapping_offset + pub(crate) fn simd_arith_offset(ptr: T, offset: U) -> T; } diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs index df26c462f93..2c20362119e 100644 --- a/crates/core_simd/tests/pointers.rs +++ b/crates/core_simd/tests/pointers.rs @@ -21,6 +21,14 @@ macro_rules! common_tests { ); } + fn wrapping_offset() { + test_helpers::test_binary_elementwise( + &Simd::<*$constness (), LANES>::wrapping_offset, + &<*$constness ()>::wrapping_offset, + &|_, _| true, + ); + } + fn wrapping_add() { test_helpers::test_binary_elementwise( &Simd::<*$constness (), LANES>::wrapping_add, @@ -28,6 +36,14 @@ macro_rules! common_tests { &|_, _| true, ); } + + fn wrapping_sub() { + test_helpers::test_binary_elementwise( + &Simd::<*$constness (), LANES>::wrapping_sub, + &<*$constness ()>::wrapping_sub, + &|_, _| true, + ); + } } } } From da25087f790e9c15fcf633a39e5de307608c9251 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 24 Jun 2022 01:26:24 -0400 Subject: [PATCH 020/130] Test a more useful pointer --- crates/core_simd/tests/pointers.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs index 2c20362119e..8eb0bd84042 100644 --- a/crates/core_simd/tests/pointers.rs +++ b/crates/core_simd/tests/pointers.rs @@ -7,40 +7,40 @@ macro_rules! common_tests { test_helpers::test_lanes! { fn is_null() { test_helpers::test_unary_mask_elementwise( - &Simd::<*$constness (), LANES>::is_null, - &<*$constness ()>::is_null, + &Simd::<*$constness u32, LANES>::is_null, + &<*$constness u32>::is_null, &|_| true, ); } fn addr() { test_helpers::test_unary_elementwise( - &Simd::<*$constness (), LANES>::addr, - &<*$constness ()>::addr, + &Simd::<*$constness u32, LANES>::addr, + &<*$constness u32>::addr, &|_| true, ); } fn wrapping_offset() { test_helpers::test_binary_elementwise( - &Simd::<*$constness (), LANES>::wrapping_offset, - &<*$constness ()>::wrapping_offset, + &Simd::<*$constness u32, LANES>::wrapping_offset, + &<*$constness u32>::wrapping_offset, &|_, _| true, ); } fn wrapping_add() { test_helpers::test_binary_elementwise( - &Simd::<*$constness (), LANES>::wrapping_add, - &<*$constness ()>::wrapping_add, + &Simd::<*$constness u32, LANES>::wrapping_add, + &<*$constness u32>::wrapping_add, &|_, _| true, ); } fn wrapping_sub() { test_helpers::test_binary_elementwise( - &Simd::<*$constness (), LANES>::wrapping_sub, - &<*$constness ()>::wrapping_sub, + &Simd::<*$constness u32, LANES>::wrapping_sub, + &<*$constness u32>::wrapping_sub, &|_, _| true, ); } From e7cc021189f1d18974057d60223bdbb5abd4dc15 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sat, 25 Jun 2022 00:00:20 -0400 Subject: [PATCH 021/130] Fix casts --- crates/core_simd/src/cast.rs | 158 +++++++-------------- crates/core_simd/src/elements/const_ptr.rs | 33 ++++- crates/core_simd/src/elements/mut_ptr.rs | 30 +++- crates/core_simd/src/vector.rs | 5 +- 4 files changed, 115 insertions(+), 111 deletions(-) diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs index d62d3f6635d..ddcc786afa4 100644 --- a/crates/core_simd/src/cast.rs +++ b/crates/core_simd/src/cast.rs @@ -1,129 +1,79 @@ use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SupportedLaneCount}; /// Supporting trait for `Simd::cast`. Typically doesn't need to be used directly. -pub trait SimdCast: SimdElement { +pub unsafe trait SimdCast: SimdElement { #[doc(hidden)] fn cast(x: Simd) -> Simd where - LaneCount: SupportedLaneCount; + LaneCount: SupportedLaneCount, + { + // Safety: implementing this trait indicates that the types are supported by `simd_as` + unsafe { intrinsics::simd_as(x) } + } + + #[doc(hidden)] + unsafe fn cast_unchecked(x: Simd) -> Simd + where + LaneCount: SupportedLaneCount, + { + // Safety: implementing this trait indicates that the types are supported by `simd_cast` + // The caller is responsible for the conversion invariants. + unsafe { intrinsics::simd_cast(x) } + } } macro_rules! into_number { - { $from:ty, $to:ty } => { - impl SimdCast<$to> for $from { - fn cast(x: Simd) -> Simd<$to, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: simd_as can handle numeric conversions - unsafe { intrinsics::simd_as(x) } - } - } + { unsafe $from:ty as $to:ty } => { + // Safety: casting between numbers is supported by `simd_cast` and `simd_as` + unsafe impl SimdCast<$to> for $from {} }; - { $($type:ty),* } => { + { unsafe $($type:ty),* } => { $( - into_number! { $type, i8 } - into_number! { $type, i16 } - into_number! { $type, i32 } - into_number! { $type, i64 } - into_number! { $type, isize } + into_number! { unsafe $type as i8 } + into_number! { unsafe $type as i16 } + into_number! { unsafe $type as i32 } + into_number! { unsafe $type as i64 } + into_number! { unsafe $type as isize } - into_number! { $type, u8 } - into_number! { $type, u16 } - into_number! { $type, u32 } - into_number! { $type, u64 } - into_number! { $type, usize } + into_number! { unsafe $type as u8 } + into_number! { unsafe $type as u16 } + into_number! { unsafe $type as u32 } + into_number! { unsafe $type as u64 } + into_number! { unsafe $type as usize } - into_number! { $type, f32 } - into_number! { $type, f64 } + into_number! { unsafe $type as f32 } + into_number! { unsafe $type as f64 } )* } } -into_number! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, f32, f64 } +into_number! { unsafe i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, f32, f64 } +// TODO uncomment pending PR to rustc +/* macro_rules! into_pointer { - { $($type:ty),* } => { + { unsafe $($type:ty),* } => { $( - impl SimdCast<$type> for *const T { - fn cast(x: Simd) -> Simd<$type, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: transmuting isize to pointers is safe - let x: Simd = unsafe { core::mem::transmute_copy(&x) }; - x.cast() - } - } - impl SimdCast<$type> for *mut T { - fn cast(x: Simd) -> Simd<$type, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: transmuting isize to pointers is safe - let x: Simd = unsafe { core::mem::transmute_copy(&x) }; - x.cast() - } - } - impl SimdCast<*const T> for $type { - fn cast(x: Simd<$type, LANES>) -> Simd<*const T, LANES> - where - LaneCount: SupportedLaneCount, - { - let x: Simd = x.cast(); - // Safety: transmuting isize to pointers is safe - unsafe { core::mem::transmute_copy(&x) } - } - } - impl SimdCast<*mut T> for $type { - fn cast(x: Simd<$type, LANES>) -> Simd<*mut T, LANES> - where - LaneCount: SupportedLaneCount, - { - let x: Simd = x.cast(); - // Safety: transmuting isize to pointers is safe - unsafe { core::mem::transmute_copy(&x) } - } - } + // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` + unsafe impl SimdCast<$type> for *const T {} + // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` + unsafe impl SimdCast<$type> for *mut T {} + // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` + unsafe impl SimdCast<*const T> for $type {} + // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` + unsafe impl SimdCast<*mut T> for $type {} )* } } -into_pointer! { i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } +into_pointer! { unsafe i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } -impl SimdCast<*const T> for *const U { - fn cast(x: Simd<*const U, LANES>) -> Simd<*const T, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: transmuting pointers is safe - unsafe { core::mem::transmute_copy(&x) } - } -} -impl SimdCast<*const T> for *mut U { - fn cast(x: Simd<*mut U, LANES>) -> Simd<*const T, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: transmuting pointers is safe - unsafe { core::mem::transmute_copy(&x) } - } -} -impl SimdCast<*mut T> for *const U { - fn cast(x: Simd<*const U, LANES>) -> Simd<*mut T, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: transmuting pointers is safe - unsafe { core::mem::transmute_copy(&x) } - } -} -impl SimdCast<*mut T> for *mut U { - fn cast(x: Simd<*mut U, LANES>) -> Simd<*mut T, LANES> - where - LaneCount: SupportedLaneCount, - { - // Safety: transmuting pointers is safe - unsafe { core::mem::transmute_copy(&x) } - } -} +// Safety: casting between pointers is supported by `simd_cast` and `simd_as` +unsafe impl SimdCast<*const T> for *const U {} +// Safety: casting between pointers is supported by `simd_cast` and `simd_as` +unsafe impl SimdCast<*const T> for *mut U {} +// Safety: casting between pointers is supported by `simd_cast` and `simd_as` +unsafe impl SimdCast<*mut T> for *const U {} +// Safety: casting between pointers is supported by `simd_cast` and `simd_as` +unsafe impl SimdCast<*mut T> for *mut U {} +*/ diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index d10bd1481d0..5a5faad23c8 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -23,9 +23,23 @@ pub trait SimdConstPtr: Copy + Sealed { /// Gets the "address" portion of the pointer. /// + /// This method discards pointer semantic metadata, so the result cannot be + /// directly cast into a valid pointer. + /// + /// This method semantically discards *provenance* and + /// *address-space* information. To properly restore that information, use [`with_addr`]. + /// /// Equivalent to calling [`pointer::addr`] on each lane. fn addr(self) -> Self::Usize; + /// Creates a new pointer with the given address. + /// + /// This performs the same operation as a cast, but copies the *address-space* and + /// *provenance* of `self` to the new pointer. + /// + /// Equivalent to calling [`pointer::with_addr`] on each lane. + fn with_addr(self, addr: Self::Usize) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. /// /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. @@ -63,12 +77,27 @@ where #[inline] fn as_mut(self) -> Self::MutPtr { - self.cast() + unimplemented!() + //self.cast() } #[inline] fn addr(self) -> Self::Usize { - self.cast() + // Safety: Since `addr` discards provenance, this is safe. + unsafe { core::mem::transmute_copy(&self) } + + //TODO switch to casts when available + //self.cast() + } + + #[inline] + fn with_addr(self, addr: Self::Usize) -> Self { + unimplemented!() + /* + self.cast::<*const u8>() + .wrapping_offset(addr.cast::() - self.addr().cast::()) + .cast() + */ } #[inline] diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index 4fc6202e14e..d7b05af0eac 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -23,9 +23,20 @@ pub trait SimdMutPtr: Copy + Sealed { /// Gets the "address" portion of the pointer. /// + /// This method discards pointer semantic metadata, so the result cannot be + /// directly cast into a valid pointer. + /// /// Equivalent to calling [`pointer::addr`] on each lane. fn addr(self) -> Self::Usize; + /// Creates a new pointer with the given address. + /// + /// This performs the same operation as a cast, but copies the *address-space* and + /// *provenance* of `self` to the new pointer. + /// + /// Equivalent to calling [`pointer::with_addr`] on each lane. + fn with_addr(self, addr: Self::Usize) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. /// /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. @@ -61,12 +72,27 @@ where #[inline] fn as_const(self) -> Self::ConstPtr { - self.cast() + unimplemented!() + //self.cast() } #[inline] fn addr(self) -> Self::Usize { - self.cast() + // Safety: Since `addr` discards provenance, this is safe. + unsafe { core::mem::transmute_copy(&self) } + + //TODO switch to casts when available + //self.cast() + } + + #[inline] + fn with_addr(self, addr: Self::Usize) -> Self { + unimplemented!() + /* + self.cast::<*mut u8>() + .wrapping_offset(addr.cast::() - self.addr().cast::()) + .cast() + */ } #[inline] diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 2fc090254d7..3987b7a747b 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -237,9 +237,8 @@ where T: core::convert::FloatToInt + SimdCast, I: SimdElement, { - // Safety: `self` is a vector, and `FloatToInt` ensures the type can be casted to - // an integer. - unsafe { intrinsics::simd_cast(self) } + // Safety: the caller is responsible for the invariants + unsafe { SimdCast::cast_unchecked(self) } } /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. From 8a5a5732a1527fbdffbc825ae630d911fc130e2e Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 26 Jun 2022 10:07:48 -0400 Subject: [PATCH 022/130] Clarify addr and with_addr implementations --- crates/core_simd/src/elements/const_ptr.rs | 14 +++++++++----- crates/core_simd/src/elements/mut_ptr.rs | 14 +++++++++----- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index 5a5faad23c8..3485d31e44d 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -83,17 +83,21 @@ where #[inline] fn addr(self) -> Self::Usize { - // Safety: Since `addr` discards provenance, this is safe. + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). unsafe { core::mem::transmute_copy(&self) } - - //TODO switch to casts when available - //self.cast() } #[inline] - fn with_addr(self, addr: Self::Usize) -> Self { + fn with_addr(self, _addr: Self::Usize) -> Self { unimplemented!() /* + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. self.cast::<*const u8>() .wrapping_offset(addr.cast::() - self.addr().cast::()) .cast() diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index d7b05af0eac..39fe9f35621 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -78,17 +78,21 @@ where #[inline] fn addr(self) -> Self::Usize { - // Safety: Since `addr` discards provenance, this is safe. + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // SAFETY: Pointer-to-integer transmutes are valid (if you are okay with losing the + // provenance). unsafe { core::mem::transmute_copy(&self) } - - //TODO switch to casts when available - //self.cast() } #[inline] - fn with_addr(self, addr: Self::Usize) -> Self { + fn with_addr(self, _addr: Self::Usize) -> Self { unimplemented!() /* + // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. + // + // In the mean-time, this operation is defined to be "as if" it was + // a wrapping_offset, so we can emulate it as such. This should properly + // restore pointer provenance even under today's compiler. self.cast::<*mut u8>() .wrapping_offset(addr.cast::() - self.addr().cast::()) .cast() From 176cc81324d008bd58e28136aa8e60b537caa3ce Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 4 Aug 2022 19:31:50 -0400 Subject: [PATCH 023/130] Update for new intrinsics --- crates/core_simd/src/cast.rs | 92 +++++----------------- crates/core_simd/src/elements/const_ptr.rs | 31 ++++++-- crates/core_simd/src/elements/mut_ptr.rs | 28 +++++-- crates/core_simd/src/intrinsics.rs | 23 ++++++ crates/core_simd/src/vector.rs | 30 ++++--- 5 files changed, 107 insertions(+), 97 deletions(-) diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs index ddcc786afa4..d14b0de5d5e 100644 --- a/crates/core_simd/src/cast.rs +++ b/crates/core_simd/src/cast.rs @@ -1,79 +1,23 @@ -use crate::simd::{intrinsics, LaneCount, Simd, SimdElement, SupportedLaneCount}; +use crate::simd::SimdElement; /// Supporting trait for `Simd::cast`. Typically doesn't need to be used directly. -pub unsafe trait SimdCast: SimdElement { - #[doc(hidden)] - fn cast(x: Simd) -> Simd - where - LaneCount: SupportedLaneCount, - { - // Safety: implementing this trait indicates that the types are supported by `simd_as` - unsafe { intrinsics::simd_as(x) } - } +pub unsafe trait SimdCast: SimdElement {} - #[doc(hidden)] - unsafe fn cast_unchecked(x: Simd) -> Simd - where - LaneCount: SupportedLaneCount, - { - // Safety: implementing this trait indicates that the types are supported by `simd_cast` - // The caller is responsible for the conversion invariants. - unsafe { intrinsics::simd_cast(x) } - } -} +unsafe impl SimdCast for i8 {} +unsafe impl SimdCast for i16 {} +unsafe impl SimdCast for i32 {} +unsafe impl SimdCast for i64 {} +unsafe impl SimdCast for isize {} +unsafe impl SimdCast for u8 {} +unsafe impl SimdCast for u16 {} +unsafe impl SimdCast for u32 {} +unsafe impl SimdCast for u64 {} +unsafe impl SimdCast for usize {} +unsafe impl SimdCast for f32 {} +unsafe impl SimdCast for f64 {} -macro_rules! into_number { - { unsafe $from:ty as $to:ty } => { - // Safety: casting between numbers is supported by `simd_cast` and `simd_as` - unsafe impl SimdCast<$to> for $from {} - }; - { unsafe $($type:ty),* } => { - $( - into_number! { unsafe $type as i8 } - into_number! { unsafe $type as i16 } - into_number! { unsafe $type as i32 } - into_number! { unsafe $type as i64 } - into_number! { unsafe $type as isize } +/// Supporting trait for `Simd::cast_ptr`. Typically doesn't need to be used directly. +pub unsafe trait SimdCastPtr: SimdElement {} - into_number! { unsafe $type as u8 } - into_number! { unsafe $type as u16 } - into_number! { unsafe $type as u32 } - into_number! { unsafe $type as u64 } - into_number! { unsafe $type as usize } - - into_number! { unsafe $type as f32 } - into_number! { unsafe $type as f64 } - )* - } -} - -into_number! { unsafe i8, i16, i32, i64, isize, u8, u16, u32, u64, usize, f32, f64 } - -// TODO uncomment pending PR to rustc -/* -macro_rules! into_pointer { - { unsafe $($type:ty),* } => { - $( - // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` - unsafe impl SimdCast<$type> for *const T {} - // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` - unsafe impl SimdCast<$type> for *mut T {} - // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` - unsafe impl SimdCast<*const T> for $type {} - // Safety: casting between numbers and pointers is supported by `simd_cast` and `simd_as` - unsafe impl SimdCast<*mut T> for $type {} - )* - } -} - -into_pointer! { unsafe i8, i16, i32, i64, isize, u8, u16, u32, u64, usize } - -// Safety: casting between pointers is supported by `simd_cast` and `simd_as` -unsafe impl SimdCast<*const T> for *const U {} -// Safety: casting between pointers is supported by `simd_cast` and `simd_as` -unsafe impl SimdCast<*const T> for *mut U {} -// Safety: casting between pointers is supported by `simd_cast` and `simd_as` -unsafe impl SimdCast<*mut T> for *const U {} -// Safety: casting between pointers is supported by `simd_cast` and `simd_as` -unsafe impl SimdCast<*mut T> for *mut U {} -*/ +unsafe impl SimdCastPtr for *const T {} +unsafe impl SimdCastPtr for *mut T {} diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index 3485d31e44d..27b41019dc8 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -40,6 +40,15 @@ pub trait SimdConstPtr: Copy + Sealed { /// Equivalent to calling [`pointer::with_addr`] on each lane. fn with_addr(self, addr: Self::Usize) -> Self; + /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use + /// in [`from_exposed_addr`]. + fn expose_addr(self) -> Self::Usize; + + /// Convert an address back to a pointer, picking up a previously "exposed" provenance. + /// + /// Equivalent to calling [`pointer::from_exposed_addr`] on each lane. + fn from_exposed_addr(addr: Self::Usize) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. /// /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. @@ -77,8 +86,7 @@ where #[inline] fn as_mut(self) -> Self::MutPtr { - unimplemented!() - //self.cast() + unsafe { intrinsics::simd_cast_ptr(self) } } #[inline] @@ -90,18 +98,25 @@ where } #[inline] - fn with_addr(self, _addr: Self::Usize) -> Self { - unimplemented!() - /* + fn with_addr(self, addr: Self::Usize) -> Self { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. // // In the mean-time, this operation is defined to be "as if" it was // a wrapping_offset, so we can emulate it as such. This should properly // restore pointer provenance even under today's compiler. - self.cast::<*const u8>() + self.cast_ptr::<*const u8>() .wrapping_offset(addr.cast::() - self.addr().cast::()) - .cast() - */ + .cast_ptr() + } + + #[inline] + fn expose_addr(self) -> Self::Usize { + unsafe { intrinsics::simd_expose_addr(self) } + } + + #[inline] + fn from_exposed_addr(addr: Self::Usize) -> Self { + unsafe { intrinsics::simd_from_exposed_addr(addr) } } #[inline] diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index 39fe9f35621..59a8b6293b5 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -37,6 +37,15 @@ pub trait SimdMutPtr: Copy + Sealed { /// Equivalent to calling [`pointer::with_addr`] on each lane. fn with_addr(self, addr: Self::Usize) -> Self; + /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use + /// in [`from_exposed_addr`]. + fn expose_addr(self) -> Self::Usize; + + /// Convert an address back to a pointer, picking up a previously "exposed" provenance. + /// + /// Equivalent to calling [`pointer::from_exposed_addr`] on each lane. + fn from_exposed_addr(addr: Self::Usize) -> Self; + /// Calculates the offset from a pointer using wrapping arithmetic. /// /// Equivalent to calling [`pointer::wrapping_offset`] on each lane. @@ -85,18 +94,25 @@ where } #[inline] - fn with_addr(self, _addr: Self::Usize) -> Self { - unimplemented!() - /* + fn with_addr(self, addr: Self::Usize) -> Self { // FIXME(strict_provenance_magic): I am magic and should be a compiler intrinsic. // // In the mean-time, this operation is defined to be "as if" it was // a wrapping_offset, so we can emulate it as such. This should properly // restore pointer provenance even under today's compiler. - self.cast::<*mut u8>() + self.cast_ptr::<*mut u8>() .wrapping_offset(addr.cast::() - self.addr().cast::()) - .cast() - */ + .cast_ptr() + } + + #[inline] + fn expose_addr(self) -> Self::Usize { + unsafe { intrinsics::simd_expose_addr(self) } + } + + #[inline] + fn from_exposed_addr(addr: Self::Usize) -> Self { + unsafe { intrinsics::simd_from_exposed_addr(addr) } } #[inline] diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index 41128cd1481..c0fbae2db08 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -154,4 +154,27 @@ extern "platform-intrinsic" { // equivalent to wrapping_offset pub(crate) fn simd_arith_offset(ptr: T, offset: U) -> T; + + /* + /// equivalent to `T as U` semantics, specifically for pointers + pub(crate) fn simd_cast_ptr(ptr: T) -> U; + + /// expose a pointer as an address + pub(crate) fn simd_expose_addr(ptr: T) -> U; + + /// convert an exposed address back to a pointer + pub(crate) fn simd_from_exposed_addr(addr: T) -> U; + */ +} + +pub(crate) unsafe fn simd_cast_ptr(_ptr: T) -> U { + unimplemented!() +} + +pub(crate) unsafe fn simd_expose_addr(_ptr: T) -> U { + unimplemented!() +} + +pub(crate) unsafe fn simd_from_exposed_addr(_addr: T) -> U { + unimplemented!() } diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 3987b7a747b..3c435c4c805 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -1,6 +1,6 @@ use crate::simd::{ - intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdConstPtr, SimdMutPtr, SimdPartialOrd, - SupportedLaneCount, Swizzle, + intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdCastPtr, SimdConstPtr, SimdMutPtr, + SimdPartialOrd, SupportedLaneCount, Swizzle, }; /// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. @@ -209,11 +209,23 @@ where #[must_use] #[inline] #[cfg(not(bootstrap))] - pub fn cast(self) -> Simd + pub fn cast(self) -> Simd where - T: SimdCast, + T: SimdCast, { - SimdCast::cast(self) + // Safety: supported types are guaranteed by SimdCast + unsafe { intrinsics::simd_as(self) } + } + + /// Lanewise casts pointers to another pointer type. + #[must_use] + #[inline] + pub fn cast_ptr(self) -> Simd + where + T: SimdCastPtr, + { + // Safety: supported types are guaranteed by SimdCastPtr + unsafe { intrinsics::simd_cast_ptr(self) } } /// Rounds toward zero and converts to the same-width integer type, assuming that @@ -234,11 +246,11 @@ where #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn to_int_unchecked(self) -> Simd where - T: core::convert::FloatToInt + SimdCast, - I: SimdElement, + T: core::convert::FloatToInt + SimdCast, + I: SimdCast, { - // Safety: the caller is responsible for the invariants - unsafe { SimdCast::cast_unchecked(self) } + // Safety: supported types are guaranteed by SimdCast, the caller is responsible for the extra invariants + unsafe { intrinsics::simd_cast(self) } } /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. From dadf98a290e4f52d02a469f97931b90e953a98cf Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 4 Aug 2022 19:38:56 -0400 Subject: [PATCH 024/130] Remove duplicate intrinsic --- crates/core_simd/src/intrinsics.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index c0fbae2db08..45f01fa0f77 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -61,9 +61,6 @@ extern "platform-intrinsic" { /// xor pub(crate) fn simd_xor(x: T, y: T) -> T; - /// getelementptr (without inbounds) - pub(crate) fn simd_arith_offset(ptrs: T, offsets: U) -> T; - /// fptoui/fptosi/uitofp/sitofp /// casting floats to integers is truncating, so it is safe to convert values like e.g. 1.5 /// but the truncated value must fit in the target type or the result is poison. @@ -152,7 +149,8 @@ extern "platform-intrinsic" { #[allow(unused)] pub(crate) fn simd_select_bitmask(m: M, yes: T, no: T) -> T; - // equivalent to wrapping_offset + /// getelementptr (without inbounds) + /// equivalent to wrapping_offset pub(crate) fn simd_arith_offset(ptr: T, offset: U) -> T; /* From e5db1ecc8209e90982cc4603514028ef2210e592 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 4 Aug 2022 19:46:39 -0400 Subject: [PATCH 025/130] Fix documentation --- crates/core_simd/src/elements/const_ptr.rs | 6 +++--- crates/core_simd/src/elements/mut_ptr.rs | 4 ++-- crates/core_simd/src/lib.rs | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index 27b41019dc8..0a3d4ec4087 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -27,7 +27,7 @@ pub trait SimdConstPtr: Copy + Sealed { /// directly cast into a valid pointer. /// /// This method semantically discards *provenance* and - /// *address-space* information. To properly restore that information, use [`with_addr`]. + /// *address-space* information. To properly restore that information, use [`Self::with_addr`]. /// /// Equivalent to calling [`pointer::addr`] on each lane. fn addr(self) -> Self::Usize; @@ -41,12 +41,12 @@ pub trait SimdConstPtr: Copy + Sealed { fn with_addr(self, addr: Self::Usize) -> Self; /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use - /// in [`from_exposed_addr`]. + /// in [`Self::from_exposed_addr`]. fn expose_addr(self) -> Self::Usize; /// Convert an address back to a pointer, picking up a previously "exposed" provenance. /// - /// Equivalent to calling [`pointer::from_exposed_addr`] on each lane. + /// Equivalent to calling [`core::ptr::from_exposed_addr`] on each lane. fn from_exposed_addr(addr: Self::Usize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index 59a8b6293b5..e6aa9808f37 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -38,12 +38,12 @@ pub trait SimdMutPtr: Copy + Sealed { fn with_addr(self, addr: Self::Usize) -> Self; /// Gets the "address" portion of the pointer, and "exposes" the provenance part for future use - /// in [`from_exposed_addr`]. + /// in [`Self::from_exposed_addr`]. fn expose_addr(self) -> Self::Usize; /// Convert an address back to a pointer, picking up a previously "exposed" provenance. /// - /// Equivalent to calling [`pointer::from_exposed_addr`] on each lane. + /// Equivalent to calling [`core::ptr::from_exposed_addr_mut`] on each lane. fn from_exposed_addr(addr: Self::Usize) -> Self; /// Calculates the offset from a pointer using wrapping arithmetic. diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 715f258f617..05ac3e9338b 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -7,7 +7,8 @@ repr_simd, simd_ffi, staged_api, - stdsimd + stdsimd, + strict_provenance )] #![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))] #![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))] From 0fcc4069c12a4cffa69397388a0be42d45afdd49 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Thu, 4 Aug 2022 20:17:16 -0400 Subject: [PATCH 026/130] Fix pointer mutability casts and safety lints --- crates/core_simd/src/cast.rs | 22 ++++++++++++++++++++++ crates/core_simd/src/elements/const_ptr.rs | 4 +++- crates/core_simd/src/elements/mut_ptr.rs | 5 +++-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs index d14b0de5d5e..33878581e0b 100644 --- a/crates/core_simd/src/cast.rs +++ b/crates/core_simd/src/cast.rs @@ -1,23 +1,45 @@ use crate::simd::SimdElement; /// Supporting trait for `Simd::cast`. Typically doesn't need to be used directly. +/// +/// # Safety +/// Implementing this trait asserts that the type is a valid vector element for the `simd_cast` or +/// `simd_as` intrinsics. pub unsafe trait SimdCast: SimdElement {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for i8 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for i16 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for i32 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for i64 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for isize {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for u8 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for u16 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for u32 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for u64 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for usize {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for f32 {} +// Safety: primitive number types can be cast to other primitive number types unsafe impl SimdCast for f64 {} /// Supporting trait for `Simd::cast_ptr`. Typically doesn't need to be used directly. +/// +/// # Safety +/// Implementing this trait asserts that the type is a valid vector element for the `simd_cast_ptr` +/// intrinsic. pub unsafe trait SimdCastPtr: SimdElement {} +// Safety: pointers can be cast to other pointer types unsafe impl SimdCastPtr for *const T {} +// Safety: pointers can be cast to other pointer types unsafe impl SimdCastPtr for *mut T {} diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index 0a3d4ec4087..7c856fd4332 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -86,7 +86,7 @@ where #[inline] fn as_mut(self) -> Self::MutPtr { - unsafe { intrinsics::simd_cast_ptr(self) } + self.cast_ptr() } #[inline] @@ -111,11 +111,13 @@ where #[inline] fn expose_addr(self) -> Self::Usize { + // Safety: `self` is a pointer vector unsafe { intrinsics::simd_expose_addr(self) } } #[inline] fn from_exposed_addr(addr: Self::Usize) -> Self { + // Safety: `self` is a pointer vector unsafe { intrinsics::simd_from_exposed_addr(addr) } } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index e6aa9808f37..5e904d24a42 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -81,8 +81,7 @@ where #[inline] fn as_const(self) -> Self::ConstPtr { - unimplemented!() - //self.cast() + self.cast_ptr() } #[inline] @@ -107,11 +106,13 @@ where #[inline] fn expose_addr(self) -> Self::Usize { + // Safety: `self` is a pointer vector unsafe { intrinsics::simd_expose_addr(self) } } #[inline] fn from_exposed_addr(addr: Self::Usize) -> Self { + // Safety: `self` is a pointer vector unsafe { intrinsics::simd_from_exposed_addr(addr) } } From a79718ffa8cdfb5ee7ab3d9281b162fe37eb0606 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 18 Sep 2022 16:48:51 -0400 Subject: [PATCH 027/130] Use new intrinsics --- crates/core_simd/src/intrinsics.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index 45f01fa0f77..d5466822b93 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -153,7 +153,6 @@ extern "platform-intrinsic" { /// equivalent to wrapping_offset pub(crate) fn simd_arith_offset(ptr: T, offset: U) -> T; - /* /// equivalent to `T as U` semantics, specifically for pointers pub(crate) fn simd_cast_ptr(ptr: T) -> U; @@ -162,17 +161,4 @@ extern "platform-intrinsic" { /// convert an exposed address back to a pointer pub(crate) fn simd_from_exposed_addr(addr: T) -> U; - */ -} - -pub(crate) unsafe fn simd_cast_ptr(_ptr: T) -> U { - unimplemented!() -} - -pub(crate) unsafe fn simd_expose_addr(_ptr: T) -> U { - unimplemented!() -} - -pub(crate) unsafe fn simd_from_exposed_addr(_addr: T) -> U { - unimplemented!() } From 078cb58e766c20a2705f22f7a6f9bc0cf451e16d Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 18 Sep 2022 22:47:34 -0400 Subject: [PATCH 028/130] Apply suggestions from code review Co-authored-by: Jacob Lifshay --- crates/core_simd/src/elements/const_ptr.rs | 2 +- crates/core_simd/src/elements/mut_ptr.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index 7c856fd4332..f7227a56d58 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -61,7 +61,7 @@ pub trait SimdConstPtr: Copy + Sealed { /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. fn wrapping_sub(self, count: Self::Usize) -> Self; } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index 5e904d24a42..e2fd438ef8f 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -58,7 +58,7 @@ pub trait SimdMutPtr: Copy + Sealed { /// Calculates the offset from a pointer using wrapping arithmetic. /// - /// Equivalent to calling [`pointer::wrapping_add`] on each lane. + /// Equivalent to calling [`pointer::wrapping_sub`] on each lane. fn wrapping_sub(self, count: Self::Usize) -> Self; } From 469c620bded61d265ef020b2442b1f639b2d8c10 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 21 Oct 2022 21:43:48 -0400 Subject: [PATCH 029/130] Account for pointer metadata in pointer bounds --- crates/core_simd/src/cast.rs | 16 +++++++++++++--- crates/core_simd/src/lib.rs | 3 ++- crates/core_simd/src/vector.rs | 23 +++++++++++++++++------ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/crates/core_simd/src/cast.rs b/crates/core_simd/src/cast.rs index 33878581e0b..65a3f845ffc 100644 --- a/crates/core_simd/src/cast.rs +++ b/crates/core_simd/src/cast.rs @@ -37,9 +37,19 @@ unsafe impl SimdCast for f64 {} /// # Safety /// Implementing this trait asserts that the type is a valid vector element for the `simd_cast_ptr` /// intrinsic. -pub unsafe trait SimdCastPtr: SimdElement {} +pub unsafe trait SimdCastPtr {} // Safety: pointers can be cast to other pointer types -unsafe impl SimdCastPtr for *const T {} +unsafe impl SimdCastPtr for *const U +where + U: core::ptr::Pointee, + T: core::ptr::Pointee, +{ +} // Safety: pointers can be cast to other pointer types -unsafe impl SimdCastPtr for *mut T {} +unsafe impl SimdCastPtr for *mut U +where + U: core::ptr::Pointee, + T: core::ptr::Pointee, +{ +} diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 05ac3e9338b..82873162969 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -8,7 +8,8 @@ simd_ffi, staged_api, stdsimd, - strict_provenance + strict_provenance, + ptr_metadata )] #![cfg_attr(feature = "generic_const_exprs", feature(generic_const_exprs))] #![cfg_attr(feature = "generic_const_exprs", allow(incomplete_features))] diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 3c435c4c805..c5d68f1b921 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -220,9 +220,10 @@ where /// Lanewise casts pointers to another pointer type. #[must_use] #[inline] - pub fn cast_ptr(self) -> Simd + pub fn cast_ptr(self) -> Simd where - T: SimdCastPtr, + T: SimdCastPtr, + U: SimdElement, { // Safety: supported types are guaranteed by SimdCastPtr unsafe { intrinsics::simd_cast_ptr(self) } @@ -753,14 +754,24 @@ unsafe impl SimdElement for f64 { impl Sealed for *const T {} -// Safety: const pointers are valid SIMD element types, and are supported by this API -unsafe impl SimdElement for *const T { +// Safety: (thin) const pointers are valid SIMD element types, and are supported by this API +// +// Fat pointers may be supported in the future. +unsafe impl SimdElement for *const T +where + T: core::ptr::Pointee, +{ type Mask = isize; } impl Sealed for *mut T {} -// Safety: mut pointers are valid SIMD element types, and are supported by this API -unsafe impl SimdElement for *mut T { +// Safety: (thin) mut pointers are valid SIMD element types, and are supported by this API +// +// Fat pointers may be supported in the future. +unsafe impl SimdElement for *mut T +where + T: core::ptr::Pointee, +{ type Mask = isize; } From de30820035cb42d05f49575811a9f33661985e67 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sat, 29 Oct 2022 21:39:08 -0400 Subject: [PATCH 030/130] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 80313157ea2..e8ac600debe 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,10 @@ The supported element types are as follows: * **Floating Point:** `f32`, `f64` * **Signed Integers:** `i8`, `i16`, `i32`, `i64`, `isize` (`i128` excluded) * **Unsigned Integers:** `u8`, `u16`, `u32`, `u64`, `usize` (`u128` excluded) +* **Pointers:** `*const T` and `*mut T` (zero-sized metadata only) * **Masks:** 8-bit, 16-bit, 32-bit, 64-bit, and `usize`-sized masks -Floating point, signed integers, and unsigned integers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to. +Floating point, signed integers, unsigned integers, and pointers are the [primitive types](https://doc.rust-lang.org/core/primitive/index.html) you're already used to. The mask types have elements that are "truthy" values, like `bool`, but have an unspecified layout because different architectures prefer different layouts for mask types. [simd-guide]: ./beginners-guide.md From 572122a95da6f8aaf513f53c426732f4c0a91325 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Wed, 9 Nov 2022 21:28:38 -0500 Subject: [PATCH 031/130] Add missing pointer tests and rename pointer cast fns to match scalars --- crates/core_simd/src/elements/const_ptr.rs | 6 ++- crates/core_simd/src/elements/mut_ptr.rs | 6 ++- crates/core_simd/tests/pointers.rs | 52 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/crates/core_simd/src/elements/const_ptr.rs b/crates/core_simd/src/elements/const_ptr.rs index f7227a56d58..0ef9802b5e2 100644 --- a/crates/core_simd/src/elements/const_ptr.rs +++ b/crates/core_simd/src/elements/const_ptr.rs @@ -19,7 +19,9 @@ pub trait SimdConstPtr: Copy + Sealed { fn is_null(self) -> Self::Mask; /// Changes constness without changing the type. - fn as_mut(self) -> Self::MutPtr; + /// + /// Equivalent to calling [`pointer::cast_mut`] on each lane. + fn cast_mut(self) -> Self::MutPtr; /// Gets the "address" portion of the pointer. /// @@ -85,7 +87,7 @@ where } #[inline] - fn as_mut(self) -> Self::MutPtr { + fn cast_mut(self) -> Self::MutPtr { self.cast_ptr() } diff --git a/crates/core_simd/src/elements/mut_ptr.rs b/crates/core_simd/src/elements/mut_ptr.rs index e2fd438ef8f..d87986b4a09 100644 --- a/crates/core_simd/src/elements/mut_ptr.rs +++ b/crates/core_simd/src/elements/mut_ptr.rs @@ -19,7 +19,9 @@ pub trait SimdMutPtr: Copy + Sealed { fn is_null(self) -> Self::Mask; /// Changes constness without changing the type. - fn as_const(self) -> Self::ConstPtr; + /// + /// Equivalent to calling [`pointer::cast_const`] on each lane. + fn cast_const(self) -> Self::ConstPtr; /// Gets the "address" portion of the pointer. /// @@ -80,7 +82,7 @@ where } #[inline] - fn as_const(self) -> Self::ConstPtr { + fn cast_const(self) -> Self::ConstPtr { self.cast_ptr() } diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs index 8eb0bd84042..2b0008624ad 100644 --- a/crates/core_simd/tests/pointers.rs +++ b/crates/core_simd/tests/pointers.rs @@ -21,6 +21,22 @@ macro_rules! common_tests { ); } + fn with_addr() { + test_helpers::test_binary_elementwise( + &Simd::<*$constness u32, LANES>::with_addr, + &<*$constness u32>::with_addr, + &|_, _| true, + ); + } + + fn expose_addr() { + test_helpers::test_unary_elementwise( + &Simd::<*$constness u32, LANES>::expose_addr, + &<*$constness u32>::expose_addr, + &|_| true, + ); + } + fn wrapping_offset() { test_helpers::test_binary_elementwise( &Simd::<*$constness u32, LANES>::wrapping_offset, @@ -51,9 +67,45 @@ macro_rules! common_tests { mod const_ptr { use super::*; common_tests! { const } + + test_helpers::test_lanes! { + fn cast_mut() { + test_helpers::test_unary_elementwise( + &Simd::<*const u32, LANES>::cast_mut, + &<*const u32>::cast_mut, + &|_| true, + ); + } + + fn from_exposed_addr() { + test_helpers::test_unary_elementwise( + &Simd::<*const u32, LANES>::from_exposed_addr, + &core::ptr::from_exposed_addr::, + &|_| true, + ); + } + } } mod mut_ptr { use super::*; common_tests! { mut } + + test_helpers::test_lanes! { + fn cast_const() { + test_helpers::test_unary_elementwise( + &Simd::<*mut u32, LANES>::cast_const, + &<*mut u32>::cast_const, + &|_| true, + ); + } + + fn from_exposed_addr() { + test_helpers::test_unary_elementwise( + &Simd::<*mut u32, LANES>::from_exposed_addr, + &core::ptr::from_exposed_addr_mut::, + &|_| true, + ); + } + } } From 7ac1fbbcb14c05f778cf1c550e2b30f00606bb97 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 11 Nov 2022 17:32:48 -0500 Subject: [PATCH 032/130] impl TryFrom<&[T]> for Simd --- crates/core_simd/src/vector.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index c5d68f1b921..0095ed1648f 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -650,6 +650,30 @@ where } } +impl TryFrom<&[T]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + type Error = core::array::TryFromSliceError; + + fn try_from(slice: &[T]) -> Result { + Ok(Self::from_array(slice.try_into()?)) + } +} + +impl TryFrom<&mut [T]> for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement, +{ + type Error = core::array::TryFromSliceError; + + fn try_from(slice: &mut [T]) -> Result { + Ok(Self::from_array(slice.try_into()?)) + } +} + mod sealed { pub trait Sealed {} } From 9dc690c48265bae58ca6e307d8f35a1f74b921e3 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 11 Nov 2022 18:10:51 -0500 Subject: [PATCH 033/130] Add TryFrom<&[T]> tests --- crates/core_simd/tests/try_from_slice.rs | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 crates/core_simd/tests/try_from_slice.rs diff --git a/crates/core_simd/tests/try_from_slice.rs b/crates/core_simd/tests/try_from_slice.rs new file mode 100644 index 00000000000..189c18c6039 --- /dev/null +++ b/crates/core_simd/tests/try_from_slice.rs @@ -0,0 +1,25 @@ +#![feature(portable_simd)] + +#[cfg(target_arch = "wasm32")] +use wasm_bindgen_test::*; + +#[cfg(target_arch = "wasm32")] +wasm_bindgen_test_configure!(run_in_browser); + +use core_simd::i32x4; + +#[test] +#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] +fn try_from_slice() { + // Equal length + assert_eq!( + i32x4::try_from([1, 2, 3, 4].as_slice()).unwrap(), + i32x4::from_array([1, 2, 3, 4]) + ); + + // Slice length > vector length + assert!(i32x4::try_from([1, 2, 3, 4, 5].as_slice()).is_err()); + + // Slice length < vector length + assert!(i32x4::try_from([1, 2, 3].as_slice()).is_err()); +} From fd53445d05874d7662682b00d81cf073cfdbe505 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 11 Nov 2022 19:48:27 -0500 Subject: [PATCH 034/130] Add pointer scatter/gather --- crates/core_simd/src/vector.rs | 68 ++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index c5d68f1b921..850a517c799 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -364,8 +364,44 @@ where let base_ptr = Simd::<*const T, LANES>::splat(slice.as_ptr()); // Ferris forgive me, I have done pointer arithmetic here. let ptrs = base_ptr.wrapping_add(idxs); - // Safety: The ptrs have been bounds-masked to prevent memory-unsafe reads insha'allah - unsafe { intrinsics::simd_gather(or, ptrs, enable.to_int()) } + // Safety: The caller is responsible for determining the indices are okay to read + unsafe { Self::gather_select_ptr(ptrs, enable, or) } + } + + /// Read pointers elementwise into a SIMD vector vector. + /// + /// # Safety + /// + /// Each read must satisfy the same conditions as [`core::ptr::read`]. + #[must_use] + #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub unsafe fn gather_ptr(source: Simd<*const T, LANES>) -> Self + where + T: Default, + { + // TODO: add an intrinsic that doesn't use a passthru vector, and remove the T: Default bound + // Safety: The caller is responsible for upholding all invariants + unsafe { Self::gather_select_ptr(source, Mask::splat(true), Self::default()) } + } + + /// Conditionally read pointers elementwise into a SIMD vector vector. + /// The mask `enable`s all `true` lanes and disables all `false` lanes. + /// If a lane is disabled, the lane is selected from the `or` vector and no read is performed. + /// + /// # Safety + /// + /// Enabled lanes must satisfy the same conditions as [`core::ptr::read`]. + #[must_use] + #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub unsafe fn gather_select_ptr( + source: Simd<*const T, LANES>, + enable: Mask, + or: Self, + ) -> Self { + // Safety: The caller is responsible for upholding all invariants + unsafe { intrinsics::simd_gather(or, source, enable.to_int()) } } /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`. @@ -473,10 +509,36 @@ where // Ferris forgive me, I have done pointer arithmetic here. let ptrs = base_ptr.wrapping_add(idxs); // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah - intrinsics::simd_scatter(self, ptrs, enable.to_int()) + self.scatter_select_ptr(ptrs, enable); // Cleared ☢️ *mut T Zone } } + + /// Write pointers elementwise into a SIMD vector vector. + /// + /// # Safety + /// + /// Each write must satisfy the same conditions as [`core::ptr::write`]. + #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, LANES>) { + // Safety: The caller is responsible for upholding all invariants + unsafe { self.scatter_select_ptr(dest, Mask::splat(true)) } + } + + /// Conditionally write pointers elementwise into a SIMD vector vector. + /// The mask `enable`s all `true` lanes and disables all `false` lanes. + /// If a lane is disabled, the writing that lane is skipped. + /// + /// # Safety + /// + /// Enabled lanes must satisfy the same conditions as [`core::ptr::write`]. + #[inline] + #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces + pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, LANES>, enable: Mask) { + // Safety: The caller is responsible for upholding all invariants + unsafe { intrinsics::simd_scatter(self, dest, enable.to_int()) } + } } impl Copy for Simd From bef4c41fc0051444034ad9b488b06b2d512bfd17 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Fri, 11 Nov 2022 21:31:05 -0500 Subject: [PATCH 035/130] Add test examples --- crates/core_simd/src/vector.rs | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 850a517c799..52ed5490519 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -373,6 +373,19 @@ where /// # Safety /// /// Each read must satisfy the same conditions as [`core::ptr::read`]. + /// + /// # Example + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdConstPtr}; + /// let values = [6, 2, 4, 9]; + /// let offsets = Simd::from_array([1, 0, 0, 3]); + /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets); + /// let gathered = unsafe { Simd::gather_ptr(source) }; + /// assert_eq!(gathered, Simd::from_array([2, 6, 6, 9])); + /// ``` #[must_use] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces @@ -392,6 +405,20 @@ where /// # Safety /// /// Enabled lanes must satisfy the same conditions as [`core::ptr::read`]. + /// + /// # Example + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Mask, Simd, SimdConstPtr}; + /// let values = [6, 2, 4, 9]; + /// let enable = Mask::from_array([true, true, false, true]); + /// let offsets = Simd::from_array([1, 0, 0, 3]); + /// let source = Simd::splat(values.as_ptr()).wrapping_add(offsets); + /// let gathered = unsafe { Simd::gather_select_ptr(source, enable, Simd::splat(0)) }; + /// assert_eq!(gathered, Simd::from_array([2, 6, 0, 9])); + /// ``` #[must_use] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces @@ -519,6 +546,19 @@ where /// # Safety /// /// Each write must satisfy the same conditions as [`core::ptr::write`]. + /// + /// # Example + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Simd, SimdMutPtr}; + /// let mut values = [0; 4]; + /// let offset = Simd::from_array([3, 2, 1, 0]); + /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset); + /// unsafe { Simd::from_array([6, 3, 5, 7]).scatter_ptr(ptrs); } + /// assert_eq!(values, [7, 5, 3, 6]); + /// ``` #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, LANES>) { @@ -533,6 +573,20 @@ where /// # Safety /// /// Enabled lanes must satisfy the same conditions as [`core::ptr::write`]. + /// + /// # Example + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::{Mask, Simd, SimdMutPtr}; + /// let mut values = [0; 4]; + /// let offset = Simd::from_array([3, 2, 1, 0]); + /// let ptrs = Simd::splat(values.as_mut_ptr()).wrapping_add(offset); + /// let enable = Mask::from_array([true, true, false, false]); + /// unsafe { Simd::from_array([6, 3, 5, 7]).scatter_select_ptr(ptrs, enable); } + /// assert_eq!(values, [0, 0, 3, 6]); + /// ``` #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, LANES>, enable: Mask) { From c247915eb88af33302b2dc393fa7b488ee680a5f Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sat, 12 Nov 2022 22:39:54 -0500 Subject: [PATCH 036/130] Update crates/core_simd/src/vector.rs Co-authored-by: Jubilee <46493976+workingjubilee@users.noreply.github.com> --- crates/core_simd/src/vector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 52ed5490519..f25505f7c59 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -568,7 +568,7 @@ where /// Conditionally write pointers elementwise into a SIMD vector vector. /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If a lane is disabled, the writing that lane is skipped. + /// If a lane is disabled, the write to that lane is skipped. /// /// # Safety /// From 7e614f0438324b60af24554699977757228d7acd Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sat, 12 Nov 2022 22:41:44 -0500 Subject: [PATCH 037/130] Fix typo typo --- crates/core_simd/src/vector.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index f25505f7c59..0ddc3e1b395 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -368,7 +368,7 @@ where unsafe { Self::gather_select_ptr(ptrs, enable, or) } } - /// Read pointers elementwise into a SIMD vector vector. + /// Read pointers elementwise into a SIMD vector. /// /// # Safety /// @@ -398,7 +398,7 @@ where unsafe { Self::gather_select_ptr(source, Mask::splat(true), Self::default()) } } - /// Conditionally read pointers elementwise into a SIMD vector vector. + /// Conditionally read pointers elementwise into a SIMD vector. /// The mask `enable`s all `true` lanes and disables all `false` lanes. /// If a lane is disabled, the lane is selected from the `or` vector and no read is performed. /// @@ -541,7 +541,7 @@ where } } - /// Write pointers elementwise into a SIMD vector vector. + /// Write pointers elementwise into a SIMD vector. /// /// # Safety /// @@ -566,7 +566,7 @@ where unsafe { self.scatter_select_ptr(dest, Mask::splat(true)) } } - /// Conditionally write pointers elementwise into a SIMD vector vector. + /// Conditionally write pointers elementwise into a SIMD vector. /// The mask `enable`s all `true` lanes and disables all `false` lanes. /// If a lane is disabled, the write to that lane is skipped. /// From db8b23cea5ac9b45fafef65d95702f41cc02d486 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 27 Nov 2022 23:44:20 -0500 Subject: [PATCH 038/130] Remove reexport of simd::* --- crates/core_simd/src/lib.rs | 1 - crates/core_simd/tests/autoderef.rs | 2 +- .../tests/mask_ops_impl/mask_macros.rs | 2 +- crates/core_simd/tests/masks.rs | 59 ++++++++++--------- crates/core_simd/tests/ops_macros.rs | 14 ++--- crates/core_simd/tests/pointers.rs | 2 +- crates/core_simd/tests/round.rs | 2 +- crates/core_simd/tests/swizzle.rs | 2 +- crates/core_simd/tests/to_bytes.rs | 2 +- crates/core_simd/tests/try_from_slice.rs | 2 +- crates/test_helpers/src/lib.rs | 4 +- 11 files changed, 47 insertions(+), 45 deletions(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 82873162969..a6359d1e0be 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -21,4 +21,3 @@ #[path = "mod.rs"] mod core_simd; pub use self::core_simd::simd; -pub use simd::*; diff --git a/crates/core_simd/tests/autoderef.rs b/crates/core_simd/tests/autoderef.rs index 9359da16ee5..3181826ef59 100644 --- a/crates/core_simd/tests/autoderef.rs +++ b/crates/core_simd/tests/autoderef.rs @@ -1,6 +1,6 @@ // Test that we handle all our "auto-deref" cases correctly. #![feature(portable_simd)] -use core_simd::f32x4; +use core_simd::simd::f32x4; #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::*; diff --git a/crates/core_simd/tests/mask_ops_impl/mask_macros.rs b/crates/core_simd/tests/mask_ops_impl/mask_macros.rs index 795f9e27c44..faafa5fa51f 100644 --- a/crates/core_simd/tests/mask_ops_impl/mask_macros.rs +++ b/crates/core_simd/tests/mask_ops_impl/mask_macros.rs @@ -2,7 +2,7 @@ macro_rules! mask_tests { { $vector:ident, $lanes:literal } => { #[cfg(test)] mod $vector { - use core_simd::$vector as Vector; + use core_simd::simd::$vector as Vector; const LANES: usize = $lanes; #[cfg(target_arch = "wasm32")] diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 673d0db93fe..9f8bad1c36c 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -13,11 +13,13 @@ macro_rules! test_mask_api { #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::*; + use core_simd::simd::Mask; + #[test] #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] fn set_and_test() { let values = [true, false, false, true, false, false, true, false]; - let mut mask = core_simd::Mask::<$type, 8>::splat(false); + let mut mask = Mask::<$type, 8>::splat(false); for (lane, value) in values.iter().copied().enumerate() { mask.set(lane, value); } @@ -29,7 +31,7 @@ macro_rules! test_mask_api { #[test] #[should_panic] fn set_invalid_lane() { - let mut mask = core_simd::Mask::<$type, 8>::splat(false); + let mut mask = Mask::<$type, 8>::splat(false); mask.set(8, true); let _ = mask; } @@ -37,24 +39,24 @@ macro_rules! test_mask_api { #[test] #[should_panic] fn test_invalid_lane() { - let mask = core_simd::Mask::<$type, 8>::splat(false); + let mask = Mask::<$type, 8>::splat(false); let _ = mask.test(8); } #[test] fn any() { - assert!(!core_simd::Mask::<$type, 8>::splat(false).any()); - assert!(core_simd::Mask::<$type, 8>::splat(true).any()); - let mut v = core_simd::Mask::<$type, 8>::splat(false); + assert!(!Mask::<$type, 8>::splat(false).any()); + assert!(Mask::<$type, 8>::splat(true).any()); + let mut v = Mask::<$type, 8>::splat(false); v.set(2, true); assert!(v.any()); } #[test] fn all() { - assert!(!core_simd::Mask::<$type, 8>::splat(false).all()); - assert!(core_simd::Mask::<$type, 8>::splat(true).all()); - let mut v = core_simd::Mask::<$type, 8>::splat(false); + assert!(!Mask::<$type, 8>::splat(false).all()); + assert!(Mask::<$type, 8>::splat(true).all()); + let mut v = Mask::<$type, 8>::splat(false); v.set(2, true); assert!(!v.all()); } @@ -62,57 +64,57 @@ macro_rules! test_mask_api { #[test] fn roundtrip_int_conversion() { let values = [true, false, false, true, false, false, true, false]; - let mask = core_simd::Mask::<$type, 8>::from_array(values); + let mask = Mask::<$type, 8>::from_array(values); let int = mask.to_int(); assert_eq!(int.to_array(), [-1, 0, 0, -1, 0, 0, -1, 0]); - assert_eq!(core_simd::Mask::<$type, 8>::from_int(int), mask); + assert_eq!(Mask::<$type, 8>::from_int(int), mask); } #[test] fn roundtrip_bitmask_conversion() { - use core_simd::ToBitMask; + use core_simd::simd::ToBitMask; let values = [ true, false, false, true, false, false, true, false, true, true, false, false, false, false, false, true, ]; - let mask = core_simd::Mask::<$type, 16>::from_array(values); + let mask = Mask::<$type, 16>::from_array(values); let bitmask = mask.to_bitmask(); assert_eq!(bitmask, 0b1000001101001001); - assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask(bitmask), mask); + assert_eq!(Mask::<$type, 16>::from_bitmask(bitmask), mask); } #[test] fn roundtrip_bitmask_conversion_short() { - use core_simd::ToBitMask; + use core_simd::simd::ToBitMask; let values = [ false, false, false, true, ]; - let mask = core_simd::Mask::<$type, 4>::from_array(values); + let mask = Mask::<$type, 4>::from_array(values); let bitmask = mask.to_bitmask(); assert_eq!(bitmask, 0b1000); - assert_eq!(core_simd::Mask::<$type, 4>::from_bitmask(bitmask), mask); + assert_eq!(Mask::<$type, 4>::from_bitmask(bitmask), mask); let values = [true, false]; - let mask = core_simd::Mask::<$type, 2>::from_array(values); + let mask = Mask::<$type, 2>::from_array(values); let bitmask = mask.to_bitmask(); assert_eq!(bitmask, 0b01); - assert_eq!(core_simd::Mask::<$type, 2>::from_bitmask(bitmask), mask); + assert_eq!(Mask::<$type, 2>::from_bitmask(bitmask), mask); } #[test] fn cast() { - fn cast_impl() + fn cast_impl() where - core_simd::Mask<$type, 8>: Into>, + Mask<$type, 8>: Into>, { let values = [true, false, false, true, false, false, true, false]; - let mask = core_simd::Mask::<$type, 8>::from_array(values); + let mask = Mask::<$type, 8>::from_array(values); let cast_mask = mask.cast::(); assert_eq!(values, cast_mask.to_array()); - let into_mask: core_simd::Mask = mask.into(); + let into_mask: Mask = mask.into(); assert_eq!(values, into_mask.to_array()); } @@ -126,15 +128,15 @@ macro_rules! test_mask_api { #[cfg(feature = "generic_const_exprs")] #[test] fn roundtrip_bitmask_array_conversion() { - use core_simd::ToBitMaskArray; + use core_simd::simd::ToBitMaskArray; let values = [ true, false, false, true, false, false, true, false, true, true, false, false, false, false, false, true, ]; - let mask = core_simd::Mask::<$type, 16>::from_array(values); + let mask = Mask::<$type, 16>::from_array(values); let bitmask = mask.to_bitmask_array(); assert_eq!(bitmask, [0b01001001, 0b10000011]); - assert_eq!(core_simd::Mask::<$type, 16>::from_bitmask_array(bitmask), mask); + assert_eq!(Mask::<$type, 16>::from_bitmask_array(bitmask), mask); } } } @@ -150,9 +152,10 @@ mod mask_api { #[test] fn convert() { + use core_simd::simd::Mask; let values = [true, false, false, true, false, false, true, false]; assert_eq!( - core_simd::Mask::::from_array(values), - core_simd::Mask::::from_array(values).into() + Mask::::from_array(values), + Mask::::from_array(values).into() ); } diff --git a/crates/core_simd/tests/ops_macros.rs b/crates/core_simd/tests/ops_macros.rs index f759394d075..3a02f3f01e1 100644 --- a/crates/core_simd/tests/ops_macros.rs +++ b/crates/core_simd/tests/ops_macros.rs @@ -7,7 +7,7 @@ macro_rules! impl_unary_op_test { test_helpers::test_lanes! { fn $fn() { test_helpers::test_unary_elementwise( - & as core::ops::$trait>::$fn, + & as core::ops::$trait>::$fn, &$scalar_fn, &|_| true, ); @@ -27,7 +27,7 @@ macro_rules! impl_binary_op_test { { $scalar:ty, $trait:ident :: $fn:ident, $trait_assign:ident :: $fn_assign:ident, $scalar_fn:expr } => { mod $fn { use super::*; - use core_simd::Simd; + use core_simd::simd::Simd; test_helpers::test_lanes! { fn normal() { @@ -64,7 +64,7 @@ macro_rules! impl_binary_checked_op_test { { $scalar:ty, $trait:ident :: $fn:ident, $trait_assign:ident :: $fn_assign:ident, $scalar_fn:expr, $check_fn:expr } => { mod $fn { use super::*; - use core_simd::Simd; + use core_simd::simd::Simd; test_helpers::test_lanes! { fn normal() { @@ -173,7 +173,7 @@ macro_rules! impl_signed_tests { { $scalar:tt } => { mod $scalar { use core_simd::simd::SimdInt; - type Vector = core_simd::Simd; + type Vector = core_simd::simd::Simd; type Scalar = $scalar; impl_common_integer_tests! { Vector, Scalar } @@ -314,7 +314,7 @@ macro_rules! impl_unsigned_tests { { $scalar:tt } => { mod $scalar { use core_simd::simd::SimdUint; - type Vector = core_simd::Simd; + type Vector = core_simd::simd::Simd; type Scalar = $scalar; impl_common_integer_tests! { Vector, Scalar } @@ -348,8 +348,8 @@ macro_rules! impl_unsigned_tests { macro_rules! impl_float_tests { { $scalar:tt, $int_scalar:tt } => { mod $scalar { - use core_simd::SimdFloat; - type Vector = core_simd::Simd; + use core_simd::simd::SimdFloat; + type Vector = core_simd::simd::Simd; type Scalar = $scalar; impl_unary_op_test!(Scalar, Neg::neg); diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs index 2b0008624ad..0ae8f83b8b9 100644 --- a/crates/core_simd/tests/pointers.rs +++ b/crates/core_simd/tests/pointers.rs @@ -1,6 +1,6 @@ #![feature(portable_simd, strict_provenance)] -use core_simd::{Simd, SimdConstPtr, SimdMutPtr}; +use core_simd::simd::{Simd, SimdConstPtr, SimdMutPtr}; macro_rules! common_tests { { $constness:ident } => { diff --git a/crates/core_simd/tests/round.rs b/crates/core_simd/tests/round.rs index 484fd5bf47d..8b9638ad466 100644 --- a/crates/core_simd/tests/round.rs +++ b/crates/core_simd/tests/round.rs @@ -5,7 +5,7 @@ macro_rules! float_rounding_test { mod $scalar { use std_float::StdFloat; - type Vector = core_simd::Simd<$scalar, LANES>; + type Vector = core_simd::simd::Simd<$scalar, LANES>; type Scalar = $scalar; type IntScalar = $int_scalar; diff --git a/crates/core_simd/tests/swizzle.rs b/crates/core_simd/tests/swizzle.rs index 33a7becb421..8cd7c33e823 100644 --- a/crates/core_simd/tests/swizzle.rs +++ b/crates/core_simd/tests/swizzle.rs @@ -1,5 +1,5 @@ #![feature(portable_simd)] -use core_simd::{Simd, Swizzle}; +use core_simd::simd::{Simd, Swizzle}; #[cfg(target_arch = "wasm32")] use wasm_bindgen_test::*; diff --git a/crates/core_simd/tests/to_bytes.rs b/crates/core_simd/tests/to_bytes.rs index debb4335e2c..be0ee4349c5 100644 --- a/crates/core_simd/tests/to_bytes.rs +++ b/crates/core_simd/tests/to_bytes.rs @@ -2,7 +2,7 @@ #![allow(incomplete_features)] #![cfg(feature = "generic_const_exprs")] -use core_simd::Simd; +use core_simd::simd::Simd; #[test] fn byte_convert() { diff --git a/crates/core_simd/tests/try_from_slice.rs b/crates/core_simd/tests/try_from_slice.rs index 189c18c6039..859e3b94f2c 100644 --- a/crates/core_simd/tests/try_from_slice.rs +++ b/crates/core_simd/tests/try_from_slice.rs @@ -6,7 +6,7 @@ use wasm_bindgen_test::*; #[cfg(target_arch = "wasm32")] wasm_bindgen_test_configure!(run_in_browser); -use core_simd::i32x4; +use core_simd::simd::i32x4; #[test] #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test)] diff --git a/crates/test_helpers/src/lib.rs b/crates/test_helpers/src/lib.rs index 5f2a928b5e4..b26cdc311a2 100644 --- a/crates/test_helpers/src/lib.rs +++ b/crates/test_helpers/src/lib.rs @@ -401,7 +401,7 @@ macro_rules! test_lanes { fn implementation() where - core_simd::LaneCount<$lanes>: core_simd::SupportedLaneCount, + core_simd::simd::LaneCount<$lanes>: core_simd::simd::SupportedLaneCount, $body #[cfg(target_arch = "wasm32")] @@ -508,7 +508,7 @@ macro_rules! test_lanes_panic { fn implementation() where - core_simd::LaneCount<$lanes>: core_simd::SupportedLaneCount, + core_simd::simd::LaneCount<$lanes>: core_simd::simd::SupportedLaneCount, $body $crate::test_lanes_helper!( From 54b6f6923e281ba68d13269b43faa927c6df83d5 Mon Sep 17 00:00:00 2001 From: Thom Chiovoloni Date: Mon, 28 Nov 2022 06:03:32 -0800 Subject: [PATCH 039/130] Avoid a scalar loop in `Simd::from_slice` --- crates/core_simd/src/lib.rs | 1 + crates/core_simd/src/vector.rs | 11 ++++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 82873162969..34b79e630a4 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -1,5 +1,6 @@ #![no_std] #![feature( + const_ptr_read, convert_float_to_int, decl_macro, intra_doc_pointers, diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index d109087eaa6..51b0d999a81 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -174,13 +174,10 @@ where slice.len() >= LANES, "slice length must be at least the number of lanes" ); - let mut array = [slice[0]; LANES]; - let mut i = 0; - while i < LANES { - array[i] = slice[i]; - i += 1; - } - Self(array) + // Safety: + // - We've checked the length is sufficient. + // - `T` and `Simd` are Copy types. + unsafe { slice.as_ptr().cast::().read_unaligned() } } /// Performs lanewise conversion of a SIMD vector's elements to another SIMD-valid type. From df3a63906c44b23de7065d60c20bf99e2571ccc8 Mon Sep 17 00:00:00 2001 From: miguel raz Date: Fri, 4 Jun 2021 14:24:47 -0500 Subject: [PATCH 040/130] add dot_product example --- crates/core_simd/examples/dot_product.rs | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 crates/core_simd/examples/dot_product.rs diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs new file mode 100644 index 00000000000..812b0b23eeb --- /dev/null +++ b/crates/core_simd/examples/dot_product.rs @@ -0,0 +1,31 @@ +// Code taken from the `packed_simd` crate +// Run this code with `cargo test --example dot_product` +#![feature(array_chunks)] +use core_simd::*; + +pub fn dot_prod(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + + // TODO handle remainder when a.len() % 4 != 0 + a.array_chunks::<4>() + .map(|&a| f32x4::from_array(a)) + .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .map(|(a, b)| (a * b).horizontal_sum()) + .sum() +} + +fn main() { + // Empty main to make cargo happy +} + +#[cfg(test)] +mod tests { + #[test] + fn test() { + use super::*; + let a: Vec = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; + let b: Vec = vec![-8.0, -7.0, -6.0, -5.0, 4.0, 3.0, 2.0, 1.0]; + + assert_eq!(0.0, dot_prod(&a, &b)); + } +} From c08a4d1f10473bfbdddf3d2eefc40e1194a633a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Sat, 26 Mar 2022 14:04:37 -0600 Subject: [PATCH 041/130] add more basic dot products and comments, README --- crates/core_simd/examples/README.md | 19 ++++++++++++++++ crates/core_simd/examples/dot_product.rs | 29 +++++++++++++++++++++--- 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 crates/core_simd/examples/README.md diff --git a/crates/core_simd/examples/README.md b/crates/core_simd/examples/README.md new file mode 100644 index 00000000000..b37dffa8eaa --- /dev/null +++ b/crates/core_simd/examples/README.md @@ -0,0 +1,19 @@ +### `stdsimd` examples + +This crate is a port of example uses of `stdsimd`, mostly taken from the `packed_simd` crate. + +The examples contain, as in the case of `dot_product.rs`, multiple ways of solving the problem, in order to show idiomatic uses of SIMD and iteration of performance designs. + +Run the tests with the command + +``` +cargo run --example dot_product +``` + +and the benchmarks via the command + +``` +cargo run --example --benchmark ??? +``` + +and measure the timings on your local system. diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 812b0b23eeb..3e415fc4471 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -3,7 +3,27 @@ #![feature(array_chunks)] use core_simd::*; -pub fn dot_prod(a: &[f32], b: &[f32]) -> f32 { +/// This is your barebones dot product implementation: +/// Take 2 vectors, multiply them element wise and *then* +/// add up the result. In the next example we will see if there +/// is any difference to adding as we go along multiplying. +pub fn dot_prod_0(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + + a.iter() + .zip(b.iter()) + .map(|a, b| a * b) + .sum() +} + +pub fn dot_prod_1(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + a.iter() + .zip(b.iter()) + .fold(0.0, |a, b| a * b) +} + +pub fn dot_prod_simd_0(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); // TODO handle remainder when a.len() % 4 != 0 @@ -21,11 +41,14 @@ fn main() { #[cfg(test)] mod tests { #[test] - fn test() { + fn smoke_test() { use super::*; let a: Vec = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let b: Vec = vec![-8.0, -7.0, -6.0, -5.0, 4.0, 3.0, 2.0, 1.0]; - assert_eq!(0.0, dot_prod(&a, &b)); + assert_eq!(0.0, dot_prod_0(&a, &b)); + assert_eq!(0.0, dot_prod_1(&a, &b)); + assert_eq!(0.0, dot_prod_simd_0(&a, &b)); + assert_eq!(0.0, dot_prod_simd_1(&a, &b)); } } From 4615805ec2ce44c37792df3b5b179a795f57542b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Sat, 26 Mar 2022 16:10:25 -0600 Subject: [PATCH 042/130] add remainder dot_product and cleanup cleanup dot_product and README.md --- crates/core_simd/examples/README.md | 8 +- crates/core_simd/examples/dot_product.rs | 106 ++++++++++++++++++++--- 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/crates/core_simd/examples/README.md b/crates/core_simd/examples/README.md index b37dffa8eaa..82747f1b5a6 100644 --- a/crates/core_simd/examples/README.md +++ b/crates/core_simd/examples/README.md @@ -10,10 +10,4 @@ Run the tests with the command cargo run --example dot_product ``` -and the benchmarks via the command - -``` -cargo run --example --benchmark ??? -``` - -and measure the timings on your local system. +and verify the code for `dot_product.rs` on your machine. diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 3e415fc4471..ed210192e2a 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -1,39 +1,113 @@ // Code taken from the `packed_simd` crate // Run this code with `cargo test --example dot_product` +//use std::iter::zip; + #![feature(array_chunks)] +#![feature(slice_as_chunks)] +// Add these imports to use the stdsimd library +#![feature(portable_simd)] use core_simd::*; -/// This is your barebones dot product implementation: -/// Take 2 vectors, multiply them element wise and *then* -/// add up the result. In the next example we will see if there -/// is any difference to adding as we go along multiplying. +// This is your barebones dot product implementation: +// Take 2 vectors, multiply them element wise and *then* +// go along the resulting array and add up the result. +// In the next example we will see if there +// is any difference to adding and multiplying in tandem. pub fn dot_prod_0(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); - a.iter() - .zip(b.iter()) - .map(|a, b| a * b) - .sum() + a.iter().zip(b.iter()).map(|(a, b)| a * b).sum() } +// When dealing with SIMD, it is very important to think about the amount +// of data movement and when it happens. We're going over simple computation examples here, and yet +// it is not trivial to understand what may or may not contribute to performance +// changes. Eventually, you will need tools to inspect the generated assembly and confirm your +// hypothesis and benchmarks - we will mention them later on. +// With the use of `fold`, we're doing a multiplication, +// and then adding it to the sum, one element from both vectors at a time. pub fn dot_prod_1(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); a.iter() - .zip(b.iter()) - .fold(0.0, |a, b| a * b) + .zip(b.iter()) + .fold(0.0, |a, zipped| a + zipped.0 * zipped.1) } +// We now move on to the SIMD implementations: notice the following constructs: +// `array_chunks::<4>`: mapping this over the vector will let use construct SIMD vectors +// `f32x4::from_array`: construct the SIMD vector from a slice +// `(a * b).reduce_sum()`: Multiply both f32x4 vectors together, and then reduce them. +// This approach essentially uses SIMD to produce a vector of length N/4 of all the products, +// and then add those with `sum()`. This is suboptimal. +// TODO: ASCII diagrams pub fn dot_prod_simd_0(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); - // TODO handle remainder when a.len() % 4 != 0 a.array_chunks::<4>() .map(|&a| f32x4::from_array(a)) .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) - .map(|(a, b)| (a * b).horizontal_sum()) + .map(|(a, b)| (a * b).reduce_sum()) .sum() } +// There's some simple ways to improve the previous code: +// 1. Make a `zero` `f32x4` SIMD vector that we will be accumulating into +// So that there is only one `sum()` reduction when the last `f32x4` has been processed +// 2. Exploit Fused Multiply Add so that the multiplication, addition and sinking into the reduciton +// happen in the same step. +// If the arrays are large, minimizing the data shuffling will lead to great perf. +// If the arrays are small, handling the remainder elements when the length isn't a multiple of 4 +// Can become a problem. +pub fn dot_prod_simd_1(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + // TODO handle remainder when a.len() % 4 != 0 + a.array_chunks::<4>() + .map(|&a| f32x4::from_array(a)) + .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .fold(f32x4::splat(0.0), |acc, zipped| acc + zipped.0 * zipped.1) + .reduce_sum() +} + +// A lot of knowledgeable use of SIMD comes from knowing specific instructions that are +// available - let's try to use the `mul_add` instruction, which is the fused-multiply-add we were looking for. +use std_float::StdFloat; +pub fn dot_prod_simd_2(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + // TODO handle remainder when a.len() % 4 != 0 + let mut res = f32x4::splat(0.0); + a.array_chunks::<4>() + .map(|&a| f32x4::from_array(a)) + .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .for_each(|(a, b)| { + res = a.mul_add(b, res); + }); + res.reduce_sum() +} + +// Finally, we will write the same operation but handling the loop remainder. +const LANES: usize = 4; +pub fn dot_prod_simd_3(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + + let (a_extra, a_chunks) = a.as_rchunks(); + let (b_extra, b_chunks) = b.as_rchunks(); + + // These are always true, but for emphasis: + assert_eq!(a_chunks.len(), b_chunks.len()); + assert_eq!(a_extra.len(), b_extra.len()); + + let mut sums = [0.0; LANES]; + for ((x, y), d) in std::iter::zip(a_extra, b_extra).zip(&mut sums) { + *d = x * y; + } + + let mut sums = f32x4::from_array(sums); + std::iter::zip(a_chunks, b_chunks).for_each(|(x, y)| { + sums += f32x4::from_array(*x) * f32x4::from_array(*y); + }); + + sums.reduce_sum() +} fn main() { // Empty main to make cargo happy } @@ -45,10 +119,18 @@ mod tests { use super::*; let a: Vec = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let b: Vec = vec![-8.0, -7.0, -6.0, -5.0, 4.0, 3.0, 2.0, 1.0]; + let x: Vec = [0.5; 1003].to_vec(); + let y: Vec = [2.0; 1003].to_vec(); + // Basic check assert_eq!(0.0, dot_prod_0(&a, &b)); assert_eq!(0.0, dot_prod_1(&a, &b)); assert_eq!(0.0, dot_prod_simd_0(&a, &b)); assert_eq!(0.0, dot_prod_simd_1(&a, &b)); + assert_eq!(0.0, dot_prod_simd_2(&a, &b)); + assert_eq!(0.0, dot_prod_simd_3(&a, &b)); + + // We can handle vectors that are non-multiples of 4 + assert_eq!(1003.0, dot_prod_simd_3(&x, &y)); } } From 4ddfd2f3f8c547fa7c42a0f9a5979665262a30c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Tue, 29 Mar 2022 16:52:54 -0600 Subject: [PATCH 043/130] non allocating fold simd allocating fold with std::ops::Add::add --- crates/core_simd/examples/dot_product.rs | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index ed210192e2a..75d628ee392 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -108,6 +108,37 @@ pub fn dot_prod_simd_3(a: &[f32], b: &[f32]) -> f32 { sums.reduce_sum() } + +// Finally, we present an iterator version for handling remainders in a scalar fashion at the end of the loop. +// Unfortunately, this is allocating 1 `XMM` register on the order of `~len(a)` - we'll see how we can get around it in the +// next example. +pub fn dot_prod_simd_4(a: &[f32], b: &[f32]) -> f32 { + let mut sum = a + .array_chunks::<4>() + .map(|&a| f32x4::from_array(a)) + .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .map(|(a, b)| a * b) + .fold(f32x4::splat(0.0), std::ops::Add::add) + .reduce_sum(); + let remain = a.len() - (a.len() % 4); + sum += a[remain..] + .iter() + .zip(&b[remain..]) + .map(|(a, b)| a * b) + .sum::(); + sum +} + +// This version allocates a single `XMM` register for accumulation, and the folds don't allocate on top of that. +// Notice the the use of `mul_add`, which can do a multiply and an add operation ber iteration. +pub fn dot_prod_simd_5(a: &[f32], b: &[f32]) -> f32 { + a.array_chunks::<4>() + .map(|&a| f32x4::from_array(a)) + .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) + .fold(f32x4::splat(0.), |acc, (a, b)| acc.mul_add(a, b)) + .reduce_sum() +} + fn main() { // Empty main to make cargo happy } From aeac9ed37339c463a6a155b12135b7f167611e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Tue, 29 Mar 2022 17:36:47 -0600 Subject: [PATCH 044/130] proper mul_add arg order, added tests --- crates/core_simd/examples/dot_product.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 75d628ee392..84824c2e5c4 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -135,7 +135,7 @@ pub fn dot_prod_simd_5(a: &[f32], b: &[f32]) -> f32 { a.array_chunks::<4>() .map(|&a| f32x4::from_array(a)) .zip(b.array_chunks::<4>().map(|&b| f32x4::from_array(b))) - .fold(f32x4::splat(0.), |acc, (a, b)| acc.mul_add(a, b)) + .fold(f32x4::splat(0.), |acc, (a, b)| a.mul_add(b, acc)) .reduce_sum() } @@ -160,6 +160,8 @@ mod tests { assert_eq!(0.0, dot_prod_simd_1(&a, &b)); assert_eq!(0.0, dot_prod_simd_2(&a, &b)); assert_eq!(0.0, dot_prod_simd_3(&a, &b)); + assert_eq!(0.0, dot_prod_simd_4(&a, &b)); + assert_eq!(0.0, dot_prod_simd_5(&a, &b)); // We can handle vectors that are non-multiples of 4 assert_eq!(1003.0, dot_prod_simd_3(&x, &y)); From 64247a327d30a2d5fe7ad3d98f527bff1cc8fb85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Raz=20Guzm=C3=A1n=20Macedo?= Date: Wed, 30 Mar 2022 17:45:59 -0600 Subject: [PATCH 045/130] add _scalar names for dot_product examples --- crates/core_simd/examples/dot_product.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 84824c2e5c4..936741a2ceb 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -13,7 +13,7 @@ use core_simd::*; // go along the resulting array and add up the result. // In the next example we will see if there // is any difference to adding and multiplying in tandem. -pub fn dot_prod_0(a: &[f32], b: &[f32]) -> f32 { +pub fn dot_prod_scalar_0(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); a.iter().zip(b.iter()).map(|(a, b)| a * b).sum() @@ -26,7 +26,7 @@ pub fn dot_prod_0(a: &[f32], b: &[f32]) -> f32 { // hypothesis and benchmarks - we will mention them later on. // With the use of `fold`, we're doing a multiplication, // and then adding it to the sum, one element from both vectors at a time. -pub fn dot_prod_1(a: &[f32], b: &[f32]) -> f32 { +pub fn dot_prod_scalar_1(a: &[f32], b: &[f32]) -> f32 { assert_eq!(a.len(), b.len()); a.iter() .zip(b.iter()) @@ -154,8 +154,8 @@ mod tests { let y: Vec = [2.0; 1003].to_vec(); // Basic check - assert_eq!(0.0, dot_prod_0(&a, &b)); - assert_eq!(0.0, dot_prod_1(&a, &b)); + assert_eq!(0.0, dot_prod_scalar_0(&a, &b)); + assert_eq!(0.0, dot_prod_scalar_1(&a, &b)); assert_eq!(0.0, dot_prod_simd_0(&a, &b)); assert_eq!(0.0, dot_prod_simd_1(&a, &b)); assert_eq!(0.0, dot_prod_simd_2(&a, &b)); From da3bd6d3a04f84ebc7fc6314f2e1f8a74e379018 Mon Sep 17 00:00:00 2001 From: The Atelier Date: Sat, 3 Dec 2022 18:40:07 -0800 Subject: [PATCH 046/130] Update dot_product example import --- crates/core_simd/examples/dot_product.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/examples/dot_product.rs b/crates/core_simd/examples/dot_product.rs index 936741a2ceb..391f08f55a0 100644 --- a/crates/core_simd/examples/dot_product.rs +++ b/crates/core_simd/examples/dot_product.rs @@ -6,7 +6,7 @@ #![feature(slice_as_chunks)] // Add these imports to use the stdsimd library #![feature(portable_simd)] -use core_simd::*; +use core_simd::simd::*; // This is your barebones dot product implementation: // Take 2 vectors, multiply them element wise and *then* From e3ef226f7b33e7257d0e549046bed44cabfd5585 Mon Sep 17 00:00:00 2001 From: Yang Hau Date: Mon, 23 Jan 2023 11:00:35 +0700 Subject: [PATCH 047/130] Fix the typo --- crates/core_simd/src/masks/to_bitmask.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/masks/to_bitmask.rs b/crates/core_simd/src/masks/to_bitmask.rs index 46914dfe0d9..fc7d6b781f2 100644 --- a/crates/core_simd/src/masks/to_bitmask.rs +++ b/crates/core_simd/src/masks/to_bitmask.rs @@ -72,7 +72,7 @@ impl_integer_intrinsic! { impl ToBitMask for Mask<_, 64> } -/// Returns the minimum numnber of bytes in a bitmask with `lanes` lanes. +/// Returns the minimum number of bytes in a bitmask with `lanes` lanes. #[cfg(feature = "generic_const_exprs")] pub const fn bitmask_len(lanes: usize) -> usize { (lanes + 7) / 8 From 0fd7c8e138db1362e3cba9cdb40403dc7a83364b Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 19 Feb 2023 12:21:27 -0500 Subject: [PATCH 048/130] Add copy_to_slice --- crates/core_simd/src/vector.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 51b0d999a81..870c2eefee1 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -159,7 +159,7 @@ where /// /// Panics if the slice's length is less than the vector's `Simd::LANES`. /// - /// # Examples + /// # Example /// /// ``` /// # #![feature(portable_simd)] @@ -180,6 +180,35 @@ where unsafe { slice.as_ptr().cast::().read_unaligned() } } + /// Writes a SIMD vector to the first `LANES` elements of a slice. + /// + /// # Panics + /// + /// Panics if the slice's length is less than the vector's `Simd::LANES`. + /// + /// # Example + /// + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd; + /// # use simd::u32x4; + /// let mut dest = vec![0; 6]; + /// let v = u32x4::from_array([1, 2, 3, 4]); + /// v.copy_to_slice(&mut dest); + /// assert_eq!(&dest, &[1, 2, 3, 4, 0, 0]); + /// ``` + pub fn copy_to_slice(self, slice: &mut [T]) { + assert!( + slice.len() >= LANES, + "slice length must be at least the number of lanes" + ); + // Safety: + // - We've checked the length is sufficient + // - `T` and `Simd` are Copy types. + unsafe { slice.as_mut_ptr().cast::().write_unaligned(self) } + } + /// Performs lanewise conversion of a SIMD vector's elements to another SIMD-valid type. /// /// This follows the semantics of Rust's `as` conversion for casting From 36829ddca7de02b4d8bad31bdfb0fbc83664017b Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 19 Feb 2023 15:35:36 -0500 Subject: [PATCH 049/130] Check that vectors aren't padded --- crates/core_simd/src/vector.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 870c2eefee1..3e39f1d623c 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -174,6 +174,7 @@ where slice.len() >= LANES, "slice length must be at least the number of lanes" ); + assert!(core::mem::size_of::() == LANES * core::mem::size_of::()); // Safety: // - We've checked the length is sufficient. // - `T` and `Simd` are Copy types. @@ -203,6 +204,7 @@ where slice.len() >= LANES, "slice length must be at least the number of lanes" ); + assert!(core::mem::size_of::() == LANES * core::mem::size_of::()); // Safety: // - We've checked the length is sufficient // - `T` and `Simd` are Copy types. From 65b5210bdbb3a7af57e5c39d41424ba260ee3fbc Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 26 Mar 2023 12:56:35 +0200 Subject: [PATCH 050/130] Skip building wasm-bindgen-test on non-wasm targets This reduces compilation time --- crates/core_simd/Cargo.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/core_simd/Cargo.toml b/crates/core_simd/Cargo.toml index 7435e24edd3..d1a3a515a7e 100644 --- a/crates/core_simd/Cargo.toml +++ b/crates/core_simd/Cargo.toml @@ -15,11 +15,9 @@ std = [] generic_const_exprs = [] all_lane_counts = [] -[target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen] -version = "0.2" - -[dev-dependencies.wasm-bindgen-test] -version = "0.3" +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +wasm-bindgen = "0.2" +wasm-bindgen-test = "0.3" [dev-dependencies.proptest] version = "0.10" From 90f2af774ae3149ad52ec6bb2d48649b72844a2c Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 26 Mar 2023 16:11:05 -0400 Subject: [PATCH 051/130] Fix lint --- crates/test_helpers/src/array.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/test_helpers/src/array.rs b/crates/test_helpers/src/array.rs index 5ffc9226976..984a427320d 100644 --- a/crates/test_helpers/src/array.rs +++ b/crates/test_helpers/src/array.rs @@ -41,6 +41,7 @@ where fn new_tree(&self, runner: &mut TestRunner) -> NewTree { let tree: [S::Tree; LANES] = unsafe { + #[allow(clippy::uninit_assumed_init)] let mut tree: [MaybeUninit; LANES] = MaybeUninit::uninit().assume_init(); for t in tree.iter_mut() { *t = MaybeUninit::new(self.strategy.new_tree(runner)?) @@ -60,6 +61,7 @@ impl ValueTree for ArrayValueTree<[T; LANES]> fn current(&self) -> Self::Value { unsafe { + #[allow(clippy::uninit_assumed_init)] let mut value: [MaybeUninit; LANES] = MaybeUninit::uninit().assume_init(); for (tree_elem, value_elem) in self.tree.iter().zip(value.iter_mut()) { *value_elem = MaybeUninit::new(tree_elem.current()); From ceb26115928c5c69b10268fd2f9e500865c142d6 Mon Sep 17 00:00:00 2001 From: Jubilee <46493976+workingjubilee@users.noreply.github.com> Date: Sun, 9 Apr 2023 21:26:40 -0700 Subject: [PATCH 052/130] Remove formats `[T; N]` does not impl (rust-lang/portable-simd#337) Remove these extra formatting traits, as they are inconsistent with how arrays and slices format, and it can cause unnecessary code bloat in binaries. We can revisit this if people ever agree on doing these formatters for the other slice-y types. Prefer to dispatch to the `impl `fmt::Debug for [T]`, to reduce the chances of monomorphizing twice. Inlining it seems like a good idea for similar reasons? --- crates/core_simd/src/fmt.rs | 50 ++++++++++++------------------------- 1 file changed, 16 insertions(+), 34 deletions(-) diff --git a/crates/core_simd/src/fmt.rs b/crates/core_simd/src/fmt.rs index dbd9839c4bf..b7317969cbb 100644 --- a/crates/core_simd/src/fmt.rs +++ b/crates/core_simd/src/fmt.rs @@ -1,39 +1,21 @@ use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount}; use core::fmt; -macro_rules! impl_fmt_trait { - { $($trait:ident,)* } => { - $( - impl fmt::$trait for Simd - where - LaneCount: SupportedLaneCount, - T: SimdElement + fmt::$trait, - { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - #[repr(transparent)] - struct Wrapper<'a, T: fmt::$trait>(&'a T); - - impl fmt::Debug for Wrapper<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } - } - - f.debug_list() - .entries(self.as_array().iter().map(|x| Wrapper(x))) - .finish() - } - } - )* +impl fmt::Debug for Simd +where + LaneCount: SupportedLaneCount, + T: SimdElement + fmt::Debug, +{ + /// A `Simd` has a debug format like the one for `[T]`: + /// ``` + /// # #![feature(portable_simd)] + /// # #[cfg(feature = "as_crate")] use core_simd::simd::Simd; + /// # #[cfg(not(feature = "as_crate"))] use core::simd::Simd; + /// let floats = Simd::::splat(-1.0); + /// assert_eq!(format!("{:?}", [-1.0; 4]), format!("{:?}", floats)); + /// ``` + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + <[T] as fmt::Debug>::fmt(self.as_array(), f) } } - -impl_fmt_trait! { - Debug, - Binary, - LowerExp, - UpperExp, - Octal, - LowerHex, - UpperHex, -} From 09e1fae118bce2530ef34e50f666b46cc2070e41 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 28 Dec 2022 17:10:21 +0000 Subject: [PATCH 053/130] Support linking to rust dylibs from a staticlib --- .../rustc_metadata/src/dependency_format.rs | 23 +++++++--------- .../run-make/staticlib-dylib-linkage/Makefile | 27 +++++++++++++++++++ tests/run-make/staticlib-dylib-linkage/bar.rs | 5 ++++ tests/run-make/staticlib-dylib-linkage/foo.c | 10 +++++++ tests/run-make/staticlib-dylib-linkage/foo.rs | 13 +++++++++ 5 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 tests/run-make/staticlib-dylib-linkage/Makefile create mode 100644 tests/run-make/staticlib-dylib-linkage/bar.rs create mode 100644 tests/run-make/staticlib-dylib-linkage/foo.c create mode 100644 tests/run-make/staticlib-dylib-linkage/foo.rs diff --git a/compiler/rustc_metadata/src/dependency_format.rs b/compiler/rustc_metadata/src/dependency_format.rs index 39ef4276faf..590a7374d11 100644 --- a/compiler/rustc_metadata/src/dependency_format.rs +++ b/compiler/rustc_metadata/src/dependency_format.rs @@ -89,11 +89,12 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList { // to try to eagerly statically link all dependencies. This is normally // done for end-product dylibs, not intermediate products. // - // Treat cdylibs similarly. If `-C prefer-dynamic` is set, the caller may - // be code-size conscious, but without it, it makes sense to statically - // link a cdylib. - CrateType::Dylib | CrateType::Cdylib if !sess.opts.cg.prefer_dynamic => Linkage::Static, - CrateType::Dylib | CrateType::Cdylib => Linkage::Dynamic, + // Treat cdylibs and staticlibs similarly. If `-C prefer-dynamic` is set, + // the caller may be code-size conscious, but without it, it makes sense + // to statically link a cdylib or staticlib. + CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { + if sess.opts.cg.prefer_dynamic { Linkage::Dynamic } else { Linkage::Static } + } // If the global prefer_dynamic switch is turned off, or the final // executable will be statically linked, prefer static crate linkage. @@ -108,9 +109,6 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList { // No linkage happens with rlibs, we just needed the metadata (which we // got long ago), so don't bother with anything. CrateType::Rlib => Linkage::NotLinked, - - // staticlibs must have all static dependencies. - CrateType::Staticlib => Linkage::Static, }; match preferred_linkage { @@ -123,12 +121,11 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList { return v; } - // Staticlibs and static executables must have all static dependencies. + // Static executables must have all static dependencies. // If any are not found, generate some nice pretty errors. - if ty == CrateType::Staticlib - || (ty == CrateType::Executable - && sess.crt_static(Some(ty)) - && !sess.target.crt_static_allows_dylibs) + if ty == CrateType::Executable + && sess.crt_static(Some(ty)) + && !sess.target.crt_static_allows_dylibs { for &cnum in tcx.crates(()).iter() { if tcx.dep_kind(cnum).macros_only() { diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile new file mode 100644 index 00000000000..bf811395981 --- /dev/null +++ b/tests/run-make/staticlib-dylib-linkage/Makefile @@ -0,0 +1,27 @@ +include ../tools.mk + +TARGET_SYSROOT := $(shell $(RUSTC) --print sysroot)/lib/rustlib/$(TARGET)/lib + +ifdef IS_MSVC +LIBSTD := $(wildcard $(TARGET_SYSROOT)/libstd-*.dll.lib) +else +LIBSTD := $(wildcard $(TARGET_SYSROOT)/$(call DYLIB_GLOB,std)) +STD := $(basename $(patsubst lib%,%, $(notdir $(LIBSTD)))) +endif + +all: $(call RUN_BINFILE,foo) + $(call RUN,foo) + +ifdef IS_MSVC +CLIBS := $(TMPDIR)/foo.lib $(TMPDIR)/bar.dll.lib $(LIBSTD) +$(call RUN_BINFILE,foo): $(call STATICLIB,foo) + $(CC) $(CFLAGS) foo.c $(CLIBS) $(call OUT_EXE,foo) +else +CLIBS := $(TMPDIR)/libfoo.a -lbar -l$(STD) -L $(TMPDIR) -L $(TARGET_SYSROOT) +$(call RUN_BINFILE,foo): $(call STATICLIB,foo) + $(CC) $(CFLAGS) foo.c $(CLIBS) -o $(call RUN_BINFILE,foo) +endif + +$(call STATICLIB,foo): + $(RUSTC) -C prefer-dynamic bar.rs + $(RUSTC) foo.rs diff --git a/tests/run-make/staticlib-dylib-linkage/bar.rs b/tests/run-make/staticlib-dylib-linkage/bar.rs new file mode 100644 index 00000000000..b3a7539abae --- /dev/null +++ b/tests/run-make/staticlib-dylib-linkage/bar.rs @@ -0,0 +1,5 @@ +#![crate_type = "dylib"] + +pub fn bar() { + println!("hello!"); +} diff --git a/tests/run-make/staticlib-dylib-linkage/foo.c b/tests/run-make/staticlib-dylib-linkage/foo.c new file mode 100644 index 00000000000..154f9682ef8 --- /dev/null +++ b/tests/run-make/staticlib-dylib-linkage/foo.c @@ -0,0 +1,10 @@ +#include + +extern void foo(); +extern unsigned bar(unsigned a, unsigned b); + +int main() { + foo(); + assert(bar(1, 2) == 3); + return 0; +} diff --git a/tests/run-make/staticlib-dylib-linkage/foo.rs b/tests/run-make/staticlib-dylib-linkage/foo.rs new file mode 100644 index 00000000000..af439391c75 --- /dev/null +++ b/tests/run-make/staticlib-dylib-linkage/foo.rs @@ -0,0 +1,13 @@ +#![crate_type = "staticlib"] + +extern crate bar; + +#[no_mangle] +pub extern "C" fn foo() { + bar::bar(); +} + +#[no_mangle] +pub extern "C" fn bar(a: u32, b: u32) -> u32 { + a + b +} From 39ba9dadeed12f67e44ae00ebdcbcebc0edf3d61 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 28 Dec 2022 18:13:19 +0000 Subject: [PATCH 054/130] Support `--print native-static-libs` with rust dylibs --- compiler/rustc_codegen_ssa/src/back/link.rs | 75 +++++++++++++++++-- .../run-make/staticlib-dylib-linkage/Makefile | 35 +++------ 2 files changed, 81 insertions(+), 29 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 02e21e74fad..5144319752a 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -544,10 +544,36 @@ fn link_staticlib<'a>( ab.build(out_filename); - if !all_native_libs.is_empty() { - if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) { - print_native_static_libs(sess, &all_native_libs); + let crates = codegen_results.crate_info.used_crates.iter(); + + let fmts = codegen_results + .crate_info + .dependency_formats + .iter() + .find_map(|&(ty, ref list)| if ty == CrateType::Staticlib { Some(list) } else { None }) + .expect("no dependency formats for staticlib"); + + let mut all_rust_dylibs = vec![]; + for &cnum in crates { + match fmts.get(cnum.as_usize() - 1) { + Some(&Linkage::Dynamic) => {} + _ => continue, } + let crate_name = codegen_results.crate_info.crate_name[&cnum]; + let used_crate_source = &codegen_results.crate_info.used_crate_source[&cnum]; + if let Some((path, _)) = &used_crate_source.dylib { + all_rust_dylibs.push(&**path); + } else { + if used_crate_source.rmeta.is_some() { + sess.emit_fatal(errors::LinkRlibError::OnlyRmetaFound { crate_name }); + } else { + sess.emit_fatal(errors::LinkRlibError::NotFound { crate_name }); + } + } + } + + if sess.opts.prints.contains(&PrintRequest::NativeStaticLibs) { + print_native_static_libs(sess, &all_native_libs, &all_rust_dylibs); } Ok(()) @@ -1291,8 +1317,12 @@ enum RlibFlavor { StaticlibBase, } -fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) { - let lib_args: Vec<_> = all_native_libs +fn print_native_static_libs( + sess: &Session, + all_native_libs: &[NativeLib], + all_rust_dylibs: &[&Path], +) { + let mut lib_args: Vec<_> = all_native_libs .iter() .filter(|l| relevant_lib(sess, l)) .filter_map(|lib| { @@ -1322,6 +1352,41 @@ fn print_native_static_libs(sess: &Session, all_native_libs: &[NativeLib]) { } }) .collect(); + for path in all_rust_dylibs { + // FIXME deduplicate with add_dynamic_crate + + // Just need to tell the linker about where the library lives and + // what its name is + let parent = path.parent(); + if let Some(dir) = parent { + let dir = fix_windows_verbatim_for_gcc(dir); + if sess.target.is_like_msvc { + let mut arg = String::from("/LIBPATH:"); + arg.push_str(&dir.display().to_string()); + lib_args.push(arg); + } else { + lib_args.push("-L".to_owned()); + lib_args.push(dir.display().to_string()); + } + } + let stem = path.file_stem().unwrap().to_str().unwrap(); + // Convert library file-stem into a cc -l argument. + let prefix = if stem.starts_with("lib") && !sess.target.is_like_windows { 3 } else { 0 }; + let lib = &stem[prefix..]; + let path = parent.unwrap_or_else(|| Path::new("")); + if sess.target.is_like_msvc { + // When producing a dll, the MSVC linker may not actually emit a + // `foo.lib` file if the dll doesn't actually export any symbols, so we + // check to see if the file is there and just omit linking to it if it's + // not present. + let name = format!("{}.dll.lib", lib); + if path.join(&name).exists() { + lib_args.push(name); + } + } else { + lib_args.push(format!("-l{}", lib)); + } + } if !lib_args.is_empty() { sess.emit_note(errors::StaticLibraryNativeArtifacts); // Prefix for greppability diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile index bf811395981..faa45243104 100644 --- a/tests/run-make/staticlib-dylib-linkage/Makefile +++ b/tests/run-make/staticlib-dylib-linkage/Makefile @@ -1,27 +1,14 @@ include ../tools.mk -TARGET_SYSROOT := $(shell $(RUSTC) --print sysroot)/lib/rustlib/$(TARGET)/lib - -ifdef IS_MSVC -LIBSTD := $(wildcard $(TARGET_SYSROOT)/libstd-*.dll.lib) -else -LIBSTD := $(wildcard $(TARGET_SYSROOT)/$(call DYLIB_GLOB,std)) -STD := $(basename $(patsubst lib%,%, $(notdir $(LIBSTD)))) -endif - -all: $(call RUN_BINFILE,foo) - $(call RUN,foo) - -ifdef IS_MSVC -CLIBS := $(TMPDIR)/foo.lib $(TMPDIR)/bar.dll.lib $(LIBSTD) -$(call RUN_BINFILE,foo): $(call STATICLIB,foo) - $(CC) $(CFLAGS) foo.c $(CLIBS) $(call OUT_EXE,foo) -else -CLIBS := $(TMPDIR)/libfoo.a -lbar -l$(STD) -L $(TMPDIR) -L $(TARGET_SYSROOT) -$(call RUN_BINFILE,foo): $(call STATICLIB,foo) - $(CC) $(CFLAGS) foo.c $(CLIBS) -o $(call RUN_BINFILE,foo) -endif - -$(call STATICLIB,foo): +all: $(RUSTC) -C prefer-dynamic bar.rs - $(RUSTC) foo.rs + $(RUSTC) foo.rs --crate-type staticlib --print native-static-libs 2>&1 | grep 'note: native-static-libs: ' | sed 's/note: native-static-libs: \(.*\)/\1/' > $(TMPDIR)/libs.txt + cat $(TMPDIR)/libs.txt + +ifdef IS_MSVC + $(CC) $(CFLAGS) foo.c $(TMPDIR)/foo.lib $$(cat $(TMPDIR)/libs.txt) $(call OUT_EXE,foo) +else + $(CC) $(CFLAGS) foo.c -L $(TMPDIR) -lfoo $$(cat $(TMPDIR)/libs.txt) -o $(call RUN_BINFILE,foo) +endif + + $(call RUN,foo) From 83f96e81423b63af89d04899e4710ca883c716f2 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 4 Jan 2023 20:24:03 +0000 Subject: [PATCH 055/130] Add unstable feature flags --- .../rustc_metadata/src/dependency_format.rs | 26 ++++++++++++++----- compiler/rustc_session/src/options.rs | 4 +++ .../run-make/staticlib-dylib-linkage/Makefile | 4 ++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_metadata/src/dependency_format.rs b/compiler/rustc_metadata/src/dependency_format.rs index 590a7374d11..72b208a7132 100644 --- a/compiler/rustc_metadata/src/dependency_format.rs +++ b/compiler/rustc_metadata/src/dependency_format.rs @@ -91,9 +91,22 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList { // // Treat cdylibs and staticlibs similarly. If `-C prefer-dynamic` is set, // the caller may be code-size conscious, but without it, it makes sense - // to statically link a cdylib or staticlib. - CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { - if sess.opts.cg.prefer_dynamic { Linkage::Dynamic } else { Linkage::Static } + // to statically link a cdylib or staticlib. For staticlibs we use + // `-Z staticlib-prefer-dynamic` for now. This may be merged into + // `-C prefer-dynamic` in the future. + CrateType::Dylib | CrateType::Cdylib => { + if sess.opts.cg.prefer_dynamic { + Linkage::Dynamic + } else { + Linkage::Static + } + } + CrateType::Staticlib => { + if sess.opts.unstable_opts.staticlib_prefer_dynamic { + Linkage::Dynamic + } else { + Linkage::Static + } } // If the global prefer_dynamic switch is turned off, or the final @@ -123,9 +136,10 @@ fn calculate_type(tcx: TyCtxt<'_>, ty: CrateType) -> DependencyList { // Static executables must have all static dependencies. // If any are not found, generate some nice pretty errors. - if ty == CrateType::Executable - && sess.crt_static(Some(ty)) - && !sess.target.crt_static_allows_dylibs + if (ty == CrateType::Staticlib && !sess.opts.unstable_opts.staticlib_allow_rdylib_deps) + || (ty == CrateType::Executable + && sess.crt_static(Some(ty)) + && !sess.target.crt_static_allows_dylibs) { for &cnum in tcx.crates(()).iter() { if tcx.dep_kind(cnum).macros_only() { diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 631dd0a2146..0743392b19a 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1709,6 +1709,10 @@ options! { #[rustc_lint_opt_deny_field_access("use `Session::stack_protector` instead of this field")] stack_protector: StackProtector = (StackProtector::None, parse_stack_protector, [TRACKED], "control stack smash protection strategy (`rustc --print stack-protector-strategies` for details)"), + staticlib_allow_rdylib_deps: bool = (false, parse_bool, [TRACKED], + "allow staticlibs to have rust dylib dependencies"), + staticlib_prefer_dynamic: bool = (false, parse_bool, [TRACKED], + "prefer dynamic linking to static linking for staticlibs (default: no)"), strict_init_checks: bool = (false, parse_bool, [TRACKED], "control if mem::uninitialized and mem::zeroed panic on more UB"), strip: Strip = (Strip::None, parse_strip, [UNTRACKED], diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile index faa45243104..fd76f6c5578 100644 --- a/tests/run-make/staticlib-dylib-linkage/Makefile +++ b/tests/run-make/staticlib-dylib-linkage/Makefile @@ -2,7 +2,9 @@ include ../tools.mk all: $(RUSTC) -C prefer-dynamic bar.rs - $(RUSTC) foo.rs --crate-type staticlib --print native-static-libs 2>&1 | grep 'note: native-static-libs: ' | sed 's/note: native-static-libs: \(.*\)/\1/' > $(TMPDIR)/libs.txt + $(RUSTC) foo.rs --crate-type staticlib --print native-static-libs \ + -Z staticlib-allow-rdylib-deps 2>&1 | grep 'note: native-static-libs: ' \ + | sed 's/note: native-static-libs: \(.*\)/\1/' > $(TMPDIR)/libs.txt cat $(TMPDIR)/libs.txt ifdef IS_MSVC From d6fc3667dd3a3cd71897eba33c952077c24a65bd Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 4 Mar 2023 11:06:33 +0000 Subject: [PATCH 056/130] Ignore test on MSVC for now I can't figure out how to link with the MSVC toolchain --- tests/run-make/staticlib-dylib-linkage/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile index fd76f6c5578..c3570a46327 100644 --- a/tests/run-make/staticlib-dylib-linkage/Makefile +++ b/tests/run-make/staticlib-dylib-linkage/Makefile @@ -1,5 +1,7 @@ include ../tools.mk +# ignore-msvc FIXME(bjorn3) can't figure out how to link with the MSVC toolchain + all: $(RUSTC) -C prefer-dynamic bar.rs $(RUSTC) foo.rs --crate-type staticlib --print native-static-libs \ @@ -8,7 +10,8 @@ all: cat $(TMPDIR)/libs.txt ifdef IS_MSVC - $(CC) $(CFLAGS) foo.c $(TMPDIR)/foo.lib $$(cat $(TMPDIR)/libs.txt) $(call OUT_EXE,foo) + $(CC) $(CFLAGS) /c foo.c /Fo:$(TMPDIR)/foo.o + $(RUSTC_LINKER) $(TMPDIR)/foo.o $(TMPDIR)/foo.lib $$(cat $(TMPDIR)/libs.txt) $(call OUT_EXE,foo) else $(CC) $(CFLAGS) foo.c -L $(TMPDIR) -lfoo $$(cat $(TMPDIR)/libs.txt) -o $(call RUN_BINFILE,foo) endif From 2a3ef11e3b21d6b16b50cac5198c0ba6d70f954e Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:09:18 +0000 Subject: [PATCH 057/130] Ignore test on wasm as dylibs aren't supported --- tests/run-make/staticlib-dylib-linkage/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile index c3570a46327..08e4fa5d3aa 100644 --- a/tests/run-make/staticlib-dylib-linkage/Makefile +++ b/tests/run-make/staticlib-dylib-linkage/Makefile @@ -1,6 +1,7 @@ include ../tools.mk # ignore-msvc FIXME(bjorn3) can't figure out how to link with the MSVC toolchain +# ignore-wasm wasm doesn't support dynamic libraries all: $(RUSTC) -C prefer-dynamic bar.rs From afad9c3f644ddbfef3301f617cb9d23ca4e71fe0 Mon Sep 17 00:00:00 2001 From: Markus Everling Date: Sat, 22 Apr 2023 21:12:35 +0000 Subject: [PATCH 058/130] Don't use direct field access in `Simd` functions --- crates/core_simd/src/lib.rs | 2 ++ crates/core_simd/src/vector.rs | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 927b1654f8e..a372e2e40c4 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -1,6 +1,8 @@ #![no_std] #![feature( const_ptr_read, + const_refs_to_cell, + const_transmute_copy, convert_float_to_int, decl_macro, intra_doc_pointers, diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 3e39f1d623c..c1af4af5f57 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -135,22 +135,32 @@ where /// assert_eq!(v.as_array(), &[0, 1, 2, 3]); /// ``` pub const fn as_array(&self) -> &[T; LANES] { - &self.0 + // SAFETY: Transmuting between `Simd` and `[T; LANES]` + // is always valid and `Simd` never has a lower alignment + // than `[T; LANES]`. + unsafe { &*(self as *const Self as *const [T; LANES]) } } /// Returns a mutable array reference containing the entire SIMD vector. pub fn as_mut_array(&mut self) -> &mut [T; LANES] { - &mut self.0 + // SAFETY: Transmuting between `Simd` and `[T; LANES]` + // is always valid and `Simd` never has a lower alignment + // than `[T; LANES]`. + unsafe { &mut *(self as *mut Self as *mut [T; LANES]) } } /// Converts an array to a SIMD vector. pub const fn from_array(array: [T; LANES]) -> Self { - Self(array) + // SAFETY: Transmuting between `Simd` and `[T; LANES]` + // is always valid. + unsafe { core::mem::transmute_copy(&array) } } /// Converts a SIMD vector to an array. pub const fn to_array(self) -> [T; LANES] { - self.0 + // SAFETY: Transmuting between `Simd` and `[T; LANES]` + // is always valid. + unsafe { core::mem::transmute_copy(&self) } } /// Converts a slice to a SIMD vector containing `slice[..LANES]`. @@ -735,7 +745,7 @@ where { #[inline] fn as_ref(&self) -> &[T; LANES] { - &self.0 + self.as_array() } } @@ -746,7 +756,7 @@ where { #[inline] fn as_mut(&mut self) -> &mut [T; LANES] { - &mut self.0 + self.as_mut_array() } } @@ -758,7 +768,7 @@ where { #[inline] fn as_ref(&self) -> &[T] { - &self.0 + self.as_array() } } @@ -769,7 +779,7 @@ where { #[inline] fn as_mut(&mut self) -> &mut [T] { - &mut self.0 + self.as_mut_array() } } From 52833ccbe88ed98b73d0ccd7299f2a667439bb4b Mon Sep 17 00:00:00 2001 From: Markus Everling Date: Sat, 22 Apr 2023 23:02:45 +0000 Subject: [PATCH 059/130] Add notes to avoid direct field accesses --- crates/core_simd/src/vector.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index c1af4af5f57..eee105ff5fc 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -76,6 +76,11 @@ use crate::simd::{ /// [`read`]: pointer::read /// [`write`]: pointer::write /// [as_simd]: slice::as_simd +// +// NOTE: Accessing the inner array directly in any way (e.g. by using the `.0` field syntax) or +// directly constructing an instance of the type (i.e. `let vector = Simd(array)`) should be +// avoided, as it will likely become illegal on `#[repr(simd)]` structs in the future. It also +// causes rustc to emit illegal LLVM IR in some cases. #[repr(simd)] pub struct Simd([T; LANES]) where @@ -138,6 +143,9 @@ where // SAFETY: Transmuting between `Simd` and `[T; LANES]` // is always valid and `Simd` never has a lower alignment // than `[T; LANES]`. + // + // NOTE: This deliberately doesn't just use `&self.0`, see the comment + // on the struct definition for details. unsafe { &*(self as *const Self as *const [T; LANES]) } } @@ -146,6 +154,9 @@ where // SAFETY: Transmuting between `Simd` and `[T; LANES]` // is always valid and `Simd` never has a lower alignment // than `[T; LANES]`. + // + // NOTE: This deliberately doesn't just use `&mut self.0`, see the comment + // on the struct definition for details. unsafe { &mut *(self as *mut Self as *mut [T; LANES]) } } @@ -153,6 +164,9 @@ where pub const fn from_array(array: [T; LANES]) -> Self { // SAFETY: Transmuting between `Simd` and `[T; LANES]` // is always valid. + // + // NOTE: This deliberately doesn't just use `Self(array)`, see the comment + // on the struct definition for details. unsafe { core::mem::transmute_copy(&array) } } @@ -160,6 +174,9 @@ where pub const fn to_array(self) -> [T; LANES] { // SAFETY: Transmuting between `Simd` and `[T; LANES]` // is always valid. + // + // NOTE: This deliberately doesn't just use `self.0`, see the comment + // on the struct definition for details. unsafe { core::mem::transmute_copy(&self) } } From f1b86baf8453733c72e196ce2c08b4d85e94d81a Mon Sep 17 00:00:00 2001 From: Markus Everling Date: Sat, 22 Apr 2023 23:22:39 +0000 Subject: [PATCH 060/130] Use pointer reads for better codegen in debug mode --- crates/core_simd/src/lib.rs | 1 - crates/core_simd/src/vector.rs | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index a372e2e40c4..e054d483ca5 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -2,7 +2,6 @@ #![feature( const_ptr_read, const_refs_to_cell, - const_transmute_copy, convert_float_to_int, decl_macro, intra_doc_pointers, diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index eee105ff5fc..a38d701588c 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -163,21 +163,31 @@ where /// Converts an array to a SIMD vector. pub const fn from_array(array: [T; LANES]) -> Self { // SAFETY: Transmuting between `Simd` and `[T; LANES]` - // is always valid. + // is always valid. We need to use `read_unaligned` here, since + // the array may have a lower alignment than the vector. + // + // FIXME: We currently use a pointer read instead of `transmute_copy` because + // it results in better codegen with optimizations disabled, but we should + // probably just use `transmute` once that works on const generic types. // // NOTE: This deliberately doesn't just use `Self(array)`, see the comment // on the struct definition for details. - unsafe { core::mem::transmute_copy(&array) } + unsafe { (&array as *const [T; LANES] as *const Self).read_unaligned() } } /// Converts a SIMD vector to an array. pub const fn to_array(self) -> [T; LANES] { // SAFETY: Transmuting between `Simd` and `[T; LANES]` - // is always valid. + // is always valid. No need to use `read_unaligned` here, since + // the vector never has a lower alignment than the array. + // + // FIXME: We currently use a pointer read instead of `transmute_copy` because + // it results in better codegen with optimizations disabled, but we should + // probably just use `transmute` once that works on const generic types. // // NOTE: This deliberately doesn't just use `self.0`, see the comment // on the struct definition for details. - unsafe { core::mem::transmute_copy(&self) } + unsafe { (&self as *const Self as *const [T; LANES]).read() } } /// Converts a slice to a SIMD vector containing `slice[..LANES]`. From 71d4c368509536f7277e9a1cb6e6286ba6de7911 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Fri, 17 Mar 2023 17:56:45 -0700 Subject: [PATCH 061/130] lane -> element for core::simd::Simd A while ago we began saying T, N instead of T, LANES in reference to Simd. At some point that leaked in to us checking in code with const N: usize. After a while, we had a discussion and agreed that "lanes", while common, is unnecessary jargon for Rust learners who aren't familiar with SIMD, and is fully interchangeable with terms for arrays like element and index. But we never acted on that. Let's update the main type's docs, at least. The example tweaks also enable removing a slated-for-removal nightly fn. --- crates/core_simd/src/vector.rs | 390 +++++++++++++++++---------------- 1 file changed, 199 insertions(+), 191 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index a38d701588c..154b467752b 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -3,48 +3,55 @@ use crate::simd::{ SimdPartialOrd, SupportedLaneCount, Swizzle, }; -/// A SIMD vector of `LANES` elements of type `T`. `Simd` has the same shape as [`[T; N]`](array), but operates like `T`. +/// A SIMD vector with the shape of `[T; N]` but the operations of `T`. /// -/// Two vectors of the same type and length will, by convention, support the operators (+, *, etc.) that `T` does. -/// These take the lanes at each index on the left-hand side and right-hand side, perform the operation, -/// and return the result in the same lane in a vector of equal size. For a given operator, this is equivalent to zipping -/// the two arrays together and mapping the operator over each lane. +/// `Simd` supports the operators (+, *, etc.) that `T` does in "elementwise" fashion. +/// These take the element at each index from the left-hand side and right-hand side, +/// perform the operation, then return the result in the same index in a vector of equal size. +/// In other words, an elementwise operation is equivalent to a zip, then map. /// /// ```rust -/// # #![feature(array_zip, portable_simd)] +/// # #![feature(portable_simd)] /// # use core::simd::{Simd}; -/// let a0: [i32; 4] = [-2, 0, 2, 4]; -/// let a1 = [10, 9, 8, 7]; -/// let zm_add = a0.zip(a1).map(|(lhs, rhs)| lhs + rhs); -/// let zm_mul = a0.zip(a1).map(|(lhs, rhs)| lhs * rhs); +/// # use core::array; +/// let a: [i32; 4] = [-2, 0, 2, 4]; +/// let b = [10, 9, 8, 7]; +/// let sum = array::from_fn(|i| a[i] + b[i]); +/// let prod = array::from_fn(|i| a[i] * b[i]); /// /// // `Simd` implements `From<[T; N]> -/// let (v0, v1) = (Simd::from(a0), Simd::from(a1)); +/// let (v, w) = (Simd::from(a), Simd::from(b)); /// // Which means arrays implement `Into>`. -/// assert_eq!(v0 + v1, zm_add.into()); -/// assert_eq!(v0 * v1, zm_mul.into()); +/// assert_eq!(v + w, sum.into()); +/// assert_eq!(v * w, prod.into()); /// ``` /// -/// `Simd` with integers has the quirk that these operations are also inherently wrapping, as if `T` was [`Wrapping`]. +/// +/// `Simd` with integer elements treats operators as wrapping, as if `T` was [`Wrapping`]. /// Thus, `Simd` does not implement `wrapping_add`, because that is the default behavior. /// This means there is no warning on overflows, even in "debug" builds. /// For most applications where `Simd` is appropriate, it is "not a bug" to wrap, /// and even "debug builds" are unlikely to tolerate the loss of performance. /// You may want to consider using explicitly checked arithmetic if such is required. -/// Division by zero still causes a panic, so you may want to consider using floating point numbers if that is unacceptable. +/// Division by zero on integers still causes a panic, so +/// you may want to consider using `f32` or `f64` if that is unacceptable. /// /// [`Wrapping`]: core::num::Wrapping /// /// # Layout -/// `Simd` has a layout similar to `[T; N]` (identical "shapes"), but with a greater alignment. +/// `Simd` has a layout similar to `[T; N]` (identical "shapes"), with a greater alignment. /// `[T; N]` is aligned to `T`, but `Simd` will have an alignment based on both `T` and `N`. -/// It is thus sound to [`transmute`] `Simd` to `[T; N]`, and will typically optimize to zero cost, -/// but the reverse transmutation is more likely to require a copy the compiler cannot simply elide. +/// Thus it is sound to [`transmute`] `Simd` to `[T; N]` and should optimize to "zero cost", +/// but the reverse transmutation may require a copy the compiler cannot simply elide. /// /// # ABI "Features" -/// Due to Rust's safety guarantees, `Simd` is currently passed to and from functions via memory, not SIMD registers, -/// except as an optimization. `#[inline]` hints are recommended on functions that accept `Simd` or return it. -/// The need for this may be corrected in the future. +/// Due to Rust's safety guarantees, `Simd` is currently passed and returned via memory, +/// not SIMD registers, except as an optimization. Using `#[inline]` on functions that accept +/// `Simd` or return it is recommended, at the cost of code generation time, as +/// inlining SIMD-using functions can omit a large function prolog or epilog and thus +/// improve both speed and code size. The need for this may be corrected in the future. +/// +/// Using `#[inline(always)]` still requires additional care. /// /// # Safe SIMD with Unsafe Rust /// @@ -55,18 +62,22 @@ use crate::simd::{ /// Thus, when using `unsafe` Rust to read and write `Simd` through [raw pointers], it is a good idea to first try with /// [`read_unaligned`] and [`write_unaligned`]. This is because: /// - [`read`] and [`write`] require full alignment (in this case, `Simd`'s alignment) -/// - the likely source for reading or destination for writing `Simd` is [`[T]`](slice) and similar types, aligned to `T` -/// - combining these actions would violate the `unsafe` contract and explode the program into a puff of **undefined behavior** -/// - the compiler can implicitly adjust layouts to make unaligned reads or writes fully aligned if it sees the optimization -/// - most contemporary processors suffer no performance penalty for "unaligned" reads and writes that are aligned at runtime +/// - `Simd` is often read from or written to [`[T]`](slice) and other types aligned to `T` +/// - combining these actions violates the `unsafe` contract and explodes the program into +/// a puff of **undefined behavior** +/// - the compiler can implicitly adjust layouts to make unaligned reads or writes fully aligned +/// if it sees the optimization +/// - most contemporary processors with "aligned" and "unaligned" read and write instructions +/// exhibit no performance difference if the "unaligned" variant is aligned at runtime /// -/// By imposing less obligations, unaligned functions are less likely to make the program unsound, +/// Less obligations mean unaligned reads and writes are less likely to make the program unsound, /// and may be just as fast as stricter alternatives. -/// When trying to guarantee alignment, [`[T]::as_simd`][as_simd] is an option for converting `[T]` to `[Simd]`, -/// and allows soundly operating on an aligned SIMD body, but it may cost more time when handling the scalar head and tail. -/// If these are not sufficient, then it is most ideal to design data structures to be already aligned -/// to the `Simd` you wish to use before using `unsafe` Rust to read or write. -/// More conventional ways to compensate for these facts, like materializing `Simd` to or from an array first, +/// When trying to guarantee alignment, [`[T]::as_simd`][as_simd] is an option for +/// converting `[T]` to `[Simd]`, and allows soundly operating on an aligned SIMD body, +/// but it may cost more time when handling the scalar head and tail. +/// If these are not enough, it is most ideal to design data structures to be already aligned +/// to `mem::align_of::>()` before using `unsafe` Rust to read or write. +/// Other ways to compensate for these facts, like materializing `Simd` to or from an array first, /// are handled by safe methods like [`Simd::from_array`] and [`Simd::from_slice`]. /// /// [`transmute`]: core::mem::transmute @@ -82,20 +93,20 @@ use crate::simd::{ // avoided, as it will likely become illegal on `#[repr(simd)]` structs in the future. It also // causes rustc to emit illegal LLVM IR in some cases. #[repr(simd)] -pub struct Simd([T; LANES]) +pub struct Simd([T; N]) where - T: SimdElement, - LaneCount: SupportedLaneCount; + LaneCount: SupportedLaneCount, + T: SimdElement; -impl Simd +impl Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { - /// Number of lanes in this vector. - pub const LANES: usize = LANES; + /// Number of elements in this vector. + pub const N: usize = N; - /// Returns the number of lanes in this SIMD vector. + /// Returns the number of elements in this SIMD vector. /// /// # Examples /// @@ -106,10 +117,10 @@ where /// assert_eq!(v.lanes(), 4); /// ``` pub const fn lanes(&self) -> usize { - LANES + Self::N } - /// Constructs a new SIMD vector with all lanes set to the given value. + /// Constructs a new SIMD vector with all elements set to the given value. /// /// # Examples /// @@ -120,11 +131,11 @@ where /// assert_eq!(v.as_array(), &[8, 8, 8, 8]); /// ``` pub fn splat(value: T) -> Self { - // This is preferred over `[value; LANES]`, since it's explicitly a splat: + // This is preferred over `[value; N]`, since it's explicitly a splat: // https://github.com/rust-lang/rust/issues/97804 struct Splat; - impl Swizzle<1, LANES> for Splat { - const INDEX: [usize; LANES] = [0; LANES]; + impl Swizzle<1, N> for Splat { + const INDEX: [usize; N] = [0; N]; } Splat::swizzle(Simd::::from([value])) } @@ -139,30 +150,30 @@ where /// let v: u64x4 = Simd::from_array([0, 1, 2, 3]); /// assert_eq!(v.as_array(), &[0, 1, 2, 3]); /// ``` - pub const fn as_array(&self) -> &[T; LANES] { - // SAFETY: Transmuting between `Simd` and `[T; LANES]` - // is always valid and `Simd` never has a lower alignment - // than `[T; LANES]`. + pub const fn as_array(&self) -> &[T; N] { + // SAFETY: Transmuting between `Simd` and `[T; N]` + // is always valid and `Simd` never has a lower alignment + // than `[T; N]`. // // NOTE: This deliberately doesn't just use `&self.0`, see the comment // on the struct definition for details. - unsafe { &*(self as *const Self as *const [T; LANES]) } + unsafe { &*(self as *const Self as *const [T; N]) } } /// Returns a mutable array reference containing the entire SIMD vector. - pub fn as_mut_array(&mut self) -> &mut [T; LANES] { - // SAFETY: Transmuting between `Simd` and `[T; LANES]` - // is always valid and `Simd` never has a lower alignment - // than `[T; LANES]`. + pub fn as_mut_array(&mut self) -> &mut [T; N] { + // SAFETY: Transmuting between `Simd` and `[T; N]` + // is always valid and `Simd` never has a lower alignment + // than `[T; N]`. // // NOTE: This deliberately doesn't just use `&mut self.0`, see the comment // on the struct definition for details. - unsafe { &mut *(self as *mut Self as *mut [T; LANES]) } + unsafe { &mut *(self as *mut Self as *mut [T; N]) } } /// Converts an array to a SIMD vector. - pub const fn from_array(array: [T; LANES]) -> Self { - // SAFETY: Transmuting between `Simd` and `[T; LANES]` + pub const fn from_array(array: [T; N]) -> Self { + // SAFETY: Transmuting between `Simd` and `[T; N]` // is always valid. We need to use `read_unaligned` here, since // the array may have a lower alignment than the vector. // @@ -172,12 +183,12 @@ where // // NOTE: This deliberately doesn't just use `Self(array)`, see the comment // on the struct definition for details. - unsafe { (&array as *const [T; LANES] as *const Self).read_unaligned() } + unsafe { (&array as *const [T; N] as *const Self).read_unaligned() } } /// Converts a SIMD vector to an array. - pub const fn to_array(self) -> [T; LANES] { - // SAFETY: Transmuting between `Simd` and `[T; LANES]` + pub const fn to_array(self) -> [T; N] { + // SAFETY: Transmuting between `Simd` and `[T; N]` // is always valid. No need to use `read_unaligned` here, since // the vector never has a lower alignment than the array. // @@ -187,14 +198,14 @@ where // // NOTE: This deliberately doesn't just use `self.0`, see the comment // on the struct definition for details. - unsafe { (&self as *const Self as *const [T; LANES]).read() } + unsafe { (&self as *const Self as *const [T; N]).read() } } - /// Converts a slice to a SIMD vector containing `slice[..LANES]`. + /// Converts a slice to a SIMD vector containing `slice[..N]`. /// /// # Panics /// - /// Panics if the slice's length is less than the vector's `Simd::LANES`. + /// Panics if the slice's length is less than the vector's `Simd::N`. /// /// # Example /// @@ -208,21 +219,21 @@ where #[must_use] pub const fn from_slice(slice: &[T]) -> Self { assert!( - slice.len() >= LANES, - "slice length must be at least the number of lanes" + slice.len() >= Self::N, + "slice length must be at least the number of elements" ); - assert!(core::mem::size_of::() == LANES * core::mem::size_of::()); + assert!(core::mem::size_of::() == Self::N * core::mem::size_of::()); // Safety: // - We've checked the length is sufficient. // - `T` and `Simd` are Copy types. unsafe { slice.as_ptr().cast::().read_unaligned() } } - /// Writes a SIMD vector to the first `LANES` elements of a slice. + /// Writes a SIMD vector to the first `N` elements of a slice. /// /// # Panics /// - /// Panics if the slice's length is less than the vector's `Simd::LANES`. + /// Panics if the slice's length is less than the vector's `Simd::N`. /// /// # Example /// @@ -238,22 +249,22 @@ where /// ``` pub fn copy_to_slice(self, slice: &mut [T]) { assert!( - slice.len() >= LANES, - "slice length must be at least the number of lanes" + slice.len() >= Self::N, + "slice length must be at least the number of elements" ); - assert!(core::mem::size_of::() == LANES * core::mem::size_of::()); + assert!(core::mem::size_of::() == Self::N * core::mem::size_of::()); // Safety: // - We've checked the length is sufficient // - `T` and `Simd` are Copy types. unsafe { slice.as_mut_ptr().cast::().write_unaligned(self) } } - /// Performs lanewise conversion of a SIMD vector's elements to another SIMD-valid type. + /// Performs elementwise conversion of a SIMD vector's elements to another SIMD-valid type. /// - /// This follows the semantics of Rust's `as` conversion for casting - /// integers to unsigned integers (interpreting as the other type, so `-1` to `MAX`), - /// and from floats to integers (truncating, or saturating at the limits) for each lane, - /// or vice versa. + /// This follows the semantics of Rust's `as` conversion for casting integers between + /// signed and unsigned (interpreting integers as 2s complement, so `-1` to `U::MAX` and + /// `1 << (U::BITS -1)` becoming `I::MIN` ), and from floats to integers (truncating, + /// or saturating at the limits) for each element. /// /// # Examples /// ``` @@ -274,7 +285,7 @@ where #[must_use] #[inline] #[cfg(not(bootstrap))] - pub fn cast(self) -> Simd + pub fn cast(self) -> Simd where T: SimdCast, { @@ -282,10 +293,10 @@ where unsafe { intrinsics::simd_as(self) } } - /// Lanewise casts pointers to another pointer type. + /// Casts a vector of pointers to another pointer type. #[must_use] #[inline] - pub fn cast_ptr(self) -> Simd + pub fn cast_ptr(self) -> Simd where T: SimdCastPtr, U: SimdElement, @@ -310,7 +321,7 @@ where /// [cast]: Simd::cast #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub unsafe fn to_int_unchecked(self) -> Simd + pub unsafe fn to_int_unchecked(self) -> Simd where T: core::convert::FloatToInt + SimdCast, I: SimdCast, @@ -320,79 +331,79 @@ where } /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. - /// If an index is out-of-bounds, the lane is instead selected from the `or` vector. + /// If an index is out-of-bounds, the element is instead selected from the `or` vector. /// /// # Examples /// ``` /// # #![feature(portable_simd)] /// # use core::simd::Simd; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let idxs = Simd::from_array([9, 3, 0, 5]); // Note the index that is out-of-bounds /// let alt = Simd::from_array([-5, -4, -3, -2]); /// - /// let result = Simd::gather_or(&vec, idxs, alt); // Note the lane that is out-of-bounds. + /// let result = Simd::gather_or(&vec, idxs, alt); /// assert_eq!(result, Simd::from_array([-5, 13, 10, 15])); /// ``` #[must_use] #[inline] - pub fn gather_or(slice: &[T], idxs: Simd, or: Self) -> Self { + pub fn gather_or(slice: &[T], idxs: Simd, or: Self) -> Self { Self::gather_select(slice, Mask::splat(true), idxs, or) } - /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. - /// If an index is out-of-bounds, the lane is set to the default value for the type. + /// Reads from indices in `slice` to construct a SIMD vector. + /// If an index is out-of-bounds, the element is set to the default given by `T: Default`. /// /// # Examples /// ``` /// # #![feature(portable_simd)] /// # use core::simd::Simd; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let idxs = Simd::from_array([9, 3, 0, 5]); // Note the index that is out-of-bounds /// - /// let result = Simd::gather_or_default(&vec, idxs); // Note the lane that is out-of-bounds. + /// let result = Simd::gather_or_default(&vec, idxs); /// assert_eq!(result, Simd::from_array([0, 13, 10, 15])); /// ``` #[must_use] #[inline] - pub fn gather_or_default(slice: &[T], idxs: Simd) -> Self + pub fn gather_or_default(slice: &[T], idxs: Simd) -> Self where T: Default, { Self::gather_or(slice, idxs, Self::splat(T::default())) } - /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. - /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If an index is disabled or is out-of-bounds, the lane is selected from the `or` vector. + /// Reads from indices in `slice` to construct a SIMD vector. + /// The mask `enable`s all `true` indices and disables all `false` indices. + /// If an index is disabled or is out-of-bounds, the element is selected from the `or` vector. /// /// # Examples /// ``` /// # #![feature(portable_simd)] /// # use core::simd::{Simd, Mask}; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index /// let alt = Simd::from_array([-5, -4, -3, -2]); - /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. + /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element /// - /// let result = Simd::gather_select(&vec, enable, idxs, alt); // Note the lane that is out-of-bounds. + /// let result = Simd::gather_select(&vec, enable, idxs, alt); /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2])); /// ``` #[must_use] #[inline] pub fn gather_select( slice: &[T], - enable: Mask, - idxs: Simd, + enable: Mask, + idxs: Simd, or: Self, ) -> Self { - let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); - // Safety: We have masked-off out-of-bounds lanes. + let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); + // Safety: We have masked-off out-of-bounds indices. unsafe { Self::gather_select_unchecked(slice, enable, idxs, or) } } - /// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector. - /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If an index is disabled, the lane is selected from the `or` vector. + /// Reads from indices in `slice` to construct a SIMD vector. + /// The mask `enable`s all `true` indices and disables all `false` indices. + /// If an index is disabled, the element is selected from the `or` vector. /// /// # Safety /// @@ -406,13 +417,13 @@ where /// # #[cfg(not(feature = "as_crate"))] use core::simd; /// # use simd::{Simd, SimdPartialOrd, Mask}; /// let vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - /// let idxs = Simd::from_array([9, 3, 0, 5]); + /// let idxs = Simd::from_array([9, 3, 0, 5]); // Includes an out-of-bounds index /// let alt = Simd::from_array([-5, -4, -3, -2]); - /// let enable = Mask::from_array([true, true, true, false]); // Note the final mask lane. + /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element /// // If this mask was used to gather, it would be unsound. Let's fix that. /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len())); /// - /// // We have masked the OOB lane, so it's safe to gather now. + /// // The out-of-bounds index has been masked, so it's safe to gather now. /// let result = unsafe { Simd::gather_select_unchecked(&vec, enable, idxs, alt) }; /// assert_eq!(result, Simd::from_array([-5, 13, 10, -2])); /// ``` @@ -422,18 +433,18 @@ where #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn gather_select_unchecked( slice: &[T], - enable: Mask, - idxs: Simd, + enable: Mask, + idxs: Simd, or: Self, ) -> Self { - let base_ptr = Simd::<*const T, LANES>::splat(slice.as_ptr()); + let base_ptr = Simd::<*const T, N>::splat(slice.as_ptr()); // Ferris forgive me, I have done pointer arithmetic here. let ptrs = base_ptr.wrapping_add(idxs); // Safety: The caller is responsible for determining the indices are okay to read unsafe { Self::gather_select_ptr(ptrs, enable, or) } } - /// Read pointers elementwise into a SIMD vector. + /// Read elementwise from pointers into a SIMD vector. /// /// # Safety /// @@ -454,7 +465,7 @@ where #[must_use] #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub unsafe fn gather_ptr(source: Simd<*const T, LANES>) -> Self + pub unsafe fn gather_ptr(source: Simd<*const T, N>) -> Self where T: Default, { @@ -463,13 +474,14 @@ where unsafe { Self::gather_select_ptr(source, Mask::splat(true), Self::default()) } } - /// Conditionally read pointers elementwise into a SIMD vector. - /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If a lane is disabled, the lane is selected from the `or` vector and no read is performed. + /// Conditionally read elementwise from pointers into a SIMD vector. + /// The mask `enable`s all `true` pointers and disables all `false` pointers. + /// If a pointer is disabled, the element is selected from the `or` vector, + /// and no read is performed. /// /// # Safety /// - /// Enabled lanes must satisfy the same conditions as [`core::ptr::read`]. + /// Enabled elements must satisfy the same conditions as [`core::ptr::read`]. /// /// # Example /// ``` @@ -488,8 +500,8 @@ where #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces pub unsafe fn gather_select_ptr( - source: Simd<*const T, LANES>, - enable: Mask, + source: Simd<*const T, N>, + enable: Mask, or: Self, ) -> Self { // Safety: The caller is responsible for upholding all invariants @@ -497,30 +509,31 @@ where } /// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`. - /// If two lanes in the scattered vector would write to the same index - /// only the last lane is guaranteed to actually be written. + /// If an index is out-of-bounds, the write is suppressed without panicking. + /// If two elements in the scattered vector would write to the same index + /// only the last element is guaranteed to actually be written. /// /// # Examples /// ``` /// # #![feature(portable_simd)] /// # use core::simd::Simd; /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - /// let idxs = Simd::from_array([9, 3, 0, 0]); + /// let idxs = Simd::from_array([9, 3, 0, 0]); // Note the duplicate index. /// let vals = Simd::from_array([-27, 82, -41, 124]); /// - /// vals.scatter(&mut vec, idxs); // index 0 receives two writes. + /// vals.scatter(&mut vec, idxs); // two logical writes means the last wins. /// assert_eq!(vec, vec![124, 11, 12, 82, 14, 15, 16, 17, 18]); /// ``` #[inline] - pub fn scatter(self, slice: &mut [T], idxs: Simd) { + pub fn scatter(self, slice: &mut [T], idxs: Simd) { self.scatter_select(slice, Mask::splat(true), idxs) } - /// Writes the values in a SIMD vector to multiple potentially discontiguous indices in `slice`. - /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If an enabled index is out-of-bounds, the lane is not written. - /// If two enabled lanes in the scattered vector would write to the same index, - /// only the last lane is guaranteed to actually be written. + /// Writes values from a SIMD vector to multiple potentially discontiguous indices in `slice`. + /// The mask `enable`s all `true` indices and disables all `false` indices. + /// If an enabled index is out-of-bounds, the write is suppressed without panicking. + /// If two enabled elements in the scattered vector would write to the same index, + /// only the last element is guaranteed to actually be written. /// /// # Examples /// ``` @@ -529,29 +542,24 @@ where /// # #[cfg(not(feature = "as_crate"))] use core::simd; /// # use simd::{Simd, Mask}; /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; - /// let idxs = Simd::from_array([9, 3, 0, 0]); + /// let idxs = Simd::from_array([9, 3, 0, 0]); // Includes an out-of-bounds index /// let vals = Simd::from_array([-27, 82, -41, 124]); - /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. + /// let enable = Mask::from_array([true, true, true, false]); // Includes a masked element /// - /// vals.scatter_select(&mut vec, enable, idxs); // index 0's second write is masked, thus omitted. + /// vals.scatter_select(&mut vec, enable, idxs); // The last write is masked, thus omitted. /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]); /// ``` #[inline] - pub fn scatter_select( - self, - slice: &mut [T], - enable: Mask, - idxs: Simd, - ) { - let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); - // Safety: We have masked-off out-of-bounds lanes. + pub fn scatter_select(self, slice: &mut [T], enable: Mask, idxs: Simd) { + let enable: Mask = enable & idxs.simd_lt(Simd::splat(slice.len())); + // Safety: We have masked-off out-of-bounds indices. unsafe { self.scatter_select_unchecked(slice, enable, idxs) } } - /// Writes the values in a SIMD vector to multiple potentially discontiguous indices in `slice`. - /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If two enabled lanes in the scattered vector would write to the same index, - /// only the last lane is guaranteed to actually be written. + /// Writes values from a SIMD vector to multiple potentially discontiguous indices in `slice`. + /// The mask `enable`s all `true` indices and disables all `false` indices. + /// If two enabled elements in the scattered vector would write to the same index, + /// only the last element is guaranteed to actually be written. /// /// # Safety /// @@ -567,13 +575,13 @@ where /// let mut vec: Vec = vec![10, 11, 12, 13, 14, 15, 16, 17, 18]; /// let idxs = Simd::from_array([9, 3, 0, 0]); /// let vals = Simd::from_array([-27, 82, -41, 124]); - /// let enable = Mask::from_array([true, true, true, false]); // Note the mask of the last lane. + /// let enable = Mask::from_array([true, true, true, false]); // Masks the final index /// // If this mask was used to scatter, it would be unsound. Let's fix that. /// let enable = enable & idxs.simd_lt(Simd::splat(vec.len())); /// - /// // We have masked the OOB lane, so it's safe to scatter now. + /// // We have masked the OOB index, so it's safe to scatter now. /// unsafe { vals.scatter_select_unchecked(&mut vec, enable, idxs); } - /// // index 0's second write is masked, thus was omitted. + /// // The second write to index 0 was masked, thus omitted. /// assert_eq!(vec, vec![-41, 11, 12, 82, 14, 15, 16, 17, 18]); /// ``` /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html @@ -582,8 +590,8 @@ where pub unsafe fn scatter_select_unchecked( self, slice: &mut [T], - enable: Mask, - idxs: Simd, + enable: Mask, + idxs: Simd, ) { // Safety: This block works with *mut T derived from &mut 'a [T], // which means it is delicate in Rust's borrowing model, circa 2021: @@ -597,7 +605,7 @@ where // 3. &mut [T] which will become our base ptr. unsafe { // Now Entering ☢️ *mut T Zone - let base_ptr = Simd::<*mut T, LANES>::splat(slice.as_mut_ptr()); + let base_ptr = Simd::<*mut T, N>::splat(slice.as_mut_ptr()); // Ferris forgive me, I have done pointer arithmetic here. let ptrs = base_ptr.wrapping_add(idxs); // The ptrs have been bounds-masked to prevent memory-unsafe writes insha'allah @@ -626,18 +634,18 @@ where /// ``` #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, LANES>) { + pub unsafe fn scatter_ptr(self, dest: Simd<*mut T, N>) { // Safety: The caller is responsible for upholding all invariants unsafe { self.scatter_select_ptr(dest, Mask::splat(true)) } } /// Conditionally write pointers elementwise into a SIMD vector. - /// The mask `enable`s all `true` lanes and disables all `false` lanes. - /// If a lane is disabled, the write to that lane is skipped. + /// The mask `enable`s all `true` pointers and disables all `false` pointers. + /// If a pointer is disabled, the write to its pointee is skipped. /// /// # Safety /// - /// Enabled lanes must satisfy the same conditions as [`core::ptr::write`]. + /// Enabled pointers must satisfy the same conditions as [`core::ptr::write`]. /// /// # Example /// ``` @@ -654,32 +662,32 @@ where /// ``` #[inline] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces - pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, LANES>, enable: Mask) { + pub unsafe fn scatter_select_ptr(self, dest: Simd<*mut T, N>, enable: Mask) { // Safety: The caller is responsible for upholding all invariants unsafe { intrinsics::simd_scatter(self, dest, enable.to_int()) } } } -impl Copy for Simd +impl Copy for Simd where + LaneCount: SupportedLaneCount, T: SimdElement, - LaneCount: SupportedLaneCount, { } -impl Clone for Simd +impl Clone for Simd where + LaneCount: SupportedLaneCount, T: SimdElement, - LaneCount: SupportedLaneCount, { fn clone(&self) -> Self { *self } } -impl Default for Simd +impl Default for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + Default, { #[inline] @@ -688,20 +696,20 @@ where } } -impl PartialEq for Simd +impl PartialEq for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + PartialEq, { #[inline] fn eq(&self, other: &Self) -> bool { // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask. let mask = unsafe { - let tfvec: Simd<::Mask, LANES> = intrinsics::simd_eq(*self, *other); + let tfvec: Simd<::Mask, N> = intrinsics::simd_eq(*self, *other); Mask::from_int_unchecked(tfvec) }; - // Two vectors are equal if all lanes tested true for vertical equality. + // Two vectors are equal if they are elementwise equal mask.all() } @@ -710,18 +718,18 @@ where fn ne(&self, other: &Self) -> bool { // Safety: All SIMD vectors are SimdPartialEq, and the comparison produces a valid mask. let mask = unsafe { - let tfvec: Simd<::Mask, LANES> = intrinsics::simd_ne(*self, *other); + let tfvec: Simd<::Mask, N> = intrinsics::simd_ne(*self, *other); Mask::from_int_unchecked(tfvec) }; - // Two vectors are non-equal if any lane tested true for vertical non-equality. + // Two vectors are non-equal if they are elementwise non-equal mask.any() } } -impl PartialOrd for Simd +impl PartialOrd for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + PartialOrd, { #[inline] @@ -731,16 +739,16 @@ where } } -impl Eq for Simd +impl Eq for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + Eq, { } -impl Ord for Simd +impl Ord for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + Ord, { #[inline] @@ -750,9 +758,9 @@ where } } -impl core::hash::Hash for Simd +impl core::hash::Hash for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement + core::hash::Hash, { #[inline] @@ -765,32 +773,32 @@ where } // array references -impl AsRef<[T; LANES]> for Simd +impl AsRef<[T; N]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] - fn as_ref(&self) -> &[T; LANES] { + fn as_ref(&self) -> &[T; N] { self.as_array() } } -impl AsMut<[T; LANES]> for Simd +impl AsMut<[T; N]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] - fn as_mut(&mut self) -> &mut [T; LANES] { + fn as_mut(&mut self) -> &mut [T; N] { self.as_mut_array() } } // slice references -impl AsRef<[T]> for Simd +impl AsRef<[T]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -799,9 +807,9 @@ where } } -impl AsMut<[T]> for Simd +impl AsMut<[T]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { #[inline] @@ -811,29 +819,29 @@ where } // vector/array conversion -impl From<[T; LANES]> for Simd +impl From<[T; N]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { - fn from(array: [T; LANES]) -> Self { + fn from(array: [T; N]) -> Self { Self(array) } } -impl From> for [T; LANES] +impl From> for [T; N] where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { - fn from(vector: Simd) -> Self { + fn from(vector: Simd) -> Self { vector.to_array() } } -impl TryFrom<&[T]> for Simd +impl TryFrom<&[T]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { type Error = core::array::TryFromSliceError; @@ -843,9 +851,9 @@ where } } -impl TryFrom<&mut [T]> for Simd +impl TryFrom<&mut [T]> for Simd where - LaneCount: SupportedLaneCount, + LaneCount: SupportedLaneCount, T: SimdElement, { type Error = core::array::TryFromSliceError; From 92259a4a6c20b02e87e0589a286bef7b71cd95a9 Mon Sep 17 00:00:00 2001 From: Jubilee <46493976+workingjubilee@users.noreply.github.com> Date: Mon, 10 Apr 2023 00:11:37 -0700 Subject: [PATCH 062/130] Clarify elementwise cmp reduces Saying "elementwise (non-)equal" may suggest it returns a vector. The comments should be clear that it instead reduces to a scalar. Co-authored-by: Jacob Lifshay --- crates/core_simd/src/vector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 154b467752b..b7b5e0b002f 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -709,7 +709,7 @@ where Mask::from_int_unchecked(tfvec) }; - // Two vectors are equal if they are elementwise equal + // Two vectors are equal if all elements are equal when compared elementwise mask.all() } @@ -722,7 +722,7 @@ where Mask::from_int_unchecked(tfvec) }; - // Two vectors are non-equal if they are elementwise non-equal + // Two vectors are non-equal if any elements are non-equal when compared elementwise mask.any() } } From 4064678dafd3907253353a1efc01bc0ada78c1bc Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Mon, 10 Apr 2023 22:06:01 -0700 Subject: [PATCH 063/130] Explain why to use Simd early --- crates/core_simd/src/vector.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index b7b5e0b002f..ef67fcfeee6 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -8,7 +8,12 @@ use crate::simd::{ /// `Simd` supports the operators (+, *, etc.) that `T` does in "elementwise" fashion. /// These take the element at each index from the left-hand side and right-hand side, /// perform the operation, then return the result in the same index in a vector of equal size. -/// In other words, an elementwise operation is equivalent to a zip, then map. +/// However, `Simd` differs from normal iteration and normal arrays: +/// - `Simd` executes `N` operations in a single step with no `break`s +/// - `Simd` can have an alignment greater than `T`, for better mechanical sympathy +/// +/// By always imposing these constraints on `Simd`, it is easier to compile elementwise operations +/// into machine instructions that can themselves be executed in parallel. /// /// ```rust /// # #![feature(portable_simd)] From 2b32732d0f64a27560c9c4ca15e89bc454c482da Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Sat, 22 Apr 2023 18:22:04 -0700 Subject: [PATCH 064/130] Do not construct Simd --- crates/core_simd/src/vector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index ef67fcfeee6..106f1965959 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -830,7 +830,7 @@ where T: SimdElement, { fn from(array: [T; N]) -> Self { - Self(array) + Self::from_array(array) } } From 4f0d8225fa6e503ba785da93b0ab900d597af133 Mon Sep 17 00:00:00 2001 From: Jubilee <46493976+workingjubilee@users.noreply.github.com> Date: Sat, 22 Apr 2023 18:27:52 -0700 Subject: [PATCH 065/130] Implement dynamic byte-swizzle prototype (rust-lang/portable-simd#334) This is meant to be an example that is used to test a Rust intrinsic against, which will replace it. The interface is fairly direct and doesn't address more nuanced or interesting permutations one can do, nevermind on types other than bytes. The ultimate goal is for direct LLVM support for this. --- crates/core_simd/src/mod.rs | 2 + crates/core_simd/src/swizzle_dyn.rs | 155 ++++++++++++++++++++++++++ crates/core_simd/tests/swizzle_dyn.rs | 74 ++++++++++++ 3 files changed, 231 insertions(+) create mode 100644 crates/core_simd/src/swizzle_dyn.rs create mode 100644 crates/core_simd/tests/swizzle_dyn.rs diff --git a/crates/core_simd/src/mod.rs b/crates/core_simd/src/mod.rs index ece026a448b..35c659b7a42 100644 --- a/crates/core_simd/src/mod.rs +++ b/crates/core_simd/src/mod.rs @@ -17,6 +17,7 @@ mod masks; mod ops; mod ord; mod select; +mod swizzle_dyn; mod vector; mod vendor; @@ -32,5 +33,6 @@ pub mod simd { pub use crate::core_simd::masks::*; pub use crate::core_simd::ord::*; pub use crate::core_simd::swizzle::*; + pub use crate::core_simd::swizzle_dyn::*; pub use crate::core_simd::vector::*; } diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs new file mode 100644 index 00000000000..5c3a2c1824f --- /dev/null +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -0,0 +1,155 @@ +use crate::simd::{LaneCount, Simd, SupportedLaneCount}; +use core::mem; + +impl Simd +where + LaneCount: SupportedLaneCount, +{ + /// Swizzle a vector of bytes according to the index vector. + /// Indices within range select the appropriate byte. + /// Indices "out of bounds" instead select 0. + /// + /// Note that the current implementation is selected during build-time + /// of the standard library, so `cargo build -Zbuild-std` may be necessary + /// to unlock better performance, especially for larger vectors. + /// A planned compiler improvement will enable using `#[target_feature]` instead. + #[inline] + pub fn swizzle_dyn(self, idxs: Simd) -> Self { + #![allow(unused_imports, unused_unsafe)] + #[cfg(target_arch = "aarch64")] + use core::arch::aarch64::{uint8x8_t, vqtbl1q_u8, vtbl1_u8}; + #[cfg(all(target_arch = "arm", target_feature = "v7"))] + use core::arch::arm::{uint8x8_t, vtbl1_u8}; + #[cfg(target_arch = "wasm32")] + use core::arch::wasm32 as wasm; + #[cfg(target_arch = "x86")] + use core::arch::x86; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64 as x86; + // SAFETY: Intrinsics covered by cfg + unsafe { + match N { + #[cfg(target_feature = "neon")] + 8 => transize(vtbl1_u8, self, idxs), + #[cfg(target_feature = "ssse3")] + 16 => transize(x86::_mm_shuffle_epi8, self, idxs), + #[cfg(target_feature = "simd128")] + 16 => transize(wasm::i8x16_swizzle, self, idxs), + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + 16 => transize(vqtbl1q_u8, self, idxs), + #[cfg(all(target_feature = "avx2", not(target_feature = "avx512vbmi")))] + 32 => transize_raw(avx2_pshufb, self, idxs), + #[cfg(target_feature = "avx512vl,avx512vbmi")] + 32 => transize(x86::_mm256_permutexvar_epi8, self, idxs), + // Notable absence: avx512bw shuffle + // If avx512bw is available, odds of avx512vbmi are good + #[cfg(target_feature = "avx512vbmi")] + 64 => transize(x86::_mm512_permutexvar_epi8, self, idxs), + _ => { + let mut array = [0; N]; + for (i, k) in idxs.to_array().into_iter().enumerate() { + if (k as usize) < N { + array[i] = self[k as usize]; + }; + } + array.into() + } + } + } + } +} + +/// "vpshufb like it was meant to be" on AVX2 +/// +/// # Safety +/// This requires AVX2 to work +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "avx2")] +#[allow(unused)] +#[inline] +unsafe fn avx2_pshufb(bytes: Simd, idxs: Simd) -> Simd { + use crate::simd::SimdPartialOrd; + #[cfg(target_arch = "x86")] + use core::arch::x86; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64 as x86; + use x86::_mm256_permute2x128_si256 as avx2_cross_shuffle; + use x86::_mm256_shuffle_epi8 as avx2_half_pshufb; + let mid = Simd::splat(16u8); + let high = mid + mid; + // SAFETY: Caller promised AVX2 + unsafe { + // This is ordering sensitive, and LLVM will order these how you put them. + // Most AVX2 impls use ~5 "ports", and only 1 or 2 are capable of permutes. + // But the "compose" step will lower to ops that can also use at least 1 other port. + // So this tries to break up permutes so composition flows through "open" ports. + // Comparative benches should be done on multiple AVX2 CPUs before reordering this + + let hihi = avx2_cross_shuffle::<0x11>(bytes.into(), bytes.into()); + let hi_shuf = Simd::from(avx2_half_pshufb( + hihi, // duplicate the vector's top half + idxs.into(), // so that using only 4 bits of an index still picks bytes 16-31 + )); + // A zero-fill during the compose step gives the "all-Neon-like" OOB-is-0 semantics + let compose = idxs.simd_lt(high).select(hi_shuf, Simd::splat(0)); + let lolo = avx2_cross_shuffle::<0x00>(bytes.into(), bytes.into()); + let lo_shuf = Simd::from(avx2_half_pshufb(lolo, idxs.into())); + // Repeat, then pick indices < 16, overwriting indices 0-15 from previous compose step + let compose = idxs.simd_lt(mid).select(lo_shuf, compose); + compose + } +} + +/// This sets up a call to an architecture-specific function, and in doing so +/// it persuades rustc that everything is the correct size. Which it is. +/// This would not be needed if one could convince Rust that, by matching on N, +/// N is that value, and thus it would be valid to substitute e.g. 16. +/// +/// # Safety +/// The correctness of this function hinges on the sizes agreeing in actuality. +#[allow(dead_code)] +#[inline(always)] +unsafe fn transize( + f: unsafe fn(T, T) -> T, + bytes: Simd, + idxs: Simd, +) -> Simd +where + LaneCount: SupportedLaneCount, +{ + let idxs = zeroing_idxs(idxs); + // SAFETY: Same obligation to use this function as to use mem::transmute_copy. + unsafe { mem::transmute_copy(&f(mem::transmute_copy(&bytes), mem::transmute_copy(&idxs))) } +} + +/// Make indices that yield 0 for this architecture +#[inline(always)] +fn zeroing_idxs(idxs: Simd) -> Simd +where + LaneCount: SupportedLaneCount, +{ + // On x86, make sure the top bit is set. + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + let idxs = { + use crate::simd::SimdPartialOrd; + idxs.simd_lt(Simd::splat(N as u8)) + .select(idxs, Simd::splat(u8::MAX)) + }; + // Simply do nothing on most architectures. + idxs +} + +/// As transize but no implicit call to `zeroing_idxs`. +#[allow(dead_code)] +#[inline(always)] +unsafe fn transize_raw( + f: unsafe fn(T, T) -> T, + bytes: Simd, + idxs: Simd, +) -> Simd +where + LaneCount: SupportedLaneCount, +{ + // SAFETY: Same obligation to use this function as to use mem::transmute_copy. + unsafe { mem::transmute_copy(&f(mem::transmute_copy(&bytes), mem::transmute_copy(&idxs))) } +} diff --git a/crates/core_simd/tests/swizzle_dyn.rs b/crates/core_simd/tests/swizzle_dyn.rs new file mode 100644 index 00000000000..646cd5f3383 --- /dev/null +++ b/crates/core_simd/tests/swizzle_dyn.rs @@ -0,0 +1,74 @@ +#![feature(portable_simd)] +use core::{fmt, ops::RangeInclusive}; +use proptest; +use test_helpers::{self, biteq, make_runner, prop_assert_biteq}; + +fn swizzle_dyn_scalar_ver(values: [u8; N], idxs: [u8; N]) -> [u8; N] { + let mut array = [0; N]; + for (i, k) in idxs.into_iter().enumerate() { + if (k as usize) < N { + array[i] = values[k as usize]; + }; + } + array +} + +test_helpers::test_lanes! { + fn swizzle_dyn() { + match_simd_with_fallback( + &core_simd::simd::Simd::::swizzle_dyn, + &swizzle_dyn_scalar_ver, + &|_, _| true, + ); + } +} + +fn match_simd_with_fallback( + fv: &dyn Fn(Vector, Vector) -> VectorResult, + fs: &dyn Fn([Scalar; N], [Scalar; N]) -> [ScalarResult; N], + check: &dyn Fn([Scalar; N], [Scalar; N]) -> bool, +) where + Scalar: Copy + fmt::Debug + SwizzleStrategy, + ScalarResult: Copy + biteq::BitEq + fmt::Debug + SwizzleStrategy, + Vector: Into<[Scalar; N]> + From<[Scalar; N]> + Copy, + VectorResult: Into<[ScalarResult; N]> + From<[ScalarResult; N]> + Copy, +{ + test_swizzles_2(&|x: [Scalar; N], y: [Scalar; N]| { + proptest::prop_assume!(check(x, y)); + let result_v: [ScalarResult; N] = fv(x.into(), y.into()).into(); + let result_s: [ScalarResult; N] = fs(x, y); + crate::prop_assert_biteq!(result_v, result_s); + Ok(()) + }); +} + +fn test_swizzles_2( + f: &dyn Fn(A, B) -> proptest::test_runner::TestCaseResult, +) { + let mut runner = make_runner(); + runner + .run( + &(A::swizzled_strategy(), B::swizzled_strategy()), + |(a, b)| f(a, b), + ) + .unwrap(); +} + +pub trait SwizzleStrategy { + type Strategy: proptest::strategy::Strategy; + fn swizzled_strategy() -> Self::Strategy; +} + +impl SwizzleStrategy for u8 { + type Strategy = RangeInclusive; + fn swizzled_strategy() -> Self::Strategy { + 0..=64 + } +} + +impl SwizzleStrategy for [T; N] { + type Strategy = test_helpers::array::UniformArrayStrategy; + fn swizzled_strategy() -> Self::Strategy { + Self::Strategy::new(T::swizzled_strategy()) + } +} From 394a8845c699b5c6b47c6a17e2926a549f8801be Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Sun, 23 Apr 2023 14:52:38 -0400 Subject: [PATCH 066/130] Fix {to,from}_array UB when repr(simd) produces padding --- crates/core_simd/src/lib.rs | 2 ++ crates/core_simd/src/vector.rs | 56 +++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 14 deletions(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index e054d483ca5..31e7a3617bc 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -2,6 +2,8 @@ #![feature( const_ptr_read, const_refs_to_cell, + const_maybe_uninit_as_mut_ptr, + const_mut_refs, convert_float_to_int, decl_macro, intra_doc_pointers, diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 106f1965959..8c6c7036081 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -176,34 +176,62 @@ where unsafe { &mut *(self as *mut Self as *mut [T; N]) } } + /// Load a vector from an array of `T`. + /// + /// This function is necessary since `repr(simd)` has padding for non-power-of-2 vectors (at the time of writing). + /// With padding, `read_unaligned` will read past the end of an array of N elements. + /// + /// # Safety + /// Reading `ptr` must be safe, as if by `<*const [T; N]>::read_unaligned`. + const unsafe fn load(ptr: *const [T; N]) -> Self { + let mut tmp = core::mem::MaybeUninit::uninit(); + // SAFETY: `Simd` always contains `N` elements of type `T`. It may have padding + // which does not need to be initialized. The safety of reading `ptr` is ensured by the + // caller. + unsafe { + core::ptr::copy_nonoverlapping(ptr, tmp.as_mut_ptr() as *mut _, 1); + tmp.assume_init() + } + } + + /// Store a vector to an array of `T`. + /// + /// See `load` as to why this function is necessary. + /// + /// # Safety + /// Writing to `ptr` must be safe, as if by `<*mut [T; N]>::write_unaligned`. + const unsafe fn store(self, ptr: *mut [T; N]) { + // SAFETY: `Simd` always contains `N` elements of type `T`. The safety of writing + // `ptr` is ensured by the caller. + unsafe { core::ptr::copy_nonoverlapping(self.as_array(), ptr, 1) } + } + /// Converts an array to a SIMD vector. pub const fn from_array(array: [T; N]) -> Self { - // SAFETY: Transmuting between `Simd` and `[T; N]` - // is always valid. We need to use `read_unaligned` here, since - // the array may have a lower alignment than the vector. + // SAFETY: `&array` is safe to read. // - // FIXME: We currently use a pointer read instead of `transmute_copy` because - // it results in better codegen with optimizations disabled, but we should - // probably just use `transmute` once that works on const generic types. + // FIXME: We currently use a pointer load instead of `transmute_copy` because `repr(simd)` + // results in padding for non-power-of-2 vectors (so vectors are larger than arrays). // // NOTE: This deliberately doesn't just use `Self(array)`, see the comment // on the struct definition for details. - unsafe { (&array as *const [T; N] as *const Self).read_unaligned() } + unsafe { Self::load(&array) } } /// Converts a SIMD vector to an array. pub const fn to_array(self) -> [T; N] { - // SAFETY: Transmuting between `Simd` and `[T; N]` - // is always valid. No need to use `read_unaligned` here, since - // the vector never has a lower alignment than the array. + let mut tmp = core::mem::MaybeUninit::uninit(); + // SAFETY: writing to `tmp` is safe and initializes it. // - // FIXME: We currently use a pointer read instead of `transmute_copy` because - // it results in better codegen with optimizations disabled, but we should - // probably just use `transmute` once that works on const generic types. + // FIXME: We currently use a pointer store instead of `transmute_copy` because `repr(simd)` + // results in padding for non-power-of-2 vectors (so vectors are larger than arrays). // // NOTE: This deliberately doesn't just use `self.0`, see the comment // on the struct definition for details. - unsafe { (&self as *const Self as *const [T; N]).read() } + unsafe { + self.store(tmp.as_mut_ptr()); + tmp.assume_init() + } } /// Converts a slice to a SIMD vector containing `slice[..N]`. From c504f01abeba606a5fa7d081ed8aec25d118a486 Mon Sep 17 00:00:00 2001 From: Caleb Zulawski Date: Tue, 25 Apr 2023 21:37:04 -0400 Subject: [PATCH 067/130] Use cast and improve comments --- crates/core_simd/src/vector.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 8c6c7036081..92984f55e45 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -184,12 +184,15 @@ where /// # Safety /// Reading `ptr` must be safe, as if by `<*const [T; N]>::read_unaligned`. const unsafe fn load(ptr: *const [T; N]) -> Self { - let mut tmp = core::mem::MaybeUninit::uninit(); + // There are potentially simpler ways to write this function, but this should result in + // LLVM `load ` + + let mut tmp = core::mem::MaybeUninit::::uninit(); // SAFETY: `Simd` always contains `N` elements of type `T`. It may have padding // which does not need to be initialized. The safety of reading `ptr` is ensured by the // caller. unsafe { - core::ptr::copy_nonoverlapping(ptr, tmp.as_mut_ptr() as *mut _, 1); + core::ptr::copy_nonoverlapping(ptr, tmp.as_mut_ptr().cast(), 1); tmp.assume_init() } } @@ -201,9 +204,14 @@ where /// # Safety /// Writing to `ptr` must be safe, as if by `<*mut [T; N]>::write_unaligned`. const unsafe fn store(self, ptr: *mut [T; N]) { + // There are potentially simpler ways to write this function, but this should result in + // LLVM `store ` + + // Creating a temporary helps LLVM turn the memcpy into a store. + let tmp = self; // SAFETY: `Simd` always contains `N` elements of type `T`. The safety of writing // `ptr` is ensured by the caller. - unsafe { core::ptr::copy_nonoverlapping(self.as_array(), ptr, 1) } + unsafe { core::ptr::copy_nonoverlapping(tmp.as_array(), ptr, 1) } } /// Converts an array to a SIMD vector. From 47be0605a8bf2aec090322a0c604ebbf290206aa Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 27 Apr 2023 18:22:06 +0000 Subject: [PATCH 068/130] Ignore test when cross compiling --- tests/run-make/staticlib-dylib-linkage/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/run-make/staticlib-dylib-linkage/Makefile b/tests/run-make/staticlib-dylib-linkage/Makefile index 08e4fa5d3aa..a1e86a7ce4b 100644 --- a/tests/run-make/staticlib-dylib-linkage/Makefile +++ b/tests/run-make/staticlib-dylib-linkage/Makefile @@ -1,5 +1,6 @@ include ../tools.mk +# ignore-cross-compile # ignore-msvc FIXME(bjorn3) can't figure out how to link with the MSVC toolchain # ignore-wasm wasm doesn't support dynamic libraries From 2d5ca0ea4f896201325f9d165498118b320810a7 Mon Sep 17 00:00:00 2001 From: clubby789 Date: Sun, 30 Apr 2023 16:31:36 +0100 Subject: [PATCH 069/130] Bail out of MIR construction if `check_match` fails --- compiler/rustc_middle/src/query/mod.rs | 2 +- compiler/rustc_mir_build/src/build/mod.rs | 4 +- .../src/thir/pattern/check_match.rs | 25 ++-- .../2229_closure_analysis/bad-pattern.rs | 23 ++++ .../2229_closure_analysis/bad-pattern.stderr | 113 ++++++++++++++++++ 5 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 tests/ui/closures/2229_closure_analysis/bad-pattern.rs create mode 100644 tests/ui/closures/2229_closure_analysis/bad-pattern.stderr diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 6443c30e822..ded5c6aa3b3 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1022,7 +1022,7 @@ rustc_queries! { desc { "converting literal to mir constant" } } - query check_match(key: LocalDefId) { + query check_match(key: LocalDefId) -> Result<(), rustc_errors::ErrorGuaranteed> { desc { |tcx| "match-checking `{}`", tcx.def_path_str(key) } cache_on_disk_if { true } } diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs index 3f006765a71..20d381eddb1 100644 --- a/compiler/rustc_mir_build/src/build/mod.rs +++ b/compiler/rustc_mir_build/src/build/mod.rs @@ -42,7 +42,9 @@ fn mir_build(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> { // Ensure unsafeck and abstract const building is ran before we steal the THIR. tcx.ensure_with_value().thir_check_unsafety(def); tcx.ensure_with_value().thir_abstract_const(def); - tcx.ensure_with_value().check_match(def); + if let Err(e) = tcx.check_match(def) { + return construct_error(tcx, def, e); + } let body = match tcx.thir_body(def) { Err(error_reported) => construct_error(tcx, def, error_reported), diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs index 2b52d70af2a..c3517912cf9 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs @@ -26,8 +26,8 @@ use rustc_session::Session; use rustc_span::hygiene::DesugaringKind; use rustc_span::Span; -pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) { - let Ok((thir, expr)) = tcx.thir_body(def_id) else { return }; +pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(), ErrorGuaranteed> { + let (thir, expr) = tcx.thir_body(def_id)?; let thir = thir.borrow(); let pattern_arena = TypedArena::default(); let mut visitor = MatchVisitor { @@ -37,13 +37,16 @@ pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) { lint_level: tcx.hir().local_def_id_to_hir_id(def_id), let_source: LetSource::None, pattern_arena: &pattern_arena, + error: Ok(()), }; visitor.visit_expr(&thir[expr]); + for param in thir.params.iter() { if let Some(box ref pattern) = param.pat { visitor.check_irrefutable(pattern, "function argument", None); } } + visitor.error } fn create_e0004( @@ -77,6 +80,7 @@ struct MatchVisitor<'a, 'p, 'tcx> { lint_level: HirId, let_source: LetSource, pattern_arena: &'p TypedArena>, + error: Result<(), ErrorGuaranteed>, } impl<'a, 'tcx> Visitor<'a, 'tcx> for MatchVisitor<'a, '_, 'tcx> { @@ -276,9 +280,9 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> { let [pat_field] = &subpatterns[..] else { bug!() }; self.check_irrefutable(&pat_field.pattern, "`for` loop binding", None); } else { - non_exhaustive_match( + self.error = Err(non_exhaustive_match( &cx, self.thir, scrut_ty, scrut.span, witnesses, arms, expr_span, - ); + )); } } } @@ -409,7 +413,7 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> { } #[instrument(level = "trace", skip(self))] - fn check_irrefutable(&self, pat: &Pat<'tcx>, origin: &str, sp: Option) { + fn check_irrefutable(&mut self, pat: &Pat<'tcx>, origin: &str, sp: Option) { let mut cx = self.new_cx(self.lint_level, false); let pattern = self.lower_pattern(&mut cx, pat); @@ -478,7 +482,7 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> { AdtDefinedHere { adt_def_span, ty, variants } }; - self.tcx.sess.emit_err(PatternNotCovered { + self.error = Err(self.tcx.sess.emit_err(PatternNotCovered { span: pat.span, origin, uncovered: Uncovered::new(pat.span, &cx, witnesses), @@ -489,7 +493,7 @@ impl<'p, 'tcx> MatchVisitor<'_, 'p, 'tcx> { let_suggestion, misc_suggestion, adt_defined_here, - }); + })); } } @@ -631,7 +635,7 @@ fn non_exhaustive_match<'p, 'tcx>( witnesses: Vec>, arms: &[ArmId], expr_span: Span, -) { +) -> ErrorGuaranteed { let is_empty_match = arms.is_empty(); let non_empty_enum = match scrut_ty.kind() { ty::Adt(def, _) => def.is_enum() && !def.variants().is_empty(), @@ -643,13 +647,12 @@ fn non_exhaustive_match<'p, 'tcx>( let pattern; let patterns_len; if is_empty_match && !non_empty_enum { - cx.tcx.sess.emit_err(NonExhaustivePatternsTypeNotEmpty { + return cx.tcx.sess.emit_err(NonExhaustivePatternsTypeNotEmpty { cx, expr_span, span: sp, ty: scrut_ty, }); - return; } else { // FIXME: migration of this diagnostic will require list support let joined_patterns = joined_uncovered_patterns(cx, &witnesses); @@ -800,7 +803,7 @@ fn non_exhaustive_match<'p, 'tcx>( } else { err.help(&msg); } - err.emit(); + err.emit() } pub(crate) fn joined_uncovered_patterns<'p, 'tcx>( diff --git a/tests/ui/closures/2229_closure_analysis/bad-pattern.rs b/tests/ui/closures/2229_closure_analysis/bad-pattern.rs new file mode 100644 index 00000000000..a7bf9b67d45 --- /dev/null +++ b/tests/ui/closures/2229_closure_analysis/bad-pattern.rs @@ -0,0 +1,23 @@ +// regression test for #108683 +// edition:2021 + +enum Refutable { + A, + B, +} + +fn example(v1: u32, v2: [u32; 4], v3: Refutable) { + const PAT: u32 = 0; + let v4 = &v2[..]; + || { + let 0 = v1; //~ ERROR refutable pattern in local binding + let (0 | 1) = v1; //~ ERROR refutable pattern in local binding + let 1.. = v1; //~ ERROR refutable pattern in local binding + let [0, 0, 0, 0] = v2; //~ ERROR refutable pattern in local binding + let [0] = v4; //~ ERROR refutable pattern in local binding + let Refutable::A = v3; //~ ERROR refutable pattern in local binding + let PAT = v1; //~ ERROR refutable pattern in local binding + }; +} + +fn main() {} diff --git a/tests/ui/closures/2229_closure_analysis/bad-pattern.stderr b/tests/ui/closures/2229_closure_analysis/bad-pattern.stderr new file mode 100644 index 00000000000..ca8c2a16d32 --- /dev/null +++ b/tests/ui/closures/2229_closure_analysis/bad-pattern.stderr @@ -0,0 +1,113 @@ +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:13:13 + | +LL | let 0 = v1; + | ^ pattern `1_u32..=u32::MAX` not covered + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html + = note: the matched value is of type `u32` +help: you might want to use `if let` to ignore the variant that isn't matched + | +LL | if let 0 = v1 { todo!() }; + | ++ +++++++++++ +help: alternatively, you could prepend the pattern with an underscore to define a new named variable; identifiers cannot begin with digits + | +LL | let _0 = v1; + | + + +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:14:14 + | +LL | let (0 | 1) = v1; + | ^^^^^ pattern `2_u32..=u32::MAX` not covered + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html + = note: the matched value is of type `u32` +help: you might want to use `if let` to ignore the variant that isn't matched + | +LL | if let (0 | 1) = v1 { todo!() }; + | ++ +++++++++++ + +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:15:13 + | +LL | let 1.. = v1; + | ^^^ pattern `0_u32` not covered + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html + = note: the matched value is of type `u32` +help: you might want to use `if let` to ignore the variant that isn't matched + | +LL | if let 1.. = v1 { todo!() }; + | ++ +++++++++++ + +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:16:13 + | +LL | let [0, 0, 0, 0] = v2; + | ^^^^^^^^^^^^ pattern `[1_u32..=u32::MAX, _, _, _]` not covered + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html + = note: the matched value is of type `[u32; 4]` +help: you might want to use `if let` to ignore the variant that isn't matched + | +LL | if let [0, 0, 0, 0] = v2 { todo!() }; + | ++ +++++++++++ + +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:17:13 + | +LL | let [0] = v4; + | ^^^ patterns `&[]` and `&[_, _, ..]` not covered + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html + = note: the matched value is of type `&[u32]` +help: you might want to use `if let` to ignore the variants that aren't matched + | +LL | if let [0] = v4 { todo!() }; + | ++ +++++++++++ + +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:18:13 + | +LL | let Refutable::A = v3; + | ^^^^^^^^^^^^ pattern `Refutable::B` not covered + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html +note: `Refutable` defined here + --> $DIR/bad-pattern.rs:4:6 + | +LL | enum Refutable { + | ^^^^^^^^^ +LL | A, +LL | B, + | - not covered + = note: the matched value is of type `Refutable` +help: you might want to use `if let` to ignore the variant that isn't matched + | +LL | if let Refutable::A = v3 { todo!() }; + | ++ +++++++++++ + +error[E0005]: refutable pattern in local binding + --> $DIR/bad-pattern.rs:19:13 + | +LL | let PAT = v1; + | ^^^ + | | + | pattern `1_u32..=u32::MAX` not covered + | missing patterns are not covered because `PAT` is interpreted as a constant pattern, not a new variable + | help: introduce a variable instead: `PAT_var` + | + = note: `let` bindings require an "irrefutable pattern", like a `struct` or an `enum` with only one variant + = note: for more information, visit https://doc.rust-lang.org/book/ch18-02-refutability.html + = note: the matched value is of type `u32` + +error: aborting due to 7 previous errors + +For more information about this error, try `rustc --explain E0005`. From 879f8de4096b2db4769e64e4c1af5ffb10b53a22 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 2 May 2023 14:35:34 +0200 Subject: [PATCH 070/130] Correctly handle associated items of a trait inside a `#[doc(hidden)]` item --- src/librustdoc/formats/cache.rs | 8 ++++++- src/librustdoc/passes/strip_hidden.rs | 30 ++++++++++++++++++++++----- src/librustdoc/visit_ast.rs | 11 +++------- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/librustdoc/formats/cache.rs b/src/librustdoc/formats/cache.rs index 841abfab666..c0730e90740 100644 --- a/src/librustdoc/formats/cache.rs +++ b/src/librustdoc/formats/cache.rs @@ -195,7 +195,13 @@ impl Cache { impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { fn fold_item(&mut self, item: clean::Item) -> Option { if item.item_id.is_local() { - debug!("folding {} \"{:?}\", id {:?}", item.type_(), item.name, item.item_id); + let is_stripped = matches!(*item.kind, clean::ItemKind::StrippedItem(..)); + debug!( + "folding {} (stripped: {is_stripped:?}) \"{:?}\", id {:?}", + item.type_(), + item.name, + item.item_id + ); } // If this is a stripped module, diff --git a/src/librustdoc/passes/strip_hidden.rs b/src/librustdoc/passes/strip_hidden.rs index a688aa14863..972b0c5ec19 100644 --- a/src/librustdoc/passes/strip_hidden.rs +++ b/src/librustdoc/passes/strip_hidden.rs @@ -1,5 +1,6 @@ //! Strip all doc(hidden) items from the output. +use rustc_hir::def_id::LocalDefId; use rustc_middle::ty::TyCtxt; use rustc_span::symbol::sym; use std::mem; @@ -29,6 +30,7 @@ pub(crate) fn strip_hidden(krate: clean::Crate, cx: &mut DocContext<'_>) -> clea update_retained: true, tcx: cx.tcx, is_in_hidden_item: false, + last_reexport: None, }; stripper.fold_crate(krate) }; @@ -49,13 +51,24 @@ struct Stripper<'a, 'tcx> { update_retained: bool, tcx: TyCtxt<'tcx>, is_in_hidden_item: bool, + last_reexport: Option, } impl<'a, 'tcx> Stripper<'a, 'tcx> { + fn set_last_reexport_then_fold_item(&mut self, i: Item) -> Item { + let prev_from_reexport = self.last_reexport; + if i.inline_stmt_id.is_some() { + self.last_reexport = i.item_id.as_def_id().and_then(|def_id| def_id.as_local()); + } + let ret = self.fold_item_recur(i); + self.last_reexport = prev_from_reexport; + ret + } + fn set_is_in_hidden_item_and_fold(&mut self, is_in_hidden_item: bool, i: Item) -> Item { let prev = self.is_in_hidden_item; self.is_in_hidden_item |= is_in_hidden_item; - let ret = self.fold_item_recur(i); + let ret = self.set_last_reexport_then_fold_item(i); self.is_in_hidden_item = prev; ret } @@ -64,7 +77,7 @@ impl<'a, 'tcx> Stripper<'a, 'tcx> { /// of `is_in_hidden_item` to `true` because the impl children inherit its visibility. fn recurse_in_impl_or_exported_macro(&mut self, i: Item) -> Item { let prev = mem::replace(&mut self.is_in_hidden_item, false); - let ret = self.fold_item_recur(i); + let ret = self.set_last_reexport_then_fold_item(i); self.is_in_hidden_item = prev; ret } @@ -86,13 +99,20 @@ impl<'a, 'tcx> DocFolder for Stripper<'a, 'tcx> { if !is_impl_or_exported_macro { is_hidden = self.is_in_hidden_item || has_doc_hidden; if !is_hidden && i.inline_stmt_id.is_none() { - // We don't need to check if it's coming from a reexport since the reexport itself was - // already checked. + // `i.inline_stmt_id` is `Some` if the item is directly reexported. If it is, we + // don't need to check it, because the reexport itself was already checked. + // + // If this item is the child of a reexported module, `self.last_reexport` will be + // `Some` even though `i.inline_stmt_id` is `None`. Hiddenness inheritance needs to + // account for the possibility that an item's true parent module is hidden, but it's + // inlined into a visible module true. This code shouldn't be reachable if the + // module's reexport is itself hidden, for the same reason it doesn't need to be + // checked if `i.inline_stmt_id` is Some: hidden reexports are never inlined. is_hidden = i .item_id .as_def_id() .and_then(|def_id| def_id.as_local()) - .map(|def_id| inherits_doc_hidden(self.tcx, def_id)) + .map(|def_id| inherits_doc_hidden(self.tcx, def_id, self.last_reexport)) .unwrap_or(false); } } diff --git a/src/librustdoc/visit_ast.rs b/src/librustdoc/visit_ast.rs index a6089680fae..a6c24041380 100644 --- a/src/librustdoc/visit_ast.rs +++ b/src/librustdoc/visit_ast.rs @@ -500,19 +500,14 @@ impl<'a, 'tcx> Visitor<'tcx> for RustdocVisitor<'a, 'tcx> { fn visit_item(&mut self, i: &'tcx hir::Item<'tcx>) { self.visit_item_inner(i, None, None); - let new_value = if self.is_importable_from_parent { - matches!( + let new_value = self.is_importable_from_parent + && matches!( i.kind, hir::ItemKind::Mod(..) | hir::ItemKind::ForeignMod { .. } | hir::ItemKind::Impl(..) | hir::ItemKind::Trait(..) - ) - } else { - // Whatever the context, if it's an impl block, the items inside it can be used so they - // should be visible. - matches!(i.kind, hir::ItemKind::Impl(..)) - }; + ); let prev = mem::replace(&mut self.is_importable_from_parent, new_value); walk_item(self, i); self.is_importable_from_parent = prev; From fb160d5d3b30ad4a522149d309002fd76137b048 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 5 May 2023 15:32:56 +0200 Subject: [PATCH 071/130] Modules can be reexported and it should be handled by rustdoc --- src/librustdoc/clean/mod.rs | 23 ++++++++++- .../passes/check_doc_test_visibility.rs | 2 +- src/librustdoc/visit_ast.rs | 40 +++++++++++++++---- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 8ccdb16b784..d21a8a5477f 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -119,7 +119,28 @@ pub(crate) fn clean_doc_module<'tcx>(doc: &DocModule<'tcx>, cx: &mut DocContext< }); let kind = ModuleItem(Module { items, span }); - Item::from_def_id_and_parts(doc.def_id.to_def_id(), Some(doc.name), kind, cx) + let def_id = doc.def_id.to_def_id(); + let target_attrs = inline::load_attrs(cx, def_id); + let attrs = if let Some(import_id) = doc.import_id { + let is_inline = inline::load_attrs(cx, import_id.to_def_id()) + .lists(sym::doc) + .get_word_attr(sym::inline) + .is_some(); + let mut attrs = get_all_import_attributes(cx, import_id, doc.def_id, is_inline); + add_without_unwanted_attributes(&mut attrs, target_attrs, is_inline, None); + attrs + } else { + // We only keep the item's attributes. + target_attrs.iter().map(|attr| (Cow::Borrowed(attr), None)).collect() + }; + + let cfg = attrs.cfg(cx.tcx, &cx.cache.hidden_cfg); + let attrs = Attributes::from_ast_iter(attrs.iter().map(|(attr, did)| (&**attr, *did)), false); + + let name = doc.renamed.or_else(|| Some(doc.name)); + let mut item = Item::from_def_id_and_attrs_and_parts(def_id, name, kind, Box::new(attrs), cfg); + item.inline_stmt_id = doc.import_id.map(|local| local.to_def_id()); + item } fn clean_generic_bound<'tcx>( diff --git a/src/librustdoc/passes/check_doc_test_visibility.rs b/src/librustdoc/passes/check_doc_test_visibility.rs index 6b13e6c9581..10295cbd189 100644 --- a/src/librustdoc/passes/check_doc_test_visibility.rs +++ b/src/librustdoc/passes/check_doc_test_visibility.rs @@ -95,7 +95,7 @@ pub(crate) fn should_have_doc_example(cx: &DocContext<'_>, item: &clean::Item) - } if cx.tcx.is_doc_hidden(def_id.to_def_id()) - || inherits_doc_hidden(cx.tcx, def_id) + || inherits_doc_hidden(cx.tcx, def_id, None) || cx.tcx.def_span(def_id.to_def_id()).in_derive_expansion() { return false; diff --git a/src/librustdoc/visit_ast.rs b/src/librustdoc/visit_ast.rs index a6c24041380..f7c525042c2 100644 --- a/src/librustdoc/visit_ast.rs +++ b/src/librustdoc/visit_ast.rs @@ -27,6 +27,8 @@ pub(crate) struct Module<'hir> { pub(crate) where_inner: Span, pub(crate) mods: Vec>, pub(crate) def_id: LocalDefId, + pub(crate) renamed: Option, + pub(crate) import_id: Option, /// The key is the item `ItemId` and the value is: (item, renamed, import_id). /// We use `FxIndexMap` to keep the insert order. pub(crate) items: FxIndexMap< @@ -37,11 +39,19 @@ pub(crate) struct Module<'hir> { } impl Module<'_> { - pub(crate) fn new(name: Symbol, def_id: LocalDefId, where_inner: Span) -> Self { + pub(crate) fn new( + name: Symbol, + def_id: LocalDefId, + where_inner: Span, + renamed: Option, + import_id: Option, + ) -> Self { Module { name, def_id, where_inner, + renamed, + import_id, mods: Vec::new(), items: FxIndexMap::default(), foreigns: Vec::new(), @@ -60,9 +70,16 @@ fn def_id_to_path(tcx: TyCtxt<'_>, did: DefId) -> Vec { std::iter::once(crate_name).chain(relative).collect() } -pub(crate) fn inherits_doc_hidden(tcx: TyCtxt<'_>, mut def_id: LocalDefId) -> bool { +pub(crate) fn inherits_doc_hidden( + tcx: TyCtxt<'_>, + mut def_id: LocalDefId, + stop_at: Option, +) -> bool { let hir = tcx.hir(); while let Some(id) = tcx.opt_local_parent(def_id) { + if let Some(stop_at) = stop_at && id == stop_at { + return false; + } def_id = id; if tcx.is_doc_hidden(def_id.to_def_id()) { return true; @@ -100,6 +117,8 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> { cx.tcx.crate_name(LOCAL_CRATE), CRATE_DEF_ID, cx.tcx.hir().root_module().spans.inner_span, + None, + None, ); RustdocVisitor { @@ -260,7 +279,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> { let is_private = !self.cx.cache.effective_visibilities.is_directly_public(self.cx.tcx, ori_res_did); - let is_hidden = inherits_doc_hidden(self.cx.tcx, res_did); + let is_hidden = inherits_doc_hidden(self.cx.tcx, res_did, None); // Only inline if requested or if the item would otherwise be stripped. if (!please_inline && !is_private && !is_hidden) || is_no_inline { @@ -277,7 +296,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> { .cache .effective_visibilities .is_directly_public(self.cx.tcx, item_def_id.to_def_id()) && - !inherits_doc_hidden(self.cx.tcx, item_def_id) + !inherits_doc_hidden(self.cx.tcx, item_def_id, None) { // The imported item is public and not `doc(hidden)` so no need to inline it. return false; @@ -426,7 +445,7 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> { } } hir::ItemKind::Mod(ref m) => { - self.enter_mod(item.owner_id.def_id, m, name); + self.enter_mod(item.owner_id.def_id, m, name, renamed, import_id); } hir::ItemKind::Fn(..) | hir::ItemKind::ExternCrate(..) @@ -479,8 +498,15 @@ impl<'a, 'tcx> RustdocVisitor<'a, 'tcx> { /// This method will create a new module and push it onto the "modules stack" then call /// `visit_mod_contents`. Once done, it'll remove it from the "modules stack" and instead /// add into the list of modules of the current module. - fn enter_mod(&mut self, id: LocalDefId, m: &'tcx hir::Mod<'tcx>, name: Symbol) { - self.modules.push(Module::new(name, id, m.spans.inner_span)); + fn enter_mod( + &mut self, + id: LocalDefId, + m: &'tcx hir::Mod<'tcx>, + name: Symbol, + renamed: Option, + import_id: Option, + ) { + self.modules.push(Module::new(name, id, m.spans.inner_span, renamed, import_id)); self.visit_mod_contents(id, m); From cbc6daa5597916c12ee24a6870f7611adf989878 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 5 May 2023 15:59:41 +0200 Subject: [PATCH 072/130] Improve code to remove duplication --- src/librustdoc/clean/mod.rs | 52 +++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index d21a8a5477f..53b631b986d 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -119,14 +119,25 @@ pub(crate) fn clean_doc_module<'tcx>(doc: &DocModule<'tcx>, cx: &mut DocContext< }); let kind = ModuleItem(Module { items, span }); - let def_id = doc.def_id.to_def_id(); + generate_item_with_correct_attrs(cx, kind, doc.def_id, doc.name, doc.import_id, doc.renamed) +} + +fn generate_item_with_correct_attrs( + cx: &mut DocContext<'_>, + kind: ItemKind, + local_def_id: LocalDefId, + name: Symbol, + import_id: Option, + renamed: Option, +) -> Item { + let def_id = local_def_id.to_def_id(); let target_attrs = inline::load_attrs(cx, def_id); - let attrs = if let Some(import_id) = doc.import_id { + let attrs = if let Some(import_id) = import_id { let is_inline = inline::load_attrs(cx, import_id.to_def_id()) .lists(sym::doc) .get_word_attr(sym::inline) .is_some(); - let mut attrs = get_all_import_attributes(cx, import_id, doc.def_id, is_inline); + let mut attrs = get_all_import_attributes(cx, import_id, local_def_id, is_inline); add_without_unwanted_attributes(&mut attrs, target_attrs, is_inline, None); attrs } else { @@ -137,9 +148,9 @@ pub(crate) fn clean_doc_module<'tcx>(doc: &DocModule<'tcx>, cx: &mut DocContext< let cfg = attrs.cfg(cx.tcx, &cx.cache.hidden_cfg); let attrs = Attributes::from_ast_iter(attrs.iter().map(|(attr, did)| (&**attr, *did)), false); - let name = doc.renamed.or_else(|| Some(doc.name)); + let name = renamed.or(Some(name)); let mut item = Item::from_def_id_and_attrs_and_parts(def_id, name, kind, Box::new(attrs), cfg); - item.inline_stmt_id = doc.import_id.map(|local| local.to_def_id()); + item.inline_stmt_id = import_id.map(|local| local.to_def_id()); item } @@ -2309,29 +2320,14 @@ fn clean_maybe_renamed_item<'tcx>( _ => unreachable!("not yet converted"), }; - let target_attrs = inline::load_attrs(cx, def_id); - let attrs = if let Some(import_id) = import_id { - let is_inline = inline::load_attrs(cx, import_id.to_def_id()) - .lists(sym::doc) - .get_word_attr(sym::inline) - .is_some(); - let mut attrs = - get_all_import_attributes(cx, import_id, item.owner_id.def_id, is_inline); - add_without_unwanted_attributes(&mut attrs, target_attrs, is_inline, None); - attrs - } else { - // We only keep the item's attributes. - target_attrs.iter().map(|attr| (Cow::Borrowed(attr), None)).collect() - }; - - let cfg = attrs.cfg(cx.tcx, &cx.cache.hidden_cfg); - let attrs = - Attributes::from_ast_iter(attrs.iter().map(|(attr, did)| (&**attr, *did)), false); - - let mut item = - Item::from_def_id_and_attrs_and_parts(def_id, Some(name), kind, Box::new(attrs), cfg); - item.inline_stmt_id = import_id.map(|local| local.to_def_id()); - vec![item] + vec![generate_item_with_correct_attrs( + cx, + kind, + item.owner_id.def_id, + name, + import_id, + renamed, + )] }) } From 8de4308b43e9694384d1292f22aef384dc44a5df Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 2 May 2023 14:35:53 +0200 Subject: [PATCH 073/130] Add regression test for #111064 --- ...sue-111064-reexport-trait-from-hidden-2.rs | 31 +++++++++++++++++++ ...issue-111064-reexport-trait-from-hidden.rs | 21 +++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/rustdoc/issue-111064-reexport-trait-from-hidden-2.rs create mode 100644 tests/rustdoc/issue-111064-reexport-trait-from-hidden.rs diff --git a/tests/rustdoc/issue-111064-reexport-trait-from-hidden-2.rs b/tests/rustdoc/issue-111064-reexport-trait-from-hidden-2.rs new file mode 100644 index 00000000000..8e1029a1ca3 --- /dev/null +++ b/tests/rustdoc/issue-111064-reexport-trait-from-hidden-2.rs @@ -0,0 +1,31 @@ +#![feature(no_core)] +#![no_core] +#![crate_name = "foo"] + +// @!has 'foo/hidden/index.html' +// FIXME: add missing `@` for the two next tests once issue is fixed! +// To be done in . +// !has 'foo/hidden/inner/index.html' +// !has 'foo/hidden/inner/trait.Foo.html' +#[doc(hidden)] +pub mod hidden { + pub mod inner { + pub trait Foo { + /// Hello, world! + fn test(); + } + } +} + +// @has 'foo/visible/index.html' +// @has 'foo/visible/trait.Foo.html' +#[doc(inline)] +pub use hidden::inner as visible; + +// @has 'foo/struct.Bar.html' +// @count - '//*[@id="impl-Foo-for-Bar"]' 1 +pub struct Bar; + +impl visible::Foo for Bar { + fn test() {} +} diff --git a/tests/rustdoc/issue-111064-reexport-trait-from-hidden.rs b/tests/rustdoc/issue-111064-reexport-trait-from-hidden.rs new file mode 100644 index 00000000000..a9ce4a34507 --- /dev/null +++ b/tests/rustdoc/issue-111064-reexport-trait-from-hidden.rs @@ -0,0 +1,21 @@ +// Regression test for . +// Methods from a re-exported trait inside a `#[doc(hidden)]` item should +// be visible. + +#![crate_name = "foo"] + +// @has 'foo/index.html' +// @has - '//*[@id="main-content"]//*[@class="item-name"]/a[@href="trait.Foo.html"]' 'Foo' + +// @has 'foo/trait.Foo.html' +// @has - '//*[@id="main-content"]//*[@class="code-header"]' 'fn test()' + +#[doc(hidden)] +mod hidden { + pub trait Foo { + /// Hello, world! + fn test(); + } +} + +pub use hidden::Foo; From f4ca42f573e3a8bd9bb5099efa0285855b77367f Mon Sep 17 00:00:00 2001 From: Urgau Date: Sat, 6 May 2023 18:26:53 +0200 Subject: [PATCH 074/130] Fix --check-cfg bug with args order when parsing --- compiler/rustc_interface/src/interface.rs | 11 ++++++++++- .../order-independant.names_after.stderr | 19 +++++++++++++++++++ .../order-independant.names_before.stderr | 19 +++++++++++++++++++ tests/ui/check-cfg/order-independant.rs | 16 ++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 tests/ui/check-cfg/order-independant.names_after.stderr create mode 100644 tests/ui/check-cfg/order-independant.names_before.stderr create mode 100644 tests/ui/check-cfg/order-independant.rs diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 9d9f4ee13f4..51354c2b127 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -173,12 +173,21 @@ pub fn parse_check_cfg(specs: Vec) -> CheckCfg { let expected_values = check_cfg .expecteds .entry(ident.name.to_string()) + .and_modify(|expected_values| match expected_values { + ExpectedValues::Some(_) => {} + ExpectedValues::Any => { + // handle the case where names(...) was done + // before values by changing to a list + *expected_values = + ExpectedValues::Some(FxHashSet::default()); + } + }) .or_insert_with(|| { ExpectedValues::Some(FxHashSet::default()) }); let ExpectedValues::Some(expected_values) = expected_values else { - bug!("shoudn't be possible") + bug!("`expected_values` should be a list a values") }; for val in values { diff --git a/tests/ui/check-cfg/order-independant.names_after.stderr b/tests/ui/check-cfg/order-independant.names_after.stderr new file mode 100644 index 00000000000..91b81428b38 --- /dev/null +++ b/tests/ui/check-cfg/order-independant.names_after.stderr @@ -0,0 +1,19 @@ +warning: unexpected `cfg` condition value + --> $DIR/order-independant.rs:8:7 + | +LL | #[cfg(a)] + | ^- help: specify a config value: `= "b"` + | + = note: expected values for `a` are: `b` + = note: `#[warn(unexpected_cfgs)]` on by default + +warning: unexpected `cfg` condition value + --> $DIR/order-independant.rs:12:7 + | +LL | #[cfg(a = "unk")] + | ^^^^^^^^^ + | + = note: expected values for `a` are: `b` + +warning: 2 warnings emitted + diff --git a/tests/ui/check-cfg/order-independant.names_before.stderr b/tests/ui/check-cfg/order-independant.names_before.stderr new file mode 100644 index 00000000000..91b81428b38 --- /dev/null +++ b/tests/ui/check-cfg/order-independant.names_before.stderr @@ -0,0 +1,19 @@ +warning: unexpected `cfg` condition value + --> $DIR/order-independant.rs:8:7 + | +LL | #[cfg(a)] + | ^- help: specify a config value: `= "b"` + | + = note: expected values for `a` are: `b` + = note: `#[warn(unexpected_cfgs)]` on by default + +warning: unexpected `cfg` condition value + --> $DIR/order-independant.rs:12:7 + | +LL | #[cfg(a = "unk")] + | ^^^^^^^^^ + | + = note: expected values for `a` are: `b` + +warning: 2 warnings emitted + diff --git a/tests/ui/check-cfg/order-independant.rs b/tests/ui/check-cfg/order-independant.rs new file mode 100644 index 00000000000..ce056b8dcd6 --- /dev/null +++ b/tests/ui/check-cfg/order-independant.rs @@ -0,0 +1,16 @@ +// check-pass +// revisions: names_before names_after +// compile-flags: -Z unstable-options +// compile-flags: --check-cfg=names(names_before,names_after) +// [names_before]compile-flags: --check-cfg=names(a) --check-cfg=values(a,"b") +// [names_after]compile-flags: --check-cfg=values(a,"b") --check-cfg=names(a) + +#[cfg(a)] +//~^ WARNING unexpected `cfg` condition value +fn my_cfg() {} + +#[cfg(a = "unk")] +//~^ WARNING unexpected `cfg` condition value +fn my_cfg() {} + +fn main() {} From d548747c8531f2f0dfd3921b85aafb09ded27d49 Mon Sep 17 00:00:00 2001 From: Ali MJ Al-Nasrawy Date: Thu, 22 Dec 2022 14:36:09 +0300 Subject: [PATCH 075/130] use implied bounds when checking opaque types --- .../rustc_hir_analysis/src/check/check.rs | 17 +++++- compiler/rustc_ty_utils/src/implied_bounds.rs | 13 +++- .../wf-in-associated-type.fail.stderr | 25 ++++++++ .../wf-in-associated-type.rs | 45 ++++++++++++++ .../wf-nested.fail.stderr | 19 ++++++ .../wf-nested.pass_sound.stderr | 14 +++++ tests/ui/type-alias-impl-trait/wf-nested.rs | 60 +++++++++++++++++++ 7 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 tests/ui/type-alias-impl-trait/wf-in-associated-type.fail.stderr create mode 100644 tests/ui/type-alias-impl-trait/wf-in-associated-type.rs create mode 100644 tests/ui/type-alias-impl-trait/wf-nested.fail.stderr create mode 100644 tests/ui/type-alias-impl-trait/wf-nested.pass_sound.stderr create mode 100644 tests/ui/type-alias-impl-trait/wf-nested.rs diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index c4d4e0d6d78..5187e63f8e3 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -31,6 +31,7 @@ use rustc_target::abi::FieldIdx; use rustc_target::spec::abi::Abi; use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedDirective; use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _; +use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _; use rustc_trait_selection::traits::{self, ObligationCtxt, TraitEngine, TraitEngineExt as _}; use std::ops::ControlFlow; @@ -222,7 +223,7 @@ fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) { if check_opaque_for_cycles(tcx, item.owner_id.def_id, substs, span, &origin).is_err() { return; } - check_opaque_meets_bounds(tcx, item.owner_id.def_id, substs, span, &origin); + check_opaque_meets_bounds(tcx, item.owner_id.def_id, span, &origin); } /// Checks that an opaque type does not use `Self` or `T::Foo` projections that would result @@ -391,7 +392,6 @@ pub(super) fn check_opaque_for_cycles<'tcx>( fn check_opaque_meets_bounds<'tcx>( tcx: TyCtxt<'tcx>, def_id: LocalDefId, - substs: SubstsRef<'tcx>, span: Span, origin: &hir::OpaqueTyOrigin, ) { @@ -406,6 +406,8 @@ fn check_opaque_meets_bounds<'tcx>( .with_opaque_type_inference(DefiningAnchor::Bind(defining_use_anchor)) .build(); let ocx = ObligationCtxt::new(&infcx); + + let substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id()); let opaque_ty = tcx.mk_opaque(def_id.to_def_id(), substs); // `ReErased` regions appear in the "parent_substs" of closures/generators. @@ -448,9 +450,18 @@ fn check_opaque_meets_bounds<'tcx>( match origin { // Checked when type checking the function containing them. hir::OpaqueTyOrigin::FnReturn(..) | hir::OpaqueTyOrigin::AsyncFn(..) => {} + // Nested opaque types occur only in associated types: + // ` type Opaque = impl Trait<&'static T, AssocTy = impl Nested>; ` + // They can only be referenced as ` as Trait<&'static T>>::AssocTy`. + // We don't have to check them here because their well-formedness follows from the WF of + // the projection input types in the defining- and use-sites. + hir::OpaqueTyOrigin::TyAlias + if tcx.def_kind(tcx.parent(def_id.to_def_id())) == DefKind::OpaqueTy => {} // Can have different predicates to their defining use hir::OpaqueTyOrigin::TyAlias => { - let outlives_env = OutlivesEnvironment::new(param_env); + let wf_tys = ocx.assumed_wf_types(param_env, span, def_id); + let implied_bounds = infcx.implied_bounds_tys(param_env, def_id, wf_tys); + let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds); let _ = ocx.resolve_regions_and_report_errors(defining_use_anchor, &outlives_env); } } diff --git a/compiler/rustc_ty_utils/src/implied_bounds.rs b/compiler/rustc_ty_utils/src/implied_bounds.rs index 56d6cc28bc8..5ca5d14337c 100644 --- a/compiler/rustc_ty_utils/src/implied_bounds.rs +++ b/compiler/rustc_ty_utils/src/implied_bounds.rs @@ -31,6 +31,18 @@ fn assumed_wf_types(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List> { } } DefKind::AssocConst | DefKind::AssocTy => tcx.assumed_wf_types(tcx.parent(def_id)), + DefKind::OpaqueTy => match tcx.def_kind(tcx.parent(def_id)) { + DefKind::TyAlias => ty::List::empty(), + DefKind::AssocTy => tcx.assumed_wf_types(tcx.parent(def_id)), + // Nested opaque types only occur in associated types: + // ` type Opaque = impl Trait<&'static T, AssocTy = impl Nested>; ` + // assumed_wf_types should include those of `Opaque`, `Opaque` itself + // and `&'static T`. + DefKind::OpaqueTy => bug!("unimplemented implied bounds for neseted opaque types"), + def_kind @ _ => { + bug!("unimplemented implied bounds for opaque types with parent {def_kind:?}") + } + }, DefKind::Mod | DefKind::Struct | DefKind::Union @@ -51,7 +63,6 @@ fn assumed_wf_types(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::List> { | DefKind::ForeignMod | DefKind::AnonConst | DefKind::InlineConst - | DefKind::OpaqueTy | DefKind::ImplTraitPlaceholder | DefKind::Field | DefKind::LifetimeParam diff --git a/tests/ui/type-alias-impl-trait/wf-in-associated-type.fail.stderr b/tests/ui/type-alias-impl-trait/wf-in-associated-type.fail.stderr new file mode 100644 index 00000000000..9e96323ab54 --- /dev/null +++ b/tests/ui/type-alias-impl-trait/wf-in-associated-type.fail.stderr @@ -0,0 +1,25 @@ +error[E0309]: the parameter type `T` may not live long enough + --> $DIR/wf-in-associated-type.rs:36:23 + | +LL | type Opaque = impl Sized + 'a; + | ^^^^^^^^^^^^^^^ ...so that the type `&'a T` will meet its required lifetime bounds + | +help: consider adding an explicit lifetime bound... + | +LL | impl<'a, T: 'a> Trait<'a, T> for () { + | ++++ + +error[E0309]: the parameter type `T` may not live long enough + --> $DIR/wf-in-associated-type.rs:36:23 + | +LL | type Opaque = impl Sized + 'a; + | ^^^^^^^^^^^^^^^ ...so that the reference type `&'a T` does not outlive the data it points at + | +help: consider adding an explicit lifetime bound... + | +LL | impl<'a, T: 'a> Trait<'a, T> for () { + | ++++ + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0309`. diff --git a/tests/ui/type-alias-impl-trait/wf-in-associated-type.rs b/tests/ui/type-alias-impl-trait/wf-in-associated-type.rs new file mode 100644 index 00000000000..31fbef9f78f --- /dev/null +++ b/tests/ui/type-alias-impl-trait/wf-in-associated-type.rs @@ -0,0 +1,45 @@ +// WF check for impl Trait in associated type position. +// +// revisions: pass fail +// [pass] check-pass +// [fail] check-fail + +#![feature(impl_trait_in_assoc_type)] + +// The hidden type here (`&'a T`) requires proving `T: 'a`. +// We know it holds because of implied bounds from the impl header. +#[cfg(pass)] +mod pass { + trait Trait { + type Opaque1; + fn constrain_opaque1(req: Req) -> Self::Opaque1; + } + + impl<'a, T> Trait<&'a T> for () { + type Opaque1 = impl IntoIterator; + fn constrain_opaque1(req: &'a T) -> Self::Opaque1 { + [req] + } + } +} + +// The hidden type here (`&'a T`) requires proving `T: 'a`, +// but that is not known to hold in the impl. +#[cfg(fail)] +mod fail { + trait Trait<'a, T> { + type Opaque; + fn constrain_opaque(req: &'a T) -> Self::Opaque; + } + + impl<'a, T> Trait<'a, T> for () { + type Opaque = impl Sized + 'a; + //[fail]~^ ERROR the parameter type `T` may not live long enough + //[fail]~| ERROR the parameter type `T` may not live long enough + fn constrain_opaque(req: &'a T) -> Self::Opaque { + req + } + } +} + +fn main() {} diff --git a/tests/ui/type-alias-impl-trait/wf-nested.fail.stderr b/tests/ui/type-alias-impl-trait/wf-nested.fail.stderr new file mode 100644 index 00000000000..753a46e882e --- /dev/null +++ b/tests/ui/type-alias-impl-trait/wf-nested.fail.stderr @@ -0,0 +1,19 @@ +error[E0310]: the parameter type `T` may not live long enough + --> $DIR/wf-nested.rs:55:27 + | +LL | type InnerOpaque = impl Sized; + | ^^^^^^^^^^ ...so that the type `T` will meet its required lifetime bounds... + | +note: ...that is required by this bound + --> $DIR/wf-nested.rs:12:20 + | +LL | struct IsStatic(T); + | ^^^^^^^ +help: consider adding an explicit lifetime bound... + | +LL | type InnerOpaque = impl Sized; + | +++++++++ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0310`. diff --git a/tests/ui/type-alias-impl-trait/wf-nested.pass_sound.stderr b/tests/ui/type-alias-impl-trait/wf-nested.pass_sound.stderr new file mode 100644 index 00000000000..9ab6685a7f7 --- /dev/null +++ b/tests/ui/type-alias-impl-trait/wf-nested.pass_sound.stderr @@ -0,0 +1,14 @@ +error[E0310]: the parameter type `T` may not live long enough + --> $DIR/wf-nested.rs:46:17 + | +LL | let _ = outer.get(); + | ^^^^^^^^^^^ ...so that the type `T` will meet its required lifetime bounds + | +help: consider adding an explicit lifetime bound... + | +LL | fn test() { + | +++++++++ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0310`. diff --git a/tests/ui/type-alias-impl-trait/wf-nested.rs b/tests/ui/type-alias-impl-trait/wf-nested.rs new file mode 100644 index 00000000000..de388329489 --- /dev/null +++ b/tests/ui/type-alias-impl-trait/wf-nested.rs @@ -0,0 +1,60 @@ +// Well-formedness of nested opaque types, i.e. `impl Sized` in +// `type Outer = impl Trait`. +// See the comments below. +// +// revisions: pass pass_sound fail +// [pass] check-pass +// [pass_sound] check-fail +// [fail] check-fail + +#![feature(type_alias_impl_trait)] + +struct IsStatic(T); + +trait Trait { + type Out; + + fn get(&self) -> Result { + Err(()) + } +} + +impl Trait<&'static T> for () { + type Out = IsStatic; +} + +// The hidden type for `impl Sized` is `IsStatic`, which requires `T: 'static`. +// We know it is well-formed because it can *only* be referenced as a projection: +// as Trait<&'static T>>::Out`. +// So any instantiation of the type already requires proving `T: 'static`. +#[cfg(pass)] +mod pass { + use super::*; + type OuterOpaque = impl Trait<&'static T, Out = impl Sized>; + fn define() -> OuterOpaque {} +} + +// Test the soundness of `pass` - We should require `T: 'static` at the use site. +#[cfg(pass_sound)] +mod pass_sound { + use super::*; + type OuterOpaque = impl Trait<&'static T, Out = impl Sized>; + fn define() -> OuterOpaque {} + + fn test() { + let outer = define::(); + let _ = outer.get(); //[pass_sound]~ ERROR `T` may not live long enough + } +} + +// Similar to `pass` but here `impl Sized` can be referenced directly as +// InnerOpaque, so we require an explicit bound `T: 'static`. +#[cfg(fail)] +mod fail { + use super::*; + type InnerOpaque = impl Sized; //[fail]~ ERROR `T` may not live long enough + type OuterOpaque = impl Trait<&'static T, Out = InnerOpaque>; + fn define() -> OuterOpaque {} +} + +fn main() {} From 4967f25f6bf930a5f79d5c66f2ffc53159d43c4a Mon Sep 17 00:00:00 2001 From: Markus Everling Date: Sun, 7 May 2023 00:11:56 +0000 Subject: [PATCH 076/130] Use the new `load`/`store` functions in `{from,to}_slice` --- crates/core_simd/src/vector.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 92984f55e45..a793ae9e391 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -263,11 +263,9 @@ where slice.len() >= Self::N, "slice length must be at least the number of elements" ); - assert!(core::mem::size_of::() == Self::N * core::mem::size_of::()); - // Safety: - // - We've checked the length is sufficient. - // - `T` and `Simd` are Copy types. - unsafe { slice.as_ptr().cast::().read_unaligned() } + // SAFETY: We just checked that the slice contains + // at least `N` elements. + unsafe { Self::load(slice.as_ptr().cast()) } } /// Writes a SIMD vector to the first `N` elements of a slice. @@ -293,11 +291,9 @@ where slice.len() >= Self::N, "slice length must be at least the number of elements" ); - assert!(core::mem::size_of::() == Self::N * core::mem::size_of::()); - // Safety: - // - We've checked the length is sufficient - // - `T` and `Simd` are Copy types. - unsafe { slice.as_mut_ptr().cast::().write_unaligned(self) } + // SAFETY: We just checked that the slice contains + // at least `N` elements. + unsafe { self.store(slice.as_mut_ptr().cast()) } } /// Performs elementwise conversion of a SIMD vector's elements to another SIMD-valid type. From b246e454387ef2d80078db36975d2df5d957f9fa Mon Sep 17 00:00:00 2001 From: Markus Everling Date: Sun, 7 May 2023 00:15:18 +0000 Subject: [PATCH 077/130] Fix inaccurate safety comments --- crates/core_simd/src/vector.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 92984f55e45..ff761fc900f 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -156,9 +156,9 @@ where /// assert_eq!(v.as_array(), &[0, 1, 2, 3]); /// ``` pub const fn as_array(&self) -> &[T; N] { - // SAFETY: Transmuting between `Simd` and `[T; N]` - // is always valid and `Simd` never has a lower alignment - // than `[T; N]`. + // SAFETY: `Simd` is just an overaligned `[T; N]` with + // potential padding at the end, so pointer casting to a + // `&[T; N]` is safe. // // NOTE: This deliberately doesn't just use `&self.0`, see the comment // on the struct definition for details. @@ -167,9 +167,9 @@ where /// Returns a mutable array reference containing the entire SIMD vector. pub fn as_mut_array(&mut self) -> &mut [T; N] { - // SAFETY: Transmuting between `Simd` and `[T; N]` - // is always valid and `Simd` never has a lower alignment - // than `[T; N]`. + // SAFETY: `Simd` is just an overaligned `[T; N]` with + // potential padding at the end, so pointer casting to a + // `&mut [T; N]` is safe. // // NOTE: This deliberately doesn't just use `&mut self.0`, see the comment // on the struct definition for details. From 27a3ce29e368f7f0167761f761bf5826c83cf12c Mon Sep 17 00:00:00 2001 From: Zalathar Date: Thu, 4 May 2023 17:30:09 +1000 Subject: [PATCH 078/130] Fix instrument-coverage tests by using Python to sort instantiation groups --- tests/run-make/coverage-reports/Makefile | 23 +++------ .../expected_show_coverage.async.txt | 4 +- .../expected_show_coverage.sort_groups.txt | 49 ++++++++++++++++++ .../expected_show_coverage.uses_crate.txt | 18 +++---- ...pected_show_coverage.uses_inline_crate.txt | 14 +++--- .../coverage-reports/sort_subviews.py | 50 +++++++++++++++++++ tests/run-make/coverage/sort_groups.rs | 23 +++++++++ 7 files changed, 148 insertions(+), 33 deletions(-) create mode 100644 tests/run-make/coverage-reports/expected_show_coverage.sort_groups.txt create mode 100644 tests/run-make/coverage-reports/sort_subviews.py create mode 100644 tests/run-make/coverage/sort_groups.rs diff --git a/tests/run-make/coverage-reports/Makefile b/tests/run-make/coverage-reports/Makefile index d4ae03e590a..0ae409c4119 100644 --- a/tests/run-make/coverage-reports/Makefile +++ b/tests/run-make/coverage-reports/Makefile @@ -138,6 +138,7 @@ endif ) \ 2> "$(TMPDIR)"/show_coverage_stderr.$@.txt \ | "$(PYTHON)" $(BASEDIR)/normalize_paths.py \ + | "$(PYTHON)" $(BASEDIR)/sort_subviews.py \ > "$(TMPDIR)"/actual_show_coverage.$@.txt || \ ( status=$$? ; \ >&2 cat "$(TMPDIR)"/show_coverage_stderr.$@.txt ; \ @@ -158,23 +159,15 @@ ifdef RUSTC_BLESS_TEST else # Compare the show coverage output (`--bless` refreshes `typical` files). # - # FIXME(richkadel): None of the Rust test source samples have the - # `// ignore-llvm-cov-show-diffs` anymore. This directive exists to work around a limitation - # with `llvm-cov show`. When reporting coverage for multiple instantiations of a generic function, - # with different type substitutions, `llvm-cov show` prints these in a non-deterministic order, - # breaking the `diff` comparison. + # `llvm-cov show` normally prints instantiation groups in an unpredictable + # order, but we have used `sort_subviews.py` to sort them, so we can still + # check the output directly with `diff`. # - # A partial workaround is implemented below, with `diff --ignore-matching-lines=RE` - # to ignore each line prefixing each generic instantiation coverage code region. - # - # This workaround only works if the coverage counts are identical across all reported - # instantiations. If there is no way to ensure this, you may need to apply the - # `// ignore-llvm-cov-show-diffs` directive, and check for differences using the - # `.json` files to validate that results have not changed. (Until then, the JSON - # files are redundant, so there is no need to generate `expected_*.json` files or - # compare actual JSON results.) + # Some of the test cases are currently not working (since #110393) and have + # been marked with `// ignore-llvm-cov-show-diffs` so that they don't fail + # the build. - $(DIFF) --ignore-matching-lines='^ \| .*::<.*>.*:$$' --ignore-matching-lines='^ \| <.*>::.*:$$' \ + $(DIFF) \ expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \ ( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \ >&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \ diff --git a/tests/run-make/coverage-reports/expected_show_coverage.async.txt b/tests/run-make/coverage-reports/expected_show_coverage.async.txt index 87ccb6c43ea..93c1535b06b 100644 --- a/tests/run-make/coverage-reports/expected_show_coverage.async.txt +++ b/tests/run-make/coverage-reports/expected_show_coverage.async.txt @@ -41,9 +41,9 @@ 41| 1| // executed asynchronously. 42| 1| match x { 43| 1| y if c(x).await == y + 1 => { d().await; } - ^0 ^0 ^0 ^0 + ^0 ^0 ^0 ^0 44| 1| y if f().await == y + 1 => (), - ^0 ^0 ^0 + ^0 ^0 ^0 45| 1| _ => (), 46| | } 47| 1|} diff --git a/tests/run-make/coverage-reports/expected_show_coverage.sort_groups.txt b/tests/run-make/coverage-reports/expected_show_coverage.sort_groups.txt new file mode 100644 index 00000000000..81468cb35da --- /dev/null +++ b/tests/run-make/coverage-reports/expected_show_coverage.sort_groups.txt @@ -0,0 +1,49 @@ + 1| |// compile-flags: --edition=2021 + 2| | + 3| |// Demonstrate that `sort_subviews.py` can sort instantiation groups into a + 4| |// predictable order, while preserving their heterogeneous contents. + 5| | + 6| 1|fn main() { + 7| 1| let cond = std::env::args().len() > 1; + 8| 1| generic_fn::<()>(cond); + 9| 1| generic_fn::<&'static str>(!cond); + 10| 1| if false { + 11| 0| generic_fn::(cond); + 12| 1| } + 13| 1| generic_fn::(cond); + 14| 1| other_fn(); + 15| 1|} + 16| | + 17| 3|fn generic_fn(cond: bool) { + 18| 3| if cond { + 19| 1| println!("{}", std::any::type_name::()); + 20| 2| } + 21| 3|} + ------------------ + | Unexecuted instantiation: sort_groups::generic_fn:: + ------------------ + | sort_groups::generic_fn::<&str>: + | 17| 1|fn generic_fn(cond: bool) { + | 18| 1| if cond { + | 19| 1| println!("{}", std::any::type_name::()); + | 20| 1| } + | ^0 + | 21| 1|} + ------------------ + | sort_groups::generic_fn::<()>: + | 17| 1|fn generic_fn(cond: bool) { + | 18| 1| if cond { + | 19| 0| println!("{}", std::any::type_name::()); + | 20| 1| } + | 21| 1|} + ------------------ + | sort_groups::generic_fn::: + | 17| 1|fn generic_fn(cond: bool) { + | 18| 1| if cond { + | 19| 0| println!("{}", std::any::type_name::()); + | 20| 1| } + | 21| 1|} + ------------------ + 22| | + 23| 1|fn other_fn() {} + diff --git a/tests/run-make/coverage-reports/expected_show_coverage.uses_crate.txt b/tests/run-make/coverage-reports/expected_show_coverage.uses_crate.txt index 65eb1008dd8..412f4a93b9c 100644 --- a/tests/run-make/coverage-reports/expected_show_coverage.uses_crate.txt +++ b/tests/run-make/coverage-reports/expected_show_coverage.uses_crate.txt @@ -19,29 +19,29 @@ 18| 2| println!("used_only_from_bin_crate_generic_function with {:?}", arg); 19| 2|} ------------------ - | used_crate::used_only_from_bin_crate_generic_function::<&str>: - | 17| 1|pub fn used_only_from_bin_crate_generic_function(arg: T) { - | 18| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg); - | 19| 1|} + | Unexecuted instantiation: used_crate::used_only_from_bin_crate_generic_function::<_> ------------------ | used_crate::used_only_from_bin_crate_generic_function::<&alloc::vec::Vec>: | 17| 1|pub fn used_only_from_bin_crate_generic_function(arg: T) { | 18| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg); | 19| 1|} ------------------ - | Unexecuted instantiation: used_crate::used_only_from_bin_crate_generic_function::<_> + | used_crate::used_only_from_bin_crate_generic_function::<&str>: + | 17| 1|pub fn used_only_from_bin_crate_generic_function(arg: T) { + | 18| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg); + | 19| 1|} ------------------ 20| |// Expect for above function: `Unexecuted instantiation` (see below) 21| 2|pub fn used_only_from_this_lib_crate_generic_function(arg: T) { 22| 2| println!("used_only_from_this_lib_crate_generic_function with {:?}", arg); 23| 2|} ------------------ - | used_crate::used_only_from_this_lib_crate_generic_function::>: + | used_crate::used_only_from_this_lib_crate_generic_function::<&str>: | 21| 1|pub fn used_only_from_this_lib_crate_generic_function(arg: T) { | 22| 1| println!("used_only_from_this_lib_crate_generic_function with {:?}", arg); | 23| 1|} ------------------ - | used_crate::used_only_from_this_lib_crate_generic_function::<&str>: + | used_crate::used_only_from_this_lib_crate_generic_function::>: | 21| 1|pub fn used_only_from_this_lib_crate_generic_function(arg: T) { | 22| 1| println!("used_only_from_this_lib_crate_generic_function with {:?}", arg); | 23| 1|} @@ -51,12 +51,12 @@ 26| 2| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); 27| 2|} ------------------ - | used_crate::used_from_bin_crate_and_lib_crate_generic_function::>: + | used_crate::used_from_bin_crate_and_lib_crate_generic_function::<&str>: | 25| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function(arg: T) { | 26| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); | 27| 1|} ------------------ - | used_crate::used_from_bin_crate_and_lib_crate_generic_function::<&str>: + | used_crate::used_from_bin_crate_and_lib_crate_generic_function::>: | 25| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function(arg: T) { | 26| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); | 27| 1|} diff --git a/tests/run-make/coverage-reports/expected_show_coverage.uses_inline_crate.txt b/tests/run-make/coverage-reports/expected_show_coverage.uses_inline_crate.txt index 748343885de..66ca9e80a32 100644 --- a/tests/run-make/coverage-reports/expected_show_coverage.uses_inline_crate.txt +++ b/tests/run-make/coverage-reports/expected_show_coverage.uses_inline_crate.txt @@ -42,6 +42,8 @@ 40| 2| println!("used_only_from_bin_crate_generic_function with {:?}", arg); 41| 2|} ------------------ + | Unexecuted instantiation: used_inline_crate::used_only_from_bin_crate_generic_function::<_> + ------------------ | used_inline_crate::used_only_from_bin_crate_generic_function::<&alloc::vec::Vec>: | 39| 1|pub fn used_only_from_bin_crate_generic_function(arg: T) { | 40| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg); @@ -51,8 +53,6 @@ | 39| 1|pub fn used_only_from_bin_crate_generic_function(arg: T) { | 40| 1| println!("used_only_from_bin_crate_generic_function with {:?}", arg); | 41| 1|} - ------------------ - | Unexecuted instantiation: used_inline_crate::used_only_from_bin_crate_generic_function::<_> ------------------ 42| |// Expect for above function: `Unexecuted instantiation` (see notes in `used_crate.rs`) 43| | @@ -77,15 +77,15 @@ 51| 3| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); 52| 3|} ------------------ - | used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function::>: - | 50| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function(arg: T) { - | 51| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); - | 52| 1|} - ------------------ | used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function::<&str>: | 50| 2|pub fn used_from_bin_crate_and_lib_crate_generic_function(arg: T) { | 51| 2| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); | 52| 2|} + ------------------ + | used_inline_crate::used_from_bin_crate_and_lib_crate_generic_function::>: + | 50| 1|pub fn used_from_bin_crate_and_lib_crate_generic_function(arg: T) { + | 51| 1| println!("used_from_bin_crate_and_lib_crate_generic_function with {:?}", arg); + | 52| 1|} ------------------ 53| | 54| |#[inline(always)] diff --git a/tests/run-make/coverage-reports/sort_subviews.py b/tests/run-make/coverage-reports/sort_subviews.py new file mode 100644 index 00000000000..10cfc51d447 --- /dev/null +++ b/tests/run-make/coverage-reports/sort_subviews.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +# `llvm-cov show` prints grouped subviews (e.g. for generic functions) in an +# unstable order, which is inconvenient when checking output snapshots with +# `diff`. To work around that, this script detects consecutive subviews in its +# piped input, and sorts them while preserving their contents. + +from __future__ import print_function + +import sys + + +def main(): + subviews = [] + + def flush_subviews(): + if not subviews: + return + + # The last "subview" should be just a boundary line on its own, so + # temporarily remove it before sorting the accumulated subviews. + terminator = subviews.pop() + subviews.sort() + subviews.append(terminator) + + for view in subviews: + for line in view: + print(line, end="") + + subviews.clear() + + for line in sys.stdin: + if line.startswith(" ------------------"): + # This is a subview boundary line, so start a new subview. + subviews.append([line]) + elif line.startswith(" |"): + # Add this line to the current subview. + subviews[-1].append(line) + else: + # This line is not part of a subview, so sort and print any + # accumulated subviews, and then print the line as-is. + flush_subviews() + print(line, end="") + + flush_subviews() + assert not subviews + + +if __name__ == "__main__": + main() diff --git a/tests/run-make/coverage/sort_groups.rs b/tests/run-make/coverage/sort_groups.rs new file mode 100644 index 00000000000..f89f9f3ec61 --- /dev/null +++ b/tests/run-make/coverage/sort_groups.rs @@ -0,0 +1,23 @@ +// compile-flags: --edition=2021 + +// Demonstrate that `sort_subviews.py` can sort instantiation groups into a +// predictable order, while preserving their heterogeneous contents. + +fn main() { + let cond = std::env::args().len() > 1; + generic_fn::<()>(cond); + generic_fn::<&'static str>(!cond); + if false { + generic_fn::(cond); + } + generic_fn::(cond); + other_fn(); +} + +fn generic_fn(cond: bool) { + if cond { + println!("{}", std::any::type_name::()); + } +} + +fn other_fn() {} From 7f74ae57e8e24e87b7177af43db6ec0e8262cd5d Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Mon, 24 Apr 2023 00:47:01 +0000 Subject: [PATCH 079/130] Create a trait to abstract over the smir API --- compiler/rustc_smir/src/lib.rs | 2 + compiler/rustc_smir/src/rustc_internal/mod.rs | 47 +++++++--- compiler/rustc_smir/src/rustc_smir/mod.rs | 86 +++++++++---------- compiler/rustc_smir/src/stable_mir/mod.rs | 66 ++++++++++++-- tests/ui-fulldeps/stable-mir/crate-info.rs | 2 +- 5 files changed, 138 insertions(+), 65 deletions(-) diff --git a/compiler/rustc_smir/src/lib.rs b/compiler/rustc_smir/src/lib.rs index 54d474db038..b00f0a1c153 100644 --- a/compiler/rustc_smir/src/lib.rs +++ b/compiler/rustc_smir/src/lib.rs @@ -11,6 +11,8 @@ test(attr(allow(unused_variables), deny(warnings))) )] #![cfg_attr(not(feature = "default"), feature(rustc_private))] +#![feature(local_key_cell_methods)] +#![feature(ptr_metadata)] pub mod rustc_internal; pub mod stable_mir; diff --git a/compiler/rustc_smir/src/rustc_internal/mod.rs b/compiler/rustc_smir/src/rustc_internal/mod.rs index 5998c8b6500..e643ad28c03 100644 --- a/compiler/rustc_smir/src/rustc_internal/mod.rs +++ b/compiler/rustc_smir/src/rustc_internal/mod.rs @@ -3,30 +3,49 @@ //! For that, we define APIs that will temporarily be public to 3P that exposes rustc internal APIs //! until stable MIR is complete. -use std::sync::RwLock; - -use crate::stable_mir; +use crate::{ + rustc_smir::Tables, + stable_mir::{self, with}, +}; +use rustc_middle::ty::TyCtxt; pub use rustc_span::def_id::{CrateNum, DefId}; -static DEF_ID_MAP: RwLock> = RwLock::new(Vec::new()); +fn with_tables(mut f: impl FnMut(&mut Tables<'_>) -> R) -> R { + let mut ret = None; + with(|tables| tables.rustc_tables(&mut |t| ret = Some(f(t)))); + ret.unwrap() +} pub fn item_def_id(item: &stable_mir::CrateItem) -> DefId { - DEF_ID_MAP.read().unwrap()[item.0] + with_tables(|t| t.item_def_id(item)) } pub fn crate_item(did: DefId) -> stable_mir::CrateItem { - // FIXME: this becomes inefficient when we have too many ids - let mut map = DEF_ID_MAP.write().unwrap(); - for (i, &d) in map.iter().enumerate() { - if d == did { - return stable_mir::CrateItem(i); - } + with_tables(|t| t.crate_item(did)) +} + +impl<'tcx> Tables<'tcx> { + pub fn item_def_id(&self, item: &stable_mir::CrateItem) -> DefId { + self.def_ids[item.0] + } + + pub fn crate_item(&mut self, did: DefId) -> stable_mir::CrateItem { + // FIXME: this becomes inefficient when we have too many ids + for (i, &d) in self.def_ids.iter().enumerate() { + if d == did { + return stable_mir::CrateItem(i); + } + } + let id = self.def_ids.len(); + self.def_ids.push(did); + stable_mir::CrateItem(id) } - let id = map.len(); - map.push(did); - stable_mir::CrateItem(id) } pub fn crate_num(item: &stable_mir::Crate) -> CrateNum { item.id.into() } + +pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) { + crate::stable_mir::run(Tables { tcx, def_ids: vec![] }, f); +} diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs index 241cd182059..81ecce415d5 100644 --- a/compiler/rustc_smir/src/rustc_smir/mod.rs +++ b/compiler/rustc_smir/src/rustc_smir/mod.rs @@ -7,55 +7,36 @@ //! //! For now, we are developing everything inside `rustc`, thus, we keep this module private. -use crate::{ - rustc_internal::{crate_item, item_def_id}, - stable_mir::{self}, -}; -use rustc_middle::ty::{tls::with, TyCtxt}; -use rustc_span::def_id::{CrateNum, LOCAL_CRATE}; +use crate::stable_mir::{self, Context}; +use rustc_middle::ty::TyCtxt; +use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE}; use tracing::debug; -/// Get information about the local crate. -pub fn local_crate() -> stable_mir::Crate { - with(|tcx| smir_crate(tcx, LOCAL_CRATE)) -} +impl<'tcx> Context for Tables<'tcx> { + fn local_crate(&self) -> stable_mir::Crate { + smir_crate(self.tcx, LOCAL_CRATE) + } -/// Retrieve a list of all external crates. -pub fn external_crates() -> Vec { - with(|tcx| tcx.crates(()).iter().map(|crate_num| smir_crate(tcx, *crate_num)).collect()) -} + fn external_crates(&self) -> Vec { + self.tcx.crates(()).iter().map(|crate_num| smir_crate(self.tcx, *crate_num)).collect() + } -/// Find a crate with the given name. -pub fn find_crate(name: &str) -> Option { - with(|tcx| { - [LOCAL_CRATE].iter().chain(tcx.crates(()).iter()).find_map(|crate_num| { - let crate_name = tcx.crate_name(*crate_num).to_string(); - (name == crate_name).then(|| smir_crate(tcx, *crate_num)) + fn find_crate(&self, name: &str) -> Option { + [LOCAL_CRATE].iter().chain(self.tcx.crates(()).iter()).find_map(|crate_num| { + let crate_name = self.tcx.crate_name(*crate_num).to_string(); + (name == crate_name).then(|| smir_crate(self.tcx, *crate_num)) }) - }) -} + } -/// Retrieve all items of the local crate that have a MIR associated with them. -pub fn all_local_items() -> stable_mir::CrateItems { - with(|tcx| tcx.mir_keys(()).iter().map(|item| crate_item(item.to_def_id())).collect()) -} - -pub fn entry_fn() -> Option { - with(|tcx| Some(crate_item(tcx.entry_fn(())?.0))) -} - -/// Build a stable mir crate from a given crate number. -fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate { - let crate_name = tcx.crate_name(crate_num).to_string(); - let is_local = crate_num == LOCAL_CRATE; - debug!(?crate_name, ?crate_num, "smir_crate"); - stable_mir::Crate { id: crate_num.into(), name: crate_name, is_local } -} - -pub fn mir_body(item: &stable_mir::CrateItem) -> stable_mir::mir::Body { - with(|tcx| { - let def_id = item_def_id(item); - let mir = tcx.optimized_mir(def_id); + fn all_local_items(&mut self) -> stable_mir::CrateItems { + self.tcx.mir_keys(()).iter().map(|item| self.crate_item(item.to_def_id())).collect() + } + fn entry_fn(&mut self) -> Option { + Some(self.crate_item(self.tcx.entry_fn(())?.0)) + } + fn mir_body(&self, item: &stable_mir::CrateItem) -> stable_mir::mir::Body { + let def_id = self.item_def_id(item); + let mir = self.tcx.optimized_mir(def_id); stable_mir::mir::Body { blocks: mir .basic_blocks @@ -66,7 +47,24 @@ pub fn mir_body(item: &stable_mir::CrateItem) -> stable_mir::mir::Body { }) .collect(), } - }) + } + + fn rustc_tables(&mut self, f: &mut dyn FnMut(&mut Tables<'_>)) { + f(self) + } +} + +pub struct Tables<'tcx> { + pub tcx: TyCtxt<'tcx>, + pub def_ids: Vec, +} + +/// Build a stable mir crate from a given crate number. +fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate { + let crate_name = tcx.crate_name(crate_num).to_string(); + let is_local = crate_num == LOCAL_CRATE; + debug!(?crate_name, ?crate_num, "smir_crate"); + stable_mir::Crate { id: crate_num.into(), name: crate_name, is_local } } fn rustc_statement_to_statement( diff --git a/compiler/rustc_smir/src/stable_mir/mod.rs b/compiler/rustc_smir/src/stable_mir/mod.rs index 1d2efb5eab9..612777b9c75 100644 --- a/compiler/rustc_smir/src/stable_mir/mod.rs +++ b/compiler/rustc_smir/src/stable_mir/mod.rs @@ -11,7 +11,14 @@ //! There shouldn't be any direct references to internal compiler constructs in this module. //! If you need an internal construct, consider using `rustc_internal` or `rustc_smir`. +use std::cell::Cell; + +use crate::rustc_smir::Tables; + +use self::ty::{Ty, TyKind}; + pub mod mir; +pub mod ty; /// Use String for now but we should replace it. pub type Symbol = String; @@ -41,7 +48,7 @@ pub struct CrateItem(pub(crate) DefId); impl CrateItem { pub fn body(&self) -> mir::Body { - crate::rustc_smir::mir_body(self) + with(|cx| cx.mir_body(self)) } } @@ -49,25 +56,72 @@ impl CrateItem { /// crate defines that. This is usually `main`, but could be /// `start` if the crate is a no-std crate. pub fn entry_fn() -> Option { - crate::rustc_smir::entry_fn() + with(|cx| cx.entry_fn()) } /// Access to the local crate. pub fn local_crate() -> Crate { - crate::rustc_smir::local_crate() + with(|cx| cx.local_crate()) } /// Try to find a crate with the given name. pub fn find_crate(name: &str) -> Option { - crate::rustc_smir::find_crate(name) + with(|cx| cx.find_crate(name)) } /// Try to find a crate with the given name. pub fn external_crates() -> Vec { - crate::rustc_smir::external_crates() + with(|cx| cx.external_crates()) } /// Retrieve all items in the local crate that have a MIR associated with them. pub fn all_local_items() -> CrateItems { - crate::rustc_smir::all_local_items() + with(|cx| cx.all_local_items()) +} + +pub trait Context { + fn entry_fn(&mut self) -> Option; + /// Retrieve all items of the local crate that have a MIR associated with them. + fn all_local_items(&mut self) -> CrateItems; + fn mir_body(&mut self, item: &CrateItem) -> mir::Body; + /// Get information about the local crate. + fn local_crate(&self) -> Crate; + /// Retrieve a list of all external crates. + fn external_crates(&self) -> Vec; + + /// Find a crate with the given name. + fn find_crate(&self, name: &str) -> Option; + + /// Obtain the representation of a type. + fn ty_kind(&mut self, ty: Ty) -> TyKind; + + /// HACK: Until we have fully stable consumers, we need an escape hatch + /// to get `DefId`s out of `CrateItem`s. + fn rustc_tables(&mut self, f: &mut dyn FnMut(&mut Tables<'_>)); +} + +thread_local! { + /// A thread local variable that stores a pointer to the tables mapping between TyCtxt + /// datastructures and stable MIR datastructures. + static TLV: Cell<*mut ()> = const { Cell::new(std::ptr::null_mut()) }; +} + +pub fn run(mut context: impl Context, f: impl FnOnce()) { + assert!(TLV.get().is_null()); + fn g<'a>(mut context: &mut (dyn Context + 'a), f: impl FnOnce()) { + TLV.set(&mut context as *mut &mut _ as _); + f(); + TLV.replace(std::ptr::null_mut()); + } + g(&mut context, f); +} + +/// Loads the current context and calls a function with it. +/// Do not nest these, as that will ICE. +pub(crate) fn with(f: impl FnOnce(&mut dyn Context) -> R) -> R { + let ptr = TLV.replace(std::ptr::null_mut()) as *mut &mut dyn Context; + assert!(!ptr.is_null()); + let ret = f(unsafe { *ptr }); + TLV.set(ptr as _); + ret } diff --git a/tests/ui-fulldeps/stable-mir/crate-info.rs b/tests/ui-fulldeps/stable-mir/crate-info.rs index 1454d6dde6c..b0fcc36b0a8 100644 --- a/tests/ui-fulldeps/stable-mir/crate-info.rs +++ b/tests/ui-fulldeps/stable-mir/crate-info.rs @@ -123,7 +123,7 @@ impl Callbacks for SMirCalls { queries: &'tcx Queries<'tcx>, ) -> Compilation { queries.global_ctxt().unwrap().enter(|tcx| { - test_stable_mir(tcx); + rustc_smir::rustc_internal::run(tcx, || test_stable_mir(tcx)); }); // No need to keep going. Compilation::Stop From 5c6e2342f61ed9296fafb3781f0a89675a470eb9 Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Mon, 24 Apr 2023 01:04:44 +0000 Subject: [PATCH 080/130] Encode types in SMIR --- compiler/rustc_smir/src/rustc_internal/mod.rs | 2 +- compiler/rustc_smir/src/rustc_smir/mod.rs | 57 ++++++++++++++++++- .../rustc_smir/src/stable_mir/mir/body.rs | 3 + compiler/rustc_smir/src/stable_mir/ty.rs | 15 +++++ tests/ui-fulldeps/stable-mir/crate-info.rs | 2 + 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 compiler/rustc_smir/src/stable_mir/ty.rs diff --git a/compiler/rustc_smir/src/rustc_internal/mod.rs b/compiler/rustc_smir/src/rustc_internal/mod.rs index e643ad28c03..609a04d263c 100644 --- a/compiler/rustc_smir/src/rustc_internal/mod.rs +++ b/compiler/rustc_smir/src/rustc_internal/mod.rs @@ -47,5 +47,5 @@ pub fn crate_num(item: &stable_mir::Crate) -> CrateNum { } pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) { - crate::stable_mir::run(Tables { tcx, def_ids: vec![] }, f); + crate::stable_mir::run(Tables { tcx, def_ids: vec![], types: vec![] }, f); } diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs index 81ecce415d5..6af43f5d3f3 100644 --- a/compiler/rustc_smir/src/rustc_smir/mod.rs +++ b/compiler/rustc_smir/src/rustc_smir/mod.rs @@ -7,8 +7,8 @@ //! //! For now, we are developing everything inside `rustc`, thus, we keep this module private. -use crate::stable_mir::{self, Context}; -use rustc_middle::ty::TyCtxt; +use crate::stable_mir::{self, ty::TyKind, Context}; +use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE}; use tracing::debug; @@ -34,7 +34,7 @@ impl<'tcx> Context for Tables<'tcx> { fn entry_fn(&mut self) -> Option { Some(self.crate_item(self.tcx.entry_fn(())?.0)) } - fn mir_body(&self, item: &stable_mir::CrateItem) -> stable_mir::mir::Body { + fn mir_body(&mut self, item: &stable_mir::CrateItem) -> stable_mir::mir::Body { let def_id = self.item_def_id(item); let mir = self.tcx.optimized_mir(def_id); stable_mir::mir::Body { @@ -46,17 +46,68 @@ impl<'tcx> Context for Tables<'tcx> { statements: block.statements.iter().map(rustc_statement_to_statement).collect(), }) .collect(), + locals: mir.local_decls.iter().map(|decl| self.intern_ty(decl.ty)).collect(), } } fn rustc_tables(&mut self, f: &mut dyn FnMut(&mut Tables<'_>)) { f(self) } + + fn ty_kind(&mut self, ty: crate::stable_mir::ty::Ty) -> TyKind { + self.rustc_ty_to_ty(self.types[ty.0]) + } } pub struct Tables<'tcx> { pub tcx: TyCtxt<'tcx>, pub def_ids: Vec, + pub types: Vec>, +} + +impl<'tcx> Tables<'tcx> { + fn rustc_ty_to_ty(&mut self, ty: Ty<'tcx>) -> TyKind { + match ty.kind() { + ty::Bool => TyKind::Bool, + ty::Char => todo!(), + ty::Int(_) => todo!(), + ty::Uint(_) => todo!(), + ty::Float(_) => todo!(), + ty::Adt(_, _) => todo!(), + ty::Foreign(_) => todo!(), + ty::Str => todo!(), + ty::Array(_, _) => todo!(), + ty::Slice(_) => todo!(), + ty::RawPtr(_) => todo!(), + ty::Ref(_, _, _) => todo!(), + ty::FnDef(_, _) => todo!(), + ty::FnPtr(_) => todo!(), + ty::Placeholder(..) => todo!(), + ty::Dynamic(_, _, _) => todo!(), + ty::Closure(_, _) => todo!(), + ty::Generator(_, _, _) => todo!(), + ty::GeneratorWitness(_) => todo!(), + ty::GeneratorWitnessMIR(_, _) => todo!(), + ty::Never => todo!(), + ty::Tuple(fields) => { + TyKind::Tuple(fields.iter().map(|ty| self.intern_ty(ty)).collect()) + } + ty::Alias(_, _) => todo!(), + ty::Param(_) => todo!(), + ty::Bound(_, _) => todo!(), + ty::Infer(_) => todo!(), + ty::Error(_) => todo!(), + } + } + + fn intern_ty(&mut self, ty: Ty<'tcx>) -> stable_mir::ty::Ty { + if let Some(id) = self.types.iter().position(|&t| t == ty) { + return stable_mir::ty::Ty(id); + } + let id = self.types.len(); + self.types.push(ty); + stable_mir::ty::Ty(id) + } } /// Build a stable mir crate from a given crate number. diff --git a/compiler/rustc_smir/src/stable_mir/mir/body.rs b/compiler/rustc_smir/src/stable_mir/mir/body.rs index 4baf3f1f75e..6328c35aa59 100644 --- a/compiler/rustc_smir/src/stable_mir/mir/body.rs +++ b/compiler/rustc_smir/src/stable_mir/mir/body.rs @@ -1,6 +1,9 @@ +use crate::stable_mir::ty::Ty; + #[derive(Clone, Debug)] pub struct Body { pub blocks: Vec, + pub locals: Vec, } #[derive(Clone, Debug)] diff --git a/compiler/rustc_smir/src/stable_mir/ty.rs b/compiler/rustc_smir/src/stable_mir/ty.rs new file mode 100644 index 00000000000..f27801b0f6c --- /dev/null +++ b/compiler/rustc_smir/src/stable_mir/ty.rs @@ -0,0 +1,15 @@ +use super::with; + +#[derive(Copy, Clone, Debug)] +pub struct Ty(pub usize); + +impl Ty { + pub fn kind(&self) -> TyKind { + with(|context| context.ty_kind(*self)) + } +} + +pub enum TyKind { + Bool, + Tuple(Vec), +} diff --git a/tests/ui-fulldeps/stable-mir/crate-info.rs b/tests/ui-fulldeps/stable-mir/crate-info.rs index b0fcc36b0a8..a3db2e9ef24 100644 --- a/tests/ui-fulldeps/stable-mir/crate-info.rs +++ b/tests/ui-fulldeps/stable-mir/crate-info.rs @@ -40,6 +40,7 @@ fn test_stable_mir(tcx: TyCtxt<'_>) { let bar = get_item(tcx, &items, (DefKind::Fn, "bar")).unwrap(); let body = bar.body(); + assert_eq!(body.locals.len(), 2); assert_eq!(body.blocks.len(), 1); let block = &body.blocks[0]; assert_eq!(block.statements.len(), 1); @@ -54,6 +55,7 @@ fn test_stable_mir(tcx: TyCtxt<'_>) { let foo_bar = get_item(tcx, &items, (DefKind::Fn, "foo_bar")).unwrap(); let body = foo_bar.body(); + assert_eq!(body.locals.len(), 7); assert_eq!(body.blocks.len(), 4); let block = &body.blocks[0]; match &block.terminator { From 41a9cbeb64f486a27f5b22a6b1dafa9d693c0fb6 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Sun, 30 Apr 2023 22:00:16 +0200 Subject: [PATCH 081/130] Shrink `SelectionError` a lot `SelectionError` used to be 80 bytes (on 64 bit). That's quite big. Especially because the selection cache contained `Result<_, SelectionError>. The Ok type is only 32 bytes, so the 80 bytes significantly inflate the size of the cache. Most variants of the `SelectionError` seem to be hard errors, only `Unimplemented` shows up in practice (for cranelift-codegen, it occupies 23.4% of all cache entries). We can just box away the biggest variant, `OutputTypeParameterMismatch`, to get the size down to 16 bytes, well within the size of the Ok type inside the cache. --- .../src/fn_ctxt/adjust_fulfillment_errors.rs | 6 ++++-- compiler/rustc_hir_typeck/src/lib.rs | 1 + compiler/rustc_middle/src/traits/mod.rs | 13 ++++++++----- .../src/traits/error_reporting/mod.rs | 13 +++++++++---- .../src/traits/select/confirmation.rs | 9 ++++++++- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs index 05e5d850bf9..3efdab53438 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs @@ -278,9 +278,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { span: Span, ) -> bool { if let traits::FulfillmentErrorCode::CodeSelectionError( - traits::SelectionError::OutputTypeParameterMismatch(_, expected, _), + traits::SelectionError::OutputTypeParameterMismatch(box traits::SelectionOutputTypeParameterMismatch{ + expected_trait_ref, .. + }), ) = error.code - && let ty::Closure(def_id, _) | ty::Generator(def_id, ..) = expected.skip_binder().self_ty().kind() + && let ty::Closure(def_id, _) | ty::Generator(def_id, ..) = expected_trait_ref.skip_binder().self_ty().kind() && span.overlaps(self.tcx.def_span(*def_id)) { true diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs index a2a4362e2f5..dcc323493f4 100644 --- a/compiler/rustc_hir_typeck/src/lib.rs +++ b/compiler/rustc_hir_typeck/src/lib.rs @@ -2,6 +2,7 @@ #![feature(let_chains)] #![feature(try_blocks)] #![feature(never_type)] +#![feature(box_patterns)] #![feature(min_specialization)] #![feature(control_flow_enum)] #![feature(drain_filter)] diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs index 8366567c2c3..449c453555e 100644 --- a/compiler/rustc_middle/src/traits/mod.rs +++ b/compiler/rustc_middle/src/traits/mod.rs @@ -580,11 +580,7 @@ pub enum SelectionError<'tcx> { /// After a closure impl has selected, its "outputs" were evaluated /// (which for closures includes the "input" type params) and they /// didn't resolve. See `confirm_poly_trait_refs` for more. - OutputTypeParameterMismatch( - ty::PolyTraitRef<'tcx>, - ty::PolyTraitRef<'tcx>, - ty::error::TypeError<'tcx>, - ), + OutputTypeParameterMismatch(Box>), /// The trait pointed by `DefId` is not object safe. TraitNotObjectSafe(DefId), /// A given constant couldn't be evaluated. @@ -596,6 +592,13 @@ pub enum SelectionError<'tcx> { ErrorReporting, } +#[derive(Clone, Debug, TypeVisitable, Lift)] +pub struct SelectionOutputTypeParameterMismatch<'tcx> { + pub found_trait_ref: ty::PolyTraitRef<'tcx>, + pub expected_trait_ref: ty::PolyTraitRef<'tcx>, + pub terr: ty::error::TypeError<'tcx>, +} + /// When performing resolution, it is typically the case that there /// can be one of three outcomes: /// diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs index c9e2ed092d1..98365223923 100644 --- a/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs +++ b/compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs @@ -28,6 +28,7 @@ use rustc_hir::{GenericParam, Item, Node}; use rustc_infer::infer::error_reporting::TypeErrCtxt; use rustc_infer::infer::{InferOk, TypeTrace}; use rustc_middle::traits::select::OverflowError; +use rustc_middle::traits::SelectionOutputTypeParameterMismatch; use rustc_middle::ty::abstract_const::NotConstEvaluatable; use rustc_middle::ty::error::{ExpectedFound, TypeError}; use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable}; @@ -1087,17 +1088,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> { } } - OutputTypeParameterMismatch( + OutputTypeParameterMismatch(box SelectionOutputTypeParameterMismatch { found_trait_ref, expected_trait_ref, - terr @ TypeError::CyclicTy(_), - ) => self.report_type_parameter_mismatch_cyclic_type_error( + terr: terr @ TypeError::CyclicTy(_), + }) => self.report_type_parameter_mismatch_cyclic_type_error( &obligation, found_trait_ref, expected_trait_ref, terr, ), - OutputTypeParameterMismatch(found_trait_ref, expected_trait_ref, _) => { + OutputTypeParameterMismatch(box SelectionOutputTypeParameterMismatch { + found_trait_ref, + expected_trait_ref, + terr: _, + }) => { match self.report_type_parameter_mismatch_error( &obligation, span, diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index 616187b69dd..cf9c8b8fa23 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -10,6 +10,7 @@ use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_hir::lang_items::LangItem; use rustc_infer::infer::LateBoundRegionConversionTime::HigherRankedType; use rustc_infer::infer::{DefineOpaqueTypes, InferOk}; +use rustc_middle::traits::SelectionOutputTypeParameterMismatch; use rustc_middle::ty::{ self, Binder, GenericParamDefKind, InternalSubsts, SubstsRef, ToPolyTraitRef, ToPredicate, TraitRef, Ty, TyCtxt, TypeVisitableExt, @@ -834,7 +835,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { obligations.extend(nested); obligations }) - .map_err(|e| OutputTypeParameterMismatch(expected_trait_ref, obligation_trait_ref, e)) + .map_err(|terr| { + OutputTypeParameterMismatch(Box::new(SelectionOutputTypeParameterMismatch { + expected_trait_ref: obligation_trait_ref, + found_trait_ref: expected_trait_ref, + terr, + })) + }) } fn confirm_trait_upcasting_unsize_candidate( From e8ab6489027098dbf4a407ddd0cedd0fbe40e3a4 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Tue, 9 May 2023 07:16:59 +0000 Subject: [PATCH 082/130] Rename `expected_trait_ref` to `self_ty_trait_ref` This trait ref is derived from the self type and then equated to the trait ref from the obligation. For example, for `fn(): Fn(u32)`, `self_ty_trait_ref` is `Fn()`, which is then equated to `Fn(u32)` (which will fail, causing the obligation to fail). --- .../rustc_trait_selection/src/traits/select/confirmation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs index cf9c8b8fa23..4dc84e0ad10 100644 --- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs +++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs @@ -812,7 +812,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { fn confirm_poly_trait_refs( &mut self, obligation: &TraitObligation<'tcx>, - expected_trait_ref: ty::PolyTraitRef<'tcx>, + self_ty_trait_ref: ty::PolyTraitRef<'tcx>, ) -> Result>, SelectionError<'tcx>> { let obligation_trait_ref = obligation.predicate.to_poly_trait_ref(); // Normalize the obligation and expected trait refs together, because why not @@ -823,7 +823,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> { obligation.param_env, obligation.cause.clone(), obligation.recursion_depth + 1, - (obligation_trait_ref, expected_trait_ref), + (obligation_trait_ref, self_ty_trait_ref), ) }); From b7f570fff3e582e2ffb8bce3ea698b6bc8f4ccef Mon Sep 17 00:00:00 2001 From: Oli Scherer Date: Tue, 9 May 2023 07:54:18 +0000 Subject: [PATCH 083/130] Keep encoding attributes for closures --- compiler/rustc_metadata/src/rmeta/encoder.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 82c66b9dfb9..29cf432b8f9 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -862,6 +862,11 @@ fn should_encode_attrs(def_kind: DefKind) -> bool { | DefKind::Macro(_) | DefKind::Field | DefKind::Impl { .. } => true, + // Tools may want to be able to detect their tool lints on + // closures from upstream crates, too. This is used by + // https://github.com/model-checking/kani and is not a performance + // or maintenance issue for us. + DefKind::Closure => true, DefKind::TyParam | DefKind::ConstParam | DefKind::Ctor(..) @@ -874,7 +879,6 @@ fn should_encode_attrs(def_kind: DefKind) -> bool { | DefKind::ImplTraitPlaceholder | DefKind::LifetimeParam | DefKind::GlobalAsm - | DefKind::Closure | DefKind::Generator => false, } } From 7cab196e7cccb68454e0ee5e667628387ad06463 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Mon, 8 May 2023 13:57:20 +1000 Subject: [PATCH 084/130] Isolate coverage FFI type layouts from their underlying LLVM C++ types --- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 8 +- .../rustc_codegen_ssa/src/coverageinfo/ffi.rs | 18 +-- .../llvm-wrapper/CoverageMappingWrapper.cpp | 107 ++++++++++++++++-- 3 files changed, 117 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 61365e6dc4b..aefd5b2a13c 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -680,7 +680,9 @@ pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_v pub mod coverageinfo { use super::coverage_map; - /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) + /// Corresponds to enum `llvm::coverage::CounterMappingRegion::RegionKind`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegionKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum RegionKind { @@ -714,7 +716,9 @@ pub mod coverageinfo { /// array", encoded separately), and source location (start and end positions of the represented /// code region). /// - /// Matches LLVMRustCounterMappingRegion. + /// Corresponds to struct `llvm::coverage::CounterMappingRegion`. + /// + /// Must match the layout of `LLVMRustCounterMappingRegion`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct CounterMappingRegion { diff --git a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs index e288760a02b..1791ce4b315 100644 --- a/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_ssa/src/coverageinfo/ffi.rs @@ -1,6 +1,6 @@ use rustc_middle::mir::coverage::{CounterValueReference, MappedExpressionIndex}; -/// Aligns with [llvm::coverage::Counter::CounterKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L95) +/// Must match the layout of `LLVMRustCounterKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum CounterKind { @@ -17,8 +17,10 @@ pub enum CounterKind { /// `instrprof.increment()`) /// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of /// counter expressions. -/// Aligns with [llvm::coverage::Counter](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L102-L103) -/// Important: The Rust struct layout (order and types of fields) must match its C++ counterpart. +/// +/// Corresponds to struct `llvm::coverage::Counter`. +/// +/// Must match the layout of `LLVMRustCounter`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct Counter { @@ -59,7 +61,9 @@ impl Counter { } } -/// Aligns with [llvm::coverage::CounterExpression::ExprKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L150) +/// Corresponds to enum `llvm::coverage::CounterExpression::ExprKind`. +/// +/// Must match the layout of `LLVMRustCounterExprKind`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub enum ExprKind { @@ -67,9 +71,9 @@ pub enum ExprKind { Add = 1, } -/// Aligns with [llvm::coverage::CounterExpression](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L152) -/// Important: The Rust struct layout (order and types of fields) must match its C++ -/// counterpart. +/// Corresponds to struct `llvm::coverage::CounterExpression`. +/// +/// Must match the layout of `LLVMRustCounterExpression`. #[derive(Copy, Clone, Debug)] #[repr(C)] pub struct CounterExpression { diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index 03e6d2149e9..38fcd0527da 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -8,18 +8,100 @@ using namespace llvm; +// FFI equivalent of enum `llvm::coverage::Counter::CounterKind` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L97-L99 +enum class LLVMRustCounterKind { + Zero = 0, + CounterValueReference = 1, + Expression = 2, +}; + +// FFI equivalent of struct `llvm::coverage::Counter` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L94-L149 +struct LLVMRustCounter { + LLVMRustCounterKind CounterKind; + uint32_t ID; +}; + +static coverage::Counter fromRust(LLVMRustCounter Counter) { + switch (Counter.CounterKind) { + case LLVMRustCounterKind::Zero: + return coverage::Counter::getZero(); + case LLVMRustCounterKind::CounterValueReference: + return coverage::Counter::getCounter(Counter.ID); + case LLVMRustCounterKind::Expression: + return coverage::Counter::getExpression(Counter.ID); + } + report_fatal_error("Bad LLVMRustCounterKind!"); +} + +// FFI equivalent of enum `llvm::coverage::CounterMappingRegion::RegionKind` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L213-L234 +enum class LLVMRustCounterMappingRegionKind { + CodeRegion = 0, + ExpansionRegion = 1, + SkippedRegion = 2, + GapRegion = 3, + BranchRegion = 4, +}; + +static coverage::CounterMappingRegion::RegionKind +fromRust(LLVMRustCounterMappingRegionKind Kind) { + switch (Kind) { + case LLVMRustCounterMappingRegionKind::CodeRegion: + return coverage::CounterMappingRegion::CodeRegion; + case LLVMRustCounterMappingRegionKind::ExpansionRegion: + return coverage::CounterMappingRegion::ExpansionRegion; + case LLVMRustCounterMappingRegionKind::SkippedRegion: + return coverage::CounterMappingRegion::SkippedRegion; + case LLVMRustCounterMappingRegionKind::GapRegion: + return coverage::CounterMappingRegion::GapRegion; + case LLVMRustCounterMappingRegionKind::BranchRegion: + return coverage::CounterMappingRegion::BranchRegion; + } + report_fatal_error("Bad LLVMRustCounterMappingRegionKind!"); +} + +// FFI equivalent of struct `llvm::coverage::CounterMappingRegion` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L211-L304 struct LLVMRustCounterMappingRegion { - coverage::Counter Count; - coverage::Counter FalseCount; + LLVMRustCounter Count; + LLVMRustCounter FalseCount; uint32_t FileID; uint32_t ExpandedFileID; uint32_t LineStart; uint32_t ColumnStart; uint32_t LineEnd; uint32_t ColumnEnd; - coverage::CounterMappingRegion::RegionKind Kind; + LLVMRustCounterMappingRegionKind Kind; }; +// FFI equivalent of enum `llvm::coverage::CounterExpression::ExprKind` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L154 +enum class LLVMRustCounterExprKind { + Subtract = 0, + Add = 1, +}; + +// FFI equivalent of struct `llvm::coverage::CounterExpression` +// https://github.com/rust-lang/llvm-project/blob/ea6fa9c2/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L151-L160 +struct LLVMRustCounterExpression { + LLVMRustCounterExprKind Kind; + LLVMRustCounter LHS; + LLVMRustCounter RHS; +}; + +static coverage::CounterExpression::ExprKind +fromRust(LLVMRustCounterExprKind Kind) { + switch (Kind) { + case LLVMRustCounterExprKind::Subtract: + return coverage::CounterExpression::Subtract; + case LLVMRustCounterExprKind::Add: + return coverage::CounterExpression::Add; + } + report_fatal_error("Bad LLVMRustCounterExprKind!"); +} + extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( const char* const Filenames[], size_t FilenamesLen, @@ -37,7 +119,7 @@ extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer( extern "C" void LLVMRustCoverageWriteMappingToBuffer( const unsigned *VirtualFileMappingIDs, unsigned NumVirtualFileMappingIDs, - const coverage::CounterExpression *Expressions, + const LLVMRustCounterExpression *RustExpressions, unsigned NumExpressions, LLVMRustCounterMappingRegion *RustMappingRegions, unsigned NumMappingRegions, @@ -48,13 +130,24 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer( for (const auto &Region : ArrayRef( RustMappingRegions, NumMappingRegions)) { MappingRegions.emplace_back( - Region.Count, Region.FalseCount, Region.FileID, Region.ExpandedFileID, + fromRust(Region.Count), fromRust(Region.FalseCount), + Region.FileID, Region.ExpandedFileID, Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd, - Region.Kind); + fromRust(Region.Kind)); } + + std::vector Expressions; + Expressions.reserve(NumExpressions); + for (const auto &Expression : + ArrayRef(RustExpressions, NumExpressions)) { + Expressions.emplace_back(fromRust(Expression.Kind), + fromRust(Expression.LHS), + fromRust(Expression.RHS)); + } + auto CoverageMappingWriter = coverage::CoverageMappingWriter( ArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs), - ArrayRef(Expressions, NumExpressions), + Expressions, MappingRegions); RawRustStringOstream OS(BufferOut); CoverageMappingWriter.write(OS); From 9addf0651c2fc9174824ea7e28c2651c780ae968 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Tue, 9 May 2023 18:38:47 +1000 Subject: [PATCH 085/130] Correctly mark parameter `RustMappingRegions` as pointer-to-`const` The regions don't need to be mutable because we pass a copy of them to LLVM instead, and this matches the `*const` in the Rust-side signature. --- compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index 38fcd0527da..87906dee4d3 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -121,7 +121,7 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer( unsigned NumVirtualFileMappingIDs, const LLVMRustCounterExpression *RustExpressions, unsigned NumExpressions, - LLVMRustCounterMappingRegion *RustMappingRegions, + const LLVMRustCounterMappingRegion *RustMappingRegions, unsigned NumMappingRegions, RustStringRef BufferOut) { // Convert from FFI representation to LLVM representation. From 53e49f4c6edd28fd92a748f28d20da36cf8364ed Mon Sep 17 00:00:00 2001 From: Scott Mabin Date: Tue, 9 May 2023 12:14:45 +0100 Subject: [PATCH 086/130] Add esp-idf platform support page --- src/doc/rustc/src/SUMMARY.md | 1 + src/doc/rustc/src/platform-support.md | 2 +- src/doc/rustc/src/platform-support/esp-idf.md | 41 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 src/doc/rustc/src/platform-support/esp-idf.md diff --git a/src/doc/rustc/src/SUMMARY.md b/src/doc/rustc/src/SUMMARY.md index 48efa67191a..73343ba9df5 100644 --- a/src/doc/rustc/src/SUMMARY.md +++ b/src/doc/rustc/src/SUMMARY.md @@ -27,6 +27,7 @@ - [armv7-unknown-linux-uclibceabihf](platform-support/armv7-unknown-linux-uclibceabihf.md) - [\*-android and \*-androideabi](platform-support/android.md) - [\*-linux-ohos](platform-support/openharmony.md) + - [\*-esp-espidf](platform-support/esp-idf.md) - [\*-unknown-fuchsia](platform-support/fuchsia.md) - [\*-kmc-solid_\*](platform-support/kmc-solid.md) - [loongarch\*-unknown-linux-\*](platform-support/loongarch-linux.md) diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index 75f97c1fc1e..d22e1cf7f68 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -297,7 +297,7 @@ target | std | host | notes `riscv32gc-unknown-linux-musl` | | | RISC-V Linux (kernel 5.4, musl + RISCV32 support patches) `riscv32im-unknown-none-elf` | * | | Bare RISC-V (RV32IM ISA) [`riscv32imac-unknown-xous-elf`](platform-support/riscv32imac-unknown-xous-elf.md) | ? | | RISC-V Xous (RV32IMAC ISA) -`riscv32imc-esp-espidf` | ✓ | | RISC-V ESP-IDF +[`riscv32imc-esp-espidf`](platform-support/esp-idf.md) | ✓ | | RISC-V ESP-IDF `riscv64gc-unknown-freebsd` | | | RISC-V FreeBSD `riscv64gc-unknown-fuchsia` | | | RISC-V Fuchsia `riscv64gc-unknown-linux-musl` | | | RISC-V Linux (kernel 4.20, musl 1.2.0) diff --git a/src/doc/rustc/src/platform-support/esp-idf.md b/src/doc/rustc/src/platform-support/esp-idf.md new file mode 100644 index 00000000000..8a4ca347e22 --- /dev/null +++ b/src/doc/rustc/src/platform-support/esp-idf.md @@ -0,0 +1,41 @@ +# `*-esp-espidf` + +**Tier: 3** + +Targets for the [ESP-IDF](https://github.com/espressif/esp-idf) development framework running on RISC-V and Xtensa CPUs. + +## Target maintainers + +- Ivan Markov [@ivmarkov](https://github.com/ivmarkov) +- Scott Mabin [@MabezDev](https://github.com/MabezDev) + +## Requirements + +The target names follow this format: `$ARCH-esp-espidf`, where `$ARCH` specifies the target processor architecture. The following targets are currently defined: + +| Target name | Target CPU(s) | +|--------------------------------|-----------------------| +| `riscv32imc-esp-espidf` | [ESP32-C3](https://www.espressif.com/en/products/socs/esp32-c3) | + +The minimum supported ESP-IDF version is `v4.3`, though it is recommended to use the latest stable release if possible. + +## Building the target + +The target can be built by enabling it for a `rustc` build. The `build-std` feature is required to build the standard library for ESP-IDF. `ldproxy` is also required for linking, it can be installed from crates.io. + +```toml +[build] +target = ["$ARCH-esp-espidf"] + +[target.$ARCH-esp-espidf] +linker = "ldproxy" + +[unstable] +build-std = ["std", "panic_abort"] +``` + +The `esp-idf-sys` crate will handle the compilation of ESP-IDF, including downloading the relevant toolchains for the build. + +## Cross-compilation toolchains and C code + +`esp-idf-sys` exposes the toolchain used in the compilation of ESP-IDF, see the crate [documentation for build output propagation](https://github.com/esp-rs/esp-idf-sys#conditional-compilation) for more information. From 3c03cce341ef9be288f05e0be3e9fe56100d37ba Mon Sep 17 00:00:00 2001 From: klensy Date: Tue, 9 May 2023 18:20:13 +0300 Subject: [PATCH 087/130] bump windows crate 0.46 -> 0.48 in workspace --- Cargo.lock | 23 ++++++------------- compiler/rustc_codegen_ssa/Cargo.toml | 2 +- compiler/rustc_data_structures/Cargo.toml | 2 +- .../rustc_data_structures/src/profiling.rs | 6 +++-- compiler/rustc_driver_impl/Cargo.toml | 2 +- compiler/rustc_errors/Cargo.toml | 3 +-- compiler/rustc_errors/src/lock.rs | 3 +-- compiler/rustc_session/Cargo.toml | 2 +- compiler/rustc_session/src/filesearch.rs | 4 ++-- src/tools/compiletest/Cargo.toml | 2 +- 10 files changed, 20 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6539bd393fb..9790810902e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,7 +741,7 @@ dependencies = [ "tracing-subscriber", "unified-diff", "walkdir", - "windows 0.46.0", + "windows", ] [[package]] @@ -1647,7 +1647,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows 0.48.0", + "windows", ] [[package]] @@ -3259,7 +3259,7 @@ dependencies = [ "tempfile", "thorin-dwp", "tracing", - "windows 0.46.0", + "windows", ] [[package]] @@ -3315,7 +3315,7 @@ dependencies = [ "tempfile", "thin-vec", "tracing", - "windows 0.46.0", + "windows", ] [[package]] @@ -3376,7 +3376,7 @@ dependencies = [ "rustc_ty_utils", "serde_json", "tracing", - "windows 0.46.0", + "windows", ] [[package]] @@ -3426,7 +3426,7 @@ dependencies = [ "termize", "tracing", "unicode-width", - "windows 0.46.0", + "windows", ] [[package]] @@ -4096,7 +4096,7 @@ dependencies = [ "smallvec", "termize", "tracing", - "windows 0.46.0", + "windows", ] [[package]] @@ -5498,15 +5498,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows" version = "0.48.0" diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml index 4f73b731f5a..02be88df103 100644 --- a/compiler/rustc_codegen_ssa/Cargo.toml +++ b/compiler/rustc_codegen_ssa/Cargo.toml @@ -51,5 +51,5 @@ default-features = false features = ["read_core", "elf", "macho", "pe", "unaligned", "archive", "write"] [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.48.0" features = ["Win32_Globalization"] diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index 5e05fe463ed..c815bb2d197 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -37,7 +37,7 @@ itertools = "0.10.1" version = "0.11" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.48.0" features = [ "Win32_Foundation", "Win32_Storage_FileSystem", diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs index 5e13e7c8aaf..3c76c2b7991 100644 --- a/compiler/rustc_data_structures/src/profiling.rs +++ b/compiler/rustc_data_structures/src/profiling.rs @@ -865,14 +865,16 @@ cfg_if! { use std::mem; use windows::{ - Win32::System::ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, + // FIXME: change back to K32GetProcessMemoryInfo when windows crate + // updated to 0.49.0+ to drop dependency on psapi.dll + Win32::System::ProcessStatus::{GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS}, Win32::System::Threading::GetCurrentProcess, }; let mut pmc = PROCESS_MEMORY_COUNTERS::default(); let pmc_size = mem::size_of_val(&pmc); unsafe { - K32GetProcessMemoryInfo( + GetProcessMemoryInfo( GetCurrentProcess(), &mut pmc, pmc_size as u32, diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml index d7d97fcc3e7..67352c55c90 100644 --- a/compiler/rustc_driver_impl/Cargo.toml +++ b/compiler/rustc_driver_impl/Cargo.toml @@ -57,7 +57,7 @@ rustc_mir_transform = { path = "../rustc_mir_transform" } libc = "0.2" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.48.0" features = [ "Win32_System_Diagnostics_Debug", ] diff --git a/compiler/rustc_errors/Cargo.toml b/compiler/rustc_errors/Cargo.toml index 46ace8eb2dd..bd3033fcb3e 100644 --- a/compiler/rustc_errors/Cargo.toml +++ b/compiler/rustc_errors/Cargo.toml @@ -27,12 +27,11 @@ serde = { version = "1.0.125", features = [ "derive" ] } serde_json = "1.0.59" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.48.0" features = [ "Win32_Foundation", "Win32_Security", "Win32_System_Threading", - "Win32_System_WindowsProgramming", ] [features] diff --git a/compiler/rustc_errors/src/lock.rs b/compiler/rustc_errors/src/lock.rs index 7db262abfde..bd5cf49b56b 100644 --- a/compiler/rustc_errors/src/lock.rs +++ b/compiler/rustc_errors/src/lock.rs @@ -19,8 +19,7 @@ pub fn acquire_global_lock(name: &str) -> Box { use windows::{ core::PCSTR, Win32::Foundation::{CloseHandle, HANDLE, WAIT_ABANDONED, WAIT_OBJECT_0}, - Win32::System::Threading::{CreateMutexA, ReleaseMutex, WaitForSingleObject}, - Win32::System::WindowsProgramming::INFINITE, + Win32::System::Threading::{CreateMutexA, ReleaseMutex, WaitForSingleObject, INFINITE}, }; struct Handle(HANDLE); diff --git a/compiler/rustc_session/Cargo.toml b/compiler/rustc_session/Cargo.toml index 3477b97438c..3af83aaaaa8 100644 --- a/compiler/rustc_session/Cargo.toml +++ b/compiler/rustc_session/Cargo.toml @@ -25,7 +25,7 @@ termize = "0.1.1" libc = "0.2" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.48.0" features = [ "Win32_Foundation", "Win32_System_LibraryLoader", diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs index 7fdbd48d563..3988416d0c7 100644 --- a/compiler/rustc_session/src/filesearch.rs +++ b/compiler/rustc_session/src/filesearch.rs @@ -135,13 +135,13 @@ fn current_dll_path() -> Result { use windows::{ core::PCWSTR, - Win32::Foundation::HINSTANCE, + Win32::Foundation::HMODULE, Win32::System::LibraryLoader::{ GetModuleFileNameW, GetModuleHandleExW, GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, }, }; - let mut module = HINSTANCE::default(); + let mut module = HMODULE::default(); unsafe { GetModuleHandleExW( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS, diff --git a/src/tools/compiletest/Cargo.toml b/src/tools/compiletest/Cargo.toml index 85fd6523c82..0d42504c7f4 100644 --- a/src/tools/compiletest/Cargo.toml +++ b/src/tools/compiletest/Cargo.toml @@ -28,7 +28,7 @@ libc = "0.2" miow = "0.5" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.48.0" features = [ "Win32_Foundation", "Win32_System_Diagnostics_Debug", From a083986852aea685f17939a237f8cc297c606745 Mon Sep 17 00:00:00 2001 From: klensy Date: Tue, 9 May 2023 19:04:25 +0300 Subject: [PATCH 088/130] add windows crate to allowed tidy list (why it worked before?) --- src/tools/tidy/src/deps.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index baef4bb0140..675230b762d 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -268,6 +268,14 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "winapi-i686-pc-windows-gnu", "winapi-util", "winapi-x86_64-pc-windows-gnu", + "windows-targets", + "windows_aarch64_msvc", + "windows_aarch64_gnullvm", + "windows_x86_64_gnullvm", + "windows_i686_msvc", + "windows_i686_gnu", + "windows_x86_64_msvc", + "windows_x86_64_gnu", "writeable", // this is a false-positive: it's only used by rustfmt, but because it's enabled through a // feature, tidy thinks it's used by rustc as well. From 4f12412a7d6afe81a14a2b49731c09dca25f1e37 Mon Sep 17 00:00:00 2001 From: klensy Date: Tue, 9 May 2023 19:32:02 +0300 Subject: [PATCH 089/130] tidy: sort lists in deps.rs using tidy-alphabetical --- src/tools/tidy/src/deps.rs | 79 ++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 675230b762d..afa6bce943f 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -6,47 +6,53 @@ use std::path::Path; /// These are licenses that are allowed for all crates, including the runtime, /// rustc, tools, etc. +#[rustfmt::skip] const LICENSES: &[&str] = &[ - "MIT/Apache-2.0", - "MIT / Apache-2.0", - "Apache-2.0/MIT", + // tidy-alphabetical-start + "(MIT OR Apache-2.0) AND Unicode-DFS-2016", // unicode_ident + "0BSD OR MIT OR Apache-2.0", // adler license + "0BSD", "Apache-2.0 / MIT", - "MIT OR Apache-2.0", "Apache-2.0 OR MIT", "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT", // wasi license - "MIT", + "Apache-2.0/MIT", "ISC", - "Unlicense/MIT", + "MIT / Apache-2.0", + "MIT OR Apache-2.0 OR Zlib", // tinyvec_macros + "MIT OR Apache-2.0", + "MIT OR Zlib OR Apache-2.0", // miniz_oxide + "MIT", + "MIT/Apache-2.0", + "Unicode-DFS-2016", // tinystr and icu4x "Unlicense OR MIT", - "0BSD", - "0BSD OR MIT OR Apache-2.0", // adler license - "Zlib OR Apache-2.0 OR MIT", // tinyvec - "MIT OR Apache-2.0 OR Zlib", // tinyvec_macros - "MIT OR Zlib OR Apache-2.0", // miniz_oxide - "(MIT OR Apache-2.0) AND Unicode-DFS-2016", // unicode_ident - "Unicode-DFS-2016", // tinystr and icu4x + "Unlicense/MIT", + "Zlib OR Apache-2.0 OR MIT", // tinyvec + // tidy-alphabetical-end ]; /// These are exceptions to Rust's permissive licensing policy, and /// should be considered bugs. Exceptions are only allowed in Rust /// tooling. It is _crucial_ that no exception crates be dependencies /// of the Rust runtime (std/test). +#[rustfmt::skip] const EXCEPTIONS: &[(&str, &str)] = &[ + // tidy-alphabetical-start ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc - ("mdbook", "MPL-2.0"), // mdbook + ("codespan-reporting", "Apache-2.0"), // cxx via iana-time-zone-haiku via time, only on haiku ("colored", "MPL-2.0"), // rustfmt + ("dissimilar", "Apache-2.0"), // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps) + ("fluent-langneg", "Apache-2.0"), // rustc (fluent translations) + ("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above. + ("instant", "BSD-3-Clause"), // rustc_driver/tracing-subscriber/parking_lot + ("mdbook", "MPL-2.0"), // mdbook ("ryu", "Apache-2.0 OR BSL-1.0"), // cargo/... (because of serde) - ("codespan-reporting", "Apache-2.0"), // cxx via iana-time-zone-haiku via time, only on haiku - ("instant", "BSD-3-Clause"), // rustc_driver/tracing-subscriber/parking_lot - ("snap", "BSD-3-Clause"), // rustc - ("fluent-langneg", "Apache-2.0"), // rustc (fluent translations) - ("self_cell", "Apache-2.0"), // rustc (fluent translations) - // FIXME: this dependency violates the documentation comment above: - ("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target - ("dissimilar", "Apache-2.0"), // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps) + ("self_cell", "Apache-2.0"), // rustc (fluent translations) + ("snap", "BSD-3-Clause"), // rustc + // tidy-alphabetical-end ]; const EXCEPTIONS_CARGO: &[(&str, &str)] = &[ + // tidy-alphabetical-start ("bitmaps", "MPL-2.0+"), ("bytesize", "Apache-2.0"), ("dunce", "CC0-1.0 OR MIT-0"), @@ -62,9 +68,11 @@ const EXCEPTIONS_CARGO: &[(&str, &str)] = &[ ("sized-chunks", "MPL-2.0+"), ("subtle", "BSD-3-Clause"), ("unicode-bom", "Apache-2.0"), + // tidy-alphabetical-end ]; const EXCEPTIONS_CRANELIFT: &[(&str, &str)] = &[ + // tidy-alphabetical-start ("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"), ("cranelift-codegen", "Apache-2.0 WITH LLVM-exception"), ("cranelift-codegen-meta", "Apache-2.0 WITH LLVM-exception"), @@ -80,6 +88,7 @@ const EXCEPTIONS_CRANELIFT: &[(&str, &str)] = &[ ("regalloc2", "Apache-2.0 WITH LLVM-exception"), ("target-lexicon", "Apache-2.0 WITH LLVM-exception"), ("wasmtime-jit-icache-coherence", "Apache-2.0 WITH LLVM-exception"), + // tidy-alphabetical-end ]; const EXCEPTIONS_BOOTSTRAP: &[(&str, &str)] = &[ @@ -95,6 +104,7 @@ const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "test", "panic_abort", /// This list is here to provide a speed-bump to adding a new dependency to /// rustc. Please check with the compiler team before adding an entry. const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ + // tidy-alphabetical-start "addr2line", "adler", "ahash", @@ -113,8 +123,8 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "chalk-engine", "chalk-ir", "chalk-solve", - "convert_case", // dependency of derive_more "compiler_builtins", + "convert_case", // dependency of derive_more "cpufeatures", "crc32fast", "crossbeam-channel", @@ -187,8 +197,8 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "ppv-lite86", "proc-macro-hack", "proc-macro2", - "pulldown-cmark", "psm", + "pulldown-cmark", "punycode", "quote", "rand", @@ -227,6 +237,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "tempfile", "termcolor", "termize", + "thin-vec", "thiserror", "thiserror-impl", "thorin-dwp", @@ -234,7 +245,6 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "tinystr", "tinyvec", "tinyvec_macros", - "thin-vec", "tracing", "tracing-attributes", "tracing-core", @@ -263,37 +273,37 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "valuable", "version_check", "wasi", - "windows", "winapi", "winapi-i686-pc-windows-gnu", "winapi-util", "winapi-x86_64-pc-windows-gnu", + "windows", "windows-targets", - "windows_aarch64_msvc", "windows_aarch64_gnullvm", - "windows_x86_64_gnullvm", - "windows_i686_msvc", + "windows_aarch64_msvc", "windows_i686_gnu", - "windows_x86_64_msvc", + "windows_i686_msvc", "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", "writeable", - // this is a false-positive: it's only used by rustfmt, but because it's enabled through a - // feature, tidy thinks it's used by rustc as well. - "yansi-term", + "yansi-term", // this is a false-positive: it's only used by rustfmt, but because it's enabled through a feature, tidy thinks it's used by rustc as well. "yoke", "yoke-derive", "zerofrom", "zerofrom-derive", "zerovec", "zerovec-derive", + // tidy-alphabetical-end ]; const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ + // tidy-alphabetical-start "ahash", "anyhow", "autocfg", - "bumpalo", "bitflags", + "bumpalo", "byteorder", "cfg-if", "cranelift-bforest", @@ -332,6 +342,7 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", "windows-sys", + // tidy-alphabetical-end ]; /// Dependency checks. From 71138e99337e791eb73d73d8a2cf8aaef29960b1 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 24 Apr 2023 17:10:39 +0000 Subject: [PATCH 090/130] Make HasTop and HasBottom consts. --- .../src/framework/lattice.rs | 20 ++++------- .../rustc_mir_dataflow/src/value_analysis.rs | 34 +++++++++---------- .../src/dataflow_const_prop.rs | 4 +-- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/framework/lattice.rs b/compiler/rustc_mir_dataflow/src/framework/lattice.rs index af44a4329bf..3952f44ad48 100644 --- a/compiler/rustc_mir_dataflow/src/framework/lattice.rs +++ b/compiler/rustc_mir_dataflow/src/framework/lattice.rs @@ -75,12 +75,12 @@ pub trait MeetSemiLattice: Eq { /// A set that has a "bottom" element, which is less than or equal to any other element. pub trait HasBottom { - fn bottom() -> Self; + const BOTTOM: Self; } /// A set that has a "top" element, which is greater than or equal to any other element. pub trait HasTop { - fn top() -> Self; + const TOP: Self; } /// A `bool` is a "two-point" lattice with `true` as the top element and `false` as the bottom: @@ -113,15 +113,11 @@ impl MeetSemiLattice for bool { } impl HasBottom for bool { - fn bottom() -> Self { - false - } + const BOTTOM: Self = false; } impl HasTop for bool { - fn top() -> Self { - true - } + const TOP: Self = true; } /// A tuple (or list) of lattices is itself a lattice whose least upper bound is the concatenation @@ -274,13 +270,9 @@ impl MeetSemiLattice for FlatSet { } impl HasBottom for FlatSet { - fn bottom() -> Self { - Self::Bottom - } + const BOTTOM: Self = Self::Bottom; } impl HasTop for FlatSet { - fn top() -> Self { - Self::Top - } + const TOP: Self = Self::Top; } diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index fdff1ec788d..e830b8c7157 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -74,11 +74,11 @@ pub trait ValueAnalysis<'tcx> { StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { // StorageLive leaves the local in an uninitialized state. // StorageDead makes it UB to access the local afterwards. - state.flood_with(Place::from(*local).as_ref(), self.map(), Self::Value::bottom()); + state.flood_with(Place::from(*local).as_ref(), self.map(), Self::Value::BOTTOM); } StatementKind::Deinit(box place) => { // Deinit makes the place uninitialized. - state.flood_with(place.as_ref(), self.map(), Self::Value::bottom()); + state.flood_with(place.as_ref(), self.map(), Self::Value::BOTTOM); } StatementKind::Retag(..) => { // We don't track references. @@ -154,7 +154,7 @@ pub trait ValueAnalysis<'tcx> { Rvalue::CopyForDeref(place) => self.handle_operand(&Operand::Copy(*place), state), Rvalue::Ref(..) | Rvalue::AddressOf(..) => { // We don't track such places. - ValueOrPlace::top() + ValueOrPlace::TOP } Rvalue::Repeat(..) | Rvalue::ThreadLocalRef(..) @@ -168,7 +168,7 @@ pub trait ValueAnalysis<'tcx> { | Rvalue::Aggregate(..) | Rvalue::ShallowInitBox(..) => { // No modification is possible through these r-values. - ValueOrPlace::top() + ValueOrPlace::TOP } } } @@ -196,7 +196,7 @@ pub trait ValueAnalysis<'tcx> { self.map() .find(place.as_ref()) .map(ValueOrPlace::Place) - .unwrap_or(ValueOrPlace::top()) + .unwrap_or(ValueOrPlace::TOP) } } } @@ -214,7 +214,7 @@ pub trait ValueAnalysis<'tcx> { _constant: &Constant<'tcx>, _state: &mut State, ) -> Self::Value { - Self::Value::top() + Self::Value::TOP } /// The effect of a successful function call return should not be @@ -229,7 +229,7 @@ pub trait ValueAnalysis<'tcx> { // Effect is applied by `handle_call_return`. } TerminatorKind::Drop { place, .. } => { - state.flood_with(place.as_ref(), self.map(), Self::Value::bottom()); + state.flood_with(place.as_ref(), self.map(), Self::Value::BOTTOM); } TerminatorKind::Yield { .. } => { // They would have an effect, but are not allowed in this phase. @@ -307,7 +307,7 @@ impl<'tcx, T: ValueAnalysis<'tcx>> AnalysisDomain<'tcx> for ValueAnalysisWrapper fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) { // The initial state maps all tracked places of argument projections to ⊤ and the rest to ⊥. assert!(matches!(state.0, StateData::Unreachable)); - let values = IndexVec::from_elem_n(T::Value::bottom(), self.0.map().value_count); + let values = IndexVec::from_elem_n(T::Value::BOTTOM, self.0.map().value_count); *state = State(StateData::Reachable(values)); for arg in body.args_iter() { state.flood(PlaceRef { local: arg, projection: &[] }, self.0.map()); @@ -437,7 +437,7 @@ impl State { } pub fn flood_all(&mut self) { - self.flood_all_with(V::top()) + self.flood_all_with(V::TOP) } pub fn flood_all_with(&mut self, value: V) { @@ -455,7 +455,7 @@ impl State { } pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) { - self.flood_with(place, map, V::top()) + self.flood_with(place, map, V::TOP) } pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { @@ -468,7 +468,7 @@ impl State { } pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) { - self.flood_discr_with(place, map, V::top()) + self.flood_discr_with(place, map, V::TOP) } /// Low-level method that assigns to a place. @@ -538,14 +538,14 @@ impl State { /// Retrieve the value stored for a place, or ⊤ if it is not tracked. pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V { - map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::top()) + map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::TOP) } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V { match map.find_discr(place) { Some(place) => self.get_idx(place, map), - None => V::top(), + None => V::TOP, } } @@ -553,11 +553,11 @@ impl State { pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V { match &self.0 { StateData::Reachable(values) => { - map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::top()) + map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::TOP) } StateData::Unreachable => { // Because this is unreachable, we can return any value we want. - V::bottom() + V::BOTTOM } } } @@ -909,9 +909,7 @@ pub enum ValueOrPlace { } impl ValueOrPlace { - pub fn top() -> Self { - ValueOrPlace::Value(V::top()) - } + pub const TOP: Self = ValueOrPlace::Value(V::TOP); } /// The set of projection elements that can be used by a tracked place. diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 254b704f9fc..5ef1bd00a6f 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -208,8 +208,8 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { _ => unreachable!(), } .map(|result| ValueOrPlace::Value(self.wrap_immediate(result, *ty))) - .unwrap_or(ValueOrPlace::top()), - _ => ValueOrPlace::top(), + .unwrap_or(ValueOrPlace::TOP), + _ => ValueOrPlace::TOP, }, Rvalue::BinaryOp(op, box (left, right)) => { // Overflows must be ignored here. From 7c3d55150dc09494fda56814ff1bd529d72b6afb Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 22 Apr 2023 13:06:28 +0000 Subject: [PATCH 091/130] Create tracked places breadth first. --- .../rustc_mir_dataflow/src/value_analysis.rs | 114 ++++++++---------- 1 file changed, 49 insertions(+), 65 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index e830b8c7157..7d8fc5ffeec 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -32,6 +32,7 @@ //! Because of that, we can assume that the only way to change the value behind a tracked place is //! by direct assignment. +use std::collections::VecDeque; use std::fmt::{Debug, Formatter}; use rustc_data_structures::fx::FxHashMap; @@ -608,7 +609,7 @@ impl Map { pub fn from_filter<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - filter: impl FnMut(Ty<'tcx>) -> bool, + filter: impl Fn(Ty<'tcx>) -> bool, place_limit: Option, ) -> Self { let mut map = Self::new(); @@ -623,51 +624,67 @@ impl Map { &mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - mut filter: impl FnMut(Ty<'tcx>) -> bool, + filter: impl Fn(Ty<'tcx>) -> bool, exclude: BitSet, place_limit: Option, ) { // We use this vector as stack, pushing and popping projections. - let mut projection = Vec::new(); + let mut worklist = VecDeque::with_capacity(place_limit.unwrap_or(body.local_decls.len())); + self.locals = IndexVec::from_elem(None, &body.local_decls); for (local, decl) in body.local_decls.iter_enumerated() { - if !exclude.contains(local) { - self.register_with_filter_rec( - tcx, - local, - &mut projection, - decl.ty, - &mut filter, - place_limit, - ); + if exclude.contains(local) { + continue; } + + // Create a place for the local. + debug_assert!(self.locals[local].is_none()); + let place = self.places.push(PlaceInfo::new(None)); + self.locals[local] = Some(place); + + // And push the eventual children places to the worklist. + self.register_children(tcx, place, decl.ty, &filter, &mut worklist); + } + + // `place.elem1.elem2` with type `ty`. + while let Some((mut place, elem1, elem2, ty)) = worklist.pop_front() { + if let Some(place_limit) = place_limit && self.value_count >= place_limit { + break + } + + // Create a place for this projection. + for elem in [elem1, Some(elem2)].into_iter().flatten() { + place = *self.projections.entry((place, elem)).or_insert_with(|| { + // Prepend new child to the linked list. + let next = self.places.push(PlaceInfo::new(Some(elem))); + self.places[next].next_sibling = self.places[place].first_child; + self.places[place].first_child = Some(next); + next + }); + } + + // And push the eventual children places to the worklist. + self.register_children(tcx, place, ty, &filter, &mut worklist); } } /// Potentially register the (local, projection) place and its fields, recursively. /// /// Invariant: The projection must only contain trackable elements. - fn register_with_filter_rec<'tcx>( + fn register_children<'tcx>( &mut self, tcx: TyCtxt<'tcx>, - local: Local, - projection: &mut Vec>, + place: PlaceIndex, ty: Ty<'tcx>, - filter: &mut impl FnMut(Ty<'tcx>) -> bool, - place_limit: Option, + filter: &impl Fn(Ty<'tcx>) -> bool, + worklist: &mut VecDeque<(PlaceIndex, Option, TrackElem, Ty<'tcx>)>, ) { - if let Some(place_limit) = place_limit && self.value_count >= place_limit { - return - } - - // We know that the projection only contains trackable elements. - let place = self.make_place(local, projection).unwrap(); - // Allocate a value slot if it doesn't have one, and the user requested one. if self.places[place].value_index.is_none() && filter(ty) { self.places[place].value_index = Some(self.value_count.into()); self.value_count += 1; } + // For enums, directly create the `Discriminant`, as that's their main use. if ty.is_enum() { let discr_ty = ty.discriminant_ty(tcx); if filter(discr_ty) { @@ -692,48 +709,15 @@ impl Map { // Recurse with all fields of this place. iter_fields(ty, tcx, ty::ParamEnv::reveal_all(), |variant, field, ty| { - if let Some(variant) = variant { - projection.push(PlaceElem::Downcast(None, variant)); - let _ = self.make_place(local, projection); - projection.push(PlaceElem::Field(field, ty)); - self.register_with_filter_rec(tcx, local, projection, ty, filter, place_limit); - projection.pop(); - projection.pop(); - return; - } - projection.push(PlaceElem::Field(field, ty)); - self.register_with_filter_rec(tcx, local, projection, ty, filter, place_limit); - projection.pop(); + worklist.push_back(( + place, + variant.map(TrackElem::Variant), + TrackElem::Field(field), + ty, + )) }); } - /// Tries to add the place to the map, without allocating a value slot. - /// - /// Can fail if the projection contains non-trackable elements. - fn make_place<'tcx>( - &mut self, - local: Local, - projection: &[PlaceElem<'tcx>], - ) -> Result { - // Get the base index of the local. - let mut index = - *self.locals.get_or_insert_with(local, || self.places.push(PlaceInfo::new(None))); - - // Apply the projection. - for &elem in projection { - let elem = elem.try_into()?; - index = *self.projections.entry((index, elem)).or_insert_with(|| { - // Prepend new child to the linked list. - let next = self.places.push(PlaceInfo::new(Some(elem))); - self.places[next].next_sibling = self.places[index].first_child; - self.places[index].first_child = Some(next); - next - }); - } - - Ok(index) - } - /// Returns the number of tracked places, i.e., those for which a value can be stored. pub fn tracked_places(&self) -> usize { self.value_count @@ -750,7 +734,7 @@ impl Map { place: PlaceRef<'_>, extra: impl IntoIterator, ) -> Option { - let mut index = *self.locals.get(place.local)?.as_ref()?; + let mut index = *self.locals[place.local].as_ref()?; for &elem in place.projection { index = self.apply(index, elem.try_into().ok()?)?; @@ -794,7 +778,7 @@ impl Map { // We do not track indirect places. return; } - let Some(&Some(mut index)) = self.locals.get(place.local) else { + let Some(mut index) = self.locals[place.local] else { // The local is not tracked at all, so it does not alias anything. return; }; From 38fa676330eb938ca9e8f36397a9003509e8be07 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 23 Apr 2023 18:02:05 +0000 Subject: [PATCH 092/130] Precompute values to flood. --- .../rustc_mir_dataflow/src/value_analysis.rs | 70 ++++++++++++++----- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 7d8fc5ffeec..882f9dc11a1 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -34,6 +34,7 @@ use std::collections::VecDeque; use std::fmt::{Debug, Formatter}; +use std::ops::Range; use rustc_data_structures::fx::FxHashMap; use rustc_index::bit_set::BitSet; @@ -448,10 +449,8 @@ impl State { pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { let StateData::Reachable(values) = &mut self.0 else { return }; - map.for_each_aliasing_place(place, None, &mut |place| { - if let Some(vi) = map.places[place].value_index { - values[vi] = value.clone(); - } + map.for_each_aliasing_place(place, None, &mut |vi| { + values[vi] = value.clone(); }); } @@ -461,10 +460,8 @@ impl State { pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { let StateData::Reachable(values) = &mut self.0 else { return }; - map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |place| { - if let Some(vi) = map.places[place].value_index { - values[vi] = value.clone(); - } + map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |vi| { + values[vi] = value.clone(); }); } @@ -589,6 +586,9 @@ pub struct Map { projections: FxHashMap<(PlaceIndex, TrackElem), PlaceIndex>, places: IndexVec, value_count: usize, + // The Range corresponds to a slice into `inner_values_buffer`. + inner_values: IndexVec>, + inner_values_buffer: Vec, } impl Map { @@ -598,6 +598,8 @@ impl Map { projections: FxHashMap::default(), places: IndexVec::new(), value_count: 0, + inner_values: IndexVec::new(), + inner_values_buffer: Vec::new(), } } @@ -665,6 +667,14 @@ impl Map { // And push the eventual children places to the worklist. self.register_children(tcx, place, ty, &filter, &mut worklist); } + + self.inner_values_buffer = Vec::with_capacity(self.value_count); + self.inner_values = IndexVec::from_elem(0..0, &self.places); + for local in body.local_decls.indices() { + if let Some(place) = self.locals[local] { + self.cache_preorder_invoke(place); + } + } } /// Potentially register the (local, projection) place and its fields, recursively. @@ -718,6 +728,25 @@ impl Map { }); } + /// Precompute the list of values inside `root` and store it inside + /// as a slice within `inner_values_buffer`. + fn cache_preorder_invoke(&mut self, root: PlaceIndex) { + let start = self.inner_values_buffer.len(); + if let Some(vi) = self.places[root].value_index { + self.inner_values_buffer.push(vi); + } + + // We manually iterate instead of using `children` as we need to mutate `self`. + let mut next_child = self.places[root].first_child; + while let Some(child) = next_child { + self.cache_preorder_invoke(child); + next_child = self.places[child].next_sibling; + } + + let end = self.inner_values_buffer.len(); + self.inner_values[root] = start..end; + } + /// Returns the number of tracked places, i.e., those for which a value can be stored. pub fn tracked_places(&self) -> usize { self.value_count @@ -768,11 +797,11 @@ impl Map { /// /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track /// as such. - pub fn for_each_aliasing_place( + fn for_each_aliasing_place( &self, place: PlaceRef<'_>, tail_elem: Option, - f: &mut impl FnMut(PlaceIndex), + f: &mut impl FnMut(ValueIndex), ) { if place.is_indirect() { // We do not track indirect places. @@ -789,7 +818,9 @@ impl Map { .chain(tail_elem.map(Ok).into_iter()); for elem in elems { // A field aliases the parent place. - f(index); + if let Some(vi) = self.places[index].value_index { + f(vi); + } let Ok(elem) = elem else { return }; let sub = self.apply(index, elem); @@ -803,7 +834,7 @@ impl Map { return; } } - self.preorder_invoke(index, f); + self.for_each_value_inside(index, f); } /// Invoke the given function on all the descendants of the given place, except one branch. @@ -811,7 +842,7 @@ impl Map { &self, parent: PlaceIndex, preserved_child: Option, - f: &mut impl FnMut(PlaceIndex), + f: &mut impl FnMut(ValueIndex), ) { for sibling in self.children(parent) { let elem = self.places[sibling].proj_elem; @@ -821,16 +852,17 @@ impl Map { // Only invalidate the other variants, the current one is fine. && Some(sibling) != preserved_child { - self.preorder_invoke(sibling, f); + self.for_each_value_inside(sibling, f); } } } - /// Invoke a function on the given place and all descendants. - fn preorder_invoke(&self, root: PlaceIndex, f: &mut impl FnMut(PlaceIndex)) { - f(root); - for child in self.children(root) { - self.preorder_invoke(child, f); + /// Invoke a function on each value in the given place and all descendants. + fn for_each_value_inside(&self, root: PlaceIndex, f: &mut impl FnMut(ValueIndex)) { + let range = self.inner_values[root].clone(); + let values = &self.inner_values_buffer[range]; + for &v in values { + f(v) } } } From 2b0bf3cf59dc54b9a5721e085405b7647fe82c14 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 24 Apr 2023 16:54:11 +0000 Subject: [PATCH 093/130] Trim the places that will not be used. --- compiler/rustc_mir_dataflow/src/value_analysis.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 882f9dc11a1..ecdc6a6580e 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -675,6 +675,15 @@ impl Map { self.cache_preorder_invoke(place); } } + + // Trim useless places. + for opt_place in self.locals.iter_mut() { + if let Some(place) = *opt_place && self.inner_values[place].is_empty() { + *opt_place = None; + } + } + #[allow(rustc::potential_query_instability)] + self.projections.retain(|_, child| !self.inner_values[*child].is_empty()); } /// Potentially register the (local, projection) place and its fields, recursively. @@ -803,7 +812,7 @@ impl Map { tail_elem: Option, f: &mut impl FnMut(ValueIndex), ) { - if place.is_indirect() { + if place.has_deref() { // We do not track indirect places. return; } From add5124dceb157f1f4157bf2b0ec106e78a4943c Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 25 Apr 2023 19:47:15 +0000 Subject: [PATCH 094/130] Extract handle_set_discriminant. --- .../rustc_mir_dataflow/src/value_analysis.rs | 22 ++++++++++++-- .../src/dataflow_const_prop.rs | 30 +++++++++---------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index ecdc6a6580e..7b92eb05fba 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -67,8 +67,8 @@ pub trait ValueAnalysis<'tcx> { StatementKind::Assign(box (place, rvalue)) => { self.handle_assign(*place, rvalue, state); } - StatementKind::SetDiscriminant { box ref place, .. } => { - state.flood_discr(place.as_ref(), self.map()); + StatementKind::SetDiscriminant { box place, variant_index } => { + self.handle_set_discriminant(*place, *variant_index, state); } StatementKind::Intrinsic(box intrinsic) => { self.handle_intrinsic(intrinsic, state); @@ -94,6 +94,24 @@ pub trait ValueAnalysis<'tcx> { } } + fn handle_set_discriminant( + &self, + place: Place<'tcx>, + variant_index: VariantIdx, + state: &mut State, + ) { + self.super_set_discriminant(place, variant_index, state) + } + + fn super_set_discriminant( + &self, + place: Place<'tcx>, + _variant_index: VariantIdx, + state: &mut State, + ) { + state.flood_discr(place.as_ref(), self.map()); + } + fn handle_intrinsic( &self, intrinsic: &NonDivergingIntrinsic<'tcx>, diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 5ef1bd00a6f..7adfc9dff2a 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -79,22 +79,22 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { &self.map } - fn handle_statement(&self, statement: &Statement<'tcx>, state: &mut State) { - match statement.kind { - StatementKind::SetDiscriminant { box ref place, variant_index } => { - state.flood_discr(place.as_ref(), &self.map); - if self.map.find_discr(place.as_ref()).is_some() { - let enum_ty = place.ty(self.local_decls, self.tcx).ty; - if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) { - state.assign_discr( - place.as_ref(), - ValueOrPlace::Value(FlatSet::Elem(discr)), - &self.map, - ); - } - } + fn handle_set_discriminant( + &self, + place: Place<'tcx>, + variant_index: VariantIdx, + state: &mut State, + ) { + state.flood_discr(place.as_ref(), &self.map); + if self.map.find_discr(place.as_ref()).is_some() { + let enum_ty = place.ty(self.local_decls, self.tcx).ty; + if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) { + state.assign_discr( + place.as_ref(), + ValueOrPlace::Value(FlatSet::Elem(discr)), + &self.map, + ); } - _ => self.super_statement(statement, state), } } From 79c073746b46d621a75a5617737093db84dabee3 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 25 Apr 2023 19:50:22 +0000 Subject: [PATCH 095/130] Do not flood on copy_nonoverlapping. --- compiler/rustc_mir_dataflow/src/value_analysis.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 7b92eb05fba..2c3d4c97ed3 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -123,16 +123,18 @@ pub trait ValueAnalysis<'tcx> { fn super_intrinsic( &self, intrinsic: &NonDivergingIntrinsic<'tcx>, - state: &mut State, + _state: &mut State, ) { match intrinsic { NonDivergingIntrinsic::Assume(..) => { // Could use this, but ignoring it is sound. } - NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { dst, .. }) => { - if let Some(place) = dst.place() { - state.flood(place.as_ref(), self.map()); - } + NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { + dst: _, + src: _, + count: _, + }) => { + // This statement represents `*dst = *src`, `count` times. } } } From 2aa1c23fed118900103f70e4ab3473e75215862c Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 8 May 2023 08:46:21 +0000 Subject: [PATCH 096/130] Add a few comments. --- .../rustc_mir_dataflow/src/value_analysis.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 2c3d4c97ed3..9232ea74d36 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -632,11 +632,11 @@ impl Map { tcx: TyCtxt<'tcx>, body: &Body<'tcx>, filter: impl Fn(Ty<'tcx>) -> bool, - place_limit: Option, + value_limit: Option, ) -> Self { let mut map = Self::new(); let exclude = excluded_locals(body); - map.register_with_filter(tcx, body, filter, exclude, place_limit); + map.register_with_filter(tcx, body, filter, exclude, value_limit); debug!("registered {} places ({} nodes in total)", map.value_count, map.places.len()); map } @@ -648,10 +648,11 @@ impl Map { body: &Body<'tcx>, filter: impl Fn(Ty<'tcx>) -> bool, exclude: BitSet, - place_limit: Option, + value_limit: Option, ) { - // We use this vector as stack, pushing and popping projections. - let mut worklist = VecDeque::with_capacity(place_limit.unwrap_or(body.local_decls.len())); + let mut worklist = VecDeque::with_capacity(value_limit.unwrap_or(body.local_decls.len())); + + // Start by constructing the places for each bare local. self.locals = IndexVec::from_elem(None, &body.local_decls); for (local, decl) in body.local_decls.iter_enumerated() { if exclude.contains(local) { @@ -668,8 +669,10 @@ impl Map { } // `place.elem1.elem2` with type `ty`. + // `elem1` is either `Some(Variant(i))` or `None`. while let Some((mut place, elem1, elem2, ty)) = worklist.pop_front() { - if let Some(place_limit) = place_limit && self.value_count >= place_limit { + // The user requires a bound on the number of created values. + if let Some(value_limit) = value_limit && self.value_count >= value_limit { break } @@ -688,6 +691,9 @@ impl Map { self.register_children(tcx, place, ty, &filter, &mut worklist); } + // Pre-compute the tree of ValueIndex nested in each PlaceIndex. + // `inner_values_buffer[inner_values[place]]` is the set of all the values + // reachable by projecting `place`. self.inner_values_buffer = Vec::with_capacity(self.value_count); self.inner_values = IndexVec::from_elem(0..0, &self.places); for local in body.local_decls.indices() { From ccc1da247bf3be7e71932844484847da6e35f185 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 8 May 2023 08:48:16 +0000 Subject: [PATCH 097/130] Prevent stack overflow. --- compiler/rustc_mir_dataflow/src/value_analysis.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 9232ea74d36..b74d06e5ae8 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -37,6 +37,7 @@ use std::fmt::{Debug, Formatter}; use std::ops::Range; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_index::bit_set::BitSet; use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor}; @@ -774,7 +775,7 @@ impl Map { // We manually iterate instead of using `children` as we need to mutate `self`. let mut next_child = self.places[root].first_child; while let Some(child) = next_child { - self.cache_preorder_invoke(child); + ensure_sufficient_stack(|| self.cache_preorder_invoke(child)); next_child = self.places[child].next_sibling; } From d0766079833e70b25f081db59105888b9e986f18 Mon Sep 17 00:00:00 2001 From: Chris Denton Date: Tue, 9 May 2023 19:34:01 +0100 Subject: [PATCH 098/130] Don't force include Windows goop when documenting --- library/std/src/os/windows/io/handle.rs | 18 ++++++------ library/std/src/os/windows/io/raw.rs | 17 +++++------ library/std/src/os/windows/io/socket.rs | 39 +++++++++++++++---------- library/std/src/sys/mod.rs | 25 ---------------- 4 files changed, 40 insertions(+), 59 deletions(-) diff --git a/library/std/src/os/windows/io/handle.rs b/library/std/src/os/windows/io/handle.rs index 9b77cd8321b..280757a41a2 100644 --- a/library/std/src/os/windows/io/handle.rs +++ b/library/std/src/os/windows/io/handle.rs @@ -9,7 +9,7 @@ use crate::io; use crate::marker::PhantomData; use crate::mem::forget; use crate::ptr; -use crate::sys::c; +use crate::sys; use crate::sys::cvt; use crate::sys_common::{AsInner, FromInner, IntoInner}; @@ -190,14 +190,14 @@ impl BorrowedHandle<'_> { /// object as the existing `BorrowedHandle` instance. #[stable(feature = "io_safety", since = "1.63.0")] pub fn try_clone_to_owned(&self) -> crate::io::Result { - self.duplicate(0, false, c::DUPLICATE_SAME_ACCESS) + self.duplicate(0, false, sys::c::DUPLICATE_SAME_ACCESS) } pub(crate) fn duplicate( &self, - access: c::DWORD, + access: u32, inherit: bool, - options: c::DWORD, + options: u32, ) -> io::Result { let handle = self.as_raw_handle(); @@ -211,14 +211,14 @@ impl BorrowedHandle<'_> { let mut ret = ptr::null_mut(); cvt(unsafe { - let cur_proc = c::GetCurrentProcess(); - c::DuplicateHandle( + let cur_proc = sys::c::GetCurrentProcess(); + sys::c::DuplicateHandle( cur_proc, handle, cur_proc, &mut ret, access, - inherit as c::BOOL, + inherit as sys::c::BOOL, options, ) })?; @@ -233,7 +233,7 @@ impl TryFrom for OwnedHandle { #[inline] fn try_from(handle_or_invalid: HandleOrInvalid) -> Result { let owned_handle = handle_or_invalid.0; - if owned_handle.handle == c::INVALID_HANDLE_VALUE { + if owned_handle.handle == sys::c::INVALID_HANDLE_VALUE { // Don't call `CloseHandle`; it'd be harmless, except that it could // overwrite the `GetLastError` error. forget(owned_handle); @@ -365,7 +365,7 @@ impl Drop for OwnedHandle { #[inline] fn drop(&mut self) { unsafe { - let _ = c::CloseHandle(self.handle); + let _ = sys::c::CloseHandle(self.handle); } } } diff --git a/library/std/src/os/windows/io/raw.rs b/library/std/src/os/windows/io/raw.rs index 49e4f304f5d..1759e2e7f3f 100644 --- a/library/std/src/os/windows/io/raw.rs +++ b/library/std/src/os/windows/io/raw.rs @@ -11,7 +11,6 @@ use crate::os::windows::io::{OwnedHandle, OwnedSocket}; use crate::os::windows::raw; use crate::ptr; use crate::sys; -use crate::sys::c; use crate::sys_common::{self, AsInner, FromInner, IntoInner}; /// Raw HANDLEs. @@ -104,42 +103,42 @@ impl AsRawHandle for fs::File { #[stable(feature = "asraw_stdio", since = "1.21.0")] impl AsRawHandle for io::Stdin { fn as_raw_handle(&self) -> RawHandle { - stdio_handle(unsafe { c::GetStdHandle(c::STD_INPUT_HANDLE) as RawHandle }) + stdio_handle(unsafe { sys::c::GetStdHandle(sys::c::STD_INPUT_HANDLE) as RawHandle }) } } #[stable(feature = "asraw_stdio", since = "1.21.0")] impl AsRawHandle for io::Stdout { fn as_raw_handle(&self) -> RawHandle { - stdio_handle(unsafe { c::GetStdHandle(c::STD_OUTPUT_HANDLE) as RawHandle }) + stdio_handle(unsafe { sys::c::GetStdHandle(sys::c::STD_OUTPUT_HANDLE) as RawHandle }) } } #[stable(feature = "asraw_stdio", since = "1.21.0")] impl AsRawHandle for io::Stderr { fn as_raw_handle(&self) -> RawHandle { - stdio_handle(unsafe { c::GetStdHandle(c::STD_ERROR_HANDLE) as RawHandle }) + stdio_handle(unsafe { sys::c::GetStdHandle(sys::c::STD_ERROR_HANDLE) as RawHandle }) } } #[stable(feature = "asraw_stdio_locks", since = "1.35.0")] impl<'a> AsRawHandle for io::StdinLock<'a> { fn as_raw_handle(&self) -> RawHandle { - stdio_handle(unsafe { c::GetStdHandle(c::STD_INPUT_HANDLE) as RawHandle }) + stdio_handle(unsafe { sys::c::GetStdHandle(sys::c::STD_INPUT_HANDLE) as RawHandle }) } } #[stable(feature = "asraw_stdio_locks", since = "1.35.0")] impl<'a> AsRawHandle for io::StdoutLock<'a> { fn as_raw_handle(&self) -> RawHandle { - stdio_handle(unsafe { c::GetStdHandle(c::STD_OUTPUT_HANDLE) as RawHandle }) + stdio_handle(unsafe { sys::c::GetStdHandle(sys::c::STD_OUTPUT_HANDLE) as RawHandle }) } } #[stable(feature = "asraw_stdio_locks", since = "1.35.0")] impl<'a> AsRawHandle for io::StderrLock<'a> { fn as_raw_handle(&self) -> RawHandle { - stdio_handle(unsafe { c::GetStdHandle(c::STD_ERROR_HANDLE) as RawHandle }) + stdio_handle(unsafe { sys::c::GetStdHandle(sys::c::STD_ERROR_HANDLE) as RawHandle }) } } @@ -152,14 +151,14 @@ fn stdio_handle(raw: RawHandle) -> RawHandle { // console. In that case, return null to the user, which is consistent // with what they'd get in the parent, and which avoids the problem that // `INVALID_HANDLE_VALUE` aliases the current process handle. - if raw == c::INVALID_HANDLE_VALUE { ptr::null_mut() } else { raw } + if raw == sys::c::INVALID_HANDLE_VALUE { ptr::null_mut() } else { raw } } #[stable(feature = "from_raw_os", since = "1.1.0")] impl FromRawHandle for fs::File { #[inline] unsafe fn from_raw_handle(handle: RawHandle) -> fs::File { - let handle = handle as c::HANDLE; + let handle = handle as sys::c::HANDLE; fs::File::from_inner(sys::fs::File::from_inner(FromInner::from_inner( OwnedHandle::from_raw_handle(handle), ))) diff --git a/library/std/src/os/windows/io/socket.rs b/library/std/src/os/windows/io/socket.rs index ce34cd1a9bf..eb6097a89a6 100644 --- a/library/std/src/os/windows/io/socket.rs +++ b/library/std/src/os/windows/io/socket.rs @@ -9,7 +9,6 @@ use crate::marker::PhantomData; use crate::mem; use crate::mem::forget; use crate::sys; -use crate::sys::c; #[cfg(not(target_vendor = "uwp"))] use crate::sys::cvt; @@ -76,7 +75,7 @@ impl BorrowedSocket<'_> { #[rustc_const_stable(feature = "io_safety", since = "1.63.0")] #[stable(feature = "io_safety", since = "1.63.0")] pub const unsafe fn borrow_raw(socket: RawSocket) -> Self { - assert!(socket != c::INVALID_SOCKET as RawSocket); + assert!(socket != sys::c::INVALID_SOCKET as RawSocket); Self { socket, _phantom: PhantomData } } } @@ -94,7 +93,11 @@ impl OwnedSocket { #[cfg(not(target_vendor = "uwp"))] pub(crate) fn set_no_inherit(&self) -> io::Result<()> { cvt(unsafe { - c::SetHandleInformation(self.as_raw_socket() as c::HANDLE, c::HANDLE_FLAG_INHERIT, 0) + sys::c::SetHandleInformation( + self.as_raw_socket() as sys::c::HANDLE, + sys::c::HANDLE_FLAG_INHERIT, + 0, + ) }) .map(drop) } @@ -110,43 +113,47 @@ impl BorrowedSocket<'_> { /// object as the existing `BorrowedSocket` instance. #[stable(feature = "io_safety", since = "1.63.0")] pub fn try_clone_to_owned(&self) -> io::Result { - let mut info = unsafe { mem::zeroed::() }; + let mut info = unsafe { mem::zeroed::() }; let result = unsafe { - c::WSADuplicateSocketW(self.as_raw_socket(), c::GetCurrentProcessId(), &mut info) + sys::c::WSADuplicateSocketW( + self.as_raw_socket(), + sys::c::GetCurrentProcessId(), + &mut info, + ) }; sys::net::cvt(result)?; let socket = unsafe { - c::WSASocketW( + sys::c::WSASocketW( info.iAddressFamily, info.iSocketType, info.iProtocol, &mut info, 0, - c::WSA_FLAG_OVERLAPPED | c::WSA_FLAG_NO_HANDLE_INHERIT, + sys::c::WSA_FLAG_OVERLAPPED | sys::c::WSA_FLAG_NO_HANDLE_INHERIT, ) }; - if socket != c::INVALID_SOCKET { + if socket != sys::c::INVALID_SOCKET { unsafe { Ok(OwnedSocket::from_raw_socket(socket)) } } else { - let error = unsafe { c::WSAGetLastError() }; + let error = unsafe { sys::c::WSAGetLastError() }; - if error != c::WSAEPROTOTYPE && error != c::WSAEINVAL { + if error != sys::c::WSAEPROTOTYPE && error != sys::c::WSAEINVAL { return Err(io::Error::from_raw_os_error(error)); } let socket = unsafe { - c::WSASocketW( + sys::c::WSASocketW( info.iAddressFamily, info.iSocketType, info.iProtocol, &mut info, 0, - c::WSA_FLAG_OVERLAPPED, + sys::c::WSA_FLAG_OVERLAPPED, ) }; - if socket == c::INVALID_SOCKET { + if socket == sys::c::INVALID_SOCKET { return Err(last_error()); } @@ -161,7 +168,7 @@ impl BorrowedSocket<'_> { /// Returns the last error from the Windows socket interface. fn last_error() -> io::Error { - io::Error::from_raw_os_error(unsafe { c::WSAGetLastError() }) + io::Error::from_raw_os_error(unsafe { sys::c::WSAGetLastError() }) } #[stable(feature = "io_safety", since = "1.63.0")] @@ -194,7 +201,7 @@ impl IntoRawSocket for OwnedSocket { impl FromRawSocket for OwnedSocket { #[inline] unsafe fn from_raw_socket(socket: RawSocket) -> Self { - debug_assert_ne!(socket, c::INVALID_SOCKET as RawSocket); + debug_assert_ne!(socket, sys::c::INVALID_SOCKET as RawSocket); Self { socket } } } @@ -204,7 +211,7 @@ impl Drop for OwnedSocket { #[inline] fn drop(&mut self) { unsafe { - let _ = c::closesocket(self.socket); + let _ = sys::c::closesocket(self.socket); } } } diff --git a/library/std/src/sys/mod.rs b/library/std/src/sys/mod.rs index e767b2866cb..c72be13804d 100644 --- a/library/std/src/sys/mod.rs +++ b/library/std/src/sys/mod.rs @@ -52,31 +52,6 @@ cfg_if::cfg_if! { } } -// Import essential modules from platforms used in `std::os` when documenting. -// -// Note that on some platforms those modules don't compile -// (missing things in `libc` which is empty), so they are not included in `std::os` and can be -// omitted here as well. - -#[cfg(doc)] -#[cfg(not(any( - all(target_arch = "wasm32", not(target_os = "wasi")), - all(target_vendor = "fortanix", target_env = "sgx") -)))] -cfg_if::cfg_if! { - if #[cfg(not(windows))] { - // On non-Windows platforms (aka linux/osx/etc) pull in a "minimal" - // amount of windows goop which ends up compiling - - #[macro_use] - #[path = "windows/compat.rs"] - pub mod compat; - - #[path = "windows/c.rs"] - pub mod c; - } -} - cfg_if::cfg_if! { // Fuchsia components default to full backtrace. if #[cfg(target_os = "fuchsia")] { From 7c7b22e62cd3aa34ef60ec98b145258caa55261f Mon Sep 17 00:00:00 2001 From: Ramon de C Valle Date: Wed, 3 May 2023 22:22:24 +0000 Subject: [PATCH 099/130] CFI: Fix SIGILL reached via trait objects Fix #106547 by transforming the concrete self into a reference to a trait object before emitting type metadata identifiers for trait methods. --- compiler/rustc_codegen_llvm/src/callee.rs | 4 +- .../src/coverageinfo/mod.rs | 1 + compiler/rustc_codegen_llvm/src/declare.rs | 64 ++++++++++++----- compiler/rustc_codegen_llvm/src/intrinsic.rs | 2 +- compiler/rustc_codegen_llvm/src/mono_item.rs | 2 +- compiler/rustc_symbol_mangling/src/typeid.rs | 24 ++++++- .../src/typeid/typeid_itanium_cxx_abi.rs | 57 ++++++++++++++- compiler/rustc_target/src/abi/call/mod.rs | 8 +-- ...er-cfi-emit-type-metadata-trait-objects.rs | 44 ++++++++++++ ...r-kcfi-emit-type-metadata-trait-objects.rs | 69 +++++++++++++++++++ 10 files changed, 247 insertions(+), 28 deletions(-) create mode 100644 tests/codegen/sanitizer-cfi-emit-type-metadata-trait-objects.rs create mode 100644 tests/codegen/sanitizer-kcfi-emit-type-metadata-trait-objects.rs diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs index 30a0cf1d019..4b9ca2e7d19 100644 --- a/compiler/rustc_codegen_llvm/src/callee.rs +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -94,11 +94,11 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> // LLVM will prefix the name with `__imp_`. Ideally, we'd like the // existing logic below to set the Storage Class, but it has an // exemption for MinGW for backwards compatability. - let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi); + let llfn = cx.declare_fn(&common::i686_decorated_name(&dllimport, common::is_mingw_gnu_toolchain(&tcx.sess.target), true), fn_abi, Some(instance)); unsafe { llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); } llfn } else { - cx.declare_fn(sym, fn_abi) + cx.declare_fn(sym, fn_abi, Some(instance)) }; debug!("get_fn: not casting pointer!"); diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 3dc0ac03312..cd261293e9b 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -207,6 +207,7 @@ fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance< )), ty::List::empty(), ), + None, ); llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs index cc2a5d158be..164b12cf8d4 100644 --- a/compiler/rustc_codegen_llvm/src/declare.rs +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -19,8 +19,11 @@ use crate::llvm::AttributePlace::Function; use crate::type_::Type; use crate::value::Value; use rustc_codegen_ssa::traits::TypeMembershipMethods; -use rustc_middle::ty::Ty; -use rustc_symbol_mangling::typeid::{kcfi_typeid_for_fnabi, typeid_for_fnabi, TypeIdOptions}; +use rustc_middle::ty::{Instance, Ty}; +use rustc_symbol_mangling::typeid::{ + kcfi_typeid_for_fnabi, kcfi_typeid_for_instance, typeid_for_fnabi, typeid_for_instance, + TypeIdOptions, +}; use smallvec::SmallVec; /// Declare a function. @@ -116,7 +119,12 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { /// /// If there’s a value with the same name already declared, the function will /// update the declaration and return existing Value instead. - pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + pub fn declare_fn( + &self, + name: &str, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + instance: Option>, + ) -> &'ll Value { debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); // Function addresses in Rust are never significant, allowing functions to @@ -132,18 +140,35 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { fn_abi.apply_attrs_llfn(self, llfn); if self.tcx.sess.is_sanitizer_cfi_enabled() { - let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); - self.set_type_metadata(llfn, typeid); - let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); - self.add_type_metadata(llfn, typeid); - let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); - self.add_type_metadata(llfn, typeid); - let typeid = typeid_for_fnabi( - self.tcx, - fn_abi, - TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, - ); - self.add_type_metadata(llfn, typeid); + if let Some(instance) = instance { + let typeid = typeid_for_instance(self.tcx, &instance, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = + typeid_for_instance(self.tcx, &instance, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_instance( + self.tcx, + &instance, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } else { + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::empty()); + self.set_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::GENERALIZE_POINTERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi(self.tcx, fn_abi, TypeIdOptions::NORMALIZE_INTEGERS); + self.add_type_metadata(llfn, typeid); + let typeid = typeid_for_fnabi( + self.tcx, + fn_abi, + TypeIdOptions::GENERALIZE_POINTERS | TypeIdOptions::NORMALIZE_INTEGERS, + ); + self.add_type_metadata(llfn, typeid); + } } if self.tcx.sess.is_sanitizer_kcfi_enabled() { @@ -156,8 +181,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { options.insert(TypeIdOptions::NORMALIZE_INTEGERS); } - let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); - self.set_kcfi_type_metadata(llfn, kcfi_typeid); + if let Some(instance) = instance { + let kcfi_typeid = kcfi_typeid_for_instance(self.tcx, &instance, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } else { + let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi, options); + self.set_kcfi_type_metadata(llfn, kcfi_typeid); + } } llfn diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 00d1796f210..4e28034a850 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -772,7 +772,7 @@ fn gen_fn<'ll, 'tcx>( ) -> (&'ll Type, &'ll Value) { let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); let llty = fn_abi.llvm_type(cx); - let llfn = cx.declare_fn(name, fn_abi); + let llfn = cx.declare_fn(name, fn_abi, None); cx.set_frame_pointer_type(llfn); cx.apply_target_cpu_attr(llfn); // FIXME(eddyb) find a nicer way to do this. diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index 59bdc60830f..e8f8c321510 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -51,7 +51,7 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { assert!(!instance.substs.has_infer()); let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); - let lldecl = self.declare_fn(symbol_name, fn_abi); + let lldecl = self.declare_fn(symbol_name, fn_abi, Some(instance)); unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); base::set_link_section(lldecl, attrs); diff --git a/compiler/rustc_symbol_mangling/src/typeid.rs b/compiler/rustc_symbol_mangling/src/typeid.rs index 81dbff9ea4e..cda16e3a3f5 100644 --- a/compiler/rustc_symbol_mangling/src/typeid.rs +++ b/compiler/rustc_symbol_mangling/src/typeid.rs @@ -4,7 +4,7 @@ /// For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler, /// see design document in the tracking issue #89653. use bitflags::bitflags; -use rustc_middle::ty::{FnSig, Ty, TyCtxt}; +use rustc_middle::ty::{FnSig, Instance, Ty, TyCtxt}; use rustc_target::abi::call::FnAbi; use std::hash::Hasher; use twox_hash::XxHash64; @@ -38,6 +38,15 @@ pub fn typeid_for_fnsig<'tcx>( typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, options) } +/// Returns a type metadata identifier for the specified Instance. +pub fn typeid_for_instance<'tcx>( + tcx: TyCtxt<'tcx>, + instance: &Instance<'tcx>, + options: TypeIdOptions, +) -> String { + typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options) +} + /// Returns a KCFI type metadata identifier for the specified FnAbi. pub fn kcfi_typeid_for_fnabi<'tcx>( tcx: TyCtxt<'tcx>, @@ -63,3 +72,16 @@ pub fn kcfi_typeid_for_fnsig<'tcx>( hash.write(typeid_itanium_cxx_abi::typeid_for_fnsig(tcx, fn_sig, options).as_bytes()); hash.finish() as u32 } + +/// Returns a KCFI type metadata identifier for the specified Instance. +pub fn kcfi_typeid_for_instance<'tcx>( + tcx: TyCtxt<'tcx>, + instance: &Instance<'tcx>, + options: TypeIdOptions, +) -> u32 { + // A KCFI type metadata identifier is a 32-bit constant produced by taking the lower half of the + // xxHash64 of the type metadata identifier. (See llvm/llvm-project@cff5bef.) + let mut hash: XxHash64 = Default::default(); + hash.write(typeid_itanium_cxx_abi::typeid_for_instance(tcx, instance, options).as_bytes()); + hash.finish() as u32 +} diff --git a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs index 5310ef26da7..c281aa7e83a 100644 --- a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs +++ b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs @@ -14,8 +14,8 @@ use rustc_errors::DiagnosticMessage; use rustc_hir as hir; use rustc_middle::ty::subst::{GenericArg, GenericArgKind, SubstsRef}; use rustc_middle::ty::{ - self, Const, ExistentialPredicate, FloatTy, FnSig, IntTy, List, Region, RegionKind, TermKind, - Ty, TyCtxt, UintTy, + self, Const, ExistentialPredicate, FloatTy, FnSig, Instance, IntTy, List, Region, RegionKind, + TermKind, Ty, TyCtxt, UintTy, }; use rustc_span::def_id::DefId; use rustc_span::sym; @@ -1010,3 +1010,56 @@ pub fn typeid_for_fnsig<'tcx>( typeid } + +/// Returns a type metadata identifier for the specified Instance using the Itanium C++ ABI with +/// vendor extended type qualifiers and types for Rust types that are not used at the FFI boundary. +pub fn typeid_for_instance<'tcx>( + tcx: TyCtxt<'tcx>, + instance: &Instance<'tcx>, + options: TypeIdOptions, +) -> String { + let fn_abi = tcx + .fn_abi_of_instance(tcx.param_env(instance.def_id()).and((*instance, ty::List::empty()))) + .unwrap_or_else(|instance| { + bug!("typeid_for_instance: couldn't get fn_abi of instance {:?}", instance) + }); + + // If this instance is a method and self is a reference, get the impl it belongs to + let impl_def_id = tcx.impl_of_method(instance.def_id()); + if impl_def_id.is_some() && !fn_abi.args.is_empty() && fn_abi.args[0].layout.ty.is_ref() { + // If this impl is not an inherent impl, get the trait it implements + if let Some(trait_ref) = tcx.impl_trait_ref(impl_def_id.unwrap()) { + // Transform the concrete self into a reference to a trait object + let existential_predicate = trait_ref.map_bound(|trait_ref| { + ty::ExistentialPredicate::Trait(ty::ExistentialTraitRef::erase_self_ty( + tcx, trait_ref, + )) + }); + let existential_predicates = tcx.mk_poly_existential_predicates(&[ty::Binder::dummy( + existential_predicate.skip_binder(), + )]); + // Is the concrete self mutable? + let self_ty = if fn_abi.args[0].layout.ty.is_mutable_ptr() { + tcx.mk_mut_ref( + tcx.lifetimes.re_erased, + tcx.mk_dynamic(existential_predicates, tcx.lifetimes.re_erased, ty::Dyn), + ) + } else { + tcx.mk_imm_ref( + tcx.lifetimes.re_erased, + tcx.mk_dynamic(existential_predicates, tcx.lifetimes.re_erased, ty::Dyn), + ) + }; + + // Replace the concrete self in an fn_abi clone by the reference to a trait object + let mut fn_abi = fn_abi.clone(); + // HACK(rcvalle): It is okay to not replace or update the entire ArgAbi here because the + // other fields are never used. + fn_abi.args[0].layout.ty = self_ty; + + return typeid_for_fnabi(tcx, &fn_abi, options); + } + } + + typeid_for_fnabi(tcx, &fn_abi, options) +} diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 57011aa8a14..0aacc8ba5de 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -28,7 +28,7 @@ mod x86; mod x86_64; mod x86_win64; -#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub enum PassMode { /// Ignore the argument. /// @@ -211,7 +211,7 @@ impl Uniform { } } -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub struct CastTarget { pub prefix: [Option; 8], pub rest: Uniform, @@ -458,7 +458,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> { /// Information about how to pass an argument to, /// or return a value from, a function, under some ABI. -#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub struct ArgAbi<'a, Ty> { pub layout: TyAndLayout<'a, Ty>, pub mode: PassMode, @@ -605,7 +605,7 @@ pub enum Conv { /// /// I will do my best to describe this structure, but these /// comments are reverse-engineered and may be inaccurate. -NDM -#[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] +#[derive(Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] pub struct FnAbi<'a, Ty> { /// The LLVM types of each argument. pub args: Box<[ArgAbi<'a, Ty>]>, diff --git a/tests/codegen/sanitizer-cfi-emit-type-metadata-trait-objects.rs b/tests/codegen/sanitizer-cfi-emit-type-metadata-trait-objects.rs new file mode 100644 index 00000000000..ab5dcec7936 --- /dev/null +++ b/tests/codegen/sanitizer-cfi-emit-type-metadata-trait-objects.rs @@ -0,0 +1,44 @@ +// Verifies that type metadata identifiers for trait objects are emitted correctly. +// +// needs-sanitizer-cfi +// compile-flags: -Clto -Cno-prepopulate-passes -Ctarget-feature=-crt-static -Zsanitizer=cfi + +#![crate_type="lib"] + +trait Trait1 { + fn foo(&self); +} + +struct Type1; + +impl Trait1 for Type1 { + fn foo(&self) { + } +} + +pub fn foo() { + let a = Type1; + a.foo(); + // CHECK-LABEL: define{{.*}}foo{{.*}}!type !{{[0-9]+}} + // CHECK: call ::foo +} + +pub fn bar() { + let a = Type1; + let b = &a as &dyn Trait1; + b.foo(); + // CHECK-LABEL: define{{.*}}bar{{.*}}!type !{{[0-9]+}} + // CHECK: call i1 @llvm.type.test({{i8\*|ptr}} {{%f|%0|%1}}, metadata !"[[TYPE1:[[:print:]]+]]") +} + +pub fn baz() { + let a = Type1; + let b = &a as &dyn Trait1; + a.foo(); + b.foo(); + // CHECK-LABEL: define{{.*}}baz{{.*}}!type !{{[0-9]+}} + // CHECK: call ::foo + // CHECK: call i1 @llvm.type.test({{i8\*|ptr}} {{%f|%0|%1}}, metadata !"[[TYPE1:[[:print:]]+]]") +} + +// CHECK: !{{[0-9]+}} = !{i64 0, !"[[TYPE1]]"} diff --git a/tests/codegen/sanitizer-kcfi-emit-type-metadata-trait-objects.rs b/tests/codegen/sanitizer-kcfi-emit-type-metadata-trait-objects.rs new file mode 100644 index 00000000000..81e0d9344f7 --- /dev/null +++ b/tests/codegen/sanitizer-kcfi-emit-type-metadata-trait-objects.rs @@ -0,0 +1,69 @@ +// Verifies that type metadata identifiers for trait objects are emitted correctly. +// +// revisions: aarch64 x86_64 +// [aarch64] compile-flags: --target aarch64-unknown-none +// [aarch64] needs-llvm-components: aarch64 +// [x86_64] compile-flags: --target x86_64-unknown-none +// [x86_64] needs-llvm-components: +// compile-flags: -Cno-prepopulate-passes -Zsanitizer=kcfi -Copt-level=0 + +#![crate_type="lib"] +#![feature(arbitrary_self_types, no_core, lang_items)] +#![no_core] + +#[lang="sized"] +trait Sized { } +#[lang="copy"] +trait Copy { } +#[lang="receiver"] +trait Receiver { } +#[lang="dispatch_from_dyn"] +trait DispatchFromDyn { } +impl<'a, T: ?Sized + Unsize, U: ?Sized> DispatchFromDyn<&'a U> for &'a T {} +#[lang = "unsize"] +trait Unsize { } +#[lang = "coerce_unsized"] +pub trait CoerceUnsized { } +impl<'a, 'b: 'a, T: ?Sized + Unsize, U: ?Sized> CoerceUnsized<&'a U> for &'b T {} +#[lang="freeze"] +trait Freeze { } +#[lang="drop_in_place"] +fn drop_in_place_fn() { } + +trait Trait1 { + fn foo(&self); +} + +struct Type1; + +impl Trait1 for Type1 { + fn foo(&self) { + } +} + +pub fn foo() { + let a = Type1; + a.foo(); + // CHECK-LABEL: define{{.*}}foo{{.*}}!{{|kcfi_type}} !{{[0-9]+}} + // CHECK: call ::foo +} + +pub fn bar() { + let a = Type1; + let b = &a as &dyn Trait1; + b.foo(); + // CHECK-LABEL: define{{.*}}bar{{.*}}!{{|kcfi_type}} !{{[0-9]+}} + // CHECK: call void %0({{\{\}\*|ptr}} align 1 {{%b\.0|%_1}}){{.*}}[ "kcfi"(i32 [[TYPE1:[[:print:]]+]]) ] +} + +pub fn baz() { + let a = Type1; + let b = &a as &dyn Trait1; + a.foo(); + b.foo(); + // CHECK-LABEL: define{{.*}}baz{{.*}}!{{|kcfi_type}} !{{[0-9]+}} + // CHECK: call ::foo + // CHECK: call void %0({{\{\}\*|ptr}} align 1 {{%b\.0|%_1}}){{.*}}[ "kcfi"(i32 [[TYPE1:[[:print:]]+]]) ] +} + +// CHECK: !{{[0-9]+}} = !{i32 [[TYPE1]]} From 0dbaae4165b8ba41397a9b0f714acb2cd3610d65 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Sat, 22 Apr 2023 03:11:25 +0000 Subject: [PATCH 100/130] Make alias bounds sound in the new solver --- .../src/solve/assembly/mod.rs | 116 +++++++++++++++++- .../src/solve/project_goals.rs | 24 ++++ .../src/solve/trait_goals.rs | 24 ++++ .../traits/new-solver/alias-bound-unsound.rs | 27 ++++ .../new-solver/alias-bound-unsound.stderr | 24 ++++ .../traits/new-solver/nested-alias-bound.rs | 20 +++ 6 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 tests/ui/traits/new-solver/alias-bound-unsound.rs create mode 100644 tests/ui/traits/new-solver/alias-bound-unsound.stderr create mode 100644 tests/ui/traits/new-solver/nested-alias-bound.rs diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs index bacb0e32efc..0f42d5f1262 100644 --- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs @@ -8,6 +8,7 @@ use rustc_data_structures::fx::FxIndexSet; use rustc_hir::def_id::DefId; use rustc_infer::traits::query::NoSolution; use rustc_infer::traits::util::elaborate; +use rustc_infer::traits::Reveal; use rustc_middle::traits::solve::{CanonicalResponse, Certainty, Goal, MaybeCause, QueryResult}; use rustc_middle::ty::fast_reject::TreatProjections; use rustc_middle::ty::TypeFoldable; @@ -87,7 +88,9 @@ pub(super) enum CandidateSource { } /// Methods used to assemble candidates for either trait or projection goals. -pub(super) trait GoalKind<'tcx>: TypeFoldable> + Copy + Eq { +pub(super) trait GoalKind<'tcx>: + TypeFoldable> + Copy + Eq + std::fmt::Display +{ fn self_ty(self) -> Ty<'tcx>; fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx>; @@ -106,6 +109,16 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable> + Copy + Eq { requirements: impl IntoIterator>>, ) -> QueryResult<'tcx>; + /// Consider a bound originating from the item bounds of an alias. For this we + /// require that the well-formed requirements of the self type of the goal + /// are "satisfied from the param-env". + /// See [`EvalCtxt::validate_alias_bound_self_from_param_env`]. + fn consider_alias_bound_candidate( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + assumption: ty::Predicate<'tcx>, + ) -> QueryResult<'tcx>; + // Consider a clause specifically for a `dyn Trait` self type. This requires // additionally checking all of the supertraits and object bounds to hold, // since they're not implied by the well-formedness of the object type. @@ -463,7 +476,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { for assumption in self.tcx().item_bounds(alias_ty.def_id).subst(self.tcx(), alias_ty.substs) { - match G::consider_implied_clause(self, goal, assumption, []) { + match G::consider_alias_bound_candidate(self, goal, assumption) { Ok(result) => { candidates.push(Candidate { source: CandidateSource::AliasBound, result }) } @@ -472,6 +485,105 @@ impl<'tcx> EvalCtxt<'_, 'tcx> { } } + /// Check that we are allowed to use an alias bound originating from the self + /// type of this goal. This means something different depending on the self type's + /// alias kind. + /// + /// * Projection: Given a goal with a self type such as `::Assoc`, + /// we require that the bound `Ty: Trait` can be proven using either a nested alias + /// bound candidate, or a param-env candidate. + /// + /// * Opaque: The param-env must be in `Reveal::UserFacing` mode. Otherwise, + /// the goal should be proven by using the hidden type instead. + #[instrument(level = "debug", skip(self), ret)] + pub(super) fn validate_alias_bound_self_from_param_env>( + &mut self, + goal: Goal<'tcx, G>, + ) -> QueryResult<'tcx> { + match *goal.predicate.self_ty().kind() { + ty::Alias(ty::Projection, projection_ty) => { + let mut param_env_candidates = vec![]; + let self_trait_ref = projection_ty.trait_ref(self.tcx()); + + if self_trait_ref.self_ty().is_ty_var() { + return self + .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS); + } + + let trait_goal: Goal<'_, ty::TraitPredicate<'tcx>> = goal.with( + self.tcx(), + ty::TraitPredicate { + trait_ref: self_trait_ref, + constness: ty::BoundConstness::NotConst, + polarity: ty::ImplPolarity::Positive, + }, + ); + + self.assemble_param_env_candidates(trait_goal, &mut param_env_candidates); + // FIXME: We probably need some sort of recursion depth check here. + // Can't come up with an example yet, though, and the worst case + // we can have is a compiler stack overflow... + self.assemble_alias_bound_candidates(trait_goal, &mut param_env_candidates); + + // FIXME: We must also consider alias-bound candidates for a peculiar + // class of built-in candidates that I'll call "defaulted" built-ins. + // + // For example, we always know that `T: Pointee` is implemented, but + // we do not always know what `::Metadata` actually is, + // similar to if we had a user-defined impl with a `default type ...`. + // For these traits, since we're not able to always normalize their + // associated types to a concrete type, we must consider their alias bounds + // instead, so we can prove bounds such as `::Metadata: Copy`. + self.assemble_alias_bound_candidates_for_builtin_impl_default_items( + trait_goal, + &mut param_env_candidates, + ); + + self.merge_candidates(param_env_candidates) + } + ty::Alias(ty::Opaque, _opaque_ty) => match goal.param_env.reveal() { + Reveal::UserFacing => { + self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + } + Reveal::All => return Err(NoSolution), + }, + _ => bug!("only expected to be called on alias tys"), + } + } + + /// Assemble a subset of builtin impl candidates for a class of candidates called + /// "defaulted" built-in traits. + /// + /// For example, we always know that `T: Pointee` is implemented, but we do not + /// always know what `::Metadata` actually is! See the comment in + /// [`EvalCtxt::validate_alias_bound_self_from_param_env`] for more detail. + #[instrument(level = "debug", skip_all)] + fn assemble_alias_bound_candidates_for_builtin_impl_default_items>( + &mut self, + goal: Goal<'tcx, G>, + candidates: &mut Vec>, + ) { + let lang_items = self.tcx().lang_items(); + let trait_def_id = goal.predicate.trait_def_id(self.tcx()); + + // You probably shouldn't add anything to this list unless you + // know what you're doing. + let result = if lang_items.pointee_trait() == Some(trait_def_id) { + G::consider_builtin_pointee_candidate(self, goal) + } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) { + G::consider_builtin_discriminant_kind_candidate(self, goal) + } else { + Err(NoSolution) + }; + + match result { + Ok(result) => { + candidates.push(Candidate { source: CandidateSource::BuiltinImpl, result }) + } + Err(NoSolution) => (), + } + } + #[instrument(level = "debug", skip_all)] fn assemble_object_bound_candidates>( &mut self, diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs index e5d51064c8d..85ac7f15a2f 100644 --- a/compiler/rustc_trait_selection/src/solve/project_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs @@ -83,6 +83,30 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> { } } + fn consider_alias_bound_candidate( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + assumption: ty::Predicate<'tcx>, + ) -> QueryResult<'tcx> { + if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred() + && poly_projection_pred.projection_def_id() == goal.predicate.def_id() + { + ecx.probe(|ecx| { + let assumption_projection_pred = + ecx.instantiate_binder_with_infer(poly_projection_pred); + ecx.eq( + goal.param_env, + goal.predicate.projection_ty, + assumption_projection_pred.projection_ty, + )?; + ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)?; + ecx.validate_alias_bound_self_from_param_env(goal) + }) + } else { + Err(NoSolution) + } + } + fn consider_object_bound_candidate( ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs index 04b38edc126..0f6fd4059b4 100644 --- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs @@ -105,6 +105,30 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> { } } + fn consider_alias_bound_candidate( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + assumption: ty::Predicate<'tcx>, + ) -> QueryResult<'tcx> { + if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred() + && poly_trait_pred.def_id() == goal.predicate.def_id() + { + // FIXME: Constness and polarity + ecx.probe(|ecx| { + let assumption_trait_pred = + ecx.instantiate_binder_with_infer(poly_trait_pred); + ecx.eq( + goal.param_env, + goal.predicate.trait_ref, + assumption_trait_pred.trait_ref, + )?; + ecx.validate_alias_bound_self_from_param_env(goal) + }) + } else { + Err(NoSolution) + } + } + fn consider_object_bound_candidate( ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, diff --git a/tests/ui/traits/new-solver/alias-bound-unsound.rs b/tests/ui/traits/new-solver/alias-bound-unsound.rs new file mode 100644 index 00000000000..00294c708f1 --- /dev/null +++ b/tests/ui/traits/new-solver/alias-bound-unsound.rs @@ -0,0 +1,27 @@ +// compile-flags: -Ztrait-solver=next + +// Makes sure that alias bounds are not unsound! + +#![feature(trivial_bounds)] + +trait Foo { + type Item: Copy + where + ::Item: Copy; + + fn copy_me(x: &Self::Item) -> Self::Item { + *x + } +} + +impl Foo for () { + type Item = String where String: Copy; +} + +fn main() { + let x = String::from("hello, world"); + drop(<() as Foo>::copy_me(&x)); + //~^ ERROR `<() as Foo>::Item: Copy` is not satisfied + //~| ERROR `<() as Foo>::Item` is not well-formed + println!("{x}"); +} diff --git a/tests/ui/traits/new-solver/alias-bound-unsound.stderr b/tests/ui/traits/new-solver/alias-bound-unsound.stderr new file mode 100644 index 00000000000..9a43d2a6639 --- /dev/null +++ b/tests/ui/traits/new-solver/alias-bound-unsound.stderr @@ -0,0 +1,24 @@ +error[E0277]: the trait bound `<() as Foo>::Item: Copy` is not satisfied + --> $DIR/alias-bound-unsound.rs:23:10 + | +LL | drop(<() as Foo>::copy_me(&x)); + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Copy` is not implemented for `<() as Foo>::Item` + | +note: required by a bound in `Foo::Item` + --> $DIR/alias-bound-unsound.rs:10:30 + | +LL | type Item: Copy + | ---- required by a bound in this associated type +LL | where +LL | ::Item: Copy; + | ^^^^ required by this bound in `Foo::Item` + +error: the type `<() as Foo>::Item` is not well-formed + --> $DIR/alias-bound-unsound.rs:23:10 + | +LL | drop(<() as Foo>::copy_me(&x)); + | ^^^^^^^^^^^^^^^^^^^^^^^^ + +error: aborting due to 2 previous errors + +For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/traits/new-solver/nested-alias-bound.rs b/tests/ui/traits/new-solver/nested-alias-bound.rs new file mode 100644 index 00000000000..c365902dbe5 --- /dev/null +++ b/tests/ui/traits/new-solver/nested-alias-bound.rs @@ -0,0 +1,20 @@ +// compile-flags: -Ztrait-solver=next +// check-pass + +trait A { + type A: B; +} + +trait B { + type B: C; +} + +trait C {} + +fn needs_c() {} + +fn test() { + needs_c::<::B>(); +} + +fn main() {} From 3a863e534bcba819a4ff8d8362df442001a3dd8a Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Sat, 22 Apr 2023 04:51:35 +0000 Subject: [PATCH 101/130] Consolidate the 'match assumption' type methods in GoalKind --- .../src/solve/assembly/mod.rs | 44 +++++++++++- .../src/solve/project_goals.rs | 72 +------------------ .../src/solve/trait_goals.rs | 71 +----------------- 3 files changed, 47 insertions(+), 140 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs index 0f42d5f1262..25cc82f01d5 100644 --- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs @@ -99,6 +99,17 @@ pub(super) trait GoalKind<'tcx>: fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId; + // Try equating an assumption predicate against a goal's predicate. If it + // holds, then execute the `then` callback, which should do any additional + // work, then produce a response (typically by executing + // [`EvalCtxt::evaluate_added_goals_and_make_canonical_response`]). + fn probe_and_match_goal_against_assumption( + ecx: &mut EvalCtxt<'_, 'tcx>, + goal: Goal<'tcx, Self>, + assumption: ty::Predicate<'tcx>, + then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>, + ) -> QueryResult<'tcx>; + // Consider a clause, which consists of a "assumption" and some "requirements", // to satisfy a goal. If the requirements hold, then attempt to satisfy our // goal by equating it with the assumption. @@ -107,7 +118,12 @@ pub(super) trait GoalKind<'tcx>: goal: Goal<'tcx, Self>, assumption: ty::Predicate<'tcx>, requirements: impl IntoIterator>>, - ) -> QueryResult<'tcx>; + ) -> QueryResult<'tcx> { + Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| { + ecx.add_goals(requirements); + ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + }) + } /// Consider a bound originating from the item bounds of an alias. For this we /// require that the well-formed requirements of the self type of the goal @@ -117,7 +133,11 @@ pub(super) trait GoalKind<'tcx>: ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, assumption: ty::Predicate<'tcx>, - ) -> QueryResult<'tcx>; + ) -> QueryResult<'tcx> { + Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| { + ecx.validate_alias_bound_self_from_param_env(goal) + }) + } // Consider a clause specifically for a `dyn Trait` self type. This requires // additionally checking all of the supertraits and object bounds to hold, @@ -126,7 +146,25 @@ pub(super) trait GoalKind<'tcx>: ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, assumption: ty::Predicate<'tcx>, - ) -> QueryResult<'tcx>; + ) -> QueryResult<'tcx> { + Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| { + let tcx = ecx.tcx(); + let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else { + bug!("expected object type in `consider_object_bound_candidate`"); + }; + ecx.add_goals( + structural_traits::predicates_for_object_candidate( + &ecx, + goal.param_env, + goal.predicate.trait_ref(tcx), + bounds, + ) + .into_iter() + .map(|pred| goal.with(tcx, pred)), + ); + ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + }) + } fn consider_impl_candidate( ecx: &mut EvalCtxt<'_, 'tcx>, diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs index 85ac7f15a2f..20ce2d9416e 100644 --- a/compiler/rustc_trait_selection/src/solve/project_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs @@ -56,11 +56,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> { self.trait_def_id(tcx) } - fn consider_implied_clause( + fn probe_and_match_goal_against_assumption( ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, assumption: ty::Predicate<'tcx>, - requirements: impl IntoIterator>>, + then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>, ) -> QueryResult<'tcx> { if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred() && poly_projection_pred.projection_def_id() == goal.predicate.def_id() @@ -75,73 +75,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> { )?; ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term) .expect("expected goal term to be fully unconstrained"); - ecx.add_goals(requirements); - ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) - }) - } else { - Err(NoSolution) - } - } - - fn consider_alias_bound_candidate( - ecx: &mut EvalCtxt<'_, 'tcx>, - goal: Goal<'tcx, Self>, - assumption: ty::Predicate<'tcx>, - ) -> QueryResult<'tcx> { - if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred() - && poly_projection_pred.projection_def_id() == goal.predicate.def_id() - { - ecx.probe(|ecx| { - let assumption_projection_pred = - ecx.instantiate_binder_with_infer(poly_projection_pred); - ecx.eq( - goal.param_env, - goal.predicate.projection_ty, - assumption_projection_pred.projection_ty, - )?; - ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)?; - ecx.validate_alias_bound_self_from_param_env(goal) - }) - } else { - Err(NoSolution) - } - } - - fn consider_object_bound_candidate( - ecx: &mut EvalCtxt<'_, 'tcx>, - goal: Goal<'tcx, Self>, - assumption: ty::Predicate<'tcx>, - ) -> QueryResult<'tcx> { - if let Some(poly_projection_pred) = assumption.to_opt_poly_projection_pred() - && poly_projection_pred.projection_def_id() == goal.predicate.def_id() - { - ecx.probe(|ecx| { - let tcx = ecx.tcx(); - - let assumption_projection_pred = - ecx.instantiate_binder_with_infer(poly_projection_pred); - ecx.eq( - goal.param_env, - goal.predicate.projection_ty, - assumption_projection_pred.projection_ty, - )?; - - let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else { - bug!("expected object type in `consider_object_bound_candidate`"); - }; - ecx.add_goals( - structural_traits::predicates_for_object_candidate( - &ecx, - goal.param_env, - goal.predicate.projection_ty.trait_ref(tcx), - bounds, - ) - .into_iter() - .map(|pred| goal.with(tcx, pred)), - ); - ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term) - .expect("expected goal term to be fully unconstrained"); - ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + then(ecx) }) } else { Err(NoSolution) diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs index 0f6fd4059b4..dcfa33ae842 100644 --- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs +++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs @@ -78,11 +78,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> { }) } - fn consider_implied_clause( + fn probe_and_match_goal_against_assumption( ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, Self>, assumption: ty::Predicate<'tcx>, - requirements: impl IntoIterator>>, + then: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>, ) -> QueryResult<'tcx> { if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred() && poly_trait_pred.def_id() == goal.predicate.def_id() @@ -97,72 +97,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> { goal.predicate.trait_ref, assumption_trait_pred.trait_ref, )?; - ecx.add_goals(requirements); - ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) - }) - } else { - Err(NoSolution) - } - } - - fn consider_alias_bound_candidate( - ecx: &mut EvalCtxt<'_, 'tcx>, - goal: Goal<'tcx, Self>, - assumption: ty::Predicate<'tcx>, - ) -> QueryResult<'tcx> { - if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred() - && poly_trait_pred.def_id() == goal.predicate.def_id() - { - // FIXME: Constness and polarity - ecx.probe(|ecx| { - let assumption_trait_pred = - ecx.instantiate_binder_with_infer(poly_trait_pred); - ecx.eq( - goal.param_env, - goal.predicate.trait_ref, - assumption_trait_pred.trait_ref, - )?; - ecx.validate_alias_bound_self_from_param_env(goal) - }) - } else { - Err(NoSolution) - } - } - - fn consider_object_bound_candidate( - ecx: &mut EvalCtxt<'_, 'tcx>, - goal: Goal<'tcx, Self>, - assumption: ty::Predicate<'tcx>, - ) -> QueryResult<'tcx> { - if let Some(poly_trait_pred) = assumption.to_opt_poly_trait_pred() - && poly_trait_pred.def_id() == goal.predicate.def_id() - && poly_trait_pred.polarity() == goal.predicate.polarity - { - // FIXME: Constness and polarity - ecx.probe(|ecx| { - let assumption_trait_pred = - ecx.instantiate_binder_with_infer(poly_trait_pred); - ecx.eq( - goal.param_env, - goal.predicate.trait_ref, - assumption_trait_pred.trait_ref, - )?; - - let tcx = ecx.tcx(); - let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else { - bug!("expected object type in `consider_object_bound_candidate`"); - }; - ecx.add_goals( - structural_traits::predicates_for_object_candidate( - &ecx, - goal.param_env, - goal.predicate.trait_ref, - bounds, - ) - .into_iter() - .map(|pred| goal.with(tcx, pred)), - ); - ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes) + then(ecx) }) } else { Err(NoSolution) From 55d86b9da864ac4540b164f51b42fe05bbb4912c Mon Sep 17 00:00:00 2001 From: "Thomas M. DuBuisson" Date: Tue, 9 May 2023 14:08:13 -0700 Subject: [PATCH 102/130] Fix incorrect implication of transmuting slices transmute<&[u8]> would be useful and as a beginner it is confusing to see documents casually confuse the types of &[u8] and [u8; SZ] --- library/core/src/intrinsics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 3f2b1595d62..279a1866892 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1187,7 +1187,7 @@ extern "rust-intrinsic" { /// Below are common applications of `transmute` which can be replaced with safer /// constructs. /// - /// Turning raw bytes (`&[u8]`) into `u32`, `f64`, etc.: + /// Turning raw bytes (`[u8; SZ]`) into `u32`, `f64`, etc.: /// /// ``` /// let raw_bytes = [0x78, 0x56, 0x34, 0x12]; From 26dc139b3702e7d3d2ea9dd0915da1d307f55011 Mon Sep 17 00:00:00 2001 From: Kyle Matsuda Date: Tue, 9 May 2023 16:02:52 -0600 Subject: [PATCH 103/130] add EarlyBinder to thir_abstract_const; remove tcx.bound_abstract_const --- compiler/rustc_infer/src/infer/mod.rs | 2 +- compiler/rustc_metadata/src/rmeta/mod.rs | 2 +- compiler/rustc_middle/src/query/erase.rs | 7 ++++--- compiler/rustc_middle/src/query/mod.rs | 2 +- compiler/rustc_middle/src/ty/abstract_const.rs | 9 +-------- compiler/rustc_ty_utils/src/consts.rs | 4 ++-- 6 files changed, 10 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs index a89b9931599..9a95a9c8375 100644 --- a/compiler/rustc_infer/src/infer/mod.rs +++ b/compiler/rustc_infer/src/infer/mod.rs @@ -1530,7 +1530,7 @@ impl<'tcx> InferCtxt<'tcx> { // variables let tcx = self.tcx; if substs.has_non_region_infer() { - if let Some(ct) = tcx.bound_abstract_const(unevaluated.def)? { + if let Some(ct) = tcx.thir_abstract_const(unevaluated.def)? { let ct = tcx.expand_abstract_consts(ct.subst(tcx, substs)); if let Err(e) = ct.error_reported() { return Err(ErrorHandled::Reported(e)); diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 84f6b7f934d..e2f6acb186b 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -394,7 +394,7 @@ define_tables! { mir_for_ctfe: Table>>, mir_generator_witnesses: Table>>, promoted_mir: Table>>>, - thir_abstract_const: Table>>, + thir_abstract_const: Table>>>, impl_parent: Table, impl_polarity: Table, constness: Table, diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index 7d9aea02289..28a9c1eef1a 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -82,9 +82,10 @@ impl EraseType for Result>, rustc_errors::ErrorGuarantee [u8; size_of::>, rustc_errors::ErrorGuaranteed>>()]; } -impl EraseType for Result>, rustc_errors::ErrorGuaranteed> { - type Result = - [u8; size_of::>, rustc_errors::ErrorGuaranteed>>()]; +impl EraseType for Result>>, rustc_errors::ErrorGuaranteed> { + type Result = [u8; size_of::< + Result>>, rustc_errors::ErrorGuaranteed>, + >()]; } impl EraseType for Result, traits::query::NoSolution> { diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index d5b185e45d6..dd89283d704 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -402,7 +402,7 @@ rustc_queries! { /// Try to build an abstract representation of the given constant. query thir_abstract_const( key: DefId - ) -> Result>, ErrorGuaranteed> { + ) -> Result>>, ErrorGuaranteed> { desc { |tcx| "building an abstract representation for `{}`", tcx.def_path_str(key), } diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs index bfb740ab356..972c417cbba 100644 --- a/compiler/rustc_middle/src/ty/abstract_const.rs +++ b/compiler/rustc_middle/src/ty/abstract_const.rs @@ -4,7 +4,6 @@ use crate::ty::{ TypeVisitableExt, }; use rustc_errors::ErrorGuaranteed; -use rustc_hir::def_id::DefId; #[derive(Hash, Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq)] #[derive(TyDecodable, TyEncodable, HashStable, TypeVisitable, TypeFoldable)] @@ -35,12 +34,6 @@ TrivialTypeTraversalAndLiftImpls! { pub type BoundAbstractConst<'tcx> = Result>>, ErrorGuaranteed>; impl<'tcx> TyCtxt<'tcx> { - /// Returns a const without substs applied - pub fn bound_abstract_const(self, uv: DefId) -> BoundAbstractConst<'tcx> { - let ac = self.thir_abstract_const(uv); - Ok(ac?.map(|ac| EarlyBinder(ac))) - } - pub fn expand_abstract_consts>>(self, ac: T) -> T { struct Expander<'tcx> { tcx: TyCtxt<'tcx>, @@ -59,7 +52,7 @@ impl<'tcx> TyCtxt<'tcx> { } fn fold_const(&mut self, c: Const<'tcx>) -> Const<'tcx> { let ct = match c.kind() { - ty::ConstKind::Unevaluated(uv) => match self.tcx.bound_abstract_const(uv.def) { + ty::ConstKind::Unevaluated(uv) => match self.tcx.thir_abstract_const(uv.def) { Err(e) => self.tcx.const_error_with_guaranteed(c.ty(), e), Ok(Some(bac)) => { let substs = self.tcx.erase_regions(uv.substs); diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs index b08a92570ed..3dd1d056be2 100644 --- a/compiler/rustc_ty_utils/src/consts.rs +++ b/compiler/rustc_ty_utils/src/consts.rs @@ -394,7 +394,7 @@ impl<'a, 'tcx> visit::Visitor<'a, 'tcx> for IsThirPolymorphic<'a, 'tcx> { pub fn thir_abstract_const( tcx: TyCtxt<'_>, def: LocalDefId, -) -> Result>, ErrorGuaranteed> { +) -> Result>>, ErrorGuaranteed> { if !tcx.features().generic_const_exprs { return Ok(None); } @@ -420,7 +420,7 @@ pub fn thir_abstract_const( let root_span = body.exprs[body_id].span; - Some(recurse_build(tcx, body, body_id, root_span)).transpose() + Ok(Some(ty::EarlyBinder(recurse_build(tcx, body, body_id, root_span)?))) } pub fn provide(providers: &mut ty::query::Providers) { From 2198faeee2a1e3abf30fa71a032aa76c2545c4e3 Mon Sep 17 00:00:00 2001 From: Obei Sideg Date: Mon, 8 May 2023 23:38:54 +0300 Subject: [PATCH 104/130] Make `NonUseContext::AscribeUserTy` carry `ty::Variance` --- compiler/rustc_borrowck/src/def_use.rs | 2 +- compiler/rustc_borrowck/src/type_check/mod.rs | 2 +- compiler/rustc_middle/src/mir/visit.rs | 8 ++++---- compiler/rustc_type_ir/src/lib.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_borrowck/src/def_use.rs b/compiler/rustc_borrowck/src/def_use.rs index 74e6ce37e97..b775739fed2 100644 --- a/compiler/rustc_borrowck/src/def_use.rs +++ b/compiler/rustc_borrowck/src/def_use.rs @@ -55,7 +55,7 @@ pub fn categorize(context: PlaceContext) -> Option { // `PlaceMention` and `AscribeUserType` both evaluate the place, which must not // contain dangling references. PlaceContext::NonMutatingUse(NonMutatingUseContext::PlaceMention) | - PlaceContext::NonUse(NonUseContext::AscribeUserTy) | + PlaceContext::NonUse(NonUseContext::AscribeUserTy(_)) | PlaceContext::MutatingUse(MutatingUseContext::AddressOf) | PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf) | diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index dcabeb792be..33b24b68f7c 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -777,7 +777,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { Inspect | Copy | Move | PlaceMention | SharedBorrow | ShallowBorrow | UniqueBorrow | AddressOf | Projection, ) => ty::Covariant, - PlaceContext::NonUse(AscribeUserTy) => ty::Covariant, + PlaceContext::NonUse(AscribeUserTy(variance)) => variance, } } diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 6718605ed0b..4b7014e3109 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -64,7 +64,7 @@ use crate::mir::*; use crate::ty::subst::SubstsRef; -use crate::ty::{CanonicalUserTypeAnnotation, Ty}; +use crate::ty::{self, CanonicalUserTypeAnnotation, Ty}; use rustc_span::Span; macro_rules! make_mir_visitor { @@ -782,12 +782,12 @@ macro_rules! make_mir_visitor { fn super_ascribe_user_ty(&mut self, place: & $($mutability)? Place<'tcx>, - _variance: $(& $mutability)? ty::Variance, + variance: $(& $mutability)? ty::Variance, user_ty: & $($mutability)? UserTypeProjection, location: Location) { self.visit_place( place, - PlaceContext::NonUse(NonUseContext::AscribeUserTy), + PlaceContext::NonUse(NonUseContext::AscribeUserTy($(* &$mutability *)? variance)), location ); self.visit_user_type_projection(user_ty); @@ -1320,7 +1320,7 @@ pub enum NonUseContext { /// Ending a storage live range. StorageDead, /// User type annotation assertions for NLL. - AscribeUserTy, + AscribeUserTy(ty::Variance), /// The data of a user variable, for debug info. VarDebugInfo, } diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs index 1e91e26e2af..da043ba691c 100644 --- a/compiler/rustc_type_ir/src/lib.rs +++ b/compiler/rustc_type_ir/src/lib.rs @@ -640,7 +640,7 @@ impl UnifyKey for FloatVid { } } -#[derive(Copy, Clone, PartialEq, Decodable, Encodable, Hash, HashStable_Generic)] +#[derive(Copy, Clone, PartialEq, Eq, Decodable, Encodable, Hash, HashStable_Generic)] #[rustc_pass_by_value] pub enum Variance { Covariant, // T <: T iff A <: B -- e.g., function return type From fbe479558c6ceac9065571842ab34dcd68375184 Mon Sep 17 00:00:00 2001 From: Augie Fackler Date: Tue, 9 May 2023 05:52:20 -0400 Subject: [PATCH 105/130] vec-shrink-panik: update expectations to work on LLVM 17 For some reason, the called function is `cleanup` on LLVM 17 instead of `filter`. r? @Amanieu --- tests/codegen/vec-shrink-panik.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/codegen/vec-shrink-panik.rs b/tests/codegen/vec-shrink-panik.rs index 4e5d8dc0632..88b7edff260 100644 --- a/tests/codegen/vec-shrink-panik.rs +++ b/tests/codegen/vec-shrink-panik.rs @@ -25,7 +25,7 @@ pub fn issue71861(vec: Vec) -> Box<[u32]> { // Call to panic_cannot_unwind in case of double-panic is expected // on LLVM 16 and older, but other panics are not. - // CHECK: filter + // old: filter // old-NEXT: ; call core::panicking::panic_cannot_unwind // old-NEXT: panic_cannot_unwind @@ -40,7 +40,7 @@ pub fn issue75636<'a>(iter: &[&'a str]) -> Box<[&'a str]> { // Call to panic_cannot_unwind in case of double-panic is expected, // on LLVM 16 and older, but other panics are not. - // CHECK: filter + // old: filter // old-NEXT: ; call core::panicking::panic_cannot_unwind // old-NEXT: panic_cannot_unwind From 8f50a17c37a214632c2f5cf5b8f2833a7286883b Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Sat, 22 Apr 2023 19:27:22 -0700 Subject: [PATCH 106/130] Fixups for sync - Fix LANES over-replace - Bring in traits - Use less inference-heavy types --- crates/core_simd/src/vector.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/core_simd/src/vector.rs b/crates/core_simd/src/vector.rs index 0253f122c98..3323b92e37b 100644 --- a/crates/core_simd/src/vector.rs +++ b/crates/core_simd/src/vector.rs @@ -2,6 +2,7 @@ use crate::simd::{ intrinsics, LaneCount, Mask, MaskElement, SimdCast, SimdCastPtr, SimdConstPtr, SimdMutPtr, SimdPartialOrd, SupportedLaneCount, Swizzle, }; +use core::convert::{TryFrom, TryInto}; /// A SIMD vector with the shape of `[T; N]` but the operations of `T`. /// @@ -109,7 +110,7 @@ where T: SimdElement, { /// Number of elements in this vector. - pub const N: usize = N; + pub const LANES: usize = N; /// Returns the number of elements in this SIMD vector. /// @@ -122,7 +123,7 @@ where /// assert_eq!(v.lanes(), 4); /// ``` pub const fn lanes(&self) -> usize { - Self::N + Self::LANES } /// Constructs a new SIMD vector with all elements set to the given value. @@ -260,7 +261,7 @@ where #[must_use] pub const fn from_slice(slice: &[T]) -> Self { assert!( - slice.len() >= Self::N, + slice.len() >= Self::LANES, "slice length must be at least the number of elements" ); // SAFETY: We just checked that the slice contains @@ -288,7 +289,7 @@ where /// ``` pub fn copy_to_slice(self, slice: &mut [T]) { assert!( - slice.len() >= Self::N, + slice.len() >= Self::LANES, "slice length must be at least the number of elements" ); // SAFETY: We just checked that the slice contains @@ -883,7 +884,7 @@ where { type Error = core::array::TryFromSliceError; - fn try_from(slice: &[T]) -> Result { + fn try_from(slice: &[T]) -> Result { Ok(Self::from_array(slice.try_into()?)) } } @@ -895,7 +896,7 @@ where { type Error = core::array::TryFromSliceError; - fn try_from(slice: &mut [T]) -> Result { + fn try_from(slice: &mut [T]) -> Result { Ok(Self::from_array(slice.try_into()?)) } } From d361e4335f7e37d2409820510e059744d1c96457 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 10 May 2023 05:36:16 -0700 Subject: [PATCH 107/130] Drop const_ptr_read feature gate --- crates/core_simd/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 31e7a3617bc..e5307de2155 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -1,6 +1,5 @@ #![no_std] #![feature( - const_ptr_read, const_refs_to_cell, const_maybe_uninit_as_mut_ptr, const_mut_refs, From 852762563aa890286eda2f668b8af30f8aa84216 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 10 May 2023 05:45:24 -0700 Subject: [PATCH 108/130] Temp fix for swizzle_dyn - disable the AVX512 variant for now (flaky) - tell Clippy to knock it off --- crates/core_simd/src/swizzle_dyn.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/core_simd/src/swizzle_dyn.rs b/crates/core_simd/src/swizzle_dyn.rs index 5c3a2c1824f..3eb80d5dca1 100644 --- a/crates/core_simd/src/swizzle_dyn.rs +++ b/crates/core_simd/src/swizzle_dyn.rs @@ -43,8 +43,9 @@ where 32 => transize(x86::_mm256_permutexvar_epi8, self, idxs), // Notable absence: avx512bw shuffle // If avx512bw is available, odds of avx512vbmi are good - #[cfg(target_feature = "avx512vbmi")] - 64 => transize(x86::_mm512_permutexvar_epi8, self, idxs), + // FIXME: initial AVX512VBMI variant didn't actually pass muster + // #[cfg(target_feature = "avx512vbmi")] + // 64 => transize(x86::_mm512_permutexvar_epi8, self, idxs), _ => { let mut array = [0; N]; for (i, k) in idxs.to_array().into_iter().enumerate() { @@ -67,6 +68,7 @@ where #[target_feature(enable = "avx2")] #[allow(unused)] #[inline] +#[allow(clippy::let_and_return)] unsafe fn avx2_pshufb(bytes: Simd, idxs: Simd) -> Simd { use crate::simd::SimdPartialOrd; #[cfg(target_arch = "x86")] From 6ad0497cc07eb7ed515605348ae7c56c25c89566 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 10 May 2023 15:25:02 +0000 Subject: [PATCH 109/130] Use visit_assign to detect SSA locals. --- compiler/rustc_mir_transform/src/ssa.rs | 20 ++++++++++++------- .../copy-prop/partial_init.main.CopyProp.diff | 13 ++++++++++++ tests/mir-opt/copy-prop/partial_init.rs | 18 +++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 tests/mir-opt/copy-prop/partial_init.main.CopyProp.diff create mode 100644 tests/mir-opt/copy-prop/partial_init.rs diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 05a7b226f0c..a7b45366662 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -209,13 +209,6 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor { match ctxt { PlaceContext::MutatingUse(MutatingUseContext::Projection) | PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) => bug!(), - PlaceContext::MutatingUse(MutatingUseContext::Store) => { - self.assignments[local].insert(LocationExtended::Plain(loc)); - if let Set1::One(_) = self.assignments[local] { - // Only record if SSA-like, to avoid growing the vector needlessly. - self.assignment_order.push(local); - } - } // Anything can happen with raw pointers, so remove them. // We do not verify that all uses of the borrow dominate the assignment to `local`, // so we have to remove them too. @@ -252,6 +245,19 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor { self.visit_local(place.local, ctxt, loc); } } + + fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, loc: Location) { + if let Some(local) = place.as_local() { + self.assignments[local].insert(LocationExtended::Plain(loc)); + if let Set1::One(_) = self.assignments[local] { + // Only record if SSA-like, to avoid growing the vector needlessly. + self.assignment_order.push(local); + } + } else { + self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), loc); + } + self.visit_rvalue(rvalue, loc); + } } #[instrument(level = "trace", skip(ssa, body))] diff --git a/tests/mir-opt/copy-prop/partial_init.main.CopyProp.diff b/tests/mir-opt/copy-prop/partial_init.main.CopyProp.diff new file mode 100644 index 00000000000..5866439055e --- /dev/null +++ b/tests/mir-opt/copy-prop/partial_init.main.CopyProp.diff @@ -0,0 +1,13 @@ +- // MIR for `main` before CopyProp ++ // MIR for `main` after CopyProp + + fn main() -> () { + let mut _0: (); // return place in scope 0 at $DIR/partial_init.rs:+0:15: +0:15 + let mut _1: (isize,); // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL + + bb0: { + (_1.0: isize) = const 1_isize; // scope 0 at $DIR/partial_init.rs:+4:13: +4:20 + return; // scope 0 at $DIR/partial_init.rs:+5:13: +5:21 + } + } + diff --git a/tests/mir-opt/copy-prop/partial_init.rs b/tests/mir-opt/copy-prop/partial_init.rs new file mode 100644 index 00000000000..f5ab9974f71 --- /dev/null +++ b/tests/mir-opt/copy-prop/partial_init.rs @@ -0,0 +1,18 @@ +// unit-test: CopyProp +// Verify that we do not ICE on partial initializations. + +#![feature(custom_mir, core_intrinsics)] +extern crate core; +use core::intrinsics::mir::*; + +// EMIT_MIR partial_init.main.CopyProp.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn main() { + mir! ( + let x: (isize, ); + { + x.0 = 1; + Return() + } + ) +} From b64e9113e257976875e94360462bd2744af78f7b Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 10 May 2023 16:06:41 +0000 Subject: [PATCH 110/130] Add test. --- ...raw_then_mut_shr.ReferencePropagation.diff | 75 +++++++++++++++++++ tests/mir-opt/reference_prop.rs | 27 +++++++ ...ique_with_copies.ReferencePropagation.diff | 66 ++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff create mode 100644 tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff diff --git a/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff b/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff new file mode 100644 index 00000000000..c137705a39a --- /dev/null +++ b/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff @@ -0,0 +1,75 @@ +- // MIR for `mut_raw_then_mut_shr` before ReferencePropagation ++ // MIR for `mut_raw_then_mut_shr` after ReferencePropagation + + fn mut_raw_then_mut_shr() -> (i32, i32) { + let mut _0: (i32, i32); // return place in scope 0 at $DIR/reference_prop.rs:+0:30: +0:40 + let mut _1: i32; // in scope 0 at $DIR/reference_prop.rs:+1:9: +1:14 + let mut _4: *mut i32; // in scope 0 at $DIR/reference_prop.rs:+3:16: +3:36 + let mut _5: &mut i32; // in scope 0 at $DIR/reference_prop.rs:+3:16: +3:26 + let _8: (); // in scope 0 at $DIR/reference_prop.rs:+7:5: +7:26 + let mut _9: i32; // in scope 0 at $DIR/reference_prop.rs:+8:6: +8:7 + let mut _10: i32; // in scope 0 at $DIR/reference_prop.rs:+8:9: +8:10 + scope 1 { + debug x => _1; // in scope 1 at $DIR/reference_prop.rs:+1:9: +1:14 + let _2: &mut i32; // in scope 1 at $DIR/reference_prop.rs:+2:9: +2:13 + scope 2 { + debug xref => _2; // in scope 2 at $DIR/reference_prop.rs:+2:9: +2:13 + let _3: *mut i32; // in scope 2 at $DIR/reference_prop.rs:+3:9: +3:13 + scope 3 { + debug xraw => _3; // in scope 3 at $DIR/reference_prop.rs:+3:9: +3:13 + let _6: &i32; // in scope 3 at $DIR/reference_prop.rs:+4:9: +4:13 + scope 4 { + debug xshr => _6; // in scope 4 at $DIR/reference_prop.rs:+4:9: +4:13 + let _7: i32; // in scope 4 at $DIR/reference_prop.rs:+6:9: +6:10 + scope 5 { + debug a => _7; // in scope 5 at $DIR/reference_prop.rs:+6:9: +6:10 + scope 6 { + } + } + } + } + } + } + + bb0: { + StorageLive(_1); // scope 0 at $DIR/reference_prop.rs:+1:9: +1:14 + _1 = const 2_i32; // scope 0 at $DIR/reference_prop.rs:+1:17: +1:18 +- StorageLive(_2); // scope 1 at $DIR/reference_prop.rs:+2:9: +2:13 + _2 = &mut _1; // scope 1 at $DIR/reference_prop.rs:+2:16: +2:22 + StorageLive(_3); // scope 2 at $DIR/reference_prop.rs:+3:9: +3:13 + StorageLive(_4); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:36 +- StorageLive(_5); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 +- _5 = &mut (*_2); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 +- _4 = &raw mut (*_5); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 ++ _5 = &mut _1; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 ++ _4 = &raw mut (*_2); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 + _3 = _4; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:36 +- StorageDead(_5); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 + StorageDead(_4); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 + StorageLive(_6); // scope 3 at $DIR/reference_prop.rs:+4:9: +4:13 +- _6 = &(*_2); // scope 3 at $DIR/reference_prop.rs:+4:16: +4:22 ++ _6 = &_1; // scope 3 at $DIR/reference_prop.rs:+4:16: +4:22 + StorageLive(_7); // scope 4 at $DIR/reference_prop.rs:+6:9: +6:10 +- _7 = (*_6); // scope 4 at $DIR/reference_prop.rs:+6:13: +6:18 +- StorageLive(_8); // scope 5 at $DIR/reference_prop.rs:+7:5: +7:26 +- (*_3) = const 4_i32; // scope 6 at $DIR/reference_prop.rs:+7:14: +7:23 +- _8 = const (); // scope 6 at $DIR/reference_prop.rs:+7:5: +7:26 +- StorageDead(_8); // scope 5 at $DIR/reference_prop.rs:+7:25: +7:26 ++ _7 = (*_2); // scope 4 at $DIR/reference_prop.rs:+6:13: +6:18 ++ (*_5) = const 4_i32; // scope 6 at $DIR/reference_prop.rs:+7:14: +7:23 + StorageLive(_9); // scope 5 at $DIR/reference_prop.rs:+8:6: +8:7 + _9 = _7; // scope 5 at $DIR/reference_prop.rs:+8:6: +8:7 + StorageLive(_10); // scope 5 at $DIR/reference_prop.rs:+8:9: +8:10 + _10 = _1; // scope 5 at $DIR/reference_prop.rs:+8:9: +8:10 + _0 = (move _9, move _10); // scope 5 at $DIR/reference_prop.rs:+8:5: +8:11 + StorageDead(_10); // scope 5 at $DIR/reference_prop.rs:+8:10: +8:11 + StorageDead(_9); // scope 5 at $DIR/reference_prop.rs:+8:10: +8:11 + StorageDead(_7); // scope 4 at $DIR/reference_prop.rs:+9:1: +9:2 + StorageDead(_6); // scope 3 at $DIR/reference_prop.rs:+9:1: +9:2 + StorageDead(_3); // scope 2 at $DIR/reference_prop.rs:+9:1: +9:2 +- StorageDead(_2); // scope 1 at $DIR/reference_prop.rs:+9:1: +9:2 + StorageDead(_1); // scope 0 at $DIR/reference_prop.rs:+9:1: +9:2 + return; // scope 0 at $DIR/reference_prop.rs:+9:2: +9:2 + } + } + diff --git a/tests/mir-opt/reference_prop.rs b/tests/mir-opt/reference_prop.rs index e3e5d791464..93f8d1df8e8 100644 --- a/tests/mir-opt/reference_prop.rs +++ b/tests/mir-opt/reference_prop.rs @@ -433,6 +433,29 @@ fn maybe_dead(m: bool) { ) } +fn mut_raw_then_mut_shr() -> (i32, i32) { + let mut x = 2; + let xref = &mut x; + let xraw = &mut *xref as *mut _; + let xshr = &*xref; + // Verify that we completely replace with `x` in both cases. + let a = *xshr; + unsafe { *xraw = 4; } + (a, x) +} + +fn unique_with_copies() { + let y = { + let mut a = 0; + let x = &raw mut a; + // `*y` is not replacable below, so we must not replace `*x`. + unsafe { opaque(*x) }; + x + }; + // But rewriting as `*x` is ok. + unsafe { opaque(*y) }; +} + fn main() { let mut x = 5_usize; let mut y = 7_usize; @@ -444,6 +467,8 @@ fn main() { multiple_storage(); dominate_storage(); maybe_dead(true); + mut_raw_then_mut_shr(); + unique_with_copies(); } // EMIT_MIR reference_prop.reference_propagation.ReferencePropagation.diff @@ -454,3 +479,5 @@ fn main() { // EMIT_MIR reference_prop.multiple_storage.ReferencePropagation.diff // EMIT_MIR reference_prop.dominate_storage.ReferencePropagation.diff // EMIT_MIR reference_prop.maybe_dead.ReferencePropagation.diff +// EMIT_MIR reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff +// EMIT_MIR reference_prop.unique_with_copies.ReferencePropagation.diff diff --git a/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff b/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff new file mode 100644 index 00000000000..89aa6c7b792 --- /dev/null +++ b/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff @@ -0,0 +1,66 @@ +- // MIR for `unique_with_copies` before ReferencePropagation ++ // MIR for `unique_with_copies` after ReferencePropagation + + fn unique_with_copies() -> () { + let mut _0: (); // return place in scope 0 at $DIR/reference_prop.rs:+0:25: +0:25 + let _1: *mut i32; // in scope 0 at $DIR/reference_prop.rs:+1:9: +1:10 + let mut _2: i32; // in scope 0 at $DIR/reference_prop.rs:+2:13: +2:18 + let _4: (); // in scope 0 at $DIR/reference_prop.rs:+5:18: +5:28 + let mut _5: i32; // in scope 0 at $DIR/reference_prop.rs:+5:25: +5:27 + let _6: (); // in scope 0 at $DIR/reference_prop.rs:+9:14: +9:24 + let mut _7: i32; // in scope 0 at $DIR/reference_prop.rs:+9:21: +9:23 + scope 1 { + debug y => _1; // in scope 1 at $DIR/reference_prop.rs:+1:9: +1:10 + scope 5 { + } + } + scope 2 { + debug a => _2; // in scope 2 at $DIR/reference_prop.rs:+2:13: +2:18 + let _3: *mut i32; // in scope 2 at $DIR/reference_prop.rs:+3:13: +3:14 + scope 3 { + debug x => _3; // in scope 3 at $DIR/reference_prop.rs:+3:13: +3:14 + scope 4 { + } + } + } + + bb0: { + StorageLive(_1); // scope 0 at $DIR/reference_prop.rs:+1:9: +1:10 + StorageLive(_2); // scope 0 at $DIR/reference_prop.rs:+2:13: +2:18 + _2 = const 0_i32; // scope 0 at $DIR/reference_prop.rs:+2:21: +2:22 + StorageLive(_3); // scope 2 at $DIR/reference_prop.rs:+3:13: +3:14 + _3 = &raw mut _2; // scope 2 at $DIR/reference_prop.rs:+3:17: +3:27 + StorageLive(_4); // scope 3 at $DIR/reference_prop.rs:+5:9: +5:30 + StorageLive(_5); // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 +- _5 = (*_3); // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 ++ _5 = _2; // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 + _4 = opaque::(move _5) -> bb1; // scope 4 at $DIR/reference_prop.rs:+5:18: +5:28 + // mir::Constant + // + span: $DIR/reference_prop.rs:452:18: 452:24 + // + literal: Const { ty: fn(i32) {opaque::}, val: Value() } + } + + bb1: { + StorageDead(_5); // scope 4 at $DIR/reference_prop.rs:+5:27: +5:28 + StorageDead(_4); // scope 3 at $DIR/reference_prop.rs:+5:30: +5:31 + _1 = _3; // scope 3 at $DIR/reference_prop.rs:+6:9: +6:10 + StorageDead(_3); // scope 2 at $DIR/reference_prop.rs:+7:5: +7:6 + StorageDead(_2); // scope 0 at $DIR/reference_prop.rs:+7:5: +7:6 + StorageLive(_6); // scope 1 at $DIR/reference_prop.rs:+9:5: +9:26 + StorageLive(_7); // scope 5 at $DIR/reference_prop.rs:+9:21: +9:23 + _7 = (*_1); // scope 5 at $DIR/reference_prop.rs:+9:21: +9:23 + _6 = opaque::(move _7) -> bb2; // scope 5 at $DIR/reference_prop.rs:+9:14: +9:24 + // mir::Constant + // + span: $DIR/reference_prop.rs:456:14: 456:20 + // + literal: Const { ty: fn(i32) {opaque::}, val: Value() } + } + + bb2: { + StorageDead(_7); // scope 5 at $DIR/reference_prop.rs:+9:23: +9:24 + StorageDead(_6); // scope 1 at $DIR/reference_prop.rs:+9:26: +9:27 + _0 = const (); // scope 0 at $DIR/reference_prop.rs:+0:25: +10:2 + StorageDead(_1); // scope 0 at $DIR/reference_prop.rs:+10:1: +10:2 + return; // scope 0 at $DIR/reference_prop.rs:+10:2: +10:2 + } + } + From d0d4e0237ff708c4d41070ef946679819125a0d5 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 10 May 2023 17:17:33 +0000 Subject: [PATCH 111/130] Iteratively replace pointers. --- compiler/rustc_mir_transform/src/ref_prop.rs | 77 +++++++++++++------ ...raw_then_mut_shr.ReferencePropagation.diff | 7 +- ...read_through_raw.ReferencePropagation.diff | 7 +- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index dafd2ae23a6..ac55ce3b316 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -85,7 +85,9 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let ssa = SsaLocals::new(body); let mut replacer = compute_replacement(tcx, body, &ssa); - debug!(?replacer.targets, ?replacer.allowed_replacements, ?replacer.storage_to_remove); + debug!(?replacer.targets); + debug!(?replacer.allowed_replacements); + debug!(?replacer.storage_to_remove); replacer.visit_body_preserves_cfg(body); @@ -190,8 +192,11 @@ fn compute_replacement<'tcx>( continue; } + // Whether the current local is subject to the uniqueness rule. + let needs_unique = ty.is_mutable_ptr(); + // If this a mutable reference that we cannot fully replace, mark it as unknown. - if ty.is_mutable_ptr() && !fully_replacable_locals.contains(local) { + if needs_unique && !fully_replacable_locals.contains(local) { debug!("not fully replaceable"); continue; } @@ -217,10 +222,10 @@ fn compute_replacement<'tcx>( let mut place = *place; // Try to see through `place` in order to collapse reborrow chains. if place.projection.first() == Some(&PlaceElem::Deref) - && let Value::Pointer(target, refmut) = targets[place.local] + && let Value::Pointer(target, needs_unique) = targets[place.local] // Only see through immutable reference and pointers, as we do not know yet if // mutable references are fully replaced. - && !refmut + && !needs_unique // Only collapse chain if the pointee is definitely live. && can_perform_opt(target, location) { @@ -228,7 +233,7 @@ fn compute_replacement<'tcx>( } assert_ne!(place.local, local); if is_constant_place(place) { - targets[local] = Value::Pointer(place, ty.is_mutable_ptr()); + targets[local] = Value::Pointer(place, needs_unique); } } // We do not know what to do, so keep as not-a-pointer. @@ -276,16 +281,35 @@ fn compute_replacement<'tcx>( return; } - if let Value::Pointer(target, refmut) = self.targets[place.local] - && place.projection.first() == Some(&PlaceElem::Deref) - { - let perform_opt = (self.can_perform_opt)(target, loc); - if perform_opt { - self.allowed_replacements.insert((target.local, loc)); - } else if refmut { - // This mutable reference is not fully replacable, so drop it. - self.targets[place.local] = Value::Unknown; + if place.projection.first() != Some(&PlaceElem::Deref) { + // This is not a dereference, nothing to do. + return; + } + + let mut place = place.as_ref(); + loop { + if let Value::Pointer(target, needs_unique) = self.targets[place.local] { + let perform_opt = (self.can_perform_opt)(target, loc); + debug!(?place, ?target, ?needs_unique, ?perform_opt); + + // This a reborrow chain, recursively allow the replacement. + // + // This also allows to detect cases where `target.local` is not replacable, + // and mark it as such. + if let &[PlaceElem::Deref] = &target.projection[..] { + assert!(perform_opt); + self.allowed_replacements.insert((target.local, loc)); + place.local = target.local; + continue; + } else if perform_opt { + self.allowed_replacements.insert((target.local, loc)); + } else if needs_unique { + // This mutable reference is not fully replacable, so drop it. + self.targets[place.local] = Value::Unknown; + } } + + break; } } } @@ -326,18 +350,23 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { } fn visit_place(&mut self, place: &mut Place<'tcx>, ctxt: PlaceContext, loc: Location) { - if let Value::Pointer(target, _) = self.targets[place.local] - && place.projection.first() == Some(&PlaceElem::Deref) - { - let perform_opt = matches!(ctxt, PlaceContext::NonUse(_)) - || self.allowed_replacements.contains(&(target.local, loc)); + if place.projection.first() != Some(&PlaceElem::Deref) { + return; + } - if perform_opt { - *place = target.project_deeper(&place.projection[1..], self.tcx); - self.any_replacement = true; + loop { + if let Value::Pointer(target, _) = self.targets[place.local] { + let perform_opt = matches!(ctxt, PlaceContext::NonUse(_)) + || self.allowed_replacements.contains(&(target.local, loc)); + + if perform_opt { + *place = target.project_deeper(&place.projection[1..], self.tcx); + self.any_replacement = true; + continue; + } } - } else { - self.super_place(place, ctxt, loc); + + break; } } diff --git a/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff b/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff index c137705a39a..ed9f8c2b187 100644 --- a/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff +++ b/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff @@ -41,8 +41,7 @@ - StorageLive(_5); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 - _5 = &mut (*_2); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 - _4 = &raw mut (*_5); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 -+ _5 = &mut _1; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 -+ _4 = &raw mut (*_2); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 ++ _4 = &raw mut _1; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 _3 = _4; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:36 - StorageDead(_5); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 StorageDead(_4); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 @@ -55,8 +54,8 @@ - (*_3) = const 4_i32; // scope 6 at $DIR/reference_prop.rs:+7:14: +7:23 - _8 = const (); // scope 6 at $DIR/reference_prop.rs:+7:5: +7:26 - StorageDead(_8); // scope 5 at $DIR/reference_prop.rs:+7:25: +7:26 -+ _7 = (*_2); // scope 4 at $DIR/reference_prop.rs:+6:13: +6:18 -+ (*_5) = const 4_i32; // scope 6 at $DIR/reference_prop.rs:+7:14: +7:23 ++ _7 = _1; // scope 4 at $DIR/reference_prop.rs:+6:13: +6:18 ++ _1 = const 4_i32; // scope 6 at $DIR/reference_prop.rs:+7:14: +7:23 StorageLive(_9); // scope 5 at $DIR/reference_prop.rs:+8:6: +8:7 _9 = _7; // scope 5 at $DIR/reference_prop.rs:+8:6: +8:7 StorageLive(_10); // scope 5 at $DIR/reference_prop.rs:+8:9: +8:10 diff --git a/tests/mir-opt/reference_prop.read_through_raw.ReferencePropagation.diff b/tests/mir-opt/reference_prop.read_through_raw.ReferencePropagation.diff index a7d505c6906..75c1f8f57cc 100644 --- a/tests/mir-opt/reference_prop.read_through_raw.ReferencePropagation.diff +++ b/tests/mir-opt/reference_prop.read_through_raw.ReferencePropagation.diff @@ -9,15 +9,14 @@ let mut _5: *mut usize; // in scope 0 at $SRC_DIR/core/src/intrinsics/mir.rs:LL:COL bb0: { - _2 = &mut (*_1); // scope 0 at $DIR/reference_prop.rs:+10:13: +10:25 +- _2 = &mut (*_1); // scope 0 at $DIR/reference_prop.rs:+10:13: +10:25 - _3 = &mut (*_2); // scope 0 at $DIR/reference_prop.rs:+11:13: +11:26 - _4 = &raw mut (*_2); // scope 0 at $DIR/reference_prop.rs:+12:13: +12:30 - _5 = &raw mut (*_3); // scope 0 at $DIR/reference_prop.rs:+13:13: +13:30 - _0 = (*_4); // scope 0 at $DIR/reference_prop.rs:+15:13: +15:22 - _0 = (*_5); // scope 0 at $DIR/reference_prop.rs:+16:13: +16:22 -+ _3 = &mut (*_1); // scope 0 at $DIR/reference_prop.rs:+11:13: +11:26 -+ _0 = (*_2); // scope 0 at $DIR/reference_prop.rs:+15:13: +15:22 -+ _0 = (*_3); // scope 0 at $DIR/reference_prop.rs:+16:13: +16:22 ++ _0 = (*_1); // scope 0 at $DIR/reference_prop.rs:+15:13: +15:22 ++ _0 = (*_1); // scope 0 at $DIR/reference_prop.rs:+16:13: +16:22 return; // scope 0 at $DIR/reference_prop.rs:+17:13: +17:21 } } From aeac5555780cb11cf701a5fa69d66adcaa3e2f4a Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 10 May 2023 17:29:02 +0000 Subject: [PATCH 112/130] Do not see through copies of mutable pointers. --- compiler/rustc_mir_transform/src/ref_prop.rs | 11 ++++++----- ...rop.mut_raw_then_mut_shr.ReferencePropagation.diff | 4 ++-- ..._prop.unique_with_copies.ReferencePropagation.diff | 10 +++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index ac55ce3b316..38be7b3c7ea 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -208,13 +208,14 @@ fn compute_replacement<'tcx>( // have been visited before. Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) | Rvalue::CopyForDeref(place) => { - if let Some(rhs) = place.as_local() { + if let Some(rhs) = place.as_local() && ssa.is_ssa(rhs) { let target = targets[rhs]; - if matches!(target, Value::Pointer(..)) { + // Only see through immutable reference and pointers, as we do not know yet if + // mutable references are fully replaced. + if !needs_unique && matches!(target, Value::Pointer(..)) { targets[local] = target; - } else if ssa.is_ssa(rhs) { - let refmut = body.local_decls[rhs].ty.is_mutable_ptr(); - targets[local] = Value::Pointer(tcx.mk_place_deref(rhs.into()), refmut); + } else { + targets[local] = Value::Pointer(tcx.mk_place_deref(rhs.into()), needs_unique); } } } diff --git a/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff b/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff index ed9f8c2b187..af8ee2411d3 100644 --- a/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff +++ b/tests/mir-opt/reference_prop.mut_raw_then_mut_shr.ReferencePropagation.diff @@ -37,14 +37,14 @@ - StorageLive(_2); // scope 1 at $DIR/reference_prop.rs:+2:9: +2:13 _2 = &mut _1; // scope 1 at $DIR/reference_prop.rs:+2:16: +2:22 StorageLive(_3); // scope 2 at $DIR/reference_prop.rs:+3:9: +3:13 - StorageLive(_4); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:36 +- StorageLive(_4); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:36 - StorageLive(_5); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 - _5 = &mut (*_2); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 - _4 = &raw mut (*_5); // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 + _4 = &raw mut _1; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:26 _3 = _4; // scope 2 at $DIR/reference_prop.rs:+3:16: +3:36 - StorageDead(_5); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 - StorageDead(_4); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 +- StorageDead(_4); // scope 2 at $DIR/reference_prop.rs:+3:36: +3:37 StorageLive(_6); // scope 3 at $DIR/reference_prop.rs:+4:9: +4:13 - _6 = &(*_2); // scope 3 at $DIR/reference_prop.rs:+4:16: +4:22 + _6 = &_1; // scope 3 at $DIR/reference_prop.rs:+4:16: +4:22 diff --git a/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff b/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff index 89aa6c7b792..2cda2409e80 100644 --- a/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff +++ b/tests/mir-opt/reference_prop.unique_with_copies.ReferencePropagation.diff @@ -28,12 +28,11 @@ StorageLive(_1); // scope 0 at $DIR/reference_prop.rs:+1:9: +1:10 StorageLive(_2); // scope 0 at $DIR/reference_prop.rs:+2:13: +2:18 _2 = const 0_i32; // scope 0 at $DIR/reference_prop.rs:+2:21: +2:22 - StorageLive(_3); // scope 2 at $DIR/reference_prop.rs:+3:13: +3:14 +- StorageLive(_3); // scope 2 at $DIR/reference_prop.rs:+3:13: +3:14 _3 = &raw mut _2; // scope 2 at $DIR/reference_prop.rs:+3:17: +3:27 StorageLive(_4); // scope 3 at $DIR/reference_prop.rs:+5:9: +5:30 StorageLive(_5); // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 -- _5 = (*_3); // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 -+ _5 = _2; // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 + _5 = (*_3); // scope 4 at $DIR/reference_prop.rs:+5:25: +5:27 _4 = opaque::(move _5) -> bb1; // scope 4 at $DIR/reference_prop.rs:+5:18: +5:28 // mir::Constant // + span: $DIR/reference_prop.rs:452:18: 452:24 @@ -44,11 +43,12 @@ StorageDead(_5); // scope 4 at $DIR/reference_prop.rs:+5:27: +5:28 StorageDead(_4); // scope 3 at $DIR/reference_prop.rs:+5:30: +5:31 _1 = _3; // scope 3 at $DIR/reference_prop.rs:+6:9: +6:10 - StorageDead(_3); // scope 2 at $DIR/reference_prop.rs:+7:5: +7:6 +- StorageDead(_3); // scope 2 at $DIR/reference_prop.rs:+7:5: +7:6 StorageDead(_2); // scope 0 at $DIR/reference_prop.rs:+7:5: +7:6 StorageLive(_6); // scope 1 at $DIR/reference_prop.rs:+9:5: +9:26 StorageLive(_7); // scope 5 at $DIR/reference_prop.rs:+9:21: +9:23 - _7 = (*_1); // scope 5 at $DIR/reference_prop.rs:+9:21: +9:23 +- _7 = (*_1); // scope 5 at $DIR/reference_prop.rs:+9:21: +9:23 ++ _7 = (*_3); // scope 5 at $DIR/reference_prop.rs:+9:21: +9:23 _6 = opaque::(move _7) -> bb2; // scope 5 at $DIR/reference_prop.rs:+9:14: +9:24 // mir::Constant // + span: $DIR/reference_prop.rs:456:14: 456:20 From 9fb1c73a7309b83d495a4033c15c9e223da11f01 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 10 May 2023 17:33:58 +0000 Subject: [PATCH 113/130] Avoid shadowing. --- compiler/rustc_mir_transform/src/ref_prop.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index 38be7b3c7ea..d1bc9ee9153 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -223,10 +223,10 @@ fn compute_replacement<'tcx>( let mut place = *place; // Try to see through `place` in order to collapse reborrow chains. if place.projection.first() == Some(&PlaceElem::Deref) - && let Value::Pointer(target, needs_unique) = targets[place.local] + && let Value::Pointer(target, inner_needs_unique) = targets[place.local] // Only see through immutable reference and pointers, as we do not know yet if // mutable references are fully replaced. - && !needs_unique + && !inner_needs_unique // Only collapse chain if the pointee is definitely live. && can_perform_opt(target, location) { From a2fe9935ea6b2cef2cc9b3aca6d1fee3ae15524b Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 10 May 2023 19:46:40 +0000 Subject: [PATCH 114/130] Only warn single-use lifetime when the binders match. --- compiler/rustc_resolve/src/late.rs | 16 +++++++++++----- .../ui/associated-inherent-types/issue-109790.rs | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 2a8287d5554..d7509cbf10e 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -1482,7 +1482,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> { if let Some(&(_, res)) = rib.bindings.get(&normalized_ident) { self.record_lifetime_res(lifetime.id, res, LifetimeElisionCandidate::Named); - if let LifetimeRes::Param { param, .. } = res { + if let LifetimeRes::Param { param, binder } = res { match self.lifetime_uses.entry(param) { Entry::Vacant(v) => { debug!("First use of {:?} at {:?}", res, ident.span); @@ -1496,10 +1496,16 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> { LifetimeRibKind::Item | LifetimeRibKind::AnonymousReportError | LifetimeRibKind::ElisionFailure => Some(LifetimeUseSet::Many), - // An anonymous lifetime is legal here, go ahead. - LifetimeRibKind::AnonymousCreateParameter { .. } => { - Some(LifetimeUseSet::One { use_span: ident.span, use_ctxt }) - } + // An anonymous lifetime is legal here, and bound to the right + // place, go ahead. + LifetimeRibKind::AnonymousCreateParameter { + binder: anon_binder, + .. + } => Some(if binder == anon_binder { + LifetimeUseSet::One { use_span: ident.span, use_ctxt } + } else { + LifetimeUseSet::Many + }), // Only report if eliding the lifetime would have the same // semantics. LifetimeRibKind::Elided(r) => Some(if res == r { diff --git a/tests/ui/associated-inherent-types/issue-109790.rs b/tests/ui/associated-inherent-types/issue-109790.rs index b2be19a28f4..88327f86423 100644 --- a/tests/ui/associated-inherent-types/issue-109790.rs +++ b/tests/ui/associated-inherent-types/issue-109790.rs @@ -2,6 +2,7 @@ #![feature(inherent_associated_types)] #![allow(incomplete_features)] +#![deny(single_use_lifetimes)] struct Foo(T); From 15aa7fad7ec83db399c8a85c6b6777d0efc7bc53 Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 6 May 2023 02:00:45 -0700 Subject: [PATCH 115/130] Simplify the implementation of iterators over slices of ZSTs Currently, slice iterators over ZSTs store `end = start.wrapping_byte_add(len)`. That's slightly convenient for `is_empty`, but kinda annoying for pretty much everything else -- see bugs like 42789, for example. This PR instead changes it to just `end = ptr::invalid(len)` instead. That's easier to think about (IMHO, at least) as well as easier to represent. --- library/core/src/ptr/non_null.rs | 13 ++++++ library/core/src/slice/iter.rs | 16 +++----- library/core/src/slice/iter/macros.rs | 57 ++++++++++++++++----------- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index 61fcdf58b4f..b492d2f07bc 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -449,6 +449,19 @@ impl NonNull { // SAFETY: `self` is a `NonNull` pointer which is necessarily non-null unsafe { NonNull::new_unchecked(self.as_ptr() as *mut U) } } + + /// See [`pointer::add`] for semantics and safety requirements. + #[inline] + pub(crate) const unsafe fn add(self, delta: usize) -> Self + where + T: Sized, + { + // SAFETY: We require that the delta stays in-bounds of the object, and + // thus it cannot become null, as that would require wrapping the + // address space, which no legal objects are allowed to do. + // And the caller promised the `delta` is sound to add. + unsafe { NonNull { pointer: self.pointer.add(delta) } } + } } impl NonNull<[T]> { diff --git a/library/core/src/slice/iter.rs b/library/core/src/slice/iter.rs index 8629aab0070..67fcef0f466 100644 --- a/library/core/src/slice/iter.rs +++ b/library/core/src/slice/iter.rs @@ -13,7 +13,7 @@ use crate::iter::{ use crate::marker::{PhantomData, Send, Sized, Sync}; use crate::mem::{self, SizedTypeProperties}; use crate::num::NonZeroUsize; -use crate::ptr::NonNull; +use crate::ptr::{invalid, invalid_mut, NonNull}; use super::{from_raw_parts, from_raw_parts_mut}; @@ -67,9 +67,7 @@ pub struct Iter<'a, T: 'a> { ptr: NonNull, /// For non-ZSTs, the non-null pointer to the past-the-end element. /// - /// For ZSTs, this is `ptr.wrapping_byte_add(len)`. - /// - /// For all types, `ptr == end` tests whether the iterator is empty. + /// For ZSTs, this is `ptr::invalid(len)`. end: *const T, _marker: PhantomData<&'a T>, } @@ -94,8 +92,7 @@ impl<'a, T> Iter<'a, T> { unsafe { assume(!ptr.is_null()); - let end = - if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) }; + let end = if T::IS_ZST { invalid(slice.len()) } else { ptr.add(slice.len()) }; Self { ptr: NonNull::new_unchecked(ptr as *mut T), end, _marker: PhantomData } } @@ -193,9 +190,7 @@ pub struct IterMut<'a, T: 'a> { ptr: NonNull, /// For non-ZSTs, the non-null pointer to the past-the-end element. /// - /// For ZSTs, this is `ptr.wrapping_byte_add(len)`. - /// - /// For all types, `ptr == end` tests whether the iterator is empty. + /// For ZSTs, this is `ptr::invalid_mut(len)`. end: *mut T, _marker: PhantomData<&'a mut T>, } @@ -235,8 +230,7 @@ impl<'a, T> IterMut<'a, T> { unsafe { assume(!ptr.is_null()); - let end = - if T::IS_ZST { ptr.wrapping_byte_add(slice.len()) } else { ptr.add(slice.len()) }; + let end = if T::IS_ZST { invalid_mut(slice.len()) } else { ptr.add(slice.len()) }; Self { ptr: NonNull::new_unchecked(ptr), end, _marker: PhantomData } } diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index d2d0dd3387f..3462c0e020a 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -1,11 +1,30 @@ //! Macros used by iterators of slice. +// Shrinks the iterator when T is a ZST, setting the length to `new_len`. +// `new_len` must not exceed `self.len()`. +macro_rules! zst_set_len { + ($self: ident, $new_len: expr) => {{ + #![allow(unused_unsafe)] // we're sometimes used within an unsafe block + + // SAFETY: same as `invalid(_mut)`, but the macro doesn't know + // which versions of that function to call, so open-code it. + $self.end = unsafe { mem::transmute::($new_len) }; + }}; +} + +// Shrinks the iterator when T is a ZST, reducing the length by `n`. +// `n` must not exceed `self.len()`. +macro_rules! zst_shrink { + ($self: ident, $n: ident) => { + let new_len = $self.end.addr() - $n; + zst_set_len!($self, new_len); + }; +} + // Inlining is_empty and len makes a huge performance difference macro_rules! is_empty { - // The way we encode the length of a ZST iterator, this works both for ZST - // and non-ZST. ($self: ident) => { - $self.ptr.as_ptr() as *const T == $self.end + if T::IS_ZST { $self.end.addr() == 0 } else { $self.ptr.as_ptr() as *const _ == $self.end } }; } @@ -13,16 +32,13 @@ macro_rules! len { ($self: ident) => {{ #![allow(unused_unsafe)] // we're sometimes used within an unsafe block - let start = $self.ptr; if T::IS_ZST { - // This _cannot_ use `ptr_sub` because we depend on wrapping - // to represent the length of long ZST slice iterators. - $self.end.addr().wrapping_sub(start.as_ptr().addr()) + $self.end.addr() } else { // To get rid of some bounds checks (see `position`), we use ptr_sub instead of // offset_from (Tested by `codegen/slice-position-bounds-check`.) // SAFETY: by the type invariant pointers are aligned and `start <= end` - unsafe { $self.end.sub_ptr(start.as_ptr()) } + unsafe { $self.end.sub_ptr($self.ptr.as_ptr()) } } }}; } @@ -50,14 +66,6 @@ macro_rules! iterator { ($self: ident) => {& $( $mut_ )? *$self.pre_dec_end(1)} } - // Shrinks the iterator when T is a ZST, by moving the end of the iterator - // backwards by `n`. `n` must not exceed `self.len()`. - macro_rules! zst_shrink { - ($self: ident, $n: ident) => { - $self.end = $self.end.wrapping_byte_sub($n); - } - } - impl<'a, T> $name<'a, T> { // Helper function for creating a slice from the iterator. #[inline(always)] @@ -73,16 +81,15 @@ macro_rules! iterator { // Unsafe because the offset must not exceed `self.len()`. #[inline(always)] unsafe fn post_inc_start(&mut self, offset: usize) -> * $raw_mut T { + let old = self.ptr; if T::IS_ZST { zst_shrink!(self, offset); - self.ptr.as_ptr() } else { - let old = self.ptr.as_ptr(); // SAFETY: the caller guarantees that `offset` doesn't exceed `self.len()`, // so this new pointer is inside `self` and thus guaranteed to be non-null. - self.ptr = unsafe { NonNull::new_unchecked(self.ptr.as_ptr().add(offset)) }; - old + self.ptr = unsafe { self.ptr.add(offset) }; } + old.as_ptr() } // Helper function for moving the end of the iterator backwards by `offset` elements, @@ -155,9 +162,7 @@ macro_rules! iterator { if n >= len!(self) { // This iterator is now empty. if T::IS_ZST { - // We have to do it this way as `ptr` may never be 0, but `end` - // could be (due to wrapping). - self.end = self.ptr.as_ptr(); + zst_set_len!(self, 0); } else { // SAFETY: end can't be 0 if T isn't ZST because ptr isn't 0 and end >= ptr unsafe { @@ -356,7 +361,11 @@ macro_rules! iterator { fn nth_back(&mut self, n: usize) -> Option<$elem> { if n >= len!(self) { // This iterator is now empty. - self.end = self.ptr.as_ptr(); + if T::IS_ZST { + zst_set_len!(self, 0); + } else { + self.end = self.ptr.as_ptr(); + } return None; } // SAFETY: We are in bounds. `pre_dec_end` does the right thing even for ZSTs. From 6509c42d165eab4a6a91fd94114e3df85532fd80 Mon Sep 17 00:00:00 2001 From: Michael Goulet Date: Wed, 10 May 2023 22:49:05 +0000 Subject: [PATCH 116/130] Use proper impl self type for alias impl in rustdoc --- src/librustdoc/clean/mod.rs | 17 +++++++++-------- tests/rustdoc/impl-alias-substituted.rs | 9 +++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 tests/rustdoc/impl-alias-substituted.rs diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index c432ce3c324..59a3e631724 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -2414,14 +2414,15 @@ fn clean_impl<'tcx>( } let for_ = clean_ty(impl_.self_ty, cx); - let type_alias = for_.def_id(&cx.cache).and_then(|did| match tcx.def_kind(did) { - DefKind::TyAlias => Some(clean_middle_ty( - ty::Binder::dummy(tcx.type_of(did).subst_identity()), - cx, - Some(did), - )), - _ => None, - }); + let type_alias = + for_.def_id(&cx.cache).and_then(|alias_def_id: DefId| match tcx.def_kind(alias_def_id) { + DefKind::TyAlias => Some(clean_middle_ty( + ty::Binder::dummy(tcx.type_of(def_id).subst_identity()), + cx, + Some(def_id.to_def_id()), + )), + _ => None, + }); let mut make_item = |trait_: Option, for_: Type, items: Vec| { let kind = ImplItem(Box::new(Impl { unsafety: impl_.unsafety, diff --git a/tests/rustdoc/impl-alias-substituted.rs b/tests/rustdoc/impl-alias-substituted.rs new file mode 100644 index 00000000000..82dfffe5f1c --- /dev/null +++ b/tests/rustdoc/impl-alias-substituted.rs @@ -0,0 +1,9 @@ +pub struct Matrix([[T; N]; M]); + +pub type Vector = Matrix; + +// @has "impl_alias_substituted/struct.Matrix.html" '//*[@class="impl"]//h3[@class="code-header"]' \ +// "impl Matrix" +impl Vector { + pub fn test() {} +} From eadf3bb3110dafd41780969810164dd90f218e76 Mon Sep 17 00:00:00 2001 From: WANG Rui Date: Thu, 11 May 2023 14:26:42 +0800 Subject: [PATCH 117/130] Update cargo --- src/tools/cargo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/cargo b/src/tools/cargo index 26b73d15a68..13413c64ff8 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 26b73d15a68fb94579f6d3590585ec0e9d81d3d5 +Subproject commit 13413c64ff88dd6c2824e9eb9374fc5f10895d28 From b0b76a5db31c57aace6fd7c6302f02b3dd63fae0 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 11 May 2023 10:09:20 +0200 Subject: [PATCH 118/130] fix deny_lint test --- src/tools/miri/tests/fail/deny_lint.rs | 2 +- src/tools/miri/tests/fail/deny_lint.stderr | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/tools/miri/tests/fail/deny_lint.rs b/src/tools/miri/tests/fail/deny_lint.rs index 217d5677584..f49fa49d09d 100644 --- a/src/tools/miri/tests/fail/deny_lint.rs +++ b/src/tools/miri/tests/fail/deny_lint.rs @@ -1,6 +1,6 @@ //@error-in-other-file: miri cannot be run on programs that fail compilation -#![deny(warnings)] +#![deny(warnings, unused)] struct Foo; //~^ ERROR: struct `Foo` is never constructed diff --git a/src/tools/miri/tests/fail/deny_lint.stderr b/src/tools/miri/tests/fail/deny_lint.stderr index bb48fcc2038..d1c9b481807 100644 --- a/src/tools/miri/tests/fail/deny_lint.stderr +++ b/src/tools/miri/tests/fail/deny_lint.stderr @@ -4,7 +4,12 @@ error: struct `Foo` is never constructed LL | struct Foo; | ^^^ | - = note: `-D dead-code` implied by `-D unused` +note: the lint level is defined here + --> $DIR/deny_lint.rs:LL:CC + | +LL | #![deny(warnings, unused)] + | ^^^^^^ + = note: `#[deny(dead_code)]` implied by `#[deny(unused)]` error: miri cannot be run on programs that fail compilation From 616fb420986aad3151468c1ca5e874561b452401 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 10 May 2023 10:58:19 +0200 Subject: [PATCH 119/130] Update browser-ui-test version to 0.16.0 --- .../docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version index 7092c7c46f8..d183d4ace05 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/browser-ui-test.version @@ -1 +1 @@ -0.15.0 \ No newline at end of file +0.16.0 \ No newline at end of file From 0630283e9dca4ffe442f4541be8651fa3c676fa2 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 10 May 2023 10:56:59 +0200 Subject: [PATCH 120/130] Migrate to 0.16.0 browser-ui-test version --- src/tools/rustdoc-gui/tester.js | 3 +- tests/rustdoc-gui/check-stab-in-docblock.goml | 18 ++++--- tests/rustdoc-gui/codeblock-sub.goml | 2 +- tests/rustdoc-gui/item-info.goml | 4 +- tests/rustdoc-gui/notable-trait.goml | 8 ++-- .../scrape-examples-button-focus.goml | 6 +-- tests/rustdoc-gui/scrape-examples-layout.goml | 7 ++- tests/rustdoc-gui/search-result-display.goml | 4 +- tests/rustdoc-gui/settings.goml | 2 +- tests/rustdoc-gui/sidebar.goml | 14 +++--- tests/rustdoc-gui/source-code-page.goml | 47 ++++++------------- tests/rustdoc-gui/src-font-size.goml | 2 +- tests/rustdoc-gui/struct-fields.goml | 2 +- .../rustdoc-gui/type-declation-overflow.goml | 2 +- 14 files changed, 55 insertions(+), 66 deletions(-) diff --git a/src/tools/rustdoc-gui/tester.js b/src/tools/rustdoc-gui/tester.js index 692d5e3fcef..b26480f668b 100644 --- a/src/tools/rustdoc-gui/tester.js +++ b/src/tools/rustdoc-gui/tester.js @@ -143,7 +143,7 @@ async function runTests(opts, framework_options, files, results, status_bar, sho const tests_queue = []; for (const testPath of files) { - const callback = runTest(testPath, framework_options) + const callback = runTest(testPath, {"options": framework_options}) .then(out => { const [output, nb_failures] = out; results[nb_failures === 0 ? "successful" : "failed"].push({ @@ -323,6 +323,7 @@ async function main(argv) { if (results.failed.length > 0 || results.errored.length > 0) { process.exit(1); } + process.exit(0); } main(process.argv); diff --git a/tests/rustdoc-gui/check-stab-in-docblock.goml b/tests/rustdoc-gui/check-stab-in-docblock.goml index 2f62636211b..f25c88690e5 100644 --- a/tests/rustdoc-gui/check-stab-in-docblock.goml +++ b/tests/rustdoc-gui/check-stab-in-docblock.goml @@ -7,20 +7,26 @@ set-window-size: (786, 600) // Confirms that there 3 paragraphs. assert-count: (".top-doc .docblock p", 3) // Checking that there is no scrollable content. -store-property: (clientHeight, ".top-doc .docblock p:nth-of-type(1)", "clientHeight") -store-property: (clientWidth, ".top-doc .docblock p:nth-of-type(1)", "clientWidth") +store-property: (".top-doc .docblock p:nth-of-type(1)", { + "clientHeight": clientHeight, + "clientWidth": clientWidth, +}) assert-property: ( ".top-doc .docblock p:nth-of-type(1)", {"scrollHeight": |clientHeight|, "scrollWidth": |clientWidth|}, ) -store-property: (clientHeight, ".top-doc .docblock p:nth-of-type(2)", "clientHeight") -store-property: (clientWidth, ".top-doc .docblock p:nth-of-type(2)", "clientWidth") +store-property: (".top-doc .docblock p:nth-of-type(2)", { + "clientHeight": clientHeight, + "clientWidth": clientWidth, +}) assert-property: ( ".top-doc .docblock p:nth-of-type(2)", {"scrollHeight": |clientHeight|, "scrollWidth": |clientWidth|}, ) -store-property: (clientHeight, ".top-doc .docblock p:nth-of-type(3)", "clientHeight") -store-property: (clientWidth, ".top-doc .docblock p:nth-of-type(3)", "clientWidth") +store-property: (".top-doc .docblock p:nth-of-type(3)", { + "clientHeight": clientHeight, + "clientWidth": clientWidth, +}) assert-property: ( ".top-doc .docblock p:nth-of-type(3)", {"scrollHeight": |clientHeight|, "scrollWidth": |clientWidth|}, diff --git a/tests/rustdoc-gui/codeblock-sub.goml b/tests/rustdoc-gui/codeblock-sub.goml index 03575cc6aaa..a4b0558765a 100644 --- a/tests/rustdoc-gui/codeblock-sub.goml +++ b/tests/rustdoc-gui/codeblock-sub.goml @@ -1,5 +1,5 @@ // Test that code blocks nested within do not have a line height of 0. go-to: "file://" + |DOC_PATH| + "/test_docs/codeblock_sub/index.html" -store-property: (codeblock_sub_1, "#codeblock-sub-1", "offsetHeight") +store-property: ("#codeblock-sub-1", {"offsetHeight": codeblock_sub_1}) assert-property-false: ("#codeblock-sub-3", { "offsetHeight": |codeblock_sub_1| }) diff --git a/tests/rustdoc-gui/item-info.goml b/tests/rustdoc-gui/item-info.goml index 60fd7c4e198..030ff8f8a3e 100644 --- a/tests/rustdoc-gui/item-info.goml +++ b/tests/rustdoc-gui/item-info.goml @@ -4,8 +4,8 @@ go-to: "file://" + |DOC_PATH| + "/lib2/struct.Foo.html" // We set a fixed size so there is no chance of "random" resize. set-window-size: (1100, 800) // We check that ".item-info" is bigger than its content. -assert-css: (".item-info", {"width": "840px"}) -assert-css: (".item-info .stab", {"width": "289px"}) +assert-size: (".item-info", {"width": 840}) +assert-size: (".item-info .stab", {"width": 289}) assert-position: (".item-info .stab", {"x": 245}) // Now we ensure that they're not rendered on the same line. diff --git a/tests/rustdoc-gui/notable-trait.goml b/tests/rustdoc-gui/notable-trait.goml index f65da577478..ecb57c274a5 100644 --- a/tests/rustdoc-gui/notable-trait.goml +++ b/tests/rustdoc-gui/notable-trait.goml @@ -225,12 +225,12 @@ assert: "#method\.create_an_iterator_from_read .tooltip:focus" // Now we check that the focus isn't given back to the wrong item when opening // another popover. -store-window-property: (scroll, "scrollY") +store-window-property: {"scrollY": scroll} click: "#method\.create_an_iterator_from_read .fn" // We ensure that the scroll position changed. assert-window-property-false: {"scrollY": |scroll|} // Store the new position. -store-window-property: (scroll, "scrollY") +store-window-property: {"scrollY": scroll} click: "//*[@id='method.create_an_iterator_from_read']//*[@class='tooltip']" wait-for: "//*[@class='tooltip popover']" click: "#settings-menu a" @@ -239,12 +239,12 @@ click: ".search-input" assert-window-property-false: {"scrollY": |scroll|} // Same but with Escape handling. -store-window-property: (scroll, "scrollY") +store-window-property: {"scrollY": scroll} click: "#method\.create_an_iterator_from_read .fn" // We ensure that the scroll position changed. assert-window-property-false: {"scrollY": |scroll|} // Store the new position. -store-window-property: (scroll, "scrollY") +store-window-property: {"scrollY": scroll} click: "//*[@id='method.create_an_iterator_from_read']//*[@class='tooltip']" wait-for: "//*[@class='tooltip popover']" click: "#settings-menu a" diff --git a/tests/rustdoc-gui/scrape-examples-button-focus.goml b/tests/rustdoc-gui/scrape-examples-button-focus.goml index 77061ea2a3f..af4293dfc00 100644 --- a/tests/rustdoc-gui/scrape-examples-button-focus.goml +++ b/tests/rustdoc-gui/scrape-examples-button-focus.goml @@ -3,7 +3,7 @@ go-to: "file://" + |DOC_PATH| + "/scrape_examples/fn.test.html" // The next/prev buttons vertically scroll the code viewport between examples -store-property: (initialScrollTop, ".scraped-example-list > .scraped-example pre", "scrollTop") +store-property: (".scraped-example-list > .scraped-example pre", {"scrollTop": initialScrollTop}) focus: ".scraped-example-list > .scraped-example .next" press-key: "Enter" assert-property-false: (".scraped-example-list > .scraped-example pre", { @@ -16,7 +16,7 @@ assert-property: (".scraped-example-list > .scraped-example pre", { }, NEAR) // The expand button increases the scrollHeight of the minimized code viewport -store-property: (smallOffsetHeight, ".scraped-example-list > .scraped-example pre", "offsetHeight") +store-property: (".scraped-example-list > .scraped-example pre", {"offsetHeight": smallOffsetHeight}) assert-property-false: (".scraped-example-list > .scraped-example pre", { "scrollHeight": |smallOffsetHeight| }, NEAR) @@ -25,7 +25,7 @@ press-key: "Enter" assert-property-false: (".scraped-example-list > .scraped-example pre", { "offsetHeight": |smallOffsetHeight| }, NEAR) -store-property: (fullOffsetHeight, ".scraped-example-list > .scraped-example pre", "offsetHeight") +store-property: (".scraped-example-list > .scraped-example pre", {"offsetHeight": fullOffsetHeight}) assert-property: (".scraped-example-list > .scraped-example pre", { "scrollHeight": |fullOffsetHeight| }, NEAR) diff --git a/tests/rustdoc-gui/scrape-examples-layout.goml b/tests/rustdoc-gui/scrape-examples-layout.goml index 160056d6d05..4fc1c1ac065 100644 --- a/tests/rustdoc-gui/scrape-examples-layout.goml +++ b/tests/rustdoc-gui/scrape-examples-layout.goml @@ -9,9 +9,8 @@ assert-property-false: ( // Check that examples with very long lines have the same width as ones that don't. store-property: ( - clientWidth, ".more-scraped-examples .scraped-example:nth-child(2) .code-wrapper .src-line-numbers", - "clientWidth" + {"clientWidth": clientWidth}, ) assert-property: ( @@ -40,8 +39,8 @@ assert-property: ( store-value: (offset_y, 4) // First with desktop -assert-position: (".scraped-example .code-wrapper", {"y": 253}) -assert-position: (".scraped-example .code-wrapper .prev", {"y": 253 + |offset_y|}) +assert-position: (".scraped-example .code-wrapper", {"y": 226}) +assert-position: (".scraped-example .code-wrapper .prev", {"y": 226 + |offset_y|}) // Then with mobile set-window-size: (600, 600) diff --git a/tests/rustdoc-gui/search-result-display.goml b/tests/rustdoc-gui/search-result-display.goml index 93c71f23f24..ee5598e4b21 100644 --- a/tests/rustdoc-gui/search-result-display.goml +++ b/tests/rustdoc-gui/search-result-display.goml @@ -32,8 +32,8 @@ set-text: ( ) // Then we compare again to confirm the height didn't change. -assert-css: ("#crate-search", {"width": "527px"}) -assert-css: (".search-results-title", {"height": "50px", "width": "640px"}) +assert-size: ("#crate-search", {"width": 527}) +assert-size: (".search-results-title", {"height": 50, "width": 640}) // And we check that the `