bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #110243 - WaffleLapkin:bless_tagged_pointers🙏, r=Nilstrieb

Browse source 
Tagged pointers, now with strict provenance!

This is a big refactor of tagged pointers in rustc, with three main goals:
1. Porting the code to the strict provenance
2. Cleanup the code
3. Document the code (and safety invariants) better

This PR has grown quite a bit (almost a complete rewrite at this point...), so I'm not sure what's the best way to review this, but reviewing commit-by-commit should be fine.

r? `@Nilstrieb`
This commit is contained in:
bors 2023-04-17 21:50:13 +00:00
commit 7908a1d654
10 changed files with 665 additions and 241 deletions
Download patch file Download diff file Expand all files Collapse all files

33
compiler/rustc_data_structures/src/aligned.rs Normal file
View file

@ -0,0 +1,33 @@
use std::ptr::Alignment;
/// Returns the ABI-required minimum alignment of a type in bytes.
///
/// This is equivalent to [`mem::align_of`], but also works for some unsized
/// types (e.g. slices or rustc's `List`s).
///
/// [`mem::align_of`]: std::mem::align_of
pub const fn align_of<T: ?Sized + Aligned>() -> Alignment {
T::ALIGN
}
/// A type with a statically known alignment.
///
/// # Safety
///
/// `Self::ALIGN` must be equal to the alignment of `Self`. For sized types it
/// is [`mem::align_of<Self>()`], for unsized types it depends on the type, for
/// example `[T]` has alignment of `T`.
///
/// [`mem::align_of<Self>()`]: std::mem::align_of
pub unsafe trait Aligned {
/// Alignment of `Self`.
const ALIGN: Alignment;
}
unsafe impl<T> Aligned for T {
const ALIGN: Alignment = Alignment::of::<Self>();
}
unsafe impl<T> Aligned for [T] {
const ALIGN: Alignment = Alignment::of::<T>();
}

3
compiler/rustc_data_structures/src/lib.rs
View file

@ -29,6 +29,8 @@
#![feature(get_mut_unchecked)] #![feature(get_mut_unchecked)]
#![feature(lint_reasons)] #![feature(lint_reasons)]
#![feature(unwrap_infallible)] #![feature(unwrap_infallible)]
#![feature(strict_provenance)]
#![feature(ptr_alignment_type)]
#![allow(rustc::default_hash_types)] #![allow(rustc::default_hash_types)]
#![allow(rustc::potential_query_instability)] #![allow(rustc::potential_query_instability)]
#![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::untranslatable_diagnostic)]
@ -82,6 +84,7 @@ pub mod transitive_relation;
pub mod vec_linked_list; pub mod vec_linked_list;
pub mod work_queue; pub mod work_queue;
pub use atomic_ref::AtomicRef; pub use atomic_ref::AtomicRef;
pub mod aligned;
pub mod frozen; pub mod frozen;
pub mod owned_slice; pub mod owned_slice;
pub mod sso; pub mod sso;

276
compiler/rustc_data_structures/src/tagged_ptr.rs
View file

@ -3,166 +3,262 @@
//! In order to utilize the pointer packing, you must have two types: a pointer, //! In order to utilize the pointer packing, you must have two types: a pointer,
//! and a tag. //! and a tag.
//! //!
//! The pointer must implement the `Pointer` trait, with the primary requirement //! The pointer must implement the [`Pointer`] trait, with the primary
//! being conversion to and from a usize. Note that the pointer must be //! requirement being convertible to and from a raw pointer. Note that the
//! dereferenceable, so raw pointers generally cannot implement the `Pointer` //! pointer must be dereferenceable, so raw pointers generally cannot implement
//! trait. This implies that the pointer must also be nonzero. //! the [`Pointer`] trait. This implies that the pointer must also be non-null.
//! //!
//! Many common pointer types already implement the `Pointer` trait. //! Many common pointer types already implement the [`Pointer`] trait.
//! //!
//! The tag must implement the `Tag` trait. We assert that the tag and `Pointer` //! The tag must implement the [`Tag`] trait.
//! are compatible at compile time. //!
//! We assert that the tag and the [`Pointer`] types are compatible at compile
//! time.
use std::mem::ManuallyDrop;
use std::ops::Deref; use std::ops::Deref;
use std::ptr::NonNull;
use std::rc::Rc; use std::rc::Rc;
use std::sync::Arc; use std::sync::Arc;
use crate::aligned::Aligned;
mod copy; mod copy;
mod drop; mod drop;
pub use copy::CopyTaggedPtr; pub use copy::CopyTaggedPtr;
pub use drop::TaggedPtr; pub use drop::TaggedPtr;
/// This describes the pointer type encapsulated by TaggedPtr. /// This describes the pointer type encapsulated by [`TaggedPtr`] and
/// [`CopyTaggedPtr`].
/// ///
/// # Safety /// # Safety
/// ///
/// The usize returned from `into_usize` must be a valid, dereferenceable, /// The pointer returned from [`into_ptr`] must be a [valid], pointer to
/// pointer to `<Self as Deref>::Target`. Note that pointers to `Pointee` must /// [`<Self as Deref>::Target`].
/// be thin, even though `Pointee` may not be sized.
/// ///
/// Note that the returned pointer from `into_usize` should be castable to `&mut /// Note that if `Self` implements [`DerefMut`] the pointer returned from
/// <Self as Deref>::Target` if `Pointer: DerefMut`. /// [`into_ptr`] must be valid for writes (and thus calling [`NonNull::as_mut`]
/// on it must be safe).
/// ///
/// The BITS constant must be correct. At least `BITS` bits, least-significant, /// The [`BITS`] constant must be correct. [`BITS`] least-significant bits,
/// must be zero on all returned pointers from `into_usize`. /// must be zero on all pointers returned from [`into_ptr`].
/// ///
/// For example, if the alignment of `Pointee` is 2, then `BITS` should be 1. /// For example, if the alignment of [`Self::Target`] is 2, then `BITS` should be 1.
///
/// [`BITS`]: Pointer::BITS
/// [`into_ptr`]: Pointer::into_ptr
/// [valid]: std::ptr#safety
/// [`<Self as Deref>::Target`]: Deref::Target
/// [`Self::Target`]: Deref::Target
/// [`DerefMut`]: std::ops::DerefMut
pub unsafe trait Pointer: Deref { pub unsafe trait Pointer: Deref {
/// Number of unused (always zero) **least-significant bits** in this
/// pointer, usually related to the pointees alignment.
///
/// For example if [`BITS`] = `2`, then given `ptr = Self::into_ptr(..)`,
/// `ptr.addr() & 0b11 == 0` must be true.
///
/// Most likely the value you want to use here is the following, unless /// Most likely the value you want to use here is the following, unless
/// your Pointee type is unsized (e.g., `ty::List<T>` in rustc) in which /// your [`Self::Target`] type is unsized (e.g., `ty::List<T>` in rustc)
/// case you'll need to manually figure out what the right type to pass to /// or your pointer is over/under aligned, in which case you'll need to
/// align_of is. /// manually figure out what the right type to pass to [`bits_for`] is, or
/// what the value to set here.
/// ///
/// ```ignore UNSOLVED (what to do about the Self) /// ```rust
/// # use std::ops::Deref; /// # use std::ops::Deref;
/// std::mem::align_of::<<Self as Deref>::Target>().trailing_zeros() as usize; /// # use rustc_data_structures::tagged_ptr::bits_for;
/// # struct T;
/// # impl Deref for T { type Target = u8; fn deref(&self) -> &u8 { &0 } }
/// # impl T {
/// const BITS: u32 = bits_for::<<Self as Deref>::Target>();
/// # }
/// ``` /// ```
const BITS: usize;
fn into_usize(self) -> usize;
/// # Safety
/// ///
/// The passed `ptr` must be returned from `into_usize`. /// [`BITS`]: Pointer::BITS
/// /// [`Self::Target`]: Deref::Target
/// This acts as `ptr::read` semantically, it should not be called more than const BITS: u32;
/// once on non-`Copy` `Pointer`s.
unsafe fn from_usize(ptr: usize) -> Self;
/// This provides a reference to the `Pointer` itself, rather than the /// Turns this pointer into a raw, non-null pointer.
/// `Deref::Target`. It is used for cases where we want to call methods that ///
/// may be implement differently for the Pointer than the Pointee (e.g., /// The inverse of this function is [`from_ptr`].
/// `Rc::clone` vs cloning the inner value). ///
/// This function guarantees that the least-significant [`Self::BITS`] bits
/// are zero.
///
/// [`from_ptr`]: Pointer::from_ptr
/// [`Self::BITS`]: Pointer::BITS
fn into_ptr(self) -> NonNull<Self::Target>;
/// Re-creates the original pointer, from a raw pointer returned by [`into_ptr`].
/// ///
/// # Safety /// # Safety
/// ///
/// The passed `ptr` must be returned from `into_usize`. /// The passed `ptr` must be returned from [`into_ptr`].
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R; ///
/// This acts as [`ptr::read::<Self>()`] semantically, it should not be called more than
/// once on non-[`Copy`] `Pointer`s.
///
/// [`into_ptr`]: Pointer::into_ptr
/// [`ptr::read::<Self>()`]: std::ptr::read
unsafe fn from_ptr(ptr: NonNull<Self::Target>) -> Self;
} }
/// This describes tags that the `TaggedPtr` struct can hold. /// This describes tags that the [`TaggedPtr`] struct can hold.
/// ///
/// # Safety /// # Safety
/// ///
/// The BITS constant must be correct. /// The [`BITS`] constant must be correct.
/// ///
/// No more than `BITS` least significant bits may be set in the returned usize. /// No more than [`BITS`] least-significant bits may be set in the returned usize.
///
/// [`BITS`]: Tag::BITS
pub unsafe trait Tag: Copy { pub unsafe trait Tag: Copy {
const BITS: usize; /// Number of least-significant bits in the return value of [`into_usize`]
/// which may be non-zero. In other words this is the bit width of the
/// value.
///
/// [`into_usize`]: Tag::into_usize
const BITS: u32;
/// Turns this tag into an integer.
///
/// The inverse of this function is [`from_usize`].
///
/// This function guarantees that only the least-significant [`Self::BITS`]
/// bits can be non-zero.
///
/// [`from_usize`]: Tag::from_usize
/// [`Self::BITS`]: Tag::BITS
fn into_usize(self) -> usize; fn into_usize(self) -> usize;
/// Re-creates the tag from the integer returned by [`into_usize`].
///
/// # Safety /// # Safety
/// ///
/// The passed `tag` must be returned from `into_usize`. /// The passed `tag` must be returned from [`into_usize`].
///
/// [`into_usize`]: Tag::into_usize
unsafe fn from_usize(tag: usize) -> Self; unsafe fn from_usize(tag: usize) -> Self;
} }
unsafe impl<T> Pointer for Box<T> { unsafe impl<T: ?Sized + Aligned> Pointer for Box<T> {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; const BITS: u32 = bits_for::<Self::Target>();
#[inline] #[inline]
fn into_usize(self) -> usize { fn into_ptr(self) -> NonNull<T> {
Box::into_raw(self) as usize // Safety: pointers from `Box::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Box::into_raw(self)) }
} }
#[inline] #[inline]
unsafe fn from_usize(ptr: usize) -> Self { unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
Box::from_raw(ptr as *mut T) // Safety: `ptr` comes from `into_ptr` which calls `Box::into_raw`
} Box::from_raw(ptr.as_ptr())
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
let raw = ManuallyDrop::new(Self::from_usize(ptr));
f(&raw)
} }
} }
unsafe impl<T> Pointer for Rc<T> { unsafe impl<T: ?Sized + Aligned> Pointer for Rc<T> {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; const BITS: u32 = bits_for::<Self::Target>();
#[inline] #[inline]
fn into_usize(self) -> usize { fn into_ptr(self) -> NonNull<T> {
Rc::into_raw(self) as usize // Safety: pointers from `Rc::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Rc::into_raw(self).cast_mut()) }
} }
#[inline] #[inline]
unsafe fn from_usize(ptr: usize) -> Self { unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
Rc::from_raw(ptr as *const T) // Safety: `ptr` comes from `into_ptr` which calls `Rc::into_raw`
} Rc::from_raw(ptr.as_ptr())
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
let raw = ManuallyDrop::new(Self::from_usize(ptr));
f(&raw)
} }
} }
unsafe impl<T> Pointer for Arc<T> { unsafe impl<T: ?Sized + Aligned> Pointer for Arc<T> {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; const BITS: u32 = bits_for::<Self::Target>();
#[inline] #[inline]
fn into_usize(self) -> usize { fn into_ptr(self) -> NonNull<T> {
Arc::into_raw(self) as usize // Safety: pointers from `Arc::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Arc::into_raw(self).cast_mut()) }
} }
#[inline] #[inline]
unsafe fn from_usize(ptr: usize) -> Self { unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
Arc::from_raw(ptr as *const T) // Safety: `ptr` comes from `into_ptr` which calls `Arc::into_raw`
} Arc::from_raw(ptr.as_ptr())
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
let raw = ManuallyDrop::new(Self::from_usize(ptr));
f(&raw)
} }
} }
unsafe impl<'a, T: 'a> Pointer for &'a T { unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a T {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; const BITS: u32 = bits_for::<Self::Target>();
#[inline] #[inline]
fn into_usize(self) -> usize { fn into_ptr(self) -> NonNull<T> {
self as *const T as usize NonNull::from(self)
} }
#[inline] #[inline]
unsafe fn from_usize(ptr: usize) -> Self { unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
&*(ptr as *const T) // Safety:
} // `ptr` comes from `into_ptr` which gets the pointer from a reference
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { ptr.as_ref()
f(&*(&ptr as *const usize as *const Self))
} }
} }
unsafe impl<'a, T: 'a> Pointer for &'a mut T { unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a mut T {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize; const BITS: u32 = bits_for::<Self::Target>();
#[inline] #[inline]
fn into_ptr(self) -> NonNull<T> {
NonNull::from(self)
}
#[inline]
unsafe fn from_ptr(mut ptr: NonNull<T>) -> Self {
// Safety:
// `ptr` comes from `into_ptr` which gets the pointer from a reference
ptr.as_mut()
}
}
/// Returns the number of bits available for use for tags in a pointer to `T`
/// (this is based on `T`'s alignment).
pub const fn bits_for<T: ?Sized + Aligned>() -> u32 {
crate::aligned::align_of::<T>().as_nonzero().trailing_zeros()
}
/// A tag type used in [`CopyTaggedPtr`] and [`TaggedPtr`] tests.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[cfg(test)]
enum Tag2 {
B00 = 0b00,
B01 = 0b01,
B10 = 0b10,
B11 = 0b11,
}
#[cfg(test)]
unsafe impl Tag for Tag2 {
const BITS: u32 = 2;
fn into_usize(self) -> usize { fn into_usize(self) -> usize {
self as *mut T as usize self as _
} }
#[inline]
unsafe fn from_usize(ptr: usize) -> Self { unsafe fn from_usize(tag: usize) -> Self {
&mut *(ptr as *mut T) match tag {
} 0b00 => Tag2::B00,
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R { 0b01 => Tag2::B01,
f(&*(&ptr as *const usize as *const Self)) 0b10 => Tag2::B10,
0b11 => Tag2::B11,
_ => unreachable!(),
}
}
}
#[cfg(test)]
impl<HCX> crate::stable_hasher::HashStable<HCX> for Tag2 {
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) {
(*self as u8).hash_stable(hcx, hasher);
} }
} }

312
compiler/rustc_data_structures/src/tagged_ptr/copy.rs
View file

@ -1,78 +1,92 @@
use super::{Pointer, Tag}; use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher}; use crate::stable_hasher::{HashStable, StableHasher};
use std::fmt; use std::fmt;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData; use std::marker::PhantomData;
use std::mem::ManuallyDrop;
use std::num::NonZeroUsize; use std::num::NonZeroUsize;
use std::ops::{Deref, DerefMut};
use std::ptr::NonNull;
/// A `Copy` TaggedPtr. /// A [`Copy`] tagged pointer.
/// ///
/// You should use this instead of the `TaggedPtr` type in all cases where /// This is essentially `{ pointer: P, tag: T }` packed in a single pointer.
/// `P: Copy`. ///
/// You should use this instead of the [`TaggedPtr`] type in all cases where
/// `P` implements [`Copy`].
/// ///
/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without /// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without
/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that, /// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`];
/// wrap the TaggedPtr. /// if you want that, wrap the [`CopyTaggedPtr`].
///
/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr
pub struct CopyTaggedPtr<P, T, const COMPARE_PACKED: bool> pub struct CopyTaggedPtr<P, T, const COMPARE_PACKED: bool>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
{ {
packed: NonZeroUsize, /// This is semantically a pair of `pointer: P` and `tag: T` fields,
data: PhantomData<(P, T)>, /// however we pack them in a single pointer, to save space.
///
/// We pack the tag into the **most**-significant bits of the pointer to
/// ease retrieval of the value. A left shift is a multiplication and
/// those are embeddable in instruction encoding, for example:
///
/// ```asm
/// // (<https://godbolt.org/z/jqcYPWEr3>)
/// example::shift_read3:
/// mov eax, dword ptr [8*rdi]
/// ret
///
/// example::mask_read3:
/// and rdi, -8
/// mov eax, dword ptr [rdi]
/// ret
/// ```
///
/// This is ASM outputted by rustc for reads of values behind tagged
/// pointers for different approaches of tagging:
/// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits)
/// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits)
///
/// The shift approach thus produces less instructions and is likely faster
/// (see <https://godbolt.org/z/Y913sMdWb>).
///
/// Encoding diagram:
/// ```text
/// [ packed.addr ]
/// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits
/// ^
/// |
/// T::BITS bits
/// ```
///
/// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer
/// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`.
packed: NonNull<P::Target>,
tag_ghost: PhantomData<T>,
} }
impl<P, T, const COMPARE_PACKED: bool> Copy for CopyTaggedPtr<P, T, COMPARE_PACKED> // Note that even though `CopyTaggedPtr` is only really expected to work with
where // `P: Copy`, can't add `P: Copy` bound, because `CopyTaggedPtr` is used in the
P: Pointer, // `TaggedPtr`'s implementation.
T: Tag, impl<P, T, const CP: bool> CopyTaggedPtr<P, T, CP>
P: Copy,
{
}
impl<P, T, const COMPARE_PACKED: bool> Clone for CopyTaggedPtr<P, T, COMPARE_PACKED>
where
P: Pointer,
T: Tag,
P: Copy,
{
fn clone(&self) -> Self {
*self
}
}
// We pack the tag into the *upper* bits of the pointer to ease retrieval of the
// value; a left shift is a multiplication and those are embeddable in
// instruction encoding.
impl<P, T, const COMPARE_PACKED: bool> CopyTaggedPtr<P, T, COMPARE_PACKED>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
{ {
const TAG_BIT_SHIFT: usize = usize::BITS as usize - T::BITS; /// Tags `pointer` with `tag`.
const ASSERTION: () = { ///
assert!(T::BITS <= P::BITS); /// Note that this leaks `pointer`: it won't be dropped when
// Used for the transmute_copy's below /// `CopyTaggedPtr` is dropped. If you have a pointer with a significant
assert!(std::mem::size_of::<&P::Target>() == std::mem::size_of::<usize>()); /// drop, use [`TaggedPtr`] instead.
}; ///
/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr
pub fn new(pointer: P, tag: T) -> Self { pub fn new(pointer: P, tag: T) -> Self {
// Trigger assert! Self { packed: Self::pack(P::into_ptr(pointer), tag), tag_ghost: PhantomData }
let () = Self::ASSERTION;
let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT;
Self {
// SAFETY: We know that the pointer is non-null, as it must be
// dereferenceable per `Pointer` safety contract.
packed: unsafe {
NonZeroUsize::new_unchecked((P::into_usize(pointer) >> T::BITS) | packed_tag)
},
data: PhantomData,
}
} }
pub(super) fn pointer_raw(&self) -> usize { /// Retrieves the pointer.
self.packed.get() << T::BITS
}
pub fn pointer(self) -> P pub fn pointer(self) -> P
where where
P: Copy, P: Copy,
@ -81,66 +95,138 @@ where
// //
// Note that this isn't going to double-drop or anything because we have // Note that this isn't going to double-drop or anything because we have
// P: Copy // P: Copy
unsafe { P::from_usize(self.pointer_raw()) } unsafe { P::from_ptr(self.pointer_raw()) }
}
pub fn pointer_ref(&self) -> &P::Target {
// SAFETY: pointer_raw returns the original pointer
unsafe { std::mem::transmute_copy(&self.pointer_raw()) }
}
pub fn pointer_mut(&mut self) -> &mut P::Target
where
P: std::ops::DerefMut,
{
// SAFETY: pointer_raw returns the original pointer
unsafe { std::mem::transmute_copy(&self.pointer_raw()) }
} }
/// Retrieves the tag.
#[inline] #[inline]
pub fn tag(&self) -> T { pub fn tag(&self) -> T {
unsafe { T::from_usize(self.packed.get() >> Self::TAG_BIT_SHIFT) } // Unpack the tag, according to the `self.packed` encoding scheme
let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT;
// Safety:
// The shift retrieves the original value from `T::into_usize`,
// satisfying `T::from_usize`'s preconditions.
unsafe { T::from_usize(tag) }
} }
/// Sets the tag to a new value.
#[inline] #[inline]
pub fn set_tag(&mut self, tag: T) { pub fn set_tag(&mut self, tag: T) {
let mut packed = self.packed.get(); self.packed = Self::pack(self.pointer_raw(), tag);
let new_tag = T::into_usize(tag) << Self::TAG_BIT_SHIFT; }
let tag_mask = (1 << T::BITS) - 1;
packed &= !(tag_mask << Self::TAG_BIT_SHIFT); const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS;
packed |= new_tag; const ASSERTION: () = { assert!(T::BITS <= P::BITS) };
self.packed = unsafe { NonZeroUsize::new_unchecked(packed) };
/// Pack pointer `ptr` that comes from [`P::into_ptr`] with a `tag`,
/// according to `self.packed` encoding scheme.
///
/// [`P::into_ptr`]: Pointer::into_ptr
fn pack(ptr: NonNull<P::Target>, tag: T) -> NonNull<P::Target> {
// Trigger assert!
let () = Self::ASSERTION;
let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT;
ptr.map_addr(|addr| {
// Safety:
// - The pointer is `NonNull` => it's address is `NonZeroUsize`
// - `P::BITS` least significant bits are always zero (`Pointer` contract)
// - `T::BITS <= P::BITS` (from `Self::ASSERTION`)
//
// Thus `addr >> T::BITS` is guaranteed to be non-zero.
//
// `{non_zero} | packed_tag` can't make the value zero.
let packed = (addr.get() >> T::BITS) | packed_tag;
unsafe { NonZeroUsize::new_unchecked(packed) }
})
}
/// Retrieves the original raw pointer from `self.packed`.
pub(super) fn pointer_raw(&self) -> NonNull<P::Target> {
self.packed.map_addr(|addr| unsafe { NonZeroUsize::new_unchecked(addr.get() << T::BITS) })
}
/// This provides a reference to the `P` pointer itself, rather than the
/// `Deref::Target`. It is used for cases where we want to call methods
/// that may be implement differently for the Pointer than the Pointee
/// (e.g., `Rc::clone` vs cloning the inner value).
pub(super) fn with_pointer_ref<R>(&self, f: impl FnOnce(&P) -> R) -> R {
// Safety:
// - `self.raw.pointer_raw()` is originally returned from `P::into_ptr`
// and as such is valid for `P::from_ptr`.
// - This also allows us to not care whatever `f` panics or not.
// - Even though we create a copy of the pointer, we store it inside
// `ManuallyDrop` and only access it by-ref, so we don't double-drop.
//
// Semantically this is just `f(&self.pointer)` (where `self.pointer`
// is non-packed original pointer).
//
// Note that even though `CopyTaggedPtr` is only really expected to
// work with `P: Copy`, we have to assume `P: ?Copy`, because
// `CopyTaggedPtr` is used in the `TaggedPtr`'s implementation.
let ptr = unsafe { ManuallyDrop::new(P::from_ptr(self.pointer_raw())) };
f(&ptr)
} }
} }
impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for CopyTaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> Copy for CopyTaggedPtr<P, T, CP>
where
P: Pointer + Copy,
T: Tag,
{
}
impl<P, T, const CP: bool> Clone for CopyTaggedPtr<P, T, CP>
where
P: Pointer + Copy,
T: Tag,
{
fn clone(&self) -> Self {
*self
}
}
impl<P, T, const CP: bool> Deref for CopyTaggedPtr<P, T, CP>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
{ {
type Target = P::Target; type Target = P::Target;
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
self.pointer_ref() // Safety:
// `pointer_raw` returns the original pointer from `P::into_ptr` which,
// by the `Pointer`'s contract, must be valid.
unsafe { self.pointer_raw().as_ref() }
} }
} }
impl<P, T, const COMPARE_PACKED: bool> std::ops::DerefMut for CopyTaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> DerefMut for CopyTaggedPtr<P, T, CP>
where where
P: Pointer + std::ops::DerefMut, P: Pointer + DerefMut,
T: Tag, T: Tag,
{ {
fn deref_mut(&mut self) -> &mut Self::Target { fn deref_mut(&mut self) -> &mut Self::Target {
self.pointer_mut() // Safety:
// `pointer_raw` returns the original pointer from `P::into_ptr` which,
// by the `Pointer`'s contract, must be valid for writes if
// `P: DerefMut`.
unsafe { self.pointer_raw().as_mut() }
} }
} }
impl<P, T, const COMPARE_PACKED: bool> fmt::Debug for CopyTaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> fmt::Debug for CopyTaggedPtr<P, T, CP>
where where
P: Pointer, P: Pointer + fmt::Debug,
P::Target: fmt::Debug,
T: Tag + fmt::Debug, T: Tag + fmt::Debug,
{ {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("CopyTaggedPtr") self.with_pointer_ref(|ptr| {
.field("pointer", &self.pointer_ref()) f.debug_struct("CopyTaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish()
.field("tag", &self.tag()) })
.finish()
} }
} }
@ -161,25 +247,73 @@ where
{ {
} }
impl<P, T> std::hash::Hash for CopyTaggedPtr<P, T, true> impl<P, T> Hash for CopyTaggedPtr<P, T, true>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
{ {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) { fn hash<H: Hasher>(&self, state: &mut H) {
self.packed.hash(state); self.packed.hash(state);
} }
} }
impl<P, T, HCX, const COMPARE_PACKED: bool> HashStable<HCX> for CopyTaggedPtr<P, T, COMPARE_PACKED> impl<P, T, HCX, const CP: bool> HashStable<HCX> for CopyTaggedPtr<P, T, CP>
where where
P: Pointer + HashStable<HCX>, P: Pointer + HashStable<HCX>,
T: Tag + HashStable<HCX>, T: Tag + HashStable<HCX>,
{ {
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
unsafe { self.with_pointer_ref(|ptr| ptr.hash_stable(hcx, hasher));
Pointer::with_ref(self.pointer_raw(), |p: &P| p.hash_stable(hcx, hasher));
}
self.tag().hash_stable(hcx, hasher); self.tag().hash_stable(hcx, hasher);
} }
} }
// Safety:
// `CopyTaggedPtr<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Sync` as long as `P: Sync, T: Sync`
unsafe impl<P, T, const CP: bool> Sync for CopyTaggedPtr<P, T, CP>
where
P: Sync + Pointer,
T: Sync + Tag,
{
}
// Safety:
// `CopyTaggedPtr<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Send` as long as `P: Send, T: Send`
unsafe impl<P, T, const CP: bool> Send for CopyTaggedPtr<P, T, CP>
where
P: Send + Pointer,
T: Send + Tag,
{
}
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{CopyTaggedPtr, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = CopyTaggedPtr::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();
#[cfg(test)]
mod tests;

50
compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs Normal file
View file

@ -0,0 +1,50 @@
use std::ptr;
use crate::stable_hasher::{HashStable, StableHasher};
use crate::tagged_ptr::{CopyTaggedPtr, Pointer, Tag, Tag2};
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = tag_ptr(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
assert!(ptr::eq(ptr.pointer(), reference));
let copy = ptr;
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(copy.tag(), tag);
assert_eq!(*copy, 12);
assert!(ptr::eq(copy.pointer(), reference));
}
#[test]
fn stable_hash_hashes_as_tuple() {
let hash_packed = {
let mut hasher = StableHasher::new();
tag_ptr(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finalize()
};
let hash_tupled = {
let mut hasher = StableHasher::new();
(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finalize()
};
assert_eq!(hash_packed, hash_tupled);
}
/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter.
fn tag_ptr<P: Pointer, T: Tag>(ptr: P, tag: T) -> CopyTaggedPtr<P, T, true> {
CopyTaggedPtr::new(ptr, tag)
}

126
compiler/rustc_data_structures/src/tagged_ptr/drop.rs
View file

@ -1,14 +1,21 @@
use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
use std::fmt; use std::fmt;
use std::hash::{Hash, Hasher};
use std::ops::{Deref, DerefMut};
use super::CopyTaggedPtr; use super::CopyTaggedPtr;
use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
/// A TaggedPtr implementing `Drop`. /// A tagged pointer that supports pointers that implement [`Drop`].
///
/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer.
///
/// You should use [`CopyTaggedPtr`] instead of the this type in all cases
/// where `P` implements [`Copy`].
/// ///
/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without /// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without
/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that, /// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`];
/// wrap the TaggedPtr. /// if you want that, wrap the [`TaggedPtr`].
pub struct TaggedPtr<P, T, const COMPARE_PACKED: bool> pub struct TaggedPtr<P, T, const COMPARE_PACKED: bool>
where where
P: Pointer, P: Pointer,
@ -17,58 +24,61 @@ where
raw: CopyTaggedPtr<P, T, COMPARE_PACKED>, raw: CopyTaggedPtr<P, T, COMPARE_PACKED>,
} }
impl<P, T, const COMPARE_PACKED: bool> Clone for TaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
/// Tags `pointer` with `tag`.
pub fn new(pointer: P, tag: T) -> Self {
TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) }
}
/// Retrieves the tag.
pub fn tag(&self) -> T {
self.raw.tag()
}
/// Sets the tag to a new value.
pub fn set_tag(&mut self, tag: T) {
self.raw.set_tag(tag)
}
}
impl<P, T, const CP: bool> Clone for TaggedPtr<P, T, CP>
where where
P: Pointer + Clone, P: Pointer + Clone,
T: Tag, T: Tag,
{ {
fn clone(&self) -> Self { fn clone(&self) -> Self {
unsafe { Self::new(P::with_ref(self.raw.pointer_raw(), |p| p.clone()), self.raw.tag()) } let ptr = self.raw.with_pointer_ref(P::clone);
Self::new(ptr, self.tag())
} }
} }
// We pack the tag into the *upper* bits of the pointer to ease retrieval of the impl<P, T, const CP: bool> Deref for TaggedPtr<P, T, CP>
// value; a right shift is a multiplication and those are embeddable in
// instruction encoding.
impl<P, T, const COMPARE_PACKED: bool> TaggedPtr<P, T, COMPARE_PACKED>
where
P: Pointer,
T: Tag,
{
pub fn new(pointer: P, tag: T) -> Self {
TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) }
}
pub fn pointer_ref(&self) -> &P::Target {
self.raw.pointer_ref()
}
pub fn tag(&self) -> T {
self.raw.tag()
}
}
impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for TaggedPtr<P, T, COMPARE_PACKED>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
{ {
type Target = P::Target; type Target = P::Target;
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
self.raw.pointer_ref() self.raw.deref()
} }
} }
impl<P, T, const COMPARE_PACKED: bool> std::ops::DerefMut for TaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> DerefMut for TaggedPtr<P, T, CP>
where where
P: Pointer + std::ops::DerefMut, P: Pointer + DerefMut,
T: Tag, T: Tag,
{ {
fn deref_mut(&mut self) -> &mut Self::Target { fn deref_mut(&mut self) -> &mut Self::Target {
self.raw.pointer_mut() self.raw.deref_mut()
} }
} }
impl<P, T, const COMPARE_PACKED: bool> Drop for TaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> Drop for TaggedPtr<P, T, CP>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
@ -76,22 +86,20 @@ where
fn drop(&mut self) { fn drop(&mut self) {
// No need to drop the tag, as it's Copy // No need to drop the tag, as it's Copy
unsafe { unsafe {
drop(P::from_usize(self.raw.pointer_raw())); drop(P::from_ptr(self.raw.pointer_raw()));
} }
} }
} }
impl<P, T, const COMPARE_PACKED: bool> fmt::Debug for TaggedPtr<P, T, COMPARE_PACKED> impl<P, T, const CP: bool> fmt::Debug for TaggedPtr<P, T, CP>
where where
P: Pointer, P: Pointer + fmt::Debug,
P::Target: fmt::Debug,
T: Tag + fmt::Debug, T: Tag + fmt::Debug,
{ {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TaggedPtr") self.raw.with_pointer_ref(|ptr| {
.field("pointer", &self.pointer_ref()) f.debug_struct("TaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish()
.field("tag", &self.tag()) })
.finish()
} }
} }
@ -112,17 +120,17 @@ where
{ {
} }
impl<P, T> std::hash::Hash for TaggedPtr<P, T, true> impl<P, T> Hash for TaggedPtr<P, T, true>
where where
P: Pointer, P: Pointer,
T: Tag, T: Tag,
{ {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) { fn hash<H: Hasher>(&self, state: &mut H) {
self.raw.hash(state); self.raw.hash(state);
} }
} }
impl<P, T, HCX, const COMPARE_PACKED: bool> HashStable<HCX> for TaggedPtr<P, T, COMPARE_PACKED> impl<P, T, HCX, const CP: bool> HashStable<HCX> for TaggedPtr<P, T, CP>
where where
P: Pointer + HashStable<HCX>, P: Pointer + HashStable<HCX>,
T: Tag + HashStable<HCX>, T: Tag + HashStable<HCX>,
@ -131,3 +139,33 @@ where
self.raw.hash_stable(hcx, hasher); self.raw.hash_stable(hcx, hasher);
} }
} }
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{TaggedPtr, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = TaggedPtr::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();
#[cfg(test)]
mod tests;

71
compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs Normal file
View file

@ -0,0 +1,71 @@
use std::{ptr, sync::Arc};
use crate::tagged_ptr::{Pointer, Tag, Tag2, TaggedPtr};
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = tag_ptr(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
assert!(ptr::eq(&*ptr, &*clone))
}
#[test]
fn boxed() {
let value = 12u32;
let boxed = Box::new(value);
let tag = Tag2::B01;
let ptr = tag_ptr(boxed, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
assert!(!ptr::eq(&*ptr, &*clone))
}
#[test]
fn arclones() {
let value = 12u32;
let arc = Arc::new(value);
let tag = Tag2::B01;
let ptr = tag_ptr(arc, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert!(ptr::eq(&*ptr, &*clone))
}
/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter.
fn tag_ptr<P: Pointer, T: Tag>(ptr: P, tag: T) -> TaggedPtr<P, T, true> {
TaggedPtr::new(ptr, tag)
}

1
compiler/rustc_middle/src/lib.rs
View file

@ -59,6 +59,7 @@
#![feature(result_option_inspect)] #![feature(result_option_inspect)]
#![feature(const_option)] #![feature(const_option)]
#![feature(trait_alias)] #![feature(trait_alias)]
#![feature(ptr_alignment_type)]
#![recursion_limit = "512"] #![recursion_limit = "512"]
#![allow(rustc::potential_query_instability)] #![allow(rustc::potential_query_instability)]

28
compiler/rustc_middle/src/ty/list.rs
View file

@ -1,4 +1,5 @@
use crate::arena::Arena; use crate::arena::Arena;
use rustc_data_structures::aligned::{align_of, Aligned};
use rustc_serialize::{Encodable, Encoder}; use rustc_serialize::{Encodable, Encoder};
use std::alloc::Layout; use std::alloc::Layout;
use std::cmp::Ordering; use std::cmp::Ordering;
@ -198,22 +199,17 @@ impl<'a, T: Copy> IntoIterator for &'a List<T> {
unsafe impl<T: Sync> Sync for List<T> {} unsafe impl<T: Sync> Sync for List<T> {}
unsafe impl<'a, T: 'a> rustc_data_structures::tagged_ptr::Pointer for &'a List<T> { // Safety:
const BITS: usize = std::mem::align_of::<usize>().trailing_zeros() as usize; // Layouts of `Equivalent<T>` and `List<T>` are the same, modulo opaque tail,
// thus aligns of `Equivalent<T>` and `List<T>` must be the same.
#[inline] unsafe impl<T> Aligned for List<T> {
fn into_usize(self) -> usize { const ALIGN: ptr::Alignment = {
self as *const List<T> as usize #[repr(C)]
struct Equivalent<T> {
_len: usize,
_data: [T; 0],
} }
#[inline] align_of::<Equivalent<T>>()
unsafe fn from_usize(ptr: usize) -> &'a List<T> { };
&*(ptr as *const List<T>)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
// `Self` is `&'a List<T>` which impls `Copy`, so this is fine.
let ptr = Self::from_usize(ptr);
f(&ptr)
}
} }

4
compiler/rustc_middle/src/ty/mod.rs
View file

@ -1627,7 +1627,8 @@ struct ParamTag {
} }
unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag { unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag {
const BITS: usize = 2; const BITS: u32 = 2;
#[inline] #[inline]
fn into_usize(self) -> usize { fn into_usize(self) -> usize {
match self { match self {
@ -1637,6 +1638,7 @@ unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag {
Self { reveal: traits::Reveal::All, constness: hir::Constness::Const } => 3, Self { reveal: traits::Reveal::All, constness: hir::Constness::Const } => 3,
} }
} }
#[inline] #[inline]
unsafe fn from_usize(ptr: usize) -> Self { unsafe fn from_usize(ptr: usize) -> Self {
match ptr { match ptr {