diff --git a/src/liballoc/sync.rs b/src/liballoc/sync.rs index 127068284d3..caf727081b1 100644 --- a/src/liballoc/sync.rs +++ b/src/liballoc/sync.rs @@ -12,6 +12,7 @@ use core::sync::atomic::Ordering::{Acquire, Relaxed, Release, SeqCst}; use core::borrow; use core::fmt; use core::cmp::{self, Ordering}; +use core::iter; use core::intrinsics::abort; use core::mem::{self, align_of, align_of_val, size_of_val}; use core::ops::{Deref, Receiver, CoerceUnsized, DispatchFromDyn}; @@ -21,7 +22,7 @@ use core::marker::{Unpin, Unsize, PhantomData}; use core::hash::{Hash, Hasher}; use core::{isize, usize}; use core::convert::From; -use core::slice::from_raw_parts_mut; +use core::slice::{self, from_raw_parts_mut}; use crate::alloc::{Global, Alloc, Layout, box_free, handle_alloc_error}; use crate::boxed::Box; @@ -587,21 +588,28 @@ impl Arc { } impl Arc { - // Allocates an `ArcInner` with sufficient space for an unsized value - unsafe fn allocate_for_ptr(ptr: *const T) -> *mut ArcInner { - // Calculate layout using the given value. + // Allocates an `ArcInner` with sufficient space for + // an unsized value where the value has the layout provided. + // + // The function `mem_to_arcinner` is called with the data pointer + // and must return back a (potentially fat)-pointer for the `ArcInner`. + unsafe fn allocate_for_unsized( + value_layout: Layout, + mem_to_arcinner: impl FnOnce(*mut u8) -> *mut ArcInner + ) -> *mut ArcInner { + // Calculate layout using the given value layout. // Previously, layout was calculated on the expression // `&*(ptr as *const ArcInner)`, but this created a misaligned // reference (see #54908). let layout = Layout::new::>() - .extend(Layout::for_value(&*ptr)).unwrap().0 + .extend(value_layout).unwrap().0 .pad_to_align().unwrap(); let mem = Global.alloc(layout) .unwrap_or_else(|_| handle_alloc_error(layout)); // Initialize the ArcInner - let inner = set_data_ptr(ptr as *mut T, mem.as_ptr() as *mut u8) as *mut ArcInner; + let inner = mem_to_arcinner(mem.as_ptr()); debug_assert_eq!(Layout::for_value(&*inner), layout); ptr::write(&mut (*inner).strong, atomic::AtomicUsize::new(1)); @@ -610,6 +618,15 @@ impl Arc { inner } + // Allocates an `ArcInner` with sufficient space for an unsized value + unsafe fn allocate_for_ptr(ptr: *const T) -> *mut ArcInner { + // Allocate for the `ArcInner` using the given value. + Self::allocate_for_unsized( + Layout::for_value(&*ptr), + |mem| set_data_ptr(ptr as *mut T, mem) as *mut ArcInner, + ) + } + fn from_box(v: Box) -> Arc { unsafe { let box_unique = Box::into_unique(v); @@ -632,6 +649,32 @@ impl Arc { } } +impl Arc<[T]> { + // Allocates an `ArcInner<[T]>` with the given length. + unsafe fn allocate_for_slice(len: usize) -> *mut ArcInner<[T]> { + // FIXME(#60667): Deduplicate. + fn slice_from_raw_parts_mut(data: *mut T, len: usize) -> *mut [T] { + #[repr(C)] + union Repr { + rust_mut: *mut [T], + raw: FatPtr, + } + + #[repr(C)] + struct FatPtr { + data: *const T, + len: usize, + } + unsafe { Repr { raw: FatPtr { data, len } }.rust_mut } + } + + Self::allocate_for_unsized( + Layout::array::(len).unwrap(), + |mem| slice_from_raw_parts_mut(mem as *mut T, len) as *mut ArcInner<[T]>, + ) + } +} + // Sets the data pointer of a `?Sized` raw pointer. // // For a slice/trait object, this sets the `data` field and leaves the rest @@ -646,8 +689,7 @@ impl Arc<[T]> { // // Unsafe because the caller must either take ownership or bind `T: Copy` unsafe fn copy_from_slice(v: &[T]) -> Arc<[T]> { - let v_ptr = v as *const [T]; - let ptr = Self::allocate_for_ptr(v_ptr); + let ptr = Self::allocate_for_slice(v.len()); ptr::copy_nonoverlapping( v.as_ptr(), @@ -656,16 +698,11 @@ impl Arc<[T]> { Self::from_ptr(ptr) } -} -// Specialization trait used for From<&[T]> -trait ArcFromSlice { - fn from_slice(slice: &[T]) -> Self; -} - -impl ArcFromSlice for Arc<[T]> { - #[inline] - default fn from_slice(v: &[T]) -> Self { + /// Constructs an `Arc<[T]>` from an iterator known to be of a certain size. + /// + /// Behavior is undefined should the size be wrong. + unsafe fn from_iter_exact(iter: impl iter::Iterator, len: usize) -> Arc<[T]> { // Panic guard while cloning T elements. // In the event of a panic, elements that have been written // into the new ArcInner will be dropped, then the memory freed. @@ -687,32 +724,43 @@ impl ArcFromSlice for Arc<[T]> { } } + let ptr = Self::allocate_for_slice(len); + + let mem = ptr as *mut _ as *mut u8; + let layout = Layout::for_value(&*ptr); + + // Pointer to first element + let elems = &mut (*ptr).data as *mut [T] as *mut T; + + let mut guard = Guard { + mem: NonNull::new_unchecked(mem), + elems, + layout, + n_elems: 0, + }; + + for (i, item) in iter.enumerate() { + ptr::write(elems.add(i), item); + guard.n_elems += 1; + } + + // All clear. Forget the guard so it doesn't free the new ArcInner. + mem::forget(guard); + + Self::from_ptr(ptr) + } +} + +// Specialization trait used for From<&[T]> +trait ArcFromSlice { + fn from_slice(slice: &[T]) -> Self; +} + +impl ArcFromSlice for Arc<[T]> { + #[inline] + default fn from_slice(v: &[T]) -> Self { unsafe { - let v_ptr = v as *const [T]; - let ptr = Self::allocate_for_ptr(v_ptr); - - let mem = ptr as *mut _ as *mut u8; - let layout = Layout::for_value(&*ptr); - - // Pointer to first element - let elems = &mut (*ptr).data as *mut [T] as *mut T; - - let mut guard = Guard{ - mem: NonNull::new_unchecked(mem), - elems: elems, - layout: layout, - n_elems: 0, - }; - - for (i, item) in v.iter().enumerate() { - ptr::write(elems.add(i), item.clone()); - guard.n_elems += 1; - } - - // All clear. Forget the guard so it doesn't free the new ArcInner. - mem::forget(guard); - - Self::from_ptr(ptr) + Self::from_iter_exact(v.iter().cloned(), v.len()) } } } @@ -1792,9 +1840,88 @@ impl From> for Arc<[T]> { } #[stable(feature = "shared_from_iter", since = "1.37.0")] -impl core::iter::FromIterator for Arc<[T]> { - fn from_iter>(iter: I) -> Self { - iter.into_iter().collect::>().into() +impl iter::FromIterator for Arc<[T]> { + /// Takes each element in the `Iterator` and collects it into an `Arc<[T]>`. + /// + /// # Performance characteristics + /// + /// ## The general case + /// + /// In the general case, collecting into `Arc<[T]>` is done by first + /// collecting into a `Vec`. That is, when writing the following: + /// + /// ```rust + /// # use std::sync::Arc; + /// let evens: Arc<[u8]> = (0..10).filter(|&x| x % 2 == 0).collect(); + /// # assert_eq!(&*evens, &[0, 2, 4, 6, 8]); + /// ``` + /// + /// this behaves as if we wrote: + /// + /// ```rust + /// # use std::sync::Arc; + /// let evens: Arc<[u8]> = (0..10).filter(|&x| x % 2 == 0) + /// .collect::>() // The first set of allocations happens here. + /// .into(); // A second allocation for `Arc<[T]>` happens here. + /// # assert_eq!(&*evens, &[0, 2, 4, 6, 8]); + /// ``` + /// + /// This will allocate as many times as needed for constructing the `Vec` + /// and then it will allocate once for turning the `Vec` into the `Arc<[T]>`. + /// + /// ## Iterators of known length + /// + /// When your `Iterator` implements `TrustedLen` and is of an exact size, + /// a single allocation will be made for the `Arc<[T]>`. For example: + /// + /// ```rust + /// # use std::sync::Arc; + /// let evens: Arc<[u8]> = (0..10).collect(); // Just a single allocation happens here. + /// # assert_eq!(&*evens, &*(0..10).collect::>()); + /// ``` + fn from_iter>(iter: I) -> Self { + ArcFromIter::from_iter(iter.into_iter()) + } +} + +/// Specialization trait used for collecting into `Arc<[T]>`. +trait ArcFromIter { + fn from_iter(iter: I) -> Self; +} + +impl> ArcFromIter for Arc<[T]> { + default fn from_iter(iter: I) -> Self { + iter.collect::>().into() + } +} + +impl> ArcFromIter for Arc<[T]> { + default fn from_iter(iter: I) -> Self { + // This is the case for a `TrustedLen` iterator. + let (low, high) = iter.size_hint(); + if let Some(high) = high { + debug_assert_eq!( + low, high, + "TrustedLen iterator's size hint is not exact: {:?}", + (low, high) + ); + + unsafe { + // SAFETY: We need to ensure that the iterator has an exact length and we have. + Arc::from_iter_exact(iter, low) + } + } else { + // Fall back to normal implementation. + iter.collect::>().into() + } + } +} + +impl<'a, T: 'a + Clone> ArcFromIter<&'a T, slice::Iter<'a, T>> for Arc<[T]> { + fn from_iter(iter: slice::Iter<'a, T>) -> Self { + // Delegate to `impl From<&[T]> for Arc<[T]>` + // which will use `ptr::copy_nonoverlapping`. + iter.as_slice().into() } }