bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #100759 - fee1-dead-contrib:const_eval_select_real_intrinsic, r=oli-obk,RalfJung

Browse source 
Make `const_eval_select` a real intrinsic

This fixes issues where `track_caller` functions do not have nice panic
messages anymore when there is a call to the function, and uses the
MIR system to replace the call instead of dispatching via lang items.

Fixes #100696.
This commit is contained in:
bors 2022-09-05 01:35:01 +00:00
commit 9358d09a55
27 changed files with 440 additions and 287 deletions
Download patch file Download diff file Expand all files Collapse all files

144
library/core/src/intrinsics.rs
View file

@ -54,7 +54,9 @@
)]
#![allow(missing_docs)]
use crate::marker::{Destruct, DiscriminantKind};
#[cfg(bootstrap)]
use crate::marker::Destruct;
use crate::marker::DiscriminantKind;
use crate::mem;
// These imports are used for simplifying intra-doc links
@ -2085,6 +2087,65 @@ extern "rust-intrinsic" {
/// `ptr` must point to a vtable.
/// The intrinsic will return the alignment stored in that vtable.
pub fn vtable_align(ptr: *const ()) -> usize;
/// Selects which function to call depending on the context.
///
/// If this function is evaluated at compile-time, then a call to this
/// intrinsic will be replaced with a call to `called_in_const`. It gets
/// replaced with a call to `called_at_rt` otherwise.
///
/// # Type Requirements
///
/// The two functions must be both function items. They cannot be function
/// pointers or closures. The first function must be a `const fn`.
///
/// `arg` will be the tupled arguments that will be passed to either one of
/// the two functions, therefore, both functions must accept the same type of
/// arguments. Both functions must return RET.
///
/// # Safety
///
/// The two functions must behave observably equivalent. Safe code in other
/// crates may assume that calling a `const fn` at compile-time and at run-time
/// produces the same result. A function that produces a different result when
/// evaluated at run-time, or has any other observable side-effects, is
/// *unsound*.
///
/// Here is an example of how this could cause a problem:
/// ```no_run
/// #![feature(const_eval_select)]
/// #![feature(core_intrinsics)]
/// use std::hint::unreachable_unchecked;
/// use std::intrinsics::const_eval_select;
///
/// // Crate A
/// pub const fn inconsistent() -> i32 {
/// fn runtime() -> i32 { 1 }
/// const fn compiletime() -> i32 { 2 }
///
/// unsafe {
// // ⚠ This code violates the required equivalence of `compiletime`
/// // and `runtime`.
/// const_eval_select((), compiletime, runtime)
/// }
/// }
///
/// // Crate B
/// const X: i32 = inconsistent();
/// let x = inconsistent();
/// if x != X { unsafe { unreachable_unchecked(); }}
/// ```
///
/// This code causes Undefined Behavior when being run, since the
/// `unreachable_unchecked` is actually being reached. The bug is in *crate A*,
/// which violates the principle that a `const fn` must behave the same at
/// compile-time and at run-time. The unsafe code in crate B is fine.
#[cfg(not(bootstrap))]
#[rustc_const_unstable(feature = "const_eval_select", issue = "none")]
pub fn const_eval_select<ARG, F, G, RET>(arg: ARG, called_in_const: F, called_at_rt: G) -> RET
where
G: FnOnce<ARG, Output = RET>,
F: FnOnce<ARG, Output = RET>;
}
// Some functions are defined here because they accidentally got made
@ -2095,6 +2156,11 @@ extern "rust-intrinsic" {
/// Check that the preconditions of an unsafe function are followed, if debug_assertions are on,
/// and only at runtime.
///
/// This macro should be called as `assert_unsafe_precondition!([Generics](name: Type) => Expression)`
/// where the names specified will be moved into the macro as captured variables, and defines an item
/// to call `const_eval_select` on. The tokens inside the square brackets are used to denote generics
/// for the function declaractions and can be omitted if there is no generics.
///
/// # Safety
///
/// Invoking this macro is only sound if the following code is already UB when the passed
@ -2109,18 +2175,21 @@ extern "rust-intrinsic" {
/// the occasional mistake, and this check should help them figure things out.
#[allow_internal_unstable(const_eval_select)] // permit this to be called in stably-const fn
macro_rules! assert_unsafe_precondition {
($e:expr) => {
($([$($tt:tt)*])?($($i:ident:$ty:ty),*$(,)?) => $e:expr) => {
if cfg!(debug_assertions) {
// Use a closure so that we can capture arbitrary expressions from the invocation
let runtime = || {
// allow non_snake_case to allow capturing const generics
#[allow(non_snake_case)]
#[inline(always)]
fn runtime$(<$($tt)*>)?($($i:$ty),*) {
if !$e {
// abort instead of panicking to reduce impact on code size
::core::intrinsics::abort();
}
};
const fn comptime() {}
}
#[allow(non_snake_case)]
const fn comptime$(<$($tt)*>)?($(_:$ty),*) {}
::core::intrinsics::const_eval_select((), comptime, runtime);
::core::intrinsics::const_eval_select(($($i,)*), comptime, runtime);
}
};
}
@ -2243,7 +2312,7 @@ pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: us
// SAFETY: the safety contract for `copy_nonoverlapping` must be
// upheld by the caller.
unsafe {
assert_unsafe_precondition!(
assert_unsafe_precondition!([T](src: *const T, dst: *mut T, count: usize) =>
is_aligned_and_not_null(src)
&& is_aligned_and_not_null(dst)
&& is_nonoverlapping(src, dst, count)
@ -2329,7 +2398,8 @@ pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
// SAFETY: the safety contract for `copy` must be upheld by the caller.
unsafe {
assert_unsafe_precondition!(is_aligned_and_not_null(src) && is_aligned_and_not_null(dst));
assert_unsafe_precondition!([T](src: *const T, dst: *mut T) =>
is_aligned_and_not_null(src) && is_aligned_and_not_null(dst));
copy(src, dst, count)
}
}
@ -2397,63 +2467,12 @@ pub const unsafe fn write_bytes<T>(dst: *mut T, val: u8, count: usize) {
// SAFETY: the safety contract for `write_bytes` must be upheld by the caller.
unsafe {
assert_unsafe_precondition!(is_aligned_and_not_null(dst));
assert_unsafe_precondition!([T](dst: *mut T) => is_aligned_and_not_null(dst));
write_bytes(dst, val, count)
}
}
/// Selects which function to call depending on the context.
///
/// If this function is evaluated at compile-time, then a call to this
/// intrinsic will be replaced with a call to `called_in_const`. It gets
/// replaced with a call to `called_at_rt` otherwise.
///
/// # Type Requirements
///
/// The two functions must be both function items. They cannot be function
/// pointers or closures.
///
/// `arg` will be the arguments that will be passed to either one of the
/// two functions, therefore, both functions must accept the same type of
/// arguments. Both functions must return RET.
///
/// # Safety
///
/// The two functions must behave observably equivalent. Safe code in other
/// crates may assume that calling a `const fn` at compile-time and at run-time
/// produces the same result. A function that produces a different result when
/// evaluated at run-time, or has any other observable side-effects, is
/// *unsound*.
///
/// Here is an example of how this could cause a problem:
/// ```no_run
/// #![feature(const_eval_select)]
/// #![feature(core_intrinsics)]
/// use std::hint::unreachable_unchecked;
/// use std::intrinsics::const_eval_select;
///
/// // Crate A
/// pub const fn inconsistent() -> i32 {
/// fn runtime() -> i32 { 1 }
/// const fn compiletime() -> i32 { 2 }
///
/// unsafe {
// // ⚠ This code violates the required equivalence of `compiletime`
/// // and `runtime`.
/// const_eval_select((), compiletime, runtime)
/// }
/// }
///
/// // Crate B
/// const X: i32 = inconsistent();
/// let x = inconsistent();
/// if x != X { unsafe { unreachable_unchecked(); }}
/// ```
///
/// This code causes Undefined Behavior when being run, since the
/// `unreachable_unchecked` is actually being reached. The bug is in *crate A*,
/// which violates the principle that a `const fn` must behave the same at
/// compile-time and at run-time. The unsafe code in crate B is fine.
#[cfg(bootstrap)]
#[unstable(
feature = "const_eval_select",
issue = "none",
@ -2475,6 +2494,7 @@ where
called_at_rt.call_once(arg)
}
#[cfg(bootstrap)]
#[unstable(
feature = "const_eval_select",
issue = "none",

2
library/core/src/mem/valid_align.rs
View file

@ -28,7 +28,7 @@ impl ValidAlign {
#[inline]
pub(crate) const unsafe fn new_unchecked(align: usize) -> Self {
// SAFETY: Precondition passed to the caller.
unsafe { assert_unsafe_precondition!(align.is_power_of_two()) };
unsafe { assert_unsafe_precondition!((align: usize) => align.is_power_of_two()) };
// SAFETY: By precondition, this must be a power of two, and
// our variants encompass all possible powers of two.

24
library/core/src/num/f32.rs
View file

@ -1033,10 +1033,14 @@ impl f32 {
}
}
}
// SAFETY: `u32` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
let rt_f32_to_u32 = |rt| unsafe { mem::transmute::<f32, u32>(rt) };
#[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491
fn rt_f32_to_u32(x: f32) -> u32 {
// SAFETY: `u32` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
unsafe { mem::transmute(x) }
}
// SAFETY: We use internal implementations that either always work or fail at compile time.
unsafe { intrinsics::const_eval_select((self,), ct_f32_to_u32, rt_f32_to_u32) }
}
@ -1121,10 +1125,14 @@ impl f32 {
}
}
}
// SAFETY: `u32` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
let rt_u32_to_f32 = |rt| unsafe { mem::transmute::<u32, f32>(rt) };
#[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491
fn rt_u32_to_f32(x: u32) -> f32 {
// SAFETY: `u32` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
unsafe { mem::transmute(x) }
}
// SAFETY: We use internal implementations that either always work or fail at compile time.
unsafe { intrinsics::const_eval_select((v,), ct_u32_to_f32, rt_u32_to_f32) }
}

24
library/core/src/num/f64.rs
View file

@ -1026,10 +1026,14 @@ impl f64 {
}
}
}
// SAFETY: `u64` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
let rt_f64_to_u64 = |rt| unsafe { mem::transmute::<f64, u64>(rt) };
#[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491
fn rt_f64_to_u64(rt: f64) -> u64 {
// SAFETY: `u64` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
unsafe { mem::transmute::<f64, u64>(rt) }
}
// SAFETY: We use internal implementations that either always work or fail at compile time.
unsafe { intrinsics::const_eval_select((self,), ct_f64_to_u64, rt_f64_to_u64) }
}
@ -1119,10 +1123,14 @@ impl f64 {
}
}
}
// SAFETY: `u64` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
let rt_u64_to_f64 = |rt| unsafe { mem::transmute::<u64, f64>(rt) };
#[inline(always)] // See https://github.com/rust-lang/compiler-builtins/issues/491
fn rt_u64_to_f64(rt: u64) -> f64 {
// SAFETY: `u64` is a plain old datatype so we can always... uh...
// ...look, just pretend you forgot what you just read.
// Stability concerns.
unsafe { mem::transmute::<u64, f64>(rt) }
}
// SAFETY: We use internal implementations that either always work or fail at compile time.
unsafe { intrinsics::const_eval_select((v,), ct_u64_to_f64, rt_u64_to_f64) }
}

2
library/core/src/num/nonzero.rs
View file

@ -56,7 +56,7 @@ macro_rules! nonzero_integers {
pub const unsafe fn new_unchecked(n: $Int) -> Self {
// SAFETY: this is guaranteed to be safe by the caller.
unsafe {
core::intrinsics::assert_unsafe_precondition!(n != 0);
core::intrinsics::assert_unsafe_precondition!((n: $Int) => n != 0);
Self(n)
}
}

5
library/core/src/ptr/const_ptr.rs
View file

@ -755,9 +755,12 @@ impl<T: ?Sized> *const T {
where
T: Sized,
{
let this = self;
// SAFETY: The comparison has no side-effects, and the intrinsic
// does this check internally in the CTFE implementation.
unsafe { assert_unsafe_precondition!(self >= origin) };
unsafe {
assert_unsafe_precondition!([T](this: *const T, origin: *const T) => this >= origin)
};
let pointee_size = mem::size_of::<T>();
assert!(0 < pointee_size && pointee_size <= isize::MAX as usize);

8
library/core/src/ptr/mod.rs
View file

@ -886,7 +886,7 @@ pub const unsafe fn swap_nonoverlapping<T>(x: *mut T, y: *mut T, count: usize) {
// SAFETY: the caller must guarantee that `x` and `y` are
// valid for writes and properly aligned.
unsafe {
assert_unsafe_precondition!(
assert_unsafe_precondition!([T](x: *mut T, y: *mut T, count: usize) =>
is_aligned_and_not_null(x)
&& is_aligned_and_not_null(y)
&& is_nonoverlapping(x, y, count)
@ -983,7 +983,7 @@ pub const unsafe fn replace<T>(dst: *mut T, mut src: T) -> T {
// and cannot overlap `src` since `dst` must point to a distinct
// allocated object.
unsafe {
assert_unsafe_precondition!(is_aligned_and_not_null(dst));
assert_unsafe_precondition!([T](dst: *mut T) => is_aligned_and_not_null(dst));
mem::swap(&mut *dst, &mut src); // cannot overlap
}
src
@ -1470,7 +1470,7 @@ pub const unsafe fn write_unaligned<T>(dst: *mut T, src: T) {
pub unsafe fn read_volatile<T>(src: *const T) -> T {
// SAFETY: the caller must uphold the safety contract for `volatile_load`.
unsafe {
assert_unsafe_precondition!(is_aligned_and_not_null(src));
assert_unsafe_precondition!([T](src: *const T) => is_aligned_and_not_null(src));
intrinsics::volatile_load(src)
}
}
@ -1541,7 +1541,7 @@ pub unsafe fn read_volatile<T>(src: *const T) -> T {
pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
// SAFETY: the caller must uphold the safety contract for `volatile_store`.
unsafe {
assert_unsafe_precondition!(is_aligned_and_not_null(dst));
assert_unsafe_precondition!([T](dst: *mut T) => is_aligned_and_not_null(dst));
intrinsics::volatile_store(dst, src);
}
}

20
library/core/src/slice/index.rs
View file

@ -48,10 +48,12 @@ const fn slice_start_index_len_fail(index: usize, len: usize) -> ! {
}
// FIXME const-hack
#[track_caller]
fn slice_start_index_len_fail_rt(index: usize, len: usize) -> ! {
panic!("range start index {index} out of range for slice of length {len}");
}
#[track_caller]
const fn slice_start_index_len_fail_ct(_: usize, _: usize) -> ! {
panic!("slice start index is out of range for slice");
}
@ -69,10 +71,12 @@ const fn slice_end_index_len_fail(index: usize, len: usize) -> ! {
}
// FIXME const-hack
#[track_caller]
fn slice_end_index_len_fail_rt(index: usize, len: usize) -> ! {
panic!("range end index {index} out of range for slice of length {len}");
}
#[track_caller]
const fn slice_end_index_len_fail_ct(_: usize, _: usize) -> ! {
panic!("slice end index is out of range for slice");
}
@ -88,10 +92,12 @@ const fn slice_index_order_fail(index: usize, end: usize) -> ! {
}
// FIXME const-hack
#[track_caller]
fn slice_index_order_fail_rt(index: usize, end: usize) -> ! {
panic!("slice index starts at {index} but ends at {end}");
}
#[track_caller]
const fn slice_index_order_fail_ct(_: usize, _: usize) -> ! {
panic!("slice index start is larger than end");
}
@ -217,21 +223,23 @@ unsafe impl<T> const SliceIndex<[T]> for usize {
#[inline]
unsafe fn get_unchecked(self, slice: *const [T]) -> *const T {
let this = self;
// SAFETY: the caller guarantees that `slice` is not dangling, so it
// cannot be longer than `isize::MAX`. They also guarantee that
// `self` is in bounds of `slice` so `self` cannot overflow an `isize`,
// so the call to `add` is safe.
unsafe {
assert_unsafe_precondition!(self < slice.len());
assert_unsafe_precondition!([T](this: usize, slice: *const [T]) => this < slice.len());
slice.as_ptr().add(self)
}
}
#[inline]
unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut T {
let this = self;
// SAFETY: see comments for `get_unchecked` above.
unsafe {
assert_unsafe_precondition!(self < slice.len());
assert_unsafe_precondition!([T](this: usize, slice: *mut [T]) => this < slice.len());
slice.as_mut_ptr().add(self)
}
}
@ -276,22 +284,26 @@ unsafe impl<T> const SliceIndex<[T]> for ops::Range<usize> {
#[inline]
unsafe fn get_unchecked(self, slice: *const [T]) -> *const [T] {
let this = ops::Range { start: self.start, end: self.end };
// SAFETY: the caller guarantees that `slice` is not dangling, so it
// cannot be longer than `isize::MAX`. They also guarantee that
// `self` is in bounds of `slice` so `self` cannot overflow an `isize`,
// so the call to `add` is safe.
unsafe {
assert_unsafe_precondition!(self.end >= self.start && self.end <= slice.len());
assert_unsafe_precondition!([T](this: ops::Range<usize>, slice: *const [T]) =>
this.end >= this.start && this.end <= slice.len());
ptr::slice_from_raw_parts(slice.as_ptr().add(self.start), self.end - self.start)
}
}
#[inline]
unsafe fn get_unchecked_mut(self, slice: *mut [T]) -> *mut [T] {
let this = ops::Range { start: self.start, end: self.end };
// SAFETY: see comments for `get_unchecked` above.
unsafe {
assert_unsafe_precondition!(self.end >= self.start && self.end <= slice.len());
assert_unsafe_precondition!([T](this: ops::Range<usize>, slice: *mut [T]) =>
this.end >= this.start && this.end <= slice.len());
ptr::slice_from_raw_parts_mut(slice.as_mut_ptr().add(self.start), self.end - self.start)
}
}

15
library/core/src/slice/mod.rs
View file

@ -656,10 +656,11 @@ impl<T> [T] {
#[unstable(feature = "slice_swap_unchecked", issue = "88539")]
#[rustc_const_unstable(feature = "const_swap", issue = "83163")]
pub const unsafe fn swap_unchecked(&mut self, a: usize, b: usize) {
let ptr = self.as_mut_ptr();
let this = self;
let ptr = this.as_mut_ptr();
// SAFETY: caller has to guarantee that `a < self.len()` and `b < self.len()`
unsafe {
assert_unsafe_precondition!(a < self.len() && b < self.len());
assert_unsafe_precondition!([T](a: usize, b: usize, this: &mut [T]) => a < this.len() && b < this.len());
ptr::swap(ptr.add(a), ptr.add(b));
}
}
@ -972,9 +973,10 @@ impl<T> [T] {
#[inline]
#[must_use]
pub unsafe fn as_chunks_unchecked<const N: usize>(&self) -> &[[T; N]] {
let this = self;
// SAFETY: Caller must guarantee that `N` is nonzero and exactly divides the slice length
let new_len = unsafe {
assert_unsafe_precondition!(N != 0 && self.len() % N == 0);
assert_unsafe_precondition!([T](this: &[T], N: usize) => N != 0 && this.len() % N == 0);
exact_div(self.len(), N)
};
// SAFETY: We cast a slice of `new_len * N` elements into
@ -1111,10 +1113,11 @@ impl<T> [T] {
#[inline]
#[must_use]
pub unsafe fn as_chunks_unchecked_mut<const N: usize>(&mut self) -> &mut [[T; N]] {
let this = &*self;
// SAFETY: Caller must guarantee that `N` is nonzero and exactly divides the slice length
let new_len = unsafe {
assert_unsafe_precondition!(N != 0 && self.len() % N == 0);
exact_div(self.len(), N)
assert_unsafe_precondition!([T](this: &[T], N: usize) => N != 0 && this.len() % N == 0);
exact_div(this.len(), N)
};
// SAFETY: We cast a slice of `new_len * N` elements into
// a slice of `new_len` many `N` elements chunks.
@ -1687,7 +1690,7 @@ impl<T> [T] {
// `[ptr; mid]` and `[mid; len]` are not overlapping, so returning a mutable reference
// is fine.
unsafe {
assert_unsafe_precondition!(mid <= len);
assert_unsafe_precondition!((mid: usize, len: usize) => mid <= len);
(from_raw_parts_mut(ptr, mid), from_raw_parts_mut(ptr.add(mid), len - mid))
}
}

4
library/core/src/slice/raw.rs
View file

@ -90,7 +90,7 @@ use crate::ptr;
pub const unsafe fn from_raw_parts<'a, T>(data: *const T, len: usize) -> &'a [T] {
// SAFETY: the caller must uphold the safety contract for `from_raw_parts`.
unsafe {
assert_unsafe_precondition!(
assert_unsafe_precondition!([T](data: *const T, len: usize) =>
is_aligned_and_not_null(data)
&& crate::mem::size_of::<T>().saturating_mul(len) <= isize::MAX as usize
);
@ -134,7 +134,7 @@ pub const unsafe fn from_raw_parts<'a, T>(data: *const T, len: usize) -> &'a [T]
pub const unsafe fn from_raw_parts_mut<'a, T>(data: *mut T, len: usize) -> &'a mut [T] {
// SAFETY: the caller must uphold the safety contract for `from_raw_parts_mut`.
unsafe {
assert_unsafe_precondition!(
assert_unsafe_precondition!([T](data: *mut T, len: usize) =>
is_aligned_and_not_null(data)
&& crate::mem::size_of::<T>().saturating_mul(len) <= isize::MAX as usize
);

2
library/core/src/str/mod.rs
View file

@ -91,10 +91,12 @@ const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! {
}
}
#[track_caller]
const fn slice_error_fail_ct(_: &str, _: usize, _: usize) -> ! {
panic!("failed to slice string");
}
#[track_caller]
fn slice_error_fail_rt(s: &str, begin: usize, end: usize) -> ! {
const MAX_DISPLAY_LENGTH: usize = 256;
let trunc_len = s.floor_char_boundary(MAX_DISPLAY_LENGTH);