bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

interpret: support projecting into Place::Local without force_allocation

Browse source
This commit is contained in:
Ralf Jung 2023-07-23 21:35:54 +02:00
parent 42f5419dd2
commit a593de4fab
12 changed files with 431 additions and 279 deletions
Download patch file Download diff file Expand all files Collapse all files

2
compiler/rustc_const_eval/messages.ftl
View file

@ -423,8 +423,6 @@ const_eval_uninit_int = {$front_matter}: encountered uninitialized memory, but e
const_eval_uninit_raw_ptr = {$front_matter}: encountered uninitialized memory, but expected a raw pointer
const_eval_uninit_ref = {$front_matter}: encountered uninitialized memory, but expected a reference
const_eval_uninit_str = {$front_matter}: encountered uninitialized data in `str`
const_eval_uninit_unsized_local =
unsized local is used while uninitialized
const_eval_unreachable = entering unreachable code
const_eval_unreachable_unwind =
unwinding past a stack frame that does not allow unwinding

4
compiler/rustc_const_eval/src/errors.rs
View file

@ -835,7 +835,7 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
rustc_middle::error::middle_adjust_for_foreign_abi_error
}
InvalidProgramInfo::SizeOfUnsizedType(_) => const_eval_size_of_unsized,
InvalidProgramInfo::UninitUnsizedLocal => const_eval_uninit_unsized_local,
InvalidProgramInfo::ConstPropNonsense => panic!("We had const-prop nonsense, this should never be printed"),
}
}
fn add_args<G: EmissionGuarantee>(
@ -846,7 +846,7 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
match self {
InvalidProgramInfo::TooGeneric
| InvalidProgramInfo::AlreadyReported(_)
| InvalidProgramInfo::UninitUnsizedLocal => {}
| InvalidProgramInfo::ConstPropNonsense => {}
InvalidProgramInfo::Layout(e) => {
let diag: DiagnosticBuilder<'_, ()> = e.into_diagnostic().into_diagnostic(handler);
for (name, val) in diag.args() {

5
compiler/rustc_const_eval/src/interpret/eval_context.rs
View file

@ -1014,9 +1014,12 @@ impl<'a, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> std::fmt::Debug
{
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self.place {
Place::Local { frame, local } => {
Place::Local { frame, local, offset } => {
let mut allocs = Vec::new();
write!(fmt, "{:?}", local)?;
if let Some(offset) = offset {
write!(fmt, "+{:#x}", offset.bytes())?;
}
if frame != self.ecx.frame_idx() {
write!(fmt, " ({} frames up)", self.ecx.frame_idx() - frame)?;
}

109
compiler/rustc_const_eval/src/interpret/operand.rs
View file

@ -13,7 +13,7 @@ use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size};
use super::{
alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId,
InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, Place, PlaceTy, Pointer,
InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, PlaceTy, Pointer,
Provenance, Scalar,
};
@ -240,37 +240,69 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
let int = self.to_scalar().assert_int();
ConstInt::new(int, self.layout.ty.is_signed(), self.layout.ty.is_ptr_sized_integral())
}
/// Compute the "sub-immediate" that is located within the `base` at the given offset with the
/// given layout.
pub(super) fn offset(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> Self {
// This makes several assumptions about what layouts we will encounter; we match what
// codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
let inner_val: Immediate<_> = match (**self, self.layout.abi) {
// if the entire value is uninit, then so is the field (can happen in ConstProp)
(Immediate::Uninit, _) => Immediate::Uninit,
// the field contains no information, can be left uninit
_ if layout.is_zst() => Immediate::Uninit,
// the field covers the entire type
_ if layout.size == self.layout.size => {
assert!(match (self.layout.abi, layout.abi) {
(Abi::Scalar(..), Abi::Scalar(..)) => true,
(Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
_ => false,
});
assert!(offset.bytes() == 0);
**self
}
// extract fields from types with `ScalarPair` ABI
(Immediate::ScalarPair(a_val, b_val), Abi::ScalarPair(a, b)) => {
assert!(matches!(layout.abi, Abi::Scalar(..)));
Immediate::from(if offset.bytes() == 0 {
debug_assert_eq!(layout.size, a.size(cx));
a_val
} else {
debug_assert_eq!(offset, a.size(cx).align_to(b.align(cx).abi));
debug_assert_eq!(layout.size, b.size(cx));
b_val
})
}
// everything else is a bug
_ => bug!("invalid field access on immediate {}, layout {:#?}", self, self.layout),
};
ImmTy::from_immediate(inner_val, layout)
}
}
impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
pub fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
if self.layout.is_unsized() {
if matches!(self.op, Operand::Immediate(Immediate::Uninit)) {
// Uninit unsized places shouldn't occur. In the interpreter we have them
// temporarily for unsized arguments before their value is put in; in ConstProp they
// remain uninit and this code can actually be reached.
throw_inval!(UninitUnsizedLocal);
pub(super) fn meta(&self) -> InterpResult<'tcx, MemPlaceMeta<Prov>> {
Ok(if self.layout.is_unsized() {
if matches!(self.op, Operand::Immediate(_)) {
// Unsized immediate OpTy cannot occur. We create a MemPlace for all unsized locals during argument passing.
// However, ConstProp doesn't do that, so we can run into this nonsense situation.
throw_inval!(ConstPropNonsense);
}
// There are no unsized immediates.
self.assert_mem_place().len(cx)
self.assert_mem_place().meta
} else {
match self.layout.fields {
abi::FieldsShape::Array { count, .. } => Ok(count),
_ => bug!("len not supported on sized type {:?}", self.layout.ty),
}
}
MemPlaceMeta::None
})
}
/// Replace the layout of this operand. There's basically no sanity check that this makes sense,
/// you better know what you are doing! If this is an immediate, applying the wrong layout can
/// not just lead to invalid data, it can actually *shift the data around* since the offsets of
/// a ScalarPair are entirely determined by the layout, not the data.
pub fn transmute(&self, layout: TyAndLayout<'tcx>) -> Self {
assert_eq!(
self.layout.size, layout.size,
"transmuting with a size change, that doesn't seem right"
);
OpTy { layout, ..*self }
pub fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
self.meta()?.len(self.layout, cx)
}
/// Offset the operand in memory (if possible) and change its metadata.
@ -286,13 +318,9 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
match self.as_mplace_or_imm() {
Left(mplace) => Ok(mplace.offset_with_meta(offset, meta, layout, cx)?.into()),
Right(imm) => {
assert!(
matches!(*imm, Immediate::Uninit),
"Scalar/ScalarPair cannot be offset into"
);
assert!(!meta.has_meta()); // no place to store metadata here
// Every part of an uninit is uninit.
Ok(ImmTy::uninit(layout).into())
Ok(imm.offset(offset, layout, cx).into())
}
}
}
@ -502,13 +530,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
&self,
place: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let op = match **place {
Place::Ptr(mplace) => Operand::Indirect(mplace),
Place::Local { frame, local } => {
*self.local_to_op(&self.stack()[frame], local, None)?
match place.as_mplace_or_local() {
Left(mplace) => Ok(mplace.into()),
Right((frame, local, offset)) => {
let base = self.local_to_op(&self.stack()[frame], local, None)?;
let mut field = if let Some(offset) = offset {
// This got offset. We can be sure that the field is sized.
base.offset(offset, place.layout, self)?
} else {
assert_eq!(place.layout, base.layout);
// Unsized cases are possible here since an unsized local will be a
// `Place::Local` until the first projection calls `place_to_op` to extract the
// underlying mplace.
base
};
field.align = Some(place.align);
Ok(field)
}
};
Ok(OpTy { op, layout: place.layout, align: Some(place.align) })
}
}
/// Evaluate a place with the goal of reading from it. This lets us sometimes

225
compiler/rustc_const_eval/src/interpret/place.rs
View file

@ -2,11 +2,14 @@
//! into a place.
//! All high-level functions to write to memory work on places as destinations.
use std::assert_matches::assert_matches;
use either::{Either, Left, Right};
use rustc_ast::Mutability;
use rustc_index::IndexSlice;
use rustc_middle::mir;
use rustc_middle::mir::interpret::PointerArithmetic;
use rustc_middle::ty;
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
use rustc_middle::ty::Ty;
@ -44,6 +47,27 @@ impl<Prov: Provenance> MemPlaceMeta<Prov> {
Self::None => false,
}
}
pub(crate) fn len<'tcx>(
&self,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, u64> {
if layout.is_unsized() {
// We need to consult `meta` metadata
match layout.ty.kind() {
ty::Slice(..) | ty::Str => self.unwrap_meta().to_target_usize(cx),
_ => bug!("len not supported on unsized type {:?}", layout.ty),
}
} else {
// Go through the layout. There are lots of types that support a length,
// e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!)
match layout.fields {
abi::FieldsShape::Array { count, .. } => Ok(count),
_ => bug!("len not supported on sized type {:?}", layout.ty),
}
}
}
}
#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
@ -73,9 +97,13 @@ pub enum Place<Prov: Provenance = AllocId> {
/// A place referring to a value allocated in the `Memory` system.
Ptr(MemPlace<Prov>),
/// To support alloc-free locals, we are able to write directly to a local.
/// To support alloc-free locals, we are able to write directly to a local. The offset indicates
/// where in the local this place is located; if it is `None`, no projection has been applied.
/// Such projections are meaningful even if the offset is 0, since they can change layouts.
/// (Without that optimization, we'd just always be a `MemPlace`.)
Local { frame: usize, local: mir::Local },
/// Note that this only stores the frame index, not the thread this frame belongs to -- that is
/// implicit. This means a `Place` must never be moved across interpreter thread boundaries!
Local { frame: usize, local: mir::Local, offset: Option<Size> },
}
#[derive(Clone, Debug)]
@ -132,6 +160,11 @@ impl<Prov: Provenance> MemPlace<Prov> {
MemPlace { ptr, meta: MemPlaceMeta::None }
}
#[inline(always)]
pub fn from_ptr_with_meta(ptr: Pointer<Option<Prov>>, meta: MemPlaceMeta<Prov>) -> Self {
MemPlace { ptr, meta }
}
/// Adjust the provenance of the main pointer (metadata is unaffected).
pub fn map_provenance(self, f: impl FnOnce(Option<Prov>) -> Option<Prov>) -> Self {
MemPlace { ptr: self.ptr.map_provenance(f), ..self }
@ -150,7 +183,7 @@ impl<Prov: Provenance> MemPlace<Prov> {
}
#[inline]
pub(super) fn offset_with_meta<'tcx>(
fn offset_with_meta<'tcx>(
self,
offset: Size,
meta: MemPlaceMeta<Prov>,
@ -162,18 +195,10 @@ impl<Prov: Provenance> MemPlace<Prov> {
);
Ok(MemPlace { ptr: self.ptr.offset(offset, cx)?, meta })
}
}
impl<Prov: Provenance> Place<Prov> {
/// Asserts that this points to some local variable.
/// Returns the frame idx and the variable idx.
#[inline]
#[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
pub fn assert_local(&self) -> (usize, mir::Local) {
match self {
Place::Local { frame, local } => (*frame, *local),
_ => bug!("assert_local: expected Place::Local, got {:?}", self),
}
fn offset<'tcx>(&self, offset: Size, cx: &impl HasDataLayout) -> InterpResult<'tcx, Self> {
self.offset_with_meta(offset, MemPlaceMeta::None, cx)
}
}
@ -231,28 +256,16 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
layout: TyAndLayout<'tcx>,
meta: MemPlaceMeta<Prov>,
) -> Self {
let mut mplace = MemPlace::from_ptr(ptr);
mplace.meta = meta;
MPlaceTy { mplace, layout, align: layout.align.abi }
MPlaceTy {
mplace: MemPlace::from_ptr_with_meta(ptr, meta),
layout,
align: layout.align.abi,
}
}
#[inline]
pub(crate) fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
if self.layout.is_unsized() {
// We need to consult `meta` metadata
match self.layout.ty.kind() {
ty::Slice(..) | ty::Str => self.mplace.meta.unwrap_meta().to_target_usize(cx),
_ => bug!("len not supported on unsized type {:?}", self.layout.ty),
}
} else {
// Go through the layout. There are lots of types that support a length,
// e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!)
match self.layout.fields {
abi::FieldsShape::Array { count, .. } => Ok(count),
_ => bug!("len not supported on sized type {:?}", self.layout.ty),
}
}
self.mplace.meta.len(self.layout, cx)
}
}
@ -283,10 +296,12 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
/// A place is either an mplace or some local.
#[inline]
pub fn as_mplace_or_local(&self) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local)> {
pub fn as_mplace_or_local(
&self,
) -> Either<MPlaceTy<'tcx, Prov>, (usize, mir::Local, Option<Size>)> {
match **self {
Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout, align: self.align }),
Place::Local { frame, local } => Right((frame, local)),
Place::Local { frame, local, offset } => Right((frame, local, offset)),
}
}
@ -300,6 +315,49 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
)
})
}
/// Offset the place in memory and change its metadata.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub(crate) fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
Ok(match self.as_mplace_or_local() {
Left(mplace) => mplace.offset_with_meta(offset, meta, layout, cx)?.into(),
Right((frame, local, old_offset)) => {
assert_matches!(meta, MemPlaceMeta::None); // we couldn't store it anyway...
let new_offset = cx
.data_layout()
.offset(old_offset.unwrap_or(Size::ZERO).bytes(), offset.bytes())?;
PlaceTy {
place: Place::Local {
frame,
local,
offset: Some(Size::from_bytes(new_offset)),
},
align: self.align.restrict_for_offset(offset),
layout,
}
}
})
}
/// Offset the place in memory.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub fn offset(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert!(layout.is_sized());
self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
}
}
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
@ -308,6 +366,20 @@ where
Prov: Provenance + 'static,
M: Machine<'mir, 'tcx, Provenance = Prov>,
{
/// Get the metadata of the given place.
pub(super) fn place_meta(
&self,
place: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, MemPlaceMeta<M::Provenance>> {
if place.layout.is_unsized() {
// For `Place::Local`, the metadata is stored with the local, not the place. So we have
// to look that up first.
self.place_to_op(place)?.meta()
} else {
Ok(MemPlaceMeta::None)
}
}
/// Take a value, which represents a (thin or wide) reference, and make it a place.
/// Alignment is just based on the type. This is the inverse of `MemPlace::to_ref()`.
///
@ -327,11 +399,9 @@ where
Immediate::Uninit => throw_ub!(InvalidUninitBytes(None)),
};
let mplace = MemPlace { ptr: ptr.to_pointer(self)?, meta };
// `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced;
// we hence can't call `size_and_align_of` since that asserts more validity than we want.
let align = layout.align.abi;
Ok(MPlaceTy { mplace, layout, align })
Ok(MPlaceTy::from_aligned_ptr_with_meta(ptr.to_pointer(self)?, layout, meta))
}
/// Take an operand, representing a pointer, and dereference it to a place.
@ -422,7 +492,7 @@ where
local: mir::Local,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let layout = self.layout_of_local(&self.stack()[frame], local, None)?;
let place = Place::Local { frame, local };
let place = Place::Local { frame, local, offset: None };
Ok(PlaceTy { place, layout, align: layout.align.abi })
}
@ -430,7 +500,7 @@ where
/// place; for reading, a more efficient alternative is `eval_place_to_op`.
#[instrument(skip(self), level = "debug")]
pub fn eval_place(
&mut self,
&self,
mir_place: mir::Place<'tcx>,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let mut place = self.local_to_place(self.frame_idx(), mir_place.local)?;
@ -509,16 +579,23 @@ where
// See if we can avoid an allocation. This is the counterpart to `read_immediate_raw`,
// but not factored as a separate function.
let mplace = match dest.place {
Place::Local { frame, local } => {
match M::access_local_mut(self, frame, local)? {
Operand::Immediate(local) => {
// Local can be updated in-place.
*local = src;
return Ok(());
}
Operand::Indirect(mplace) => {
// The local is in memory, go on below.
*mplace
Place::Local { frame, local, offset } => {
if offset.is_some() {
// This has been projected to a part of this local. We could have complicated
// logic to still keep this local as an `Operand`... but it's much easier to
// just fall back to the indirect path.
*self.force_allocation(dest)?
} else {
match M::access_local_mut(self, frame, local)? {
Operand::Immediate(local) => {
// Local can be updated in-place.
*local = src;
return Ok(());
}
Operand::Indirect(mplace) => {
// The local is in memory, go on below.
*mplace
}
}
}
}
@ -593,15 +670,23 @@ where
pub fn write_uninit(&mut self, dest: &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
let mplace = match dest.as_mplace_or_local() {
Left(mplace) => mplace,
Right((frame, local)) => {
match M::access_local_mut(self, frame, local)? {
Operand::Immediate(local) => {
*local = Immediate::Uninit;
return Ok(());
}
Operand::Indirect(mplace) => {
// The local is in memory, go on below.
MPlaceTy { mplace: *mplace, layout: dest.layout, align: dest.align }
Right((frame, local, offset)) => {
if offset.is_some() {
// This has been projected to a part of this local. We could have complicated
// logic to still keep this local as an `Operand`... but it's much easier to
// just fall back to the indirect path.
// FIXME: share the logic with `write_immediate_no_validate`.
self.force_allocation(dest)?
} else {
match M::access_local_mut(self, frame, local)? {
Operand::Immediate(local) => {
*local = Immediate::Uninit;
return Ok(());
}
Operand::Indirect(mplace) => {
// The local is in memory, go on below.
MPlaceTy { mplace: *mplace, layout: dest.layout, align: dest.align }
}
}
}
}
@ -728,8 +813,8 @@ where
place: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
let mplace = match place.place {
Place::Local { frame, local } => {
match M::access_local_mut(self, frame, local)? {
Place::Local { frame, local, offset } => {
let whole_local = match M::access_local_mut(self, frame, local)? {
&mut Operand::Immediate(local_val) => {
// We need to make an allocation.
@ -742,10 +827,11 @@ where
throw_unsup_format!("unsized locals are not supported");
}
let mplace = *self.allocate(local_layout, MemoryKind::Stack)?;
// Preserve old value. (As an optimization, we can skip this if it was uninit.)
if !matches!(local_val, Immediate::Uninit) {
// Preserve old value. (As an optimization, we can skip this if it was uninit.)
// We don't have to validate as we can assume the local
// was already valid for its type.
// We don't have to validate as we can assume the local was already
// valid for its type. We must not use any part of `place` here, that
// could be a projection to a part of the local!
self.write_immediate_to_mplace_no_validate(
local_val,
local_layout,
@ -753,18 +839,25 @@ where
mplace,
)?;
}
// Now we can call `access_mut` again, asserting it goes well,
// and actually overwrite things.
// Now we can call `access_mut` again, asserting it goes well, and actually
// overwrite things. This points to the entire allocation, not just the part
// the place refers to, i.e. we do this before we apply `offset`.
*M::access_local_mut(self, frame, local).unwrap() =
Operand::Indirect(mplace);
mplace
}
&mut Operand::Indirect(mplace) => mplace, // this already was an indirect local
};
if let Some(offset) = offset {
whole_local.offset(offset, self)?
} else {
// Preserve wide place metadata, do not call `offset`.
whole_local
}
}
Place::Ptr(mplace) => mplace,
};
// Return with the original layout, so that the caller can go on
// Return with the original layout and align, so that the caller can go on
Ok(MPlaceTy { mplace, layout: place.layout, align: place.align })
}
@ -874,7 +967,7 @@ where
let vtable = self.read_pointer(&vtable)?;
let (ty, _) = self.get_ptr_vtable(vtable)?;
let layout = self.layout_of(ty)?;
let data = data.transmute(layout);
let data = data.offset(Size::ZERO, layout, self)?;
Ok((data, vtable))
}
}

326
compiler/rustc_const_eval/src/interpret/projection.rs
View file

@ -7,17 +7,15 @@
//! but we still need to do bounds checking and adjust the layout. To not duplicate that with MPlaceTy, we actually
//! implement the logic on OpTy, and MPlaceTy calls that.
use either::{Left, Right};
use rustc_middle::mir;
use rustc_middle::ty;
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
use rustc_middle::ty::Ty;
use rustc_target::abi::{self, Abi, VariantIdx};
use rustc_target::abi::Size;
use rustc_target::abi::{self, VariantIdx};
use super::{
ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, PlaceTy,
Provenance, Scalar,
InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, PlaceTy, Provenance, Scalar,
};
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
@ -28,6 +26,43 @@ where
{
//# Field access
fn project_field(
&self,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
field: usize,
) -> InterpResult<'tcx, (Size, MemPlaceMeta<M::Provenance>, TyAndLayout<'tcx>)> {
let offset = base_layout.fields.offset(field);
let field_layout = base_layout.field(self, field);
// Offset may need adjustment for unsized fields.
let (meta, offset) = if field_layout.is_unsized() {
if base_layout.is_sized() {
// An unsized field of a sized type? Sure...
// But const-prop actually feeds us such nonsense MIR!
throw_inval!(ConstPropNonsense);
}
// Re-use parent metadata to determine dynamic field layout.
// With custom DSTS, this *will* execute user-defined code, but the same
// happens at run-time so that's okay.
match self.size_and_align_of(&base_meta, &field_layout)? {
Some((_, align)) => (base_meta, offset.align_to(align)),
None => {
// For unsized types with an extern type tail we perform no adjustments.
// NOTE: keep this in sync with `PlaceRef::project_field` in the codegen backend.
assert!(matches!(base_meta, MemPlaceMeta::None));
(base_meta, offset)
}
}
} else {
// base_meta could be present; we might be accessing a sized field of an unsized
// struct.
(MemPlaceMeta::None, offset)
};
Ok((offset, meta, field_layout))
}
/// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is
/// always possible without allocating, so it can take `&self`. Also return the field's layout.
/// This supports both struct and array fields.
@ -39,28 +74,7 @@ where
base: &MPlaceTy<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
let offset = base.layout.fields.offset(field);
let field_layout = base.layout.field(self, field);
// Offset may need adjustment for unsized fields.
let (meta, offset) = if field_layout.is_unsized() {
// Re-use parent metadata to determine dynamic field layout.
// With custom DSTS, this *will* execute user-defined code, but the same
// happens at run-time so that's okay.
match self.size_and_align_of(&base.meta, &field_layout)? {
Some((_, align)) => (base.meta, offset.align_to(align)),
None => {
// For unsized types with an extern type tail we perform no adjustments.
// NOTE: keep this in sync with `PlaceRef::project_field` in the codegen backend.
assert!(matches!(base.meta, MemPlaceMeta::None));
(base.meta, offset)
}
}
} else {
// base.meta could be present; we might be accessing a sized field of an unsized
// struct.
(MemPlaceMeta::None, offset)
};
let (offset, meta, field_layout) = self.project_field(base.layout, base.meta, field)?;
// We do not look at `base.layout.align` nor `field_layout.align`, unlike
// codegen -- mostly to see if we can get away with that
@ -68,18 +82,14 @@ where
}
/// Gets the place of a field inside the place, and also the field's type.
/// Just a convenience function, but used quite a bit.
/// This is the only projection that might have a side-effect: We cannot project
/// into the field of a local `ScalarPair`, we have to first allocate it.
pub fn place_field(
&mut self,
&self,
base: &PlaceTy<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
// FIXME: We could try to be smarter and avoid allocation for fields that span the
// entire place.
let base = self.force_allocation(base)?;
Ok(self.mplace_field(&base, field)?.into())
let (offset, meta, field_layout) =
self.project_field(base.layout, self.place_meta(base)?, field)?;
base.offset_with_meta(offset, meta, field_layout, self)
}
pub fn operand_field(
@ -87,56 +97,8 @@ where
base: &OpTy<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let base = match base.as_mplace_or_imm() {
Left(ref mplace) => {
// We can reuse the mplace field computation logic for indirect operands.
let field = self.mplace_field(mplace, field)?;
return Ok(field.into());
}
Right(value) => value,
};
let field_layout = base.layout.field(self, field);
let offset = base.layout.fields.offset(field);
// This makes several assumptions about what layouts we will encounter; we match what
// codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
let field_val: Immediate<_> = match (*base, base.layout.abi) {
// if the entire value is uninit, then so is the field (can happen in ConstProp)
(Immediate::Uninit, _) => Immediate::Uninit,
// the field contains no information, can be left uninit
_ if field_layout.is_zst() => Immediate::Uninit,
// the field covers the entire type
_ if field_layout.size == base.layout.size => {
assert!(match (base.layout.abi, field_layout.abi) {
(Abi::Scalar(..), Abi::Scalar(..)) => true,
(Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
_ => false,
});
assert!(offset.bytes() == 0);
*base
}
// extract fields from types with `ScalarPair` ABI
(Immediate::ScalarPair(a_val, b_val), Abi::ScalarPair(a, b)) => {
assert!(matches!(field_layout.abi, Abi::Scalar(..)));
Immediate::from(if offset.bytes() == 0 {
debug_assert_eq!(field_layout.size, a.size(self));
a_val
} else {
debug_assert_eq!(offset, a.size(self).align_to(b.align(self).abi));
debug_assert_eq!(field_layout.size, b.size(self));
b_val
})
}
// everything else is a bug
_ => span_bug!(
self.cur_span(),
"invalid field access on immediate {}, layout {:#?}",
base,
base.layout
),
};
Ok(ImmTy::from_immediate(field_val, field_layout).into())
let (offset, meta, field_layout) = self.project_field(base.layout, base.meta()?, field)?;
base.offset_with_meta(offset, meta, field_layout, self)
}
//# Downcasting
@ -177,7 +139,36 @@ where
Ok(base)
}
//# Slice indexing
//# Slice and array indexing
/// Compute the offset and field layout for accessing the given index.
fn project_index(
&self,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
index: u64,
) -> InterpResult<'tcx, (Size, TyAndLayout<'tcx>)> {
// Not using the layout method because we want to compute on u64
match base_layout.fields {
abi::FieldsShape::Array { stride, count: _ } => {
// `count` is nonsense for slices, use the dynamic length instead.
let len = base_meta.len(base_layout, self)?;
if index >= len {
// This can only be reached in ConstProp and non-rustc-MIR.
throw_ub!(BoundsCheckFailed { len, index });
}
let offset = stride * index; // `Size` multiplication
// All fields have the same layout.
let field_layout = base_layout.field(self, 0);
Ok((offset, field_layout))
}
_ => span_bug!(
self.cur_span(),
"`mplace_index` called on non-array type {:?}",
base_layout.ty
),
}
}
#[inline(always)]
pub fn operand_index(
@ -185,42 +176,8 @@ where
base: &OpTy<'tcx, M::Provenance>,
index: u64,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
// Not using the layout method because we want to compute on u64
match base.layout.fields {
abi::FieldsShape::Array { stride, count: _ } => {
// `count` is nonsense for slices, use the dynamic length instead.
let len = base.len(self)?;
if index >= len {
// This can only be reached in ConstProp and non-rustc-MIR.
throw_ub!(BoundsCheckFailed { len, index });
}
let offset = stride * index; // `Size` multiplication
// All fields have the same layout.
let field_layout = base.layout.field(self, 0);
base.offset(offset, field_layout, self)
}
_ => span_bug!(
self.cur_span(),
"`mplace_index` called on non-array type {:?}",
base.layout.ty
),
}
}
/// Iterates over all fields of an array. Much more efficient than doing the
/// same by repeatedly calling `operand_index`.
pub fn operand_array_fields<'a>(
&self,
base: &'a OpTy<'tcx, Prov>,
) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, OpTy<'tcx, Prov>>> + 'a> {
let len = base.len(self)?; // also asserts that we have a type where this makes sense
let abi::FieldsShape::Array { stride, .. } = base.layout.fields else {
span_bug!(self.cur_span(), "operand_array_fields: expected an array layout");
};
let field_layout = base.layout.field(self, 0);
let dl = &self.tcx.data_layout;
// `Size` multiplication
Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl)))
let (offset, field_layout) = self.project_index(base.layout, base.meta()?, index)?;
base.offset(offset, field_layout, self)
}
/// Index into an array.
@ -229,31 +186,63 @@ where
base: &MPlaceTy<'tcx, M::Provenance>,
index: u64,
) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
Ok(self.operand_index(&base.into(), index)?.assert_mem_place())
let (offset, field_layout) = self.project_index(base.layout, base.meta, index)?;
base.offset(offset, field_layout, self)
}
pub fn place_index(
&mut self,
&self,
base: &PlaceTy<'tcx, M::Provenance>,
index: u64,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
// There's not a lot we can do here, since we cannot have a place to a part of a local. If
// we are accessing the only element of a 1-element array, it's still the entire local...
// that doesn't seem worth it.
let base = self.force_allocation(base)?;
Ok(self.mplace_index(&base, index)?.into())
let (offset, field_layout) =
self.project_index(base.layout, self.place_meta(base)?, index)?;
base.offset(offset, field_layout, self)
}
/// Iterates over all fields of an array. Much more efficient than doing the
/// same by repeatedly calling `operand_index`.
pub fn operand_array_fields<'a>(
&self,
base: &'a OpTy<'tcx, Prov>,
) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, OpTy<'tcx, Prov>>> + 'a> {
let abi::FieldsShape::Array { stride, .. } = base.layout.fields else {
span_bug!(self.cur_span(), "operand_array_fields: expected an array layout");
};
let len = base.len(self)?;
let field_layout = base.layout.field(self, 0);
let dl = &self.tcx.data_layout;
// `Size` multiplication
Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl)))
}
/// Iterates over all fields of an array. Much more efficient than doing the
/// same by repeatedly calling `place_index`.
pub fn place_array_fields<'a>(
&self,
base: &'a PlaceTy<'tcx, Prov>,
) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, PlaceTy<'tcx, Prov>>> + 'a> {
let abi::FieldsShape::Array { stride, .. } = base.layout.fields else {
span_bug!(self.cur_span(), "place_array_fields: expected an array layout");
};
let len = self.place_meta(base)?.len(base.layout, self)?;
let field_layout = base.layout.field(self, 0);
let dl = &self.tcx.data_layout;
// `Size` multiplication
Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl)))
}
//# ConstantIndex support
fn operand_constant_index(
fn project_constant_index(
&self,
base: &OpTy<'tcx, M::Provenance>,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
offset: u64,
min_length: u64,
from_end: bool,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let n = base.len(self)?;
) -> InterpResult<'tcx, (Size, TyAndLayout<'tcx>)> {
let n = base_meta.len(base_layout, self)?;
if n < min_length {
// This can only be reached in ConstProp and non-rustc-MIR.
throw_ub!(BoundsCheckFailed { len: min_length, index: n });
@ -267,33 +256,49 @@ where
offset
};
self.operand_index(base, index)
self.project_index(base_layout, base_meta, index)
}
fn operand_constant_index(
&self,
base: &OpTy<'tcx, M::Provenance>,
offset: u64,
min_length: u64,
from_end: bool,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let (offset, layout) =
self.project_constant_index(base.layout, base.meta()?, offset, min_length, from_end)?;
base.offset(offset, layout, self)
}
fn place_constant_index(
&mut self,
&self,
base: &PlaceTy<'tcx, M::Provenance>,
offset: u64,
min_length: u64,
from_end: bool,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let base = self.force_allocation(base)?;
Ok(self
.operand_constant_index(&base.into(), offset, min_length, from_end)?
.assert_mem_place()
.into())
let (offset, layout) = self.project_constant_index(
base.layout,
self.place_meta(base)?,
offset,
min_length,
from_end,
)?;
base.offset(offset, layout, self)
}
//# Subslicing
fn operand_subslice(
fn project_subslice(
&self,
base: &OpTy<'tcx, M::Provenance>,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
from: u64,
to: u64,
from_end: bool,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let len = base.len(self)?; // also asserts that we have a type where this makes sense
) -> InterpResult<'tcx, (Size, MemPlaceMeta<M::Provenance>, TyAndLayout<'tcx>)> {
let len = base_meta.len(base_layout, self)?; // also asserts that we have a type where this makes sense
let actual_to = if from_end {
if from.checked_add(to).map_or(true, |to| to > len) {
// This can only be reached in ConstProp and non-rustc-MIR.
@ -306,16 +311,16 @@ where
// Not using layout method because that works with usize, and does not work with slices
// (that have count 0 in their layout).
let from_offset = match base.layout.fields {
let from_offset = match base_layout.fields {
abi::FieldsShape::Array { stride, .. } => stride * from, // `Size` multiplication is checked
_ => {
span_bug!(self.cur_span(), "unexpected layout of index access: {:#?}", base.layout)
span_bug!(self.cur_span(), "unexpected layout of index access: {:#?}", base_layout)
}
};
// Compute meta and new layout
let inner_len = actual_to.checked_sub(from).unwrap();
let (meta, ty) = match base.layout.ty.kind() {
let (meta, ty) = match base_layout.ty.kind() {
// It is not nice to match on the type, but that seems to be the only way to
// implement this.
ty::Array(inner, _) => {
@ -323,25 +328,38 @@ where
}
ty::Slice(..) => {
let len = Scalar::from_target_usize(inner_len, self);
(MemPlaceMeta::Meta(len), base.layout.ty)
(MemPlaceMeta::Meta(len), base_layout.ty)
}
_ => {
span_bug!(self.cur_span(), "cannot subslice non-array type: `{:?}`", base.layout.ty)
span_bug!(self.cur_span(), "cannot subslice non-array type: `{:?}`", base_layout.ty)
}
};
let layout = self.layout_of(ty)?;
Ok((from_offset, meta, layout))
}
fn operand_subslice(
&self,
base: &OpTy<'tcx, M::Provenance>,
from: u64,
to: u64,
from_end: bool,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let (from_offset, meta, layout) =
self.project_subslice(base.layout, base.meta()?, from, to, from_end)?;
base.offset_with_meta(from_offset, meta, layout, self)
}
pub fn place_subslice(
&mut self,
&self,
base: &PlaceTy<'tcx, M::Provenance>,
from: u64,
to: u64,
from_end: bool,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let base = self.force_allocation(base)?;
Ok(self.operand_subslice(&base.into(), from, to, from_end)?.assert_mem_place().into())
let (from_offset, meta, layout) =
self.project_subslice(base.layout, self.place_meta(base)?, from, to, from_end)?;
base.offset_with_meta(from_offset, meta, layout, self)
}
//# Applying a general projection
@ -349,7 +367,7 @@ where
/// Projects into a place.
#[instrument(skip(self), level = "trace")]
pub fn place_projection(
&mut self,
&self,
base: &PlaceTy<'tcx, M::Provenance>,
proj_elem: mir::PlaceElem<'tcx>,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {

8
compiler/rustc_const_eval/src/interpret/terminator.rs
View file

@ -60,7 +60,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
pub fn fn_arg_field(
&mut self,
&self,
arg: &FnArg<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
@ -239,7 +239,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Evaluate the arguments of a function call
pub(super) fn eval_fn_call_arguments(
&mut self,
&self,
ops: &[mir::Operand<'tcx>],
) -> InterpResult<'tcx, Vec<FnArg<'tcx, M::Provenance>>> {
ops.iter()
@ -382,12 +382,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// This all has to be in memory, there are no immediate unsized values.
let src = caller_arg_copy.assert_mem_place();
// The destination cannot be one of these "spread args".
let (dest_frame, dest_local) = callee_arg.assert_local();
let (dest_frame, dest_local, dest_offset) =
callee_arg.as_mplace_or_local().right().expect("calee fn arguments must be locals");
// We are just initializing things, so there can't be anything here yet.
assert!(matches!(
*self.local_to_op(&self.stack()[dest_frame], dest_local, None)?,
Operand::Immediate(Immediate::Uninit)
));
assert_eq!(dest_offset, None);
// Allocate enough memory to hold `src`.
let Some((size, align)) = self.size_and_align_of_mplace(&src)? else {
span_bug!(

18
compiler/rustc_const_eval/src/interpret/visitor.rs
View file

@ -78,14 +78,14 @@ pub trait ValueMut<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
/// Projects to the given enum variant.
fn project_downcast(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self>;
/// Projects to the n-th field.
fn project_field(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self>;
}
@ -165,7 +165,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
#[inline(always)]
fn project_downcast(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.operand_downcast(self, variant)
@ -174,7 +174,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
#[inline(always)]
fn project_field(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.operand_field(self, field)
@ -255,7 +255,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
#[inline(always)]
fn project_downcast(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.mplace_downcast(self, variant)
@ -264,7 +264,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
#[inline(always)]
fn project_field(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.mplace_field(self, field)
@ -306,7 +306,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
#[inline(always)]
fn project_downcast(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.place_downcast(self, variant)
@ -315,7 +315,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
#[inline(always)]
fn project_field(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.place_field(self, field)
@ -508,7 +508,7 @@ macro_rules! make_value_visitor {
}
FieldsShape::Array { .. } => {
// Let's get an mplace (or immediate) first.
// This might `force_allocate` if `v` is a `PlaceTy`, but `place_index` does that anyway.
// FIXME: This might `force_allocate` if `v` is a `PlaceTy`!
let op = v.to_op_for_proj(self.ecx())?;
// Now we can go over all the fields.
// This uses the *run-time length*, i.e., if we are a slice,