bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

interpret: refactor projection code to work on a common trait, and use that for visitors

Browse source
This commit is contained in:
Ralf Jung 2023-07-24 11:44:58 +02:00
parent a593de4fab
commit a2bcafa500
44 changed files with 863 additions and 1210 deletions
Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_const_eval/src/const_eval/mod.rs
View file

@ -102,7 +102,7 @@ pub(crate) fn try_destructure_mir_constant_for_diagnostics<'tcx>(
}
ty::Adt(def, _) => {
let variant = ecx.read_discriminant(&op).ok()?.1;
let down = ecx.operand_downcast(&op, variant).ok()?;
let down = ecx.project_downcast(&op, variant).ok()?;
(def.variants()[variant].fields.len(), Some(variant), down)
}
ty::Tuple(args) => (args.len(), None, op),
@ -111,7 +111,7 @@ pub(crate) fn try_destructure_mir_constant_for_diagnostics<'tcx>(
let fields_iter = (0..field_count)
.map(|i| {
let field_op = ecx.operand_field(&down, i).ok()?;
let field_op = ecx.project_field(&down, i).ok()?;
let val = op_to_const(&ecx, &field_op);
Some((val, field_op.layout.ty))
})

20
compiler/rustc_const_eval/src/const_eval/valtrees.rs
View file

@ -2,11 +2,11 @@ use super::eval_queries::{mk_eval_cx, op_to_const};
use super::machine::CompileTimeEvalContext;
use super::{ValTreeCreationError, ValTreeCreationResult, VALTREE_MAX_NODES};
use crate::const_eval::CanAccessStatics;
use crate::interpret::MPlaceTy;
use crate::interpret::{
intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta,
MemoryKind, PlaceTy, Scalar,
MemoryKind, PlaceTy, Projectable, Scalar,
};
use crate::interpret::{MPlaceTy, Value};
use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
use rustc_span::source_map::DUMMY_SP;
use rustc_target::abi::{Align, FieldIdx, VariantIdx, FIRST_VARIANT};
@ -20,7 +20,7 @@ fn branches<'tcx>(
num_nodes: &mut usize,
) -> ValTreeCreationResult<'tcx> {
let place = match variant {
Some(variant) => ecx.mplace_downcast(&place, variant).unwrap(),
Some(variant) => ecx.project_downcast(place, variant).unwrap(),
None => *place,
};
let variant = variant.map(|variant| Some(ty::ValTree::Leaf(ScalarInt::from(variant.as_u32()))));
@ -28,7 +28,7 @@ fn branches<'tcx>(
let mut fields = Vec::with_capacity(n);
for i in 0..n {
let field = ecx.mplace_field(&place, i).unwrap();
let field = ecx.project_field(&place, i).unwrap();
let valtree = const_to_valtree_inner(ecx, &field, num_nodes)?;
fields.push(Some(valtree));
}
@ -55,13 +55,11 @@ fn slice_branches<'tcx>(
place: &MPlaceTy<'tcx>,
num_nodes: &mut usize,
) -> ValTreeCreationResult<'tcx> {
let n = place
.len(&ecx.tcx.tcx)
.unwrap_or_else(|_| panic!("expected to use len of place {:?}", place));
let n = place.len(ecx).unwrap_or_else(|_| panic!("expected to use len of place {:?}", place));
let mut elems = Vec::with_capacity(n as usize);
for i in 0..n {
let place_elem = ecx.mplace_index(place, i).unwrap();
let place_elem = ecx.project_index(place, i).unwrap();
let valtree = const_to_valtree_inner(ecx, &place_elem, num_nodes)?;
elems.push(valtree);
}
@ -386,7 +384,7 @@ fn valtree_into_mplace<'tcx>(
debug!(?variant);
(
place.project_downcast(ecx, variant_idx).unwrap(),
ecx.project_downcast(place, variant_idx).unwrap(),
&branches[1..],
Some(variant_idx),
)
@ -401,7 +399,7 @@ fn valtree_into_mplace<'tcx>(
debug!(?i, ?inner_valtree);
let mut place_inner = match ty.kind() {
ty::Str | ty::Slice(_) => ecx.mplace_index(&place, i as u64).unwrap(),
ty::Str | ty::Slice(_) => ecx.project_index(place, i as u64).unwrap(),
_ if !ty.is_sized(*ecx.tcx, ty::ParamEnv::empty())
&& i == branches.len() - 1 =>
{
@ -441,7 +439,7 @@ fn valtree_into_mplace<'tcx>(
)
.unwrap()
}
_ => ecx.mplace_field(&place_adjusted, i).unwrap(),
_ => ecx.project_field(&place_adjusted, i).unwrap(),
};
debug!(?place_inner);

14
compiler/rustc_const_eval/src/errors.rs
View file

@ -511,7 +511,8 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
InvalidUninitBytes(Some(_)) => const_eval_invalid_uninit_bytes,
DeadLocal => const_eval_dead_local,
ScalarSizeMismatch(_) => const_eval_scalar_size_mismatch,
UninhabitedEnumVariantWritten => const_eval_uninhabited_enum_variant_written,
UninhabitedEnumVariantWritten(_) => const_eval_uninhabited_enum_variant_written,
UninhabitedEnumVariantRead(_) => const_eval_uninhabited_enum_variant_read,
Validation(e) => e.diagnostic_message(),
Custom(x) => (x.msg)(),
}
@ -535,7 +536,8 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
| InvalidMeta(InvalidMetaKind::TooBig)
| InvalidUninitBytes(None)
| DeadLocal
| UninhabitedEnumVariantWritten => {}
| UninhabitedEnumVariantWritten(_)
| UninhabitedEnumVariantRead(_) => {}
BoundsCheckFailed { len, index } => {
builder.set_arg("len", len);
builder.set_arg("index", index);
@ -623,6 +625,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
UnsafeCell => const_eval_unsafe_cell,
UninhabitedVal { .. } => const_eval_uninhabited_val,
InvalidEnumTag { .. } => const_eval_invalid_enum_tag,
UninhabitedEnumTag => const_eval_uninhabited_enum_tag,
UninitEnumTag => const_eval_uninit_enum_tag,
UninitStr => const_eval_uninit_str,
Uninit { expected: ExpectedKind::Bool } => const_eval_uninit_bool,
@ -760,7 +763,8 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
| InvalidMetaSliceTooLarge { .. }
| InvalidMetaTooLarge { .. }
| DanglingPtrUseAfterFree { .. }
| DanglingPtrOutOfBounds { .. } => {}
| DanglingPtrOutOfBounds { .. }
| UninhabitedEnumTag => {}
}
}
}
@ -835,7 +839,9 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
rustc_middle::error::middle_adjust_for_foreign_abi_error
}
InvalidProgramInfo::SizeOfUnsizedType(_) => const_eval_size_of_unsized,
InvalidProgramInfo::ConstPropNonsense => panic!("We had const-prop nonsense, this should never be printed"),
InvalidProgramInfo::ConstPropNonsense => {
panic!("We had const-prop nonsense, this should never be printed")
}
}
}
fn add_args<G: EmissionGuarantee>(

4
compiler/rustc_const_eval/src/interpret/cast.rs
View file

@ -420,8 +420,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
if cast_ty_field.is_zst() {
continue;
}
let src_field = self.operand_field(src, i)?;
let dst_field = self.place_field(dest, i)?;
let src_field = self.project_field(src, i)?;
let dst_field = self.project_field(dest, i)?;
if src_field.layout.ty == cast_ty_field.ty {
self.copy_op(&src_field, &dst_field, /*allow_transmute*/ false)?;
} else {

28
compiler/rustc_const_eval/src/interpret/discriminant.rs
View file

@ -22,7 +22,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// When evaluating we will always error before even getting here, but ConstProp 'executes'
// dead code, so we cannot ICE here.
if dest.layout.for_variant(self, variant_index).abi.is_uninhabited() {
throw_ub!(UninhabitedEnumVariantWritten)
throw_ub!(UninhabitedEnumVariantWritten(variant_index))
}
match dest.layout.variants {
@ -47,7 +47,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let size = tag_layout.size(self);
let tag_val = size.truncate(discr_val);
let tag_dest = self.place_field(dest, tag_field)?;
let tag_dest = self.project_field(dest, tag_field)?;
self.write_scalar(Scalar::from_uint(tag_val, size), &tag_dest)?;
}
abi::Variants::Multiple {
@ -78,7 +78,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
&niche_start_val,
)?;
// Write result.
let niche_dest = self.place_field(dest, tag_field)?;
let niche_dest = self.project_field(dest, tag_field)?;
self.write_immediate(*tag_val, &niche_dest)?;
}
}
@ -106,6 +106,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// straight-forward (`TagEncoding::Direct`) or with a niche (`TagEncoding::Niche`).
let (tag_scalar_layout, tag_encoding, tag_field) = match op.layout.variants {
Variants::Single { index } => {
// Hilariously, `Single` is used even for 0-variant enums.
// (See https://github.com/rust-lang/rust/issues/89765).
if matches!(op.layout.ty.kind(), ty::Adt(def, ..) if def.variants().is_empty()) {
throw_ub!(UninhabitedEnumVariantRead(index))
}
let discr = match op.layout.ty.discriminant_for_variant(*self.tcx, index) {
Some(discr) => {
// This type actually has discriminants.
@ -118,6 +123,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Scalar::from_uint(index.as_u32(), discr_layout.size)
}
};
// For consisteny with `write_discriminant`, and to make sure that
// `project_downcast` cannot fail due to strange layouts, we declare immediate UB
// for uninhabited variants.
if op.layout.ty.is_enum() && op.layout.for_variant(self, index).abi.is_uninhabited() {
throw_ub!(UninhabitedEnumVariantRead(index))
}
return Ok((discr, index));
}
Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
@ -138,13 +149,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let tag_layout = self.layout_of(tag_scalar_layout.primitive().to_int_ty(*self.tcx))?;
// Read tag and sanity-check `tag_layout`.
let tag_val = self.read_immediate(&self.operand_field(op, tag_field)?)?;
let tag_val = self.read_immediate(&self.project_field(op, tag_field)?)?;
assert_eq!(tag_layout.size, tag_val.layout.size);
assert_eq!(tag_layout.abi.is_signed(), tag_val.layout.abi.is_signed());
trace!("tag value: {}", tag_val);
// Figure out which discriminant and variant this corresponds to.
Ok(match *tag_encoding {
let (discr, index) = match *tag_encoding {
TagEncoding::Direct => {
let scalar = tag_val.to_scalar();
// Generate a specific error if `tag_val` is not an integer.
@ -232,6 +243,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// encoded in the tag.
(Scalar::from_uint(variant.as_u32(), discr_layout.size), variant)
}
})
};
// For consisteny with `write_discriminant`, and to make sure that `project_downcast` cannot fail due to strange layouts, we declare immediate UB for uninhabited variants.
if op.layout.for_variant(self, index).abi.is_uninhabited() {
throw_ub!(UninhabitedEnumVariantRead(index))
}
Ok((discr, index))
}
}

127
compiler/rustc_const_eval/src/interpret/intern.rs
View file

@ -164,75 +164,6 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
&self.ecx
}
fn visit_aggregate(
&mut self,
mplace: &MPlaceTy<'tcx>,
fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>,
) -> InterpResult<'tcx> {
// We want to walk the aggregate to look for references to intern. While doing that we
// also need to take special care of interior mutability.
//
// As an optimization, however, if the allocation does not contain any references: we don't
// need to do the walk. It can be costly for big arrays for example (e.g. issue #93215).
let is_walk_needed = |mplace: &MPlaceTy<'tcx>| -> InterpResult<'tcx, bool> {
// ZSTs cannot contain pointers, we can avoid the interning walk.
if mplace.layout.is_zst() {
return Ok(false);
}
// Now, check whether this allocation could contain references.
//
// Note, this check may sometimes not be cheap, so we only do it when the walk we'd like
// to avoid could be expensive: on the potentially larger types, arrays and slices,
// rather than on all aggregates unconditionally.
if matches!(mplace.layout.ty.kind(), ty::Array(..) | ty::Slice(..)) {
let Some((size, align)) = self.ecx.size_and_align_of_mplace(&mplace)? else {
// We do the walk if we can't determine the size of the mplace: we may be
// dealing with extern types here in the future.
return Ok(true);
};
// If there is no provenance in this allocation, it does not contain references
// that point to another allocation, and we can avoid the interning walk.
if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? {
if !alloc.has_provenance() {
return Ok(false);
}
} else {
// We're encountering a ZST here, and can avoid the walk as well.
return Ok(false);
}
}
// In the general case, we do the walk.
Ok(true)
};
// If this allocation contains no references to intern, we avoid the potentially costly
// walk.
//
// We can do this before the checks for interior mutability below, because only references
// are relevant in that situation, and we're checking if there are any here.
if !is_walk_needed(mplace)? {
return Ok(());
}
if let Some(def) = mplace.layout.ty.ty_adt_def() {
if def.is_unsafe_cell() {
// We are crossing over an `UnsafeCell`, we can mutate again. This means that
// References we encounter inside here are interned as pointing to mutable
// allocations.
// Remember the `old` value to handle nested `UnsafeCell`.
let old = std::mem::replace(&mut self.inside_unsafe_cell, true);
let walked = self.walk_aggregate(mplace, fields);
self.inside_unsafe_cell = old;
return walked;
}
}
self.walk_aggregate(mplace, fields)
}
fn visit_value(&mut self, mplace: &MPlaceTy<'tcx>) -> InterpResult<'tcx> {
// Handle Reference types, as these are the only types with provenance supported by const eval.
// Raw pointers (and boxes) are handled by the `leftover_allocations` logic.
@ -315,7 +246,63 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
}
Ok(())
} else {
// Not a reference -- proceed recursively.
// Not a reference. Check if we want to recurse.
let is_walk_needed = |mplace: &MPlaceTy<'tcx>| -> InterpResult<'tcx, bool> {
// ZSTs cannot contain pointers, we can avoid the interning walk.
if mplace.layout.is_zst() {
return Ok(false);
}
// Now, check whether this allocation could contain references.
//
// Note, this check may sometimes not be cheap, so we only do it when the walk we'd like
// to avoid could be expensive: on the potentially larger types, arrays and slices,
// rather than on all aggregates unconditionally.
if matches!(mplace.layout.ty.kind(), ty::Array(..) | ty::Slice(..)) {
let Some((size, align)) = self.ecx.size_and_align_of_mplace(&mplace)? else {
// We do the walk if we can't determine the size of the mplace: we may be
// dealing with extern types here in the future.
return Ok(true);
};
// If there is no provenance in this allocation, it does not contain references
// that point to another allocation, and we can avoid the interning walk.
if let Some(alloc) = self.ecx.get_ptr_alloc(mplace.ptr, size, align)? {
if !alloc.has_provenance() {
return Ok(false);
}
} else {
// We're encountering a ZST here, and can avoid the walk as well.
return Ok(false);
}
}
// In the general case, we do the walk.
Ok(true)
};
// If this allocation contains no references to intern, we avoid the potentially costly
// walk.
//
// We can do this before the checks for interior mutability below, because only references
// are relevant in that situation, and we're checking if there are any here.
if !is_walk_needed(mplace)? {
return Ok(());
}
if let Some(def) = mplace.layout.ty.ty_adt_def() {
if def.is_unsafe_cell() {
// We are crossing over an `UnsafeCell`, we can mutate again. This means that
// References we encounter inside here are interned as pointing to mutable
// allocations.
// Remember the `old` value to handle nested `UnsafeCell`.
let old = std::mem::replace(&mut self.inside_unsafe_cell, true);
let walked = self.walk_value(mplace);
self.inside_unsafe_cell = old;
return walked;
}
}
self.walk_value(mplace)
}
}

6
compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file

@ -425,11 +425,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
);
for i in 0..dest_len {
let place = self.mplace_index(&dest, i)?;
let place = self.project_index(&dest, i)?;
let value = if i == index {
elem.clone()
} else {
self.mplace_index(&input, i)?.into()
self.project_index(&input, i)?.into()
};
self.copy_op(&value, &place.into(), /*allow_transmute*/ false)?;
}
@ -444,7 +444,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
input_len
);
self.copy_op(
&self.mplace_index(&input, index)?.into(),
&self.project_index(&input, index)?.into(),
dest,
/*allow_transmute*/ false,
)?;

6
compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
View file

@ -101,11 +101,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let location = self.allocate(loc_layout, MemoryKind::CallerLocation).unwrap();
// Initialize fields.
self.write_immediate(file.to_ref(self), &self.mplace_field(&location, 0).unwrap().into())
self.write_immediate(file.to_ref(self), &self.project_field(&location, 0).unwrap().into())
.expect("writing to memory we just allocated cannot fail");
self.write_scalar(line, &self.mplace_field(&location, 1).unwrap().into())
self.write_scalar(line, &self.project_field(&location, 1).unwrap().into())
.expect("writing to memory we just allocated cannot fail");
self.write_scalar(col, &self.mplace_field(&location, 2).unwrap().into())
self.write_scalar(col, &self.project_field(&location, 2).unwrap().into())
.expect("writing to memory we just allocated cannot fail");
location

3
compiler/rustc_const_eval/src/interpret/mod.rs
View file

@ -26,9 +26,10 @@ pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackP
pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
pub use self::operand::{ImmTy, Immediate, OpTy, Operand};
pub use self::place::{MPlaceTy, MemPlace, MemPlaceMeta, Place, PlaceTy};
pub use self::projection::Projectable;
pub use self::terminator::FnArg;
pub use self::validity::{CtfeValidationMode, RefTracking};
pub use self::visitor::{MutValueVisitor, Value, ValueVisitor};
pub use self::visitor::ValueVisitor;
pub(crate) use self::intrinsics::eval_nullary_intrinsic;
use eval_context::{from_known_layout, mir_assign_valid_types};

128
compiler/rustc_const_eval/src/interpret/operand.rs
View file

@ -1,6 +1,8 @@
//! Functions concerning immediate values and operands, and reading from operands.
//! All high-level functions to read from memory work on operands as sources.
use std::assert_matches::assert_matches;
use either::{Either, Left, Right};
use rustc_hir::def::Namespace;
@ -14,7 +16,7 @@ use rustc_target::abi::{self, Abi, Align, HasDataLayout, Size};
use super::{
alloc_range, from_known_layout, mir_assign_valid_types, AllocId, ConstValue, Frame, GlobalId,
InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, PlaceTy, Pointer,
Provenance, Scalar,
Projectable, Provenance, Scalar,
};
/// An `Immediate` represents a single immediate self-contained Rust value.
@ -199,6 +201,20 @@ impl<'tcx, Prov: Provenance> From<ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
}
}
impl<'tcx, Prov: Provenance> From<&'_ ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
#[inline(always)]
fn from(val: &ImmTy<'tcx, Prov>) -> Self {
OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None }
}
}
impl<'tcx, Prov: Provenance> From<&'_ mut ImmTy<'tcx, Prov>> for OpTy<'tcx, Prov> {
#[inline(always)]
fn from(val: &mut ImmTy<'tcx, Prov>) -> Self {
OpTy { op: Operand::Immediate(val.imm), layout: val.layout, align: None }
}
}
impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
#[inline]
pub fn from_scalar(val: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
@ -243,12 +259,8 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
/// Compute the "sub-immediate" that is located within the `base` at the given offset with the
/// given layout.
pub(super) fn offset(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> Self {
// Not called `offset` to avoid confusion with the trait method.
fn offset_(&self, offset: Size, layout: TyAndLayout<'tcx>, cx: &impl HasDataLayout) -> Self {
// This makes several assumptions about what layouts we will encounter; we match what
// codegen does as good as we can (see `extract_field` in `rustc_codegen_ssa/src/mir/operand.rs`).
let inner_val: Immediate<_> = match (**self, self.layout.abi) {
@ -256,14 +268,28 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
(Immediate::Uninit, _) => Immediate::Uninit,
// the field contains no information, can be left uninit
_ if layout.is_zst() => Immediate::Uninit,
// some fieldless enum variants can have non-zero size but still `Aggregate` ABI... try
// to detect those here and also give them no data
_ if matches!(layout.abi, Abi::Aggregate { .. })
&& matches!(&layout.fields, abi::FieldsShape::Arbitrary { offsets, .. } if offsets.len() == 0) =>
{
Immediate::Uninit
}
// the field covers the entire type
_ if layout.size == self.layout.size => {
assert!(match (self.layout.abi, layout.abi) {
(Abi::Scalar(..), Abi::Scalar(..)) => true,
(Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
_ => false,
});
assert!(offset.bytes() == 0);
assert_eq!(offset.bytes(), 0);
assert!(
match (self.layout.abi, layout.abi) {
(Abi::Scalar(..), Abi::Scalar(..)) => true,
(Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
_ => false,
},
"cannot project into {} immediate with equally-sized field {}\nouter ABI: {:#?}\nfield ABI: {:#?}",
self.layout.ty,
layout.ty,
self.layout.abi,
layout.abi,
);
**self
}
// extract fields from types with `ScalarPair` ABI
@ -286,8 +312,42 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
}
}
impl<'mir, 'tcx: 'mir, Prov: Provenance> Projectable<'mir, 'tcx, Prov> for ImmTy<'tcx, Prov> {
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
fn meta<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, MemPlaceMeta<M::Provenance>> {
assert!(self.layout.is_sized()); // unsized ImmTy can only exist temporarily and should never reach this here
Ok(MemPlaceMeta::None)
}
fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert_matches!(meta, MemPlaceMeta::None); // we can't store this anywhere anyway
Ok(self.offset_(offset, layout, cx))
}
fn to_op<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.into())
}
}
impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
pub(super) fn meta(&self) -> InterpResult<'tcx, MemPlaceMeta<Prov>> {
// Provided as inherent method since it doesn't need the `ecx` of `Projectable::meta`.
pub fn meta(&self) -> InterpResult<'tcx, MemPlaceMeta<Prov>> {
Ok(if self.layout.is_unsized() {
if matches!(self.op, Operand::Immediate(_)) {
// Unsized immediate OpTy cannot occur. We create a MemPlace for all unsized locals during argument passing.
@ -300,15 +360,24 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
MemPlaceMeta::None
})
}
}
pub fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
self.meta()?.len(self.layout, cx)
impl<'mir, 'tcx: 'mir, Prov: Provenance + 'static> Projectable<'mir, 'tcx, Prov>
for OpTy<'tcx, Prov>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
/// Offset the operand in memory (if possible) and change its metadata.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub(super) fn offset_with_meta(
fn meta<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, MemPlaceMeta<M::Provenance>> {
self.meta()
}
fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
@ -320,22 +389,16 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
Right(imm) => {
assert!(!meta.has_meta()); // no place to store metadata here
// Every part of an uninit is uninit.
Ok(imm.offset(offset, layout, cx).into())
Ok(imm.offset(offset, layout, cx)?.into())
}
}
}
/// Offset the operand in memory (if possible).
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub fn offset(
fn to_op<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert!(layout.is_sized());
self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone())
}
}
@ -525,7 +588,6 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Every place can be read from, so we can turn them into an operand.
/// This will definitely return `Indirect` if the place is a `Ptr`, i.e., this
/// will never actually read from memory.
#[inline(always)]
pub fn place_to_op(
&self,
place: &PlaceTy<'tcx, M::Provenance>,
@ -564,7 +626,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let mut op = self.local_to_op(self.frame(), mir_place.local, layout)?;
// Using `try_fold` turned out to be bad for performance, hence the loop.
for elem in mir_place.projection.iter() {
op = self.operand_projection(&op, elem)?
op = self.project(&op, elem)?
}
trace!("eval_place_to_op: got {:?}", *op);

4
compiler/rustc_const_eval/src/interpret/operator.rs
View file

@ -38,9 +38,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// With randomized layout, `(int, bool)` might cease to be a `ScalarPair`, so we have to
// do a component-wise write here. This code path is slower than the above because
// `place_field` will have to `force_allocate` locals here.
let val_field = self.place_field(&dest, 0)?;
let val_field = self.project_field(dest, 0)?;
self.write_scalar(val, &val_field)?;
let overflowed_field = self.place_field(&dest, 1)?;
let overflowed_field = self.project_field(dest, 1)?;
self.write_scalar(Scalar::from_bool(overflowed), &overflowed_field)?;
}
Ok(())

146
compiler/rustc_const_eval/src/interpret/place.rs
View file

@ -18,7 +18,7 @@ use rustc_target::abi::{self, Abi, Align, FieldIdx, HasDataLayout, Size, FIRST_V
use super::{
alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckInAllocMsg,
ConstAlloc, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, OpTy, Operand,
Pointer, Provenance, Scalar,
Pointer, Projectable, Provenance, Scalar,
};
#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
@ -183,7 +183,8 @@ impl<Prov: Provenance> MemPlace<Prov> {
}
#[inline]
fn offset_with_meta<'tcx>(
// Not called `offset_with_meta` to avoid confusion with the trait method.
fn offset_with_meta_<'tcx>(
self,
offset: Size,
meta: MemPlaceMeta<Prov>,
@ -195,11 +196,6 @@ impl<Prov: Provenance> MemPlace<Prov> {
);
Ok(MemPlace { ptr: self.ptr.offset(offset, cx)?, meta })
}
#[inline]
fn offset<'tcx>(&self, offset: Size, cx: &impl HasDataLayout) -> InterpResult<'tcx, Self> {
self.offset_with_meta(offset, MemPlaceMeta::None, cx)
}
}
impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
@ -214,37 +210,6 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
MPlaceTy { mplace: MemPlace { ptr, meta: MemPlaceMeta::None }, layout, align }
}
/// Offset the place in memory and change its metadata.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
#[inline]
pub(crate) fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
Ok(MPlaceTy {
mplace: self.mplace.offset_with_meta(offset, meta, cx)?,
align: self.align.restrict_for_offset(offset),
layout,
})
}
/// Offset the place in memory.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub fn offset(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert!(layout.is_sized());
self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
}
#[inline]
pub fn from_aligned_ptr(ptr: Pointer<Option<Prov>>, layout: TyAndLayout<'tcx>) -> Self {
MPlaceTy { mplace: MemPlace::from_ptr(ptr), layout, align: layout.align.abi }
@ -262,10 +227,42 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
align: layout.align.abi,
}
}
}
#[inline]
pub(crate) fn len(&self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
self.mplace.meta.len(self.layout, cx)
impl<'mir, 'tcx: 'mir, Prov: Provenance + 'static> Projectable<'mir, 'tcx, Prov>
for MPlaceTy<'tcx, Prov>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
fn meta<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, MemPlaceMeta<M::Provenance>> {
Ok(self.meta)
}
fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
Ok(MPlaceTy {
mplace: self.mplace.offset_with_meta_(offset, meta, cx)?,
align: self.align.restrict_for_offset(offset),
layout,
})
}
fn to_op<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.into())
}
}
@ -293,7 +290,7 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
}
}
impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
impl<'tcx, Prov: Provenance + 'static> PlaceTy<'tcx, Prov> {
/// A place is either an mplace or some local.
#[inline]
pub fn as_mplace_or_local(
@ -315,11 +312,24 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
)
})
}
}
/// Offset the place in memory and change its metadata.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub(crate) fn offset_with_meta(
impl<'mir, 'tcx: 'mir, Prov: Provenance + 'static> Projectable<'mir, 'tcx, Prov>
for PlaceTy<'tcx, Prov>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
fn meta<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, MemPlaceMeta<M::Provenance>> {
ecx.place_meta(self)
}
fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
@ -346,17 +356,11 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
})
}
/// Offset the place in memory.
///
/// This can go wrong very easily if you give the wrong layout for the new place!
pub fn offset(
fn to_op<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert!(layout.is_sized());
self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
ecx.place_to_op(self)
}
}
@ -506,7 +510,7 @@ where
let mut place = self.local_to_place(self.frame_idx(), mir_place.local)?;
// Using `try_fold` turned out to be bad for performance, hence the loop.
for elem in mir_place.projection.iter() {
place = self.place_projection(&place, elem)?
place = self.project(&place, elem)?
}
trace!("{:?}", self.dump_place(place.place));
@ -849,7 +853,7 @@ where
&mut Operand::Indirect(mplace) => mplace, // this already was an indirect local
};
if let Some(offset) = offset {
whole_local.offset(offset, self)?
whole_local.offset_with_meta_(offset, MemPlaceMeta::None, self)?
} else {
// Preserve wide place metadata, do not call `offset`.
whole_local
@ -902,7 +906,7 @@ where
self.write_uninit(&dest)?;
let (variant_index, variant_dest, active_field_index) = match *kind {
mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
let variant_dest = self.place_downcast(&dest, variant_index)?;
let variant_dest = self.project_downcast(dest, variant_index)?;
(variant_index, variant_dest, active_field_index)
}
_ => (FIRST_VARIANT, dest.clone(), None),
@ -912,7 +916,7 @@ where
}
for (field_index, operand) in operands.iter_enumerated() {
let field_index = active_field_index.unwrap_or(field_index);
let field_dest = self.place_field(&variant_dest, field_index.as_usize())?;
let field_dest = self.project_field(&variant_dest, field_index.as_usize())?;
let op = self.eval_operand(operand, Some(field_dest.layout))?;
self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
}
@ -952,22 +956,24 @@ where
Ok((mplace, vtable))
}
/// Turn an operand with a `dyn* Trait` type into an operand with the actual dynamic type.
/// Aso returns the vtable.
pub(super) fn unpack_dyn_star(
/// Turn a `dyn* Trait` type into an value with the actual dynamic type.
/// Also returns the vtable.
pub(super) fn unpack_dyn_star<P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
op: &OpTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, (OpTy<'tcx, M::Provenance>, Pointer<Option<M::Provenance>>)> {
val: &P,
) -> InterpResult<'tcx, (P, Pointer<Option<M::Provenance>>)> {
assert!(
matches!(op.layout.ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
matches!(val.layout().ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
"`unpack_dyn_star` only makes sense on `dyn*` types"
);
let data = self.operand_field(&op, 0)?;
let vtable = self.operand_field(&op, 1)?;
let vtable = self.read_pointer(&vtable)?;
let data = self.project_field(val, 0)?;
let vtable = self.project_field(val, 1)?;
let vtable = self.read_pointer(&vtable.to_op(self)?)?;
let (ty, _) = self.get_ptr_vtable(vtable)?;
let layout = self.layout_of(ty)?;
let data = data.offset(Size::ZERO, layout, self)?;
// `data` is already the right thing but has the wrong type. So we transmute it, by
// projecting with offset 0.
let data = data.transmute(layout, self)?;
Ok((data, vtable))
}
}

426
compiler/rustc_const_eval/src/interpret/projection.rs
View file

@ -11,12 +11,67 @@ use rustc_middle::mir;
use rustc_middle::ty;
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
use rustc_middle::ty::Ty;
use rustc_middle::ty::TyCtxt;
use rustc_target::abi::HasDataLayout;
use rustc_target::abi::Size;
use rustc_target::abi::{self, VariantIdx};
use super::{
InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, PlaceTy, Provenance, Scalar,
};
use super::MPlaceTy;
use super::{InterpCx, InterpResult, Machine, MemPlaceMeta, OpTy, Provenance, Scalar};
/// A thing that we can project into, and that has a layout.
pub trait Projectable<'mir, 'tcx: 'mir, Prov: Provenance>: Sized {
/// Get the layout.
fn layout(&self) -> TyAndLayout<'tcx>;
/// Get the metadata of a wide value.
fn meta<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, MemPlaceMeta<M::Provenance>>;
fn len<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, u64> {
self.meta(ecx)?.len(self.layout(), ecx)
}
/// Offset the value by the given amount, replacing the layout and metadata.
fn offset_with_meta(
&self,
offset: Size,
meta: MemPlaceMeta<Prov>,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self>;
fn offset(
&self,
offset: Size,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert!(layout.is_sized());
self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
}
fn transmute(
&self,
layout: TyAndLayout<'tcx>,
cx: &impl HasDataLayout,
) -> InterpResult<'tcx, Self> {
assert_eq!(self.layout().size, layout.size);
self.offset_with_meta(Size::ZERO, MemPlaceMeta::None, layout, cx)
}
/// Convert this to an `OpTy`. This might be an irreversible transformation, but is useful for
/// reading from this thing.
fn to_op<M: Machine<'mir, 'tcx, Provenance = Prov>>(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>;
}
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
impl<'mir, 'tcx: 'mir, Prov, M> InterpCx<'mir, 'tcx, M>
@ -24,24 +79,33 @@ where
Prov: Provenance + 'static,
M: Machine<'mir, 'tcx, Provenance = Prov>,
{
//# Field access
fn project_field(
/// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is
/// always possible without allocating, so it can take `&self`. Also return the field's layout.
/// This supports both struct and array fields, but not slices!
///
/// This also works for arrays, but then the `usize` index type is restricting.
/// For indexing into arrays, use `mplace_index`.
pub fn project_field<P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
base: &P,
field: usize,
) -> InterpResult<'tcx, (Size, MemPlaceMeta<M::Provenance>, TyAndLayout<'tcx>)> {
let offset = base_layout.fields.offset(field);
let field_layout = base_layout.field(self, field);
) -> InterpResult<'tcx, P> {
// Slices nominally have length 0, so they will panic somewhere in `fields.offset`.
debug_assert!(
!matches!(base.layout().ty.kind(), ty::Slice(..)),
"`field` projection called on a slice -- call `index` projection instead"
);
let offset = base.layout().fields.offset(field);
let field_layout = base.layout().field(self, field);
// Offset may need adjustment for unsized fields.
let (meta, offset) = if field_layout.is_unsized() {
if base_layout.is_sized() {
if base.layout().is_sized() {
// An unsized field of a sized type? Sure...
// But const-prop actually feeds us such nonsense MIR!
throw_inval!(ConstPropNonsense);
}
let base_meta = base.meta(self)?;
// Re-use parent metadata to determine dynamic field layout.
// With custom DSTS, this *will* execute user-defined code, but the same
// happens at run-time so that's okay.
@ -60,189 +124,68 @@ where
(MemPlaceMeta::None, offset)
};
Ok((offset, meta, field_layout))
}
/// Offset a pointer to project to a field of a struct/union. Unlike `place_field`, this is
/// always possible without allocating, so it can take `&self`. Also return the field's layout.
/// This supports both struct and array fields.
///
/// This also works for arrays, but then the `usize` index type is restricting.
/// For indexing into arrays, use `mplace_index`.
pub fn mplace_field(
&self,
base: &MPlaceTy<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
let (offset, meta, field_layout) = self.project_field(base.layout, base.meta, field)?;
// We do not look at `base.layout.align` nor `field_layout.align`, unlike
// codegen -- mostly to see if we can get away with that
base.offset_with_meta(offset, meta, field_layout, self)
}
/// Gets the place of a field inside the place, and also the field's type.
pub fn place_field(
/// Downcasting to an enum variant.
pub fn project_downcast<P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
base: &PlaceTy<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let (offset, meta, field_layout) =
self.project_field(base.layout, self.place_meta(base)?, field)?;
base.offset_with_meta(offset, meta, field_layout, self)
}
pub fn operand_field(
&self,
base: &OpTy<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let (offset, meta, field_layout) = self.project_field(base.layout, base.meta()?, field)?;
base.offset_with_meta(offset, meta, field_layout, self)
}
//# Downcasting
pub fn mplace_downcast(
&self,
base: &MPlaceTy<'tcx, M::Provenance>,
base: &P,
variant: VariantIdx,
) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
) -> InterpResult<'tcx, P> {
assert!(!base.meta(self)?.has_meta());
// Downcasts only change the layout.
// (In particular, no check about whether this is even the active variant -- that's by design,
// see https://github.com/rust-lang/rust/issues/93688#issuecomment-1032929496.)
assert!(!base.meta.has_meta());
let mut base = *base;
base.layout = base.layout.for_variant(self, variant);
Ok(base)
// So we just "offset" by 0.
let layout = base.layout().for_variant(self, variant);
if layout.abi.is_uninhabited() {
// `read_discriminant` should have excluded uninhabited variants... but ConstProp calls
// us on dead code.
throw_inval!(ConstPropNonsense)
}
// This cannot be `transmute` as variants *can* have a smaller size than the entire enum.
base.offset(Size::ZERO, layout, self)
}
pub fn place_downcast(
&self,
base: &PlaceTy<'tcx, M::Provenance>,
variant: VariantIdx,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
// Downcast just changes the layout
let mut base = base.clone();
base.layout = base.layout.for_variant(self, variant);
Ok(base)
}
pub fn operand_downcast(
&self,
base: &OpTy<'tcx, M::Provenance>,
variant: VariantIdx,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
// Downcast just changes the layout
let mut base = base.clone();
base.layout = base.layout.for_variant(self, variant);
Ok(base)
}
//# Slice and array indexing
/// Compute the offset and field layout for accessing the given index.
fn project_index(
pub fn project_index<P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
base: &P,
index: u64,
) -> InterpResult<'tcx, (Size, TyAndLayout<'tcx>)> {
) -> InterpResult<'tcx, P> {
// Not using the layout method because we want to compute on u64
match base_layout.fields {
let (offset, field_layout) = match base.layout().fields {
abi::FieldsShape::Array { stride, count: _ } => {
// `count` is nonsense for slices, use the dynamic length instead.
let len = base_meta.len(base_layout, self)?;
let len = base.len(self)?;
if index >= len {
// This can only be reached in ConstProp and non-rustc-MIR.
throw_ub!(BoundsCheckFailed { len, index });
}
let offset = stride * index; // `Size` multiplication
// All fields have the same layout.
let field_layout = base_layout.field(self, 0);
Ok((offset, field_layout))
let field_layout = base.layout().field(self, 0);
(offset, field_layout)
}
_ => span_bug!(
self.cur_span(),
"`mplace_index` called on non-array type {:?}",
base_layout.ty
base.layout().ty
),
}
}
#[inline(always)]
pub fn operand_index(
&self,
base: &OpTy<'tcx, M::Provenance>,
index: u64,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let (offset, field_layout) = self.project_index(base.layout, base.meta()?, index)?;
base.offset(offset, field_layout, self)
}
/// Index into an array.
pub fn mplace_index(
&self,
base: &MPlaceTy<'tcx, M::Provenance>,
index: u64,
) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
let (offset, field_layout) = self.project_index(base.layout, base.meta, index)?;
base.offset(offset, field_layout, self)
}
pub fn place_index(
&self,
base: &PlaceTy<'tcx, M::Provenance>,
index: u64,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let (offset, field_layout) =
self.project_index(base.layout, self.place_meta(base)?, index)?;
base.offset(offset, field_layout, self)
}
/// Iterates over all fields of an array. Much more efficient than doing the
/// same by repeatedly calling `operand_index`.
pub fn operand_array_fields<'a>(
&self,
base: &'a OpTy<'tcx, Prov>,
) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, OpTy<'tcx, Prov>>> + 'a> {
let abi::FieldsShape::Array { stride, .. } = base.layout.fields else {
span_bug!(self.cur_span(), "operand_array_fields: expected an array layout");
};
let len = base.len(self)?;
let field_layout = base.layout.field(self, 0);
let dl = &self.tcx.data_layout;
// `Size` multiplication
Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl)))
base.offset(offset, field_layout, self)
}
/// Iterates over all fields of an array. Much more efficient than doing the
/// same by repeatedly calling `place_index`.
pub fn place_array_fields<'a>(
fn project_constant_index<P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
base: &'a PlaceTy<'tcx, Prov>,
) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, PlaceTy<'tcx, Prov>>> + 'a> {
let abi::FieldsShape::Array { stride, .. } = base.layout.fields else {
span_bug!(self.cur_span(), "place_array_fields: expected an array layout");
};
let len = self.place_meta(base)?.len(base.layout, self)?;
let field_layout = base.layout.field(self, 0);
let dl = &self.tcx.data_layout;
// `Size` multiplication
Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl)))
}
//# ConstantIndex support
fn project_constant_index(
&self,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
base: &P,
offset: u64,
min_length: u64,
from_end: bool,
) -> InterpResult<'tcx, (Size, TyAndLayout<'tcx>)> {
let n = base_meta.len(base_layout, self)?;
) -> InterpResult<'tcx, P> {
let n = base.len(self)?;
if n < min_length {
// This can only be reached in ConstProp and non-rustc-MIR.
throw_ub!(BoundsCheckFailed { len: min_length, index: n });
@ -256,49 +199,39 @@ where
offset
};
self.project_index(base_layout, base_meta, index)
self.project_index(base, index)
}
fn operand_constant_index(
/// Iterates over all fields of an array. Much more efficient than doing the
/// same by repeatedly calling `operand_index`.
pub fn project_array_fields<'a, P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
base: &OpTy<'tcx, M::Provenance>,
offset: u64,
min_length: u64,
from_end: bool,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let (offset, layout) =
self.project_constant_index(base.layout, base.meta()?, offset, min_length, from_end)?;
base.offset(offset, layout, self)
base: &'a P,
) -> InterpResult<'tcx, impl Iterator<Item = InterpResult<'tcx, P>> + 'a>
where
'tcx: 'a,
{
let abi::FieldsShape::Array { stride, .. } = base.layout().fields else {
span_bug!(self.cur_span(), "operand_array_fields: expected an array layout");
};
let len = base.len(self)?;
let field_layout = base.layout().field(self, 0);
let tcx: TyCtxt<'tcx> = *self.tcx;
// `Size` multiplication
Ok((0..len).map(move |i| {
base.offset_with_meta(stride * i, MemPlaceMeta::None, field_layout, &tcx)
}))
}
fn place_constant_index(
/// Subslicing
fn project_subslice<P: Projectable<'mir, 'tcx, M::Provenance>>(
&self,
base: &PlaceTy<'tcx, M::Provenance>,
offset: u64,
min_length: u64,
from_end: bool,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let (offset, layout) = self.project_constant_index(
base.layout,
self.place_meta(base)?,
offset,
min_length,
from_end,
)?;
base.offset(offset, layout, self)
}
//# Subslicing
fn project_subslice(
&self,
base_layout: TyAndLayout<'tcx>,
base_meta: MemPlaceMeta<M::Provenance>,
base: &P,
from: u64,
to: u64,
from_end: bool,
) -> InterpResult<'tcx, (Size, MemPlaceMeta<M::Provenance>, TyAndLayout<'tcx>)> {
let len = base_meta.len(base_layout, self)?; // also asserts that we have a type where this makes sense
) -> InterpResult<'tcx, P> {
let len = base.len(self)?; // also asserts that we have a type where this makes sense
let actual_to = if from_end {
if from.checked_add(to).map_or(true, |to| to > len) {
// This can only be reached in ConstProp and non-rustc-MIR.
@ -311,16 +244,20 @@ where
// Not using layout method because that works with usize, and does not work with slices
// (that have count 0 in their layout).
let from_offset = match base_layout.fields {
let from_offset = match base.layout().fields {
abi::FieldsShape::Array { stride, .. } => stride * from, // `Size` multiplication is checked
_ => {
span_bug!(self.cur_span(), "unexpected layout of index access: {:#?}", base_layout)
span_bug!(
self.cur_span(),
"unexpected layout of index access: {:#?}",
base.layout()
)
}
};
// Compute meta and new layout
let inner_len = actual_to.checked_sub(from).unwrap();
let (meta, ty) = match base_layout.ty.kind() {
let (meta, ty) = match base.layout().ty.kind() {
// It is not nice to match on the type, but that seems to be the only way to
// implement this.
ty::Array(inner, _) => {
@ -328,98 +265,45 @@ where
}
ty::Slice(..) => {
let len = Scalar::from_target_usize(inner_len, self);
(MemPlaceMeta::Meta(len), base_layout.ty)
(MemPlaceMeta::Meta(len), base.layout().ty)
}
_ => {
span_bug!(self.cur_span(), "cannot subslice non-array type: `{:?}`", base_layout.ty)
span_bug!(
self.cur_span(),
"cannot subslice non-array type: `{:?}`",
base.layout().ty
)
}
};
let layout = self.layout_of(ty)?;
Ok((from_offset, meta, layout))
}
fn operand_subslice(
&self,
base: &OpTy<'tcx, M::Provenance>,
from: u64,
to: u64,
from_end: bool,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
let (from_offset, meta, layout) =
self.project_subslice(base.layout, base.meta()?, from, to, from_end)?;
base.offset_with_meta(from_offset, meta, layout, self)
}
pub fn place_subslice(
&self,
base: &PlaceTy<'tcx, M::Provenance>,
from: u64,
to: u64,
from_end: bool,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
let (from_offset, meta, layout) =
self.project_subslice(base.layout, self.place_meta(base)?, from, to, from_end)?;
base.offset_with_meta(from_offset, meta, layout, self)
}
//# Applying a general projection
/// Projects into a place.
/// Applying a general projection
#[instrument(skip(self), level = "trace")]
pub fn place_projection(
&self,
base: &PlaceTy<'tcx, M::Provenance>,
proj_elem: mir::PlaceElem<'tcx>,
) -> InterpResult<'tcx, PlaceTy<'tcx, M::Provenance>> {
pub fn project<P>(&self, base: &P, proj_elem: mir::PlaceElem<'tcx>) -> InterpResult<'tcx, P>
where
P: Projectable<'mir, 'tcx, M::Provenance>
+ From<MPlaceTy<'tcx, M::Provenance>>
+ std::fmt::Debug,
{
use rustc_middle::mir::ProjectionElem::*;
Ok(match proj_elem {
OpaqueCast(ty) => {
let mut place = base.clone();
place.layout = self.layout_of(ty)?;
place
}
Field(field, _) => self.place_field(base, field.index())?,
Downcast(_, variant) => self.place_downcast(base, variant)?,
Deref => self.deref_operand(&self.place_to_op(base)?)?.into(),
OpaqueCast(ty) => base.transmute(self.layout_of(ty)?, self)?,
Field(field, _) => self.project_field(base, field.index())?,
Downcast(_, variant) => self.project_downcast(base, variant)?,
Deref => self.deref_operand(&base.to_op(self)?)?.into(),
Index(local) => {
let layout = self.layout_of(self.tcx.types.usize)?;
let n = self.local_to_op(self.frame(), local, Some(layout))?;
let n = self.read_target_usize(&n)?;
self.place_index(base, n)?
self.project_index(base, n)?
}
ConstantIndex { offset, min_length, from_end } => {
self.place_constant_index(base, offset, min_length, from_end)?
self.project_constant_index(base, offset, min_length, from_end)?
}
Subslice { from, to, from_end } => self.place_subslice(base, from, to, from_end)?,
})
}
#[instrument(skip(self), level = "trace")]
pub fn operand_projection(
&self,
base: &OpTy<'tcx, M::Provenance>,
proj_elem: mir::PlaceElem<'tcx>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
use rustc_middle::mir::ProjectionElem::*;
Ok(match proj_elem {
OpaqueCast(ty) => {
let mut op = base.clone();
op.layout = self.layout_of(ty)?;
op
}
Field(field, _) => self.operand_field(base, field.index())?,
Downcast(_, variant) => self.operand_downcast(base, variant)?,
Deref => self.deref_operand(base)?.into(),
Index(local) => {
let layout = self.layout_of(self.tcx.types.usize)?;
let n = self.local_to_op(self.frame(), local, Some(layout))?;
let n = self.read_target_usize(&n)?;
self.operand_index(base, n)?
}
ConstantIndex { offset, min_length, from_end } => {
self.operand_constant_index(base, offset, min_length, from_end)?
}
Subslice { from, to, from_end } => self.operand_subslice(base, from, to, from_end)?,
Subslice { from, to, from_end } => self.project_subslice(base, from, to, from_end)?,
})
}
}

4
compiler/rustc_const_eval/src/interpret/step.rs
View file

@ -8,7 +8,7 @@ use rustc_middle::mir;
use rustc_middle::mir::interpret::{InterpResult, Scalar};
use rustc_middle::ty::layout::LayoutOf;
use super::{ImmTy, InterpCx, Machine};
use super::{ImmTy, InterpCx, Machine, Projectable};
use crate::util;
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
@ -197,7 +197,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.get_place_alloc_mut(&dest)?;
} else {
// Write the src to the first element.
let first = self.mplace_field(&dest, 0)?;
let first = self.project_index(&dest, 0)?;
self.copy_op(&src, &first.into(), /*allow_transmute*/ false)?;
// This is performance-sensitive code for big static/const arrays! So we

19
compiler/rustc_const_eval/src/interpret/terminator.rs
View file

@ -65,8 +65,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
field: usize,
) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
Ok(match arg {
FnArg::Copy(op) => FnArg::Copy(self.operand_field(op, field)?),
FnArg::InPlace(place) => FnArg::InPlace(self.place_field(place, field)?),
FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?),
FnArg::InPlace(place) => FnArg::InPlace(self.project_field(place, field)?),
})
}
@ -382,8 +382,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// This all has to be in memory, there are no immediate unsized values.
let src = caller_arg_copy.assert_mem_place();
// The destination cannot be one of these "spread args".
let (dest_frame, dest_local, dest_offset) =
callee_arg.as_mplace_or_local().right().expect("calee fn arguments must be locals");
let (dest_frame, dest_local, dest_offset) = callee_arg
.as_mplace_or_local()
.right()
.expect("callee fn arguments must be locals");
// We are just initializing things, so there can't be anything here yet.
assert!(matches!(
*self.local_to_op(&self.stack()[dest_frame], dest_local, None)?,
@ -597,7 +599,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
if Some(local) == body.spread_arg {
// Must be a tuple
for i in 0..dest.layout.fields.count() {
let dest = self.place_field(&dest, i)?;
let dest = self.project_field(&dest, i)?;
let callee_abi = callee_args_abis.next().unwrap();
self.pass_argument(&mut caller_args, callee_abi, &dest)?;
}
@ -679,7 +681,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// Not there yet, search for the only non-ZST field.
let mut non_zst_field = None;
for i in 0..receiver.layout.fields.count() {
let field = self.operand_field(&receiver, i)?;
let field = self.project_field(&receiver, i)?;
let zst =
field.layout.is_zst() && field.layout.align.abi.bytes() == 1;
if !zst {
@ -705,12 +707,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let (vptr, dyn_ty, adjusted_receiver) = if let ty::Dynamic(data, _, ty::DynStar) =
receiver_place.layout.ty.kind()
{
let (recv, vptr) = self.unpack_dyn_star(&receiver_place.into())?;
let (recv, vptr) = self.unpack_dyn_star(&receiver_place)?;
let (dyn_ty, dyn_trait) = self.get_ptr_vtable(vptr)?;
if dyn_trait != data.principal() {
throw_ub_custom!(fluent::const_eval_dyn_star_call_vtable_mismatch);
}
let recv = recv.assert_mem_place(); // we passed an MPlaceTy to `unpack_dyn_star` so we definitely still have one
(vptr, dyn_ty, recv.ptr)
} else {
@ -838,7 +839,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
ty::Dynamic(_, _, ty::DynStar) => {
// Dropping a `dyn*`. Need to find actual drop fn.
self.unpack_dyn_star(&place.into())?.0.assert_mem_place()
self.unpack_dyn_star(&place)?.0
}
_ => {
debug_assert_eq!(

116
compiler/rustc_const_eval/src/interpret/validity.rs
View file

@ -29,7 +29,7 @@ use std::hash::Hash;
use super::UndefinedBehaviorInfo::*;
use super::{
AllocId, CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult, MPlaceTy,
Machine, MemPlaceMeta, OpTy, Pointer, Scalar, ValueVisitor,
Machine, MemPlaceMeta, OpTy, Pointer, Projectable, Scalar, ValueVisitor,
};
macro_rules! throw_validation_failure {
@ -462,6 +462,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
/// Check if this is a value of primitive type, and if yes check the validity of the value
/// at that type. Return `true` if the type is indeed primitive.
///
/// Note that not all of these have `FieldsShape::Primitive`, e.g. wide references.
fn try_visit_primitive(
&mut self,
value: &OpTy<'tcx, M::Provenance>,
@ -655,15 +657,14 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
) -> InterpResult<'tcx, VariantIdx> {
self.with_elem(PathElem::EnumTag, move |this| {
Ok(try_validation!(
this.ecx.read_discriminant(op),
this.ecx.read_discriminant(op).map(|(_, idx)| idx),
this.path,
InvalidTag(val) => InvalidEnumTag {
value: format!("{val:x}"),
},
UninhabitedEnumVariantRead(_) => UninhabitedEnumTag,
InvalidUninitBytes(None) => UninitEnumTag,
)
.1)
))
})
}
@ -733,60 +734,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
}
}
// Recursively walk the value at its type.
self.walk_value(op)?;
// *After* all of this, check the ABI. We need to check the ABI to handle
// types like `NonNull` where the `Scalar` info is more restrictive than what
// the fields say (`rustc_layout_scalar_valid_range_start`).
// But in most cases, this will just propagate what the fields say,
// and then we want the error to point at the field -- so, first recurse,
// then check ABI.
//
// FIXME: We could avoid some redundant checks here. For newtypes wrapping
// scalars, we do the same check on every "level" (e.g., first we check
// MyNewtype and then the scalar in there).
match op.layout.abi {
Abi::Uninhabited => {
let ty = op.layout.ty;
throw_validation_failure!(self.path, UninhabitedVal { ty });
}
Abi::Scalar(scalar_layout) => {
if !scalar_layout.is_uninit_valid() {
// There is something to check here.
let scalar = self.read_scalar(op, ExpectedKind::InitScalar)?;
self.visit_scalar(scalar, scalar_layout)?;
}
}
Abi::ScalarPair(a_layout, b_layout) => {
// We can only proceed if *both* scalars need to be initialized.
// FIXME: find a way to also check ScalarPair when one side can be uninit but
// the other must be init.
if !a_layout.is_uninit_valid() && !b_layout.is_uninit_valid() {
let (a, b) =
self.read_immediate(op, ExpectedKind::InitScalar)?.to_scalar_pair();
self.visit_scalar(a, a_layout)?;
self.visit_scalar(b, b_layout)?;
}
}
Abi::Vector { .. } => {
// No checks here, we assume layout computation gets this right.
// (This is harder to check since Miri does not represent these as `Immediate`. We
// also cannot use field projections since this might be a newtype around a vector.)
}
Abi::Aggregate { .. } => {
// Nothing to do.
}
}
Ok(())
}
fn visit_aggregate(
&mut self,
op: &OpTy<'tcx, M::Provenance>,
fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>,
) -> InterpResult<'tcx> {
// Recursively walk the value at its type. Apply optimizations for some large types.
match op.layout.ty.kind() {
ty::Str => {
let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate
@ -874,12 +822,58 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
// ZST type, so either validation fails for all elements or none.
ty::Array(tys, ..) | ty::Slice(tys) if self.ecx.layout_of(*tys)?.is_zst() => {
// Validate just the first element (if any).
self.walk_aggregate(op, fields.take(1))?
if op.len(self.ecx)? > 0 {
self.visit_field(op, 0, &self.ecx.project_index(op, 0)?)?;
}
}
_ => {
self.walk_aggregate(op, fields)? // default handler
self.walk_value(op)?; // default handler
}
}
// *After* all of this, check the ABI. We need to check the ABI to handle
// types like `NonNull` where the `Scalar` info is more restrictive than what
// the fields say (`rustc_layout_scalar_valid_range_start`).
// But in most cases, this will just propagate what the fields say,
// and then we want the error to point at the field -- so, first recurse,
// then check ABI.
//
// FIXME: We could avoid some redundant checks here. For newtypes wrapping
// scalars, we do the same check on every "level" (e.g., first we check
// MyNewtype and then the scalar in there).
match op.layout.abi {
Abi::Uninhabited => {
let ty = op.layout.ty;
throw_validation_failure!(self.path, UninhabitedVal { ty });
}
Abi::Scalar(scalar_layout) => {
if !scalar_layout.is_uninit_valid() {
// There is something to check here.
let scalar = self.read_scalar(op, ExpectedKind::InitScalar)?;
self.visit_scalar(scalar, scalar_layout)?;
}
}
Abi::ScalarPair(a_layout, b_layout) => {
// We can only proceed if *both* scalars need to be initialized.
// FIXME: find a way to also check ScalarPair when one side can be uninit but
// the other must be init.
if !a_layout.is_uninit_valid() && !b_layout.is_uninit_valid() {
let (a, b) =
self.read_immediate(op, ExpectedKind::InitScalar)?.to_scalar_pair();
self.visit_scalar(a, a_layout)?;
self.visit_scalar(b, b_layout)?;
}
}
Abi::Vector { .. } => {
// No checks here, we assume layout computation gets this right.
// (This is harder to check since Miri does not represent these as `Immediate`. We
// also cannot use field projections since this might be a newtype around a vector.)
}
Abi::Aggregate { .. } => {
// Nothing to do.
}
}
Ok(())
}
}

676
compiler/rustc_const_eval/src/interpret/visitor.rs
View file

@ -1,544 +1,204 @@
//! Visitor for a run-time value with a given layout: Traverse enums, structs and other compound
//! types until we arrive at the leaves, with custom handling for primitive types.
use rustc_index::IndexVec;
use rustc_middle::mir::interpret::InterpResult;
use rustc_middle::ty;
use rustc_middle::ty::layout::TyAndLayout;
use rustc_target::abi::FieldIdx;
use rustc_target::abi::{FieldsShape, VariantIdx, Variants};
use std::num::NonZeroUsize;
use super::{InterpCx, MPlaceTy, Machine, OpTy, PlaceTy};
use super::{InterpCx, MPlaceTy, Machine, Projectable};
/// A thing that we can project into, and that has a layout.
/// This wouldn't have to depend on `Machine` but with the current type inference,
/// that's just more convenient to work with (avoids repeating all the `Machine` bounds).
pub trait Value<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
/// Gets this value's layout.
fn layout(&self) -> TyAndLayout<'tcx>;
/// How to traverse a value and what to do when we are at the leaves.
pub trait ValueVisitor<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
type V: Projectable<'mir, 'tcx, M::Provenance>
+ From<MPlaceTy<'tcx, M::Provenance>>
+ std::fmt::Debug;
/// Makes this into an `OpTy`, in a cheap way that is good for reading.
fn to_op_for_read(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>;
/// The visitor must have an `InterpCx` in it.
fn ecx(&self) -> &InterpCx<'mir, 'tcx, M>;
/// Makes this into an `OpTy`, in a potentially more expensive way that is good for projections.
fn to_op_for_proj(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
self.to_op_for_read(ecx)
/// `read_discriminant` can be hooked for better error messages.
#[inline(always)]
fn read_discriminant(&mut self, v: &Self::V) -> InterpResult<'tcx, VariantIdx> {
Ok(self.ecx().read_discriminant(&v.to_op(self.ecx())?)?.1)
}
/// Creates this from an `OpTy`.
/// This function provides the chance to reorder the order in which fields are visited for
/// `FieldsShape::Aggregate`: The order of fields will be
/// `(0..num_fields).map(aggregate_field_order)`.
///
/// If `to_op_for_proj` only ever produces `Indirect` operands, then this one is definitely `Indirect`.
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self;
/// Projects to the given enum variant.
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self>;
/// Projects to the n-th field.
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self>;
}
/// A thing that we can project into given *mutable* access to `ecx`, and that has a layout.
/// This wouldn't have to depend on `Machine` but with the current type inference,
/// that's just more convenient to work with (avoids repeating all the `Machine` bounds).
pub trait ValueMut<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
/// Gets this value's layout.
fn layout(&self) -> TyAndLayout<'tcx>;
/// Makes this into an `OpTy`, in a cheap way that is good for reading.
fn to_op_for_read(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>;
/// Makes this into an `OpTy`, in a potentially more expensive way that is good for projections.
fn to_op_for_proj(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>>;
/// Creates this from an `OpTy`.
///
/// If `to_op_for_proj` only ever produces `Indirect` operands, then this one is definitely `Indirect`.
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self;
/// Projects to the given enum variant.
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self>;
/// Projects to the n-th field.
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self>;
}
// We cannot have a general impl which shows that Value implies ValueMut. (When we do, it says we
// cannot `impl ValueMut for PlaceTy` because some downstream crate could `impl Value for PlaceTy`.)
// So we have some copy-paste here. (We could have a macro but since we only have 2 types with this
// double-impl, that would barely make the code shorter, if at all.)
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::Provenance> {
/// The default means we iterate in source declaration order; alternative this can do an inverse
/// lookup in `memory_index` to use memory field order instead.
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
fn aggregate_field_order(_memory_index: &IndexVec<FieldIdx, u32>, idx: usize) -> usize {
idx
}
// Recursive actions, ready to be overloaded.
/// Visits the given value, dispatching as appropriate to more specialized visitors.
#[inline(always)]
fn to_op_for_read(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone())
fn visit_value(&mut self, v: &Self::V) -> InterpResult<'tcx> {
self.walk_value(v)
}
/// Visits the given value as a union. No automatic recursion can happen here.
#[inline(always)]
fn visit_union(&mut self, _v: &Self::V, _fields: NonZeroUsize) -> InterpResult<'tcx> {
Ok(())
}
/// Visits the given value as the pointer of a `Box`. There is nothing to recurse into.
/// The type of `v` will be a raw pointer, but this is a field of `Box<T>` and the
/// pointee type is the actual `T`.
#[inline(always)]
fn visit_box(&mut self, _v: &Self::V) -> InterpResult<'tcx> {
Ok(())
}
/// Called each time we recurse down to a field of a "product-like" aggregate
/// (structs, tuples, arrays and the like, but not enums), passing in old (outer)
/// and new (inner) value.
/// This gives the visitor the chance to track the stack of nested fields that
/// we are descending through.
#[inline(always)]
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self {
op.clone()
fn visit_field(
&mut self,
_old_val: &Self::V,
_field: usize,
new_val: &Self::V,
) -> InterpResult<'tcx> {
self.visit_value(new_val)
}
/// Called when recursing into an enum variant.
/// This gives the visitor the chance to track the stack of nested fields that
/// we are descending through.
#[inline(always)]
fn visit_variant(
&mut self,
_old_val: &Self::V,
_variant: VariantIdx,
new_val: &Self::V,
) -> InterpResult<'tcx> {
self.visit_value(new_val)
}
#[inline(always)]
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.operand_downcast(self, variant)
}
fn walk_value(&mut self, v: &Self::V) -> InterpResult<'tcx> {
let ty = v.layout().ty;
trace!("walk_value: type: {ty}");
#[inline(always)]
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.operand_field(self, field)
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
for OpTy<'tcx, M::Provenance>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
#[inline(always)]
fn to_op_for_read(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone())
}
#[inline(always)]
fn to_op_for_proj(
&self,
_ecx: &mut InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone())
}
#[inline(always)]
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self {
op.clone()
}
#[inline(always)]
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.operand_downcast(self, variant)
}
#[inline(always)]
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.operand_field(self, field)
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
for MPlaceTy<'tcx, M::Provenance>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
#[inline(always)]
fn to_op_for_read(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.into())
}
#[inline(always)]
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self {
// assert is justified because our `to_op_for_read` only ever produces `Indirect` operands.
op.assert_mem_place()
}
#[inline(always)]
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.mplace_downcast(self, variant)
}
#[inline(always)]
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.mplace_field(self, field)
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
for MPlaceTy<'tcx, M::Provenance>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
#[inline(always)]
fn to_op_for_read(
&self,
_ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.into())
}
#[inline(always)]
fn to_op_for_proj(
&self,
_ecx: &mut InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.into())
}
#[inline(always)]
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self {
// assert is justified because our `to_op_for_proj` only ever produces `Indirect` operands.
op.assert_mem_place()
}
#[inline(always)]
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.mplace_downcast(self, variant)
}
#[inline(always)]
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.mplace_field(self, field)
}
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
for PlaceTy<'tcx, M::Provenance>
{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
}
#[inline(always)]
fn to_op_for_read(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
// No need for `force_allocation` since we are just going to read from this.
ecx.place_to_op(self)
}
#[inline(always)]
fn to_op_for_proj(
&self,
ecx: &mut InterpCx<'mir, 'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
// We `force_allocation` here so that `from_op` below can work.
Ok(ecx.force_allocation(self)?.into())
}
#[inline(always)]
fn from_op(op: &OpTy<'tcx, M::Provenance>) -> Self {
// assert is justified because our `to_op` only ever produces `Indirect` operands.
op.assert_mem_place().into()
}
#[inline(always)]
fn project_downcast(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
variant: VariantIdx,
) -> InterpResult<'tcx, Self> {
ecx.place_downcast(self, variant)
}
#[inline(always)]
fn project_field(
&self,
ecx: &InterpCx<'mir, 'tcx, M>,
field: usize,
) -> InterpResult<'tcx, Self> {
ecx.place_field(self, field)
}
}
macro_rules! make_value_visitor {
($visitor_trait:ident, $value_trait:ident, $($mutability:ident)?) => {
/// How to traverse a value and what to do when we are at the leaves.
pub trait $visitor_trait<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
type V: $value_trait<'mir, 'tcx, M>;
/// The visitor must have an `InterpCx` in it.
fn ecx(&$($mutability)? self)
-> &$($mutability)? InterpCx<'mir, 'tcx, M>;
/// `read_discriminant` can be hooked for better error messages.
#[inline(always)]
fn read_discriminant(
&mut self,
op: &OpTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, VariantIdx> {
Ok(self.ecx().read_discriminant(op)?.1)
// Special treatment for special types, where the (static) layout is not sufficient.
match *ty.kind() {
// If it is a trait object, switch to the real type that was used to create it.
ty::Dynamic(_, _, ty::Dyn) => {
// Dyn types. This is unsized, and the actual dynamic type of the data is given by the
// vtable stored in the place metadata.
// unsized values are never immediate, so we can assert_mem_place
let op = v.to_op(self.ecx())?;
let dest = op.assert_mem_place();
let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.0;
trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout);
// recurse with the inner type
return self.visit_field(&v, 0, &inner_mplace.into());
}
ty::Dynamic(_, _, ty::DynStar) => {
// DynStar types. Very different from a dyn type (but strangely part of the
// same variant in `TyKind`): These are pairs where the 2nd component is the
// vtable, and the first component is the data (which must be ptr-sized).
let data = self.ecx().unpack_dyn_star(v)?.0;
return self.visit_field(&v, 0, &data);
}
// Slices do not need special handling here: they have `Array` field
// placement with length 0, so we enter the `Array` case below which
// indirectly uses the metadata to determine the actual length.
// Recursive actions, ready to be overloaded.
/// Visits the given value, dispatching as appropriate to more specialized visitors.
#[inline(always)]
fn visit_value(&mut self, v: &Self::V) -> InterpResult<'tcx>
{
self.walk_value(v)
}
/// Visits the given value as a union. No automatic recursion can happen here.
#[inline(always)]
fn visit_union(&mut self, _v: &Self::V, _fields: NonZeroUsize) -> InterpResult<'tcx>
{
Ok(())
}
/// Visits the given value as the pointer of a `Box`. There is nothing to recurse into.
/// The type of `v` will be a raw pointer, but this is a field of `Box<T>` and the
/// pointee type is the actual `T`.
#[inline(always)]
fn visit_box(&mut self, _v: &Self::V) -> InterpResult<'tcx>
{
Ok(())
}
/// Visits this value as an aggregate, you are getting an iterator yielding
/// all the fields (still in an `InterpResult`, you have to do error handling yourself).
/// Recurses into the fields.
#[inline(always)]
fn visit_aggregate(
&mut self,
v: &Self::V,
fields: impl Iterator<Item=InterpResult<'tcx, Self::V>>,
) -> InterpResult<'tcx> {
self.walk_aggregate(v, fields)
}
// However, `Box`... let's talk about `Box`.
ty::Adt(def, ..) if def.is_box() => {
// `Box` is a hybrid primitive-library-defined type that one the one hand is
// a dereferenceable pointer, on the other hand has *basically arbitrary
// user-defined layout* since the user controls the 'allocator' field. So it
// cannot be treated like a normal pointer, since it does not fit into an
// `Immediate`. Yeah, it is quite terrible. But many visitors want to do
// something with "all boxed pointers", so we handle this mess for them.
//
// When we hit a `Box`, we do not do the usual field recursion; instead,
// we (a) call `visit_box` on the pointer value, and (b) recurse on the
// allocator field. We also assert tons of things to ensure we do not miss
// any other fields.
/// Called each time we recurse down to a field of a "product-like" aggregate
/// (structs, tuples, arrays and the like, but not enums), passing in old (outer)
/// and new (inner) value.
/// This gives the visitor the chance to track the stack of nested fields that
/// we are descending through.
#[inline(always)]
fn visit_field(
&mut self,
_old_val: &Self::V,
_field: usize,
new_val: &Self::V,
) -> InterpResult<'tcx> {
self.visit_value(new_val)
}
/// Called when recursing into an enum variant.
/// This gives the visitor the chance to track the stack of nested fields that
/// we are descending through.
#[inline(always)]
fn visit_variant(
&mut self,
_old_val: &Self::V,
_variant: VariantIdx,
new_val: &Self::V,
) -> InterpResult<'tcx> {
self.visit_value(new_val)
}
// `Box` has two fields: the pointer we care about, and the allocator.
assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields");
let (unique_ptr, alloc) =
(self.ecx().project_field(v, 0)?, self.ecx().project_field(v, 1)?);
// Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`...
// (which means another 2 fields, the second of which is a `PhantomData`)
assert_eq!(unique_ptr.layout().fields.count(), 2);
let (nonnull_ptr, phantom) = (
self.ecx().project_field(&unique_ptr, 0)?,
self.ecx().project_field(&unique_ptr, 1)?,
);
assert!(
phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()),
"2nd field of `Unique` should be PhantomData but is {:?}",
phantom.layout().ty,
);
// ... that contains a `NonNull`... (gladly, only a single field here)
assert_eq!(nonnull_ptr.layout().fields.count(), 1);
let raw_ptr = self.ecx().project_field(&nonnull_ptr, 0)?; // the actual raw ptr
// ... whose only field finally is a raw ptr we can dereference.
self.visit_box(&raw_ptr)?;
// Default recursors. Not meant to be overloaded.
fn walk_aggregate(
&mut self,
v: &Self::V,
fields: impl Iterator<Item=InterpResult<'tcx, Self::V>>,
) -> InterpResult<'tcx> {
// Now iterate over it.
for (idx, field_val) in fields.enumerate() {
self.visit_field(v, idx, &field_val?)?;
// The second `Box` field is the allocator, which we recursively check for validity
// like in regular structs.
self.visit_field(v, 1, &alloc)?;
// We visited all parts of this one.
return Ok(());
}
_ => {}
};
// Visit the fields of this value.
match &v.layout().fields {
FieldsShape::Primitive => {}
&FieldsShape::Union(fields) => {
self.visit_union(v, fields)?;
}
FieldsShape::Arbitrary { offsets, memory_index } => {
for idx in 0..offsets.len() {
let idx = Self::aggregate_field_order(memory_index, idx);
let field = self.ecx().project_field(v, idx)?;
self.visit_field(v, idx, &field)?;
}
Ok(())
}
fn walk_value(&mut self, v: &Self::V) -> InterpResult<'tcx>
{
let ty = v.layout().ty;
trace!("walk_value: type: {ty}");
// Special treatment for special types, where the (static) layout is not sufficient.
match *ty.kind() {
// If it is a trait object, switch to the real type that was used to create it.
ty::Dynamic(_, _, ty::Dyn) => {
// Dyn types. This is unsized, and the actual dynamic type of the data is given by the
// vtable stored in the place metadata.
// unsized values are never immediate, so we can assert_mem_place
let op = v.to_op_for_read(self.ecx())?;
let dest = op.assert_mem_place();
let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.0;
trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout);
// recurse with the inner type
return self.visit_field(&v, 0, &$value_trait::from_op(&inner_mplace.into()));
},
ty::Dynamic(_, _, ty::DynStar) => {
// DynStar types. Very different from a dyn type (but strangely part of the
// same variant in `TyKind`): These are pairs where the 2nd component is the
// vtable, and the first component is the data (which must be ptr-sized).
let op = v.to_op_for_proj(self.ecx())?;
let data = self.ecx().unpack_dyn_star(&op)?.0;
return self.visit_field(&v, 0, &$value_trait::from_op(&data));
}
// Slices do not need special handling here: they have `Array` field
// placement with length 0, so we enter the `Array` case below which
// indirectly uses the metadata to determine the actual length.
// However, `Box`... let's talk about `Box`.
ty::Adt(def, ..) if def.is_box() => {
// `Box` is a hybrid primitive-library-defined type that one the one hand is
// a dereferenceable pointer, on the other hand has *basically arbitrary
// user-defined layout* since the user controls the 'allocator' field. So it
// cannot be treated like a normal pointer, since it does not fit into an
// `Immediate`. Yeah, it is quite terrible. But many visitors want to do
// something with "all boxed pointers", so we handle this mess for them.
//
// When we hit a `Box`, we do not do the usual `visit_aggregate`; instead,
// we (a) call `visit_box` on the pointer value, and (b) recurse on the
// allocator field. We also assert tons of things to ensure we do not miss
// any other fields.
// `Box` has two fields: the pointer we care about, and the allocator.
assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields");
let (unique_ptr, alloc) =
(v.project_field(self.ecx(), 0)?, v.project_field(self.ecx(), 1)?);
// Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`...
// (which means another 2 fields, the second of which is a `PhantomData`)
assert_eq!(unique_ptr.layout().fields.count(), 2);
let (nonnull_ptr, phantom) = (
unique_ptr.project_field(self.ecx(), 0)?,
unique_ptr.project_field(self.ecx(), 1)?,
);
assert!(
phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()),
"2nd field of `Unique` should be PhantomData but is {:?}",
phantom.layout().ty,
);
// ... that contains a `NonNull`... (gladly, only a single field here)
assert_eq!(nonnull_ptr.layout().fields.count(), 1);
let raw_ptr = nonnull_ptr.project_field(self.ecx(), 0)?; // the actual raw ptr
// ... whose only field finally is a raw ptr we can dereference.
self.visit_box(&raw_ptr)?;
// The second `Box` field is the allocator, which we recursively check for validity
// like in regular structs.
self.visit_field(v, 1, &alloc)?;
// We visited all parts of this one.
return Ok(());
}
_ => {},
};
// Visit the fields of this value.
match &v.layout().fields {
FieldsShape::Primitive => {}
&FieldsShape::Union(fields) => {
self.visit_union(v, fields)?;
}
FieldsShape::Arbitrary { offsets, .. } => {
// FIXME: We collect in a vec because otherwise there are lifetime
// errors: Projecting to a field needs access to `ecx`.
let fields: Vec<InterpResult<'tcx, Self::V>> =
(0..offsets.len()).map(|i| {
v.project_field(self.ecx(), i)
})
.collect();
self.visit_aggregate(v, fields.into_iter())?;
}
FieldsShape::Array { .. } => {
// Let's get an mplace (or immediate) first.
// FIXME: This might `force_allocate` if `v` is a `PlaceTy`!
let op = v.to_op_for_proj(self.ecx())?;
// Now we can go over all the fields.
// This uses the *run-time length*, i.e., if we are a slice,
// the dynamic info from the metadata is used.
let iter = self.ecx().operand_array_fields(&op)?
.map(|f| f.and_then(|f| {
Ok($value_trait::from_op(&f))
}));
self.visit_aggregate(v, iter)?;
}
}
match v.layout().variants {
// If this is a multi-variant layout, find the right variant and proceed
// with *its* fields.
Variants::Multiple { .. } => {
let op = v.to_op_for_read(self.ecx())?;
let idx = self.read_discriminant(&op)?;
let inner = v.project_downcast(self.ecx(), idx)?;
trace!("walk_value: variant layout: {:#?}", inner.layout());
// recurse with the inner type
self.visit_variant(v, idx, &inner)
}
// For single-variant layouts, we already did anything there is to do.
Variants::Single { .. } => Ok(())
FieldsShape::Array { .. } => {
for (idx, field) in self.ecx().project_array_fields(v)?.enumerate() {
self.visit_field(v, idx, &field?)?;
}
}
}
match v.layout().variants {
// If this is a multi-variant layout, find the right variant and proceed
// with *its* fields.
Variants::Multiple { .. } => {
let idx = self.read_discriminant(v)?;
// There are 3 cases where downcasts can turn a Scalar/ScalarPair into a different ABI which
// could be a problem for `ImmTy` (see layout_sanity_check):
// - variant.size == Size::ZERO: works fine because `ImmTy::offset` has a special case for
// zero-sized layouts.
// - variant.fields.count() == 0: works fine because `ImmTy::offset` has a special case for
// zero-field aggregates.
// - variant.abi.is_uninhabited(): triggers UB in `read_discriminant` so we never get here.
let inner = self.ecx().project_downcast(v, idx)?;
trace!("walk_value: variant layout: {:#?}", inner.layout());
// recurse with the inner type
self.visit_variant(v, idx, &inner)?;
}
// For single-variant layouts, we already did anything there is to do.
Variants::Single { .. } => {}
}
Ok(())
}
}
make_value_visitor!(ValueVisitor, Value,);
make_value_visitor!(MutValueVisitor, ValueMut, mut);