Rollup merge of #99259 - RalfJung:visit-a-place, r=oli-obk

interpret/visitor: support visiting with a PlaceTy

Finally we can visit a `PlaceTy` in a way that will only do `force_allocation` when needed ti visit a field. :)

r? `@oli-obk`

compiler/rustc_const_eval/src/const_eval/valtrees.rs

@@ -436,7 +436,7 @@ fn valtree_into_mplace<'tcx>(
 
                         let offset = place_adjusted.layout.fields.offset(i);
                         place
-                            .offset(
+                            .offset_with_meta(
                                 offset,
                                 MemPlaceMeta::Meta(Scalar::from_machine_usize(
                                     num_elems as u64,

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -297,7 +297,7 @@ impl<'tcx, Tag: Provenance> OpTy<'tcx, Tag> {
         }
     }
 
-    pub fn offset(
+    pub fn offset_with_meta(
         &self,
         offset: Size,
         meta: MemPlaceMeta<Tag>,
@@ -305,7 +305,7 @@ impl<'tcx, Tag: Provenance> OpTy<'tcx, Tag> {
         cx: &impl HasDataLayout,
     ) -> InterpResult<'tcx, Self> {
         match self.try_as_mplace() {
-            Ok(mplace) => Ok(mplace.offset(offset, meta, layout, cx)?.into()),
+            Ok(mplace) => Ok(mplace.offset_with_meta(offset, meta, layout, cx)?.into()),
             Err(imm) => {
                 assert!(
                     matches!(*imm, Immediate::Uninit),
@@ -317,6 +317,16 @@ impl<'tcx, Tag: Provenance> OpTy<'tcx, Tag> {
             }
         }
     }
+
+    pub fn offset(
+        &self,
+        offset: Size,
+        layout: TyAndLayout<'tcx>,
+        cx: &impl HasDataLayout,
+    ) -> InterpResult<'tcx, Self> {
+        assert!(!layout.is_unsized());
+        self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
+    }
 }
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {

compiler/rustc_const_eval/src/interpret/place.rs

@@ -163,7 +163,7 @@ impl<Tag: Provenance> MemPlace<Tag> {
     }
 
     #[inline]
-    pub fn offset<'tcx>(
+    pub fn offset_with_meta<'tcx>(
         self,
         offset: Size,
         meta: MemPlaceMeta<Tag>,
@@ -199,7 +199,7 @@ impl<'tcx, Tag: Provenance> MPlaceTy<'tcx, Tag> {
     }
 
     #[inline]
-    pub fn offset(
+    pub fn offset_with_meta(
         &self,
         offset: Size,
         meta: MemPlaceMeta<Tag>,
@@ -207,12 +207,22 @@ impl<'tcx, Tag: Provenance> MPlaceTy<'tcx, Tag> {
         cx: &impl HasDataLayout,
     ) -> InterpResult<'tcx, Self> {
         Ok(MPlaceTy {
-            mplace: self.mplace.offset(offset, meta, cx)?,
+            mplace: self.mplace.offset_with_meta(offset, meta, cx)?,
             align: self.align.restrict_for_offset(offset),
             layout,
         })
     }
 
+    pub fn offset(
+        &self,
+        offset: Size,
+        layout: TyAndLayout<'tcx>,
+        cx: &impl HasDataLayout,
+    ) -> InterpResult<'tcx, Self> {
+        assert!(!layout.is_unsized());
+        self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
+    }
+
     #[inline]
     pub fn from_aligned_ptr(ptr: Pointer<Option<Tag>>, layout: TyAndLayout<'tcx>) -> Self {
         MPlaceTy { mplace: MemPlace::from_ptr(ptr), layout, align: layout.align.abi }

compiler/rustc_const_eval/src/interpret/projection.rs

@@ -63,7 +63,7 @@ where
 
         // We do not look at `base.layout.align` nor `field_layout.align`, unlike
         // codegen -- mostly to see if we can get away with that
-        base.offset(offset, meta, field_layout, self)
+        base.offset_with_meta(offset, meta, field_layout, self)
     }
 
     /// Gets the place of a field inside the place, and also the field's type.
@@ -193,9 +193,7 @@ where
                 let offset = stride * index; // `Size` multiplication
                 // All fields have the same layout.
                 let field_layout = base.layout.field(self, 0);
-                assert!(!field_layout.is_unsized());
+                base.offset(offset, field_layout, self)
-
-                base.offset(offset, MemPlaceMeta::None, field_layout, self)
             }
             _ => span_bug!(
                 self.cur_span(),
@@ -215,10 +213,10 @@ where
         let abi::FieldsShape::Array { stride, .. } = base.layout.fields else {
             span_bug!(self.cur_span(), "operand_array_fields: expected an array layout");
         };
-        let layout = base.layout.field(self, 0);
+        let field_layout = base.layout.field(self, 0);
         let dl = &self.tcx.data_layout;
         // `Size` multiplication
-        Ok((0..len).map(move |i| base.offset(stride * i, MemPlaceMeta::None, layout, dl)))
+        Ok((0..len).map(move |i| base.offset(stride * i, field_layout, dl)))
     }
 
     /// Index into an array.
@@ -326,7 +324,7 @@ where
             }
         };
         let layout = self.layout_of(ty)?;
-        base.offset(from_offset, meta, layout, self)
+        base.offset_with_meta(from_offset, meta, layout, self)
     }
 
     pub fn place_subslice(

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -853,7 +853,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
                 self.visit_scalar(scalar, scalar_layout)?;
             }
             Abi::ScalarPair(a_layout, b_layout) => {
-                // We would validate these things as we descend into the fields,
+                // There is no `rustc_layout_scalar_valid_range_start` for pairs, so
+                // we would validate these things as we descend into the fields,
                 // but that can miss bugs in layout computation. Layout computation
                 // is subtle due to enums having ScalarPair layout, where one field
                 // is the discriminant.
@@ -867,7 +868,8 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
             }
             Abi::Vector { .. } => {
                 // No checks here, we assume layout computation gets this right.
-                // (This is harder to check since Miri does not represent these as `Immediate`.)
+                // (This is harder to check since Miri does not represent these as `Immediate`. We
+                // also cannot use field projections since this might be a newtype around a vector.)
             }
             Abi::Aggregate { .. } => {
                 // Nothing to do.

compiler/rustc_const_eval/src/interpret/visitor.rs

@@ -8,23 +8,33 @@ use rustc_target::abi::{FieldsShape, VariantIdx, Variants};
 
 use std::num::NonZeroUsize;
 
-use super::{InterpCx, MPlaceTy, Machine, OpTy};
+use super::{InterpCx, MPlaceTy, Machine, OpTy, PlaceTy};
 
-// A thing that we can project into, and that has a layout.
+/// A thing that we can project into, and that has a layout.
-// This wouldn't have to depend on `Machine` but with the current type inference,
+/// This wouldn't have to depend on `Machine` but with the current type inference,
-// that's just more convenient to work with (avoids repeating all the `Machine` bounds).
+/// that's just more convenient to work with (avoids repeating all the `Machine` bounds).
 pub trait Value<'mir, 'tcx, M: Machine<'mir, 'tcx>>: Copy {
     /// Gets this value's layout.
     fn layout(&self) -> TyAndLayout<'tcx>;
 
-    /// Makes this into an `OpTy`.
+    /// Makes this into an `OpTy`, in a cheap way that is good for reading.
-    fn to_op(&self, ecx: &InterpCx<'mir, 'tcx, M>)
+    fn to_op_for_read(
-    -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>>;
+        &self,
+        ecx: &InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>>;
+
+    /// Makes this into an `OpTy`, in a potentially more expensive way that is good for projections.
+    fn to_op_for_proj(
+        &self,
+        ecx: &InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        self.to_op_for_read(ecx)
+    }
 
     /// Creates this from an `OpTy`.
     ///
-    /// If `to_op` only ever produces `Indirect` operands, then this one is definitely `Indirect`.
+    /// If `to_op_for_proj` only ever produces `Indirect` operands, then this one is definitely `Indirect`.
-    fn from_op(mplace: OpTy<'tcx, M::PointerTag>) -> Self;
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self;
 
     /// Projects to the given enum variant.
     fn project_downcast(
@@ -41,8 +51,50 @@ pub trait Value<'mir, 'tcx, M: Machine<'mir, 'tcx>>: Copy {
     ) -> InterpResult<'tcx, Self>;
 }
 
-// Operands and memory-places are both values.
+/// A thing that we can project into given *mutable* access to `ecx`, and that has a layout.
-// Places in general are not due to `place_field` having to do `force_allocation`.
+/// This wouldn't have to depend on `Machine` but with the current type inference,
+/// that's just more convenient to work with (avoids repeating all the `Machine` bounds).
+pub trait ValueMut<'mir, 'tcx, M: Machine<'mir, 'tcx>>: Copy {
+    /// Gets this value's layout.
+    fn layout(&self) -> TyAndLayout<'tcx>;
+
+    /// Makes this into an `OpTy`, in a cheap way that is good for reading.
+    fn to_op_for_read(
+        &self,
+        ecx: &InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>>;
+
+    /// Makes this into an `OpTy`, in a potentially more expensive way that is good for projections.
+    fn to_op_for_proj(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>>;
+
+    /// Creates this from an `OpTy`.
+    ///
+    /// If `to_op_for_proj` only ever produces `Indirect` operands, then this one is definitely `Indirect`.
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self;
+
+    /// Projects to the given enum variant.
+    fn project_downcast(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        variant: VariantIdx,
+    ) -> InterpResult<'tcx, Self>;
+
+    /// Projects to the n-th field.
+    fn project_field(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        field: usize,
+    ) -> InterpResult<'tcx, Self>;
+}
+
+// We cannot have a general impl which shows that Value implies ValueMut. (When we do, it says we
+// cannot `impl ValueMut for PlaceTy` because some downstream crate could `impl Value for PlaceTy`.)
+// So we have some copy-paste here. (We could have a macro but since we only have 2 types with this
+// double-impl, that would barely make the code shorter, if at all.)
+
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> {
     #[inline(always)]
     fn layout(&self) -> TyAndLayout<'tcx> {
@@ -50,7 +102,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tc
     }
 
     #[inline(always)]
-    fn to_op(
+    fn to_op_for_read(
         &self,
         _ecx: &InterpCx<'mir, 'tcx, M>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
@@ -58,8 +110,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tc
     }
 
     #[inline(always)]
-    fn from_op(op: OpTy<'tcx, M::PointerTag>) -> Self {
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self {
-        op
+        *op
     }
 
     #[inline(always)]
@@ -81,6 +133,54 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tc
     }
 }
 
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
+    for OpTy<'tcx, M::PointerTag>
+{
+    #[inline(always)]
+    fn layout(&self) -> TyAndLayout<'tcx> {
+        self.layout
+    }
+
+    #[inline(always)]
+    fn to_op_for_read(
+        &self,
+        _ecx: &InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        Ok(*self)
+    }
+
+    #[inline(always)]
+    fn to_op_for_proj(
+        &self,
+        _ecx: &mut InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        Ok(*self)
+    }
+
+    #[inline(always)]
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self {
+        *op
+    }
+
+    #[inline(always)]
+    fn project_downcast(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        variant: VariantIdx,
+    ) -> InterpResult<'tcx, Self> {
+        ecx.operand_downcast(self, variant)
+    }
+
+    #[inline(always)]
+    fn project_field(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        field: usize,
+    ) -> InterpResult<'tcx, Self> {
+        ecx.operand_field(self, field)
+    }
+}
+
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
     for MPlaceTy<'tcx, M::PointerTag>
 {
@@ -90,7 +190,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
     }
 
     #[inline(always)]
-    fn to_op(
+    fn to_op_for_read(
         &self,
         _ecx: &InterpCx<'mir, 'tcx, M>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
@@ -98,8 +198,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
     }
 
     #[inline(always)]
-    fn from_op(op: OpTy<'tcx, M::PointerTag>) -> Self {
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self {
-        // assert is justified because our `to_op` only ever produces `Indirect` operands.
+        // assert is justified because our `to_op_for_read` only ever produces `Indirect` operands.
         op.assert_mem_place()
     }
 
@@ -122,11 +222,111 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
     }
 }
 
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
+    for MPlaceTy<'tcx, M::PointerTag>
+{
+    #[inline(always)]
+    fn layout(&self) -> TyAndLayout<'tcx> {
+        self.layout
+    }
+
+    #[inline(always)]
+    fn to_op_for_read(
+        &self,
+        _ecx: &InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        Ok(self.into())
+    }
+
+    #[inline(always)]
+    fn to_op_for_proj(
+        &self,
+        _ecx: &mut InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        Ok(self.into())
+    }
+
+    #[inline(always)]
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self {
+        // assert is justified because our `to_op_for_proj` only ever produces `Indirect` operands.
+        op.assert_mem_place()
+    }
+
+    #[inline(always)]
+    fn project_downcast(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        variant: VariantIdx,
+    ) -> InterpResult<'tcx, Self> {
+        ecx.mplace_downcast(self, variant)
+    }
+
+    #[inline(always)]
+    fn project_field(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        field: usize,
+    ) -> InterpResult<'tcx, Self> {
+        ecx.mplace_field(self, field)
+    }
+}
+
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueMut<'mir, 'tcx, M>
+    for PlaceTy<'tcx, M::PointerTag>
+{
+    #[inline(always)]
+    fn layout(&self) -> TyAndLayout<'tcx> {
+        self.layout
+    }
+
+    #[inline(always)]
+    fn to_op_for_read(
+        &self,
+        ecx: &InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        // We `force_allocation` here so that `from_op` below can work.
+        ecx.place_to_op(self)
+    }
+
+    #[inline(always)]
+    fn to_op_for_proj(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+    ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
+        // We `force_allocation` here so that `from_op` below can work.
+        Ok(ecx.force_allocation(self)?.into())
+    }
+
+    #[inline(always)]
+    fn from_op(op: &OpTy<'tcx, M::PointerTag>) -> Self {
+        // assert is justified because our `to_op` only ever produces `Indirect` operands.
+        op.assert_mem_place().into()
+    }
+
+    #[inline(always)]
+    fn project_downcast(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        variant: VariantIdx,
+    ) -> InterpResult<'tcx, Self> {
+        ecx.place_downcast(self, variant)
+    }
+
+    #[inline(always)]
+    fn project_field(
+        &self,
+        ecx: &mut InterpCx<'mir, 'tcx, M>,
+        field: usize,
+    ) -> InterpResult<'tcx, Self> {
+        ecx.place_field(self, field)
+    }
+}
+
 macro_rules! make_value_visitor {
-    ($visitor_trait_name:ident, $($mutability:ident)?) => {
+    ($visitor_trait:ident, $value_trait:ident, $($mutability:ident)?) => {
         // How to traverse a value and what to do when we are at the leaves.
-        pub trait $visitor_trait_name<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
+        pub trait $visitor_trait<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
-            type V: Value<'mir, 'tcx, M>;
+            type V: $value_trait<'mir, 'tcx, M>;
 
             /// The visitor must have an `InterpCx` in it.
             fn ecx(&$($mutability)? self)
@@ -215,19 +415,20 @@ macro_rules! make_value_visitor {
             }
             fn walk_value(&mut self, v: &Self::V) -> InterpResult<'tcx>
             {
-                trace!("walk_value: type: {}", v.layout().ty);
+                let ty = v.layout().ty;
+                trace!("walk_value: type: {ty}");
 
                 // Special treatment for special types, where the (static) layout is not sufficient.
-                match *v.layout().ty.kind() {
+                match *ty.kind() {
                     // If it is a trait object, switch to the real type that was used to create it.
                     ty::Dynamic(..) => {
                         // unsized values are never immediate, so we can assert_mem_place
-                        let op = v.to_op(self.ecx())?;
+                        let op = v.to_op_for_read(self.ecx())?;
                         let dest = op.assert_mem_place();
-                        let inner = self.ecx().unpack_dyn_trait(&dest)?.1;
+                        let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.1;
-                        trace!("walk_value: dyn object layout: {:#?}", inner.layout);
+                        trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout);
                         // recurse with the inner type
-                        return self.visit_field(&v, 0, &Value::from_op(inner.into()));
+                        return self.visit_field(&v, 0, &$value_trait::from_op(&inner_mplace.into()));
                     },
                     // Slices do not need special handling here: they have `Array` field
                     // placement with length 0, so we enter the `Array` case below which
@@ -278,10 +479,10 @@ macro_rules! make_value_visitor {
 
                 // Visit the fields of this value.
                 match v.layout().fields {
-                    FieldsShape::Primitive => {},
+                    FieldsShape::Primitive => {}
                     FieldsShape::Union(fields) => {
                         self.visit_union(v, fields)?;
-                    },
+                    }
                     FieldsShape::Arbitrary { ref offsets, .. } => {
                         // FIXME: We collect in a vec because otherwise there are lifetime
                         // errors: Projecting to a field needs access to `ecx`.
@@ -291,16 +492,17 @@ macro_rules! make_value_visitor {
                             })
                             .collect();
                         self.visit_aggregate(v, fields.into_iter())?;
-                    },
+                    }
                     FieldsShape::Array { .. } => {
-                        // Let's get an mplace first.
+                        // Let's get an mplace (or immediate) first.
-                        let op = v.to_op(self.ecx())?;
+                        // This might `force_allocate` if `v` is a `PlaceTy`, but `place_index` does that anyway.
+                        let op = v.to_op_for_proj(self.ecx())?;
                         // Now we can go over all the fields.
                         // This uses the *run-time length*, i.e., if we are a slice,
                         // the dynamic info from the metadata is used.
                         let iter = self.ecx().operand_array_fields(&op)?
                             .map(|f| f.and_then(|f| {
-                                Ok(Value::from_op(f))
+                                Ok($value_trait::from_op(&f))
                             }));
                         self.visit_aggregate(v, iter)?;
                     }
@@ -310,7 +512,7 @@ macro_rules! make_value_visitor {
                     // If this is a multi-variant layout, find the right variant and proceed
                     // with *its* fields.
                     Variants::Multiple { .. } => {
-                        let op = v.to_op(self.ecx())?;
+                        let op = v.to_op_for_read(self.ecx())?;
                         let idx = self.read_discriminant(&op)?;
                         let inner = v.project_downcast(self.ecx(), idx)?;
                         trace!("walk_value: variant layout: {:#?}", inner.layout());
@@ -325,5 +527,5 @@ macro_rules! make_value_visitor {
     }
 }
 
-make_value_visitor!(ValueVisitor,);
+make_value_visitor!(ValueVisitor, Value,);
-make_value_visitor!(MutValueVisitor, mut);
+make_value_visitor!(MutValueVisitor, ValueMut, mut);