bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

optimize validation iterating over the elements of an array

Browse source 
This is still roughly 45ns slower than the old state, because it now works with
an MPlaceTy and uses the appropriate abstractions, instead of working with a
ptr-align pair directly.
This commit is contained in:
Ralf Jung 2018-08-17 17:47:37 +02:00
parent 6f5cf12894
commit 956b51f79a
4 changed files with 72 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

27
src/librustc_mir/interpret/memory.rs
View file

@ -494,6 +494,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
/// Byte accessors
impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
/// This checks alignment!
fn get_bytes_unchecked(
&self,
ptr: Pointer,
@ -514,6 +515,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
Ok(&alloc.bytes[offset..offset + size.bytes() as usize])
}
/// This checks alignment!
fn get_bytes_unchecked_mut(
&mut self,
ptr: Pointer,
@ -551,7 +553,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
) -> EvalResult<'tcx, &mut [u8]> {
assert_ne!(size.bytes(), 0);
self.clear_relocations(ptr, size)?;
self.mark_definedness(ptr.into(), size, true)?;
self.mark_definedness(ptr, size, true)?;
self.get_bytes_unchecked_mut(ptr, size, align)
}
}
@ -749,9 +751,11 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
Ok(())
}
/// Read a *non-ZST* scalar
pub fn read_scalar(&self, ptr: Pointer, ptr_align: Align, size: Size) -> EvalResult<'tcx, ScalarMaybeUndef> {
self.check_relocation_edges(ptr, size)?; // Make sure we don't read part of a pointer as a pointer
let endianness = self.endianness();
// get_bytes_unchecked tests alignment
let bytes = self.get_bytes_unchecked(ptr, size, ptr_align.min(self.int_align(size)))?;
// Undef check happens *after* we established that the alignment is correct.
// We must not return Ok() for unaligned pointers!
@ -784,16 +788,15 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
self.read_scalar(ptr, ptr_align, self.pointer_size())
}
/// Write a *non-ZST* scalar
pub fn write_scalar(
&mut self,
ptr: Scalar,
ptr: Pointer,
ptr_align: Align,
val: ScalarMaybeUndef,
type_size: Size,
type_align: Align,
) -> EvalResult<'tcx> {
let endianness = self.endianness();
self.check_align(ptr, ptr_align)?;
let val = match val {
ScalarMaybeUndef::Scalar(scalar) => scalar,
@ -806,12 +809,6 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
val.offset.bytes() as u128
}
Scalar::Bits { size: 0, .. } => {
// nothing to do for ZSTs
assert_eq!(type_size.bytes(), 0);
return Ok(());
}
Scalar::Bits { bits, size } => {
assert_eq!(size as u64, type_size.bytes());
assert_eq!(truncate(bits, Size::from_bytes(size.into())), bits,
@ -820,10 +817,9 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
},
};
let ptr = ptr.to_ptr()?;
{
let dst = self.get_bytes_mut(ptr, type_size, ptr_align.min(type_align))?;
// get_bytes_mut checks alignment
let dst = self.get_bytes_mut(ptr, type_size, ptr_align)?;
write_target_uint(endianness, dst, bytes).unwrap();
}
@ -843,7 +839,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
pub fn write_ptr_sized(&mut self, ptr: Pointer, ptr_align: Align, val: ScalarMaybeUndef) -> EvalResult<'tcx> {
let ptr_size = self.pointer_size();
self.write_scalar(ptr.into(), ptr_align, val, ptr_size, ptr_align)
self.write_scalar(ptr.into(), ptr_align, val, ptr_size)
}
fn int_align(&self, size: Size) -> Align {
@ -959,14 +955,13 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'a, 'mir, 'tcx, M> {
pub fn mark_definedness(
&mut self,
ptr: Scalar,
ptr: Pointer,
size: Size,
new_state: bool,
) -> EvalResult<'tcx> {
if size.bytes() == 0 {
return Ok(());
}
let ptr = ptr.to_ptr()?;
let alloc = self.get_mut(ptr.alloc_id)?;
alloc.undef_mask.set_range(
ptr.offset,

15
src/librustc_mir/interpret/operand.rs
View file

@ -41,6 +41,7 @@ impl<'tcx> Value {
Value::ScalarPair(val.into(), Scalar::Ptr(vtable).into())
}
#[inline]
pub fn to_scalar_or_undef(self) -> ScalarMaybeUndef {
match self {
Value::Scalar(val) => val,
@ -48,11 +49,14 @@ impl<'tcx> Value {
}
}
#[inline]
pub fn to_scalar(self) -> EvalResult<'tcx, Scalar> {
self.to_scalar_or_undef().not_undef()
}
/// Convert the value into a pointer (or a pointer-sized integer).
/// Throws away the second half of a ScalarPair!
#[inline]
pub fn to_scalar_ptr(self) -> EvalResult<'tcx, Scalar> {
match self {
Value::Scalar(ptr) |
@ -89,6 +93,7 @@ pub struct ValTy<'tcx> {
impl<'tcx> ::std::ops::Deref for ValTy<'tcx> {
type Target = Value;
#[inline(always)]
fn deref(&self) -> &Value {
&self.value
}
@ -141,12 +146,14 @@ pub struct OpTy<'tcx> {
impl<'tcx> ::std::ops::Deref for OpTy<'tcx> {
type Target = Operand;
#[inline(always)]
fn deref(&self) -> &Operand {
&self.op
}
}
impl<'tcx> From<MPlaceTy<'tcx>> for OpTy<'tcx> {
#[inline(always)]
fn from(mplace: MPlaceTy<'tcx>) -> Self {
OpTy {
op: Operand::Indirect(*mplace),
@ -156,6 +163,7 @@ impl<'tcx> From<MPlaceTy<'tcx>> for OpTy<'tcx> {
}
impl<'tcx> From<ValTy<'tcx>> for OpTy<'tcx> {
#[inline(always)]
fn from(val: ValTy<'tcx>) -> Self {
OpTy {
op: Operand::Immediate(val.value),
@ -192,14 +200,15 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
return Ok(None);
}
let (ptr, ptr_align) = mplace.to_scalar_ptr_align();
self.memory.check_align(ptr, ptr_align)?;
if mplace.layout.size.bytes() == 0 {
// Not all ZSTs have a layout we would handle below, so just short-circuit them
// all here.
self.memory.check_align(ptr, ptr_align)?;
return Ok(Some(Value::Scalar(Scalar::zst().into())));
}
let ptr = ptr.to_ptr()?;
match mplace.layout.abi {
layout::Abi::Scalar(..) => {
let scalar = self.memory.read_scalar(ptr, ptr_align, mplace.layout.size)?;
@ -264,7 +273,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
// This decides which types we will use the Immediate optimization for, and hence should
// match what `try_read_value` and `eval_place_to_op` support.
if layout.is_zst() {
return Ok(Operand::Immediate(Value::Scalar(ScalarMaybeUndef::Undef)));
return Ok(Operand::Immediate(Value::Scalar(Scalar::zst().into())));
}
Ok(match layout.abi {

51
src/librustc_mir/interpret/place.rs
View file

@ -54,6 +54,7 @@ pub struct PlaceTy<'tcx> {
impl<'tcx> ::std::ops::Deref for PlaceTy<'tcx> {
type Target = Place;
#[inline(always)]
fn deref(&self) -> &Place {
&self.place
}
@ -68,12 +69,14 @@ pub struct MPlaceTy<'tcx> {
impl<'tcx> ::std::ops::Deref for MPlaceTy<'tcx> {
type Target = MemPlace;
#[inline(always)]
fn deref(&self) -> &MemPlace {
&self.mplace
}
}
impl<'tcx> From<MPlaceTy<'tcx>> for PlaceTy<'tcx> {
#[inline(always)]
fn from(mplace: MPlaceTy<'tcx>) -> Self {
PlaceTy {
place: Place::Ptr(mplace.mplace),
@ -160,6 +163,7 @@ impl<'tcx> PartialEq for MPlaceTy<'tcx> {
impl<'tcx> Eq for MPlaceTy<'tcx> {}
impl<'tcx> OpTy<'tcx> {
#[inline(always)]
pub fn try_as_mplace(self) -> Result<MPlaceTy<'tcx>, Value> {
match *self {
Operand::Indirect(mplace) => Ok(MPlaceTy { mplace, layout: self.layout }),
@ -167,7 +171,7 @@ impl<'tcx> OpTy<'tcx> {
}
}
#[inline]
#[inline(always)]
pub fn to_mem_place(self) -> MPlaceTy<'tcx> {
self.try_as_mplace().unwrap()
}
@ -311,6 +315,28 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
Ok(MPlaceTy { mplace: MemPlace { ptr, align, extra }, layout: field_layout })
}
// Iterates over all fields of an array. Much more efficient than doing the
// same by repeatedly calling `mplace_array`.
pub fn mplace_array_fields(
&self,
base: MPlaceTy<'tcx>,
) -> EvalResult<'tcx, impl Iterator<Item=EvalResult<'tcx, MPlaceTy<'tcx>>> + 'a> {
let len = base.len();
let stride = match base.layout.fields {
layout::FieldPlacement::Array { stride, .. } => stride,
_ => bug!("mplace_array_fields: expected an array layout"),
};
let layout = base.layout.field(self, 0)?;
let dl = &self.tcx.data_layout;
Ok((0..len).map(move |i| {
let ptr = base.ptr.ptr_offset(i * stride, dl)?;
Ok(MPlaceTy {
mplace: MemPlace { ptr, align: base.align, extra: PlaceExtra::None },
layout
})
}))
}
pub fn mplace_subslice(
&self,
base: MPlaceTy<'tcx>,
@ -545,14 +571,22 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
value: Value,
dest: MPlaceTy<'tcx>,
) -> EvalResult<'tcx> {
assert_eq!(dest.extra, PlaceExtra::None);
let (ptr, ptr_align) = dest.to_scalar_ptr_align();
// Note that it is really important that the type here is the right one, and matches the type things are read at.
// In case `src_val` is a `ScalarPair`, we don't do any magic here to handle padding properly, which is only
// correct if we never look at this data with the wrong type.
// Nothing to do for ZSTs, other than checking alignment
if dest.layout.size.bytes() == 0 {
self.memory.check_align(ptr, ptr_align)?;
return Ok(());
}
let ptr = ptr.to_ptr()?;
match value {
Value::Scalar(scalar) => {
self.memory.write_scalar(
dest.ptr, dest.align, scalar, dest.layout.size, dest.layout.align
ptr, ptr_align.min(dest.layout.align), scalar, dest.layout.size
)
}
Value::ScalarPair(a_val, b_val) => {
@ -562,12 +596,11 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
};
let (a_size, b_size) = (a.size(&self), b.size(&self));
let (a_align, b_align) = (a.align(&self), b.align(&self));
let a_ptr = dest.ptr;
let b_offset = a_size.abi_align(b_align);
let b_ptr = a_ptr.ptr_offset(b_offset, &self)?.into();
let b_ptr = ptr.offset(b_offset, &self)?.into();
self.memory.write_scalar(a_ptr, dest.align, a_val, a_size, a_align)?;
self.memory.write_scalar(b_ptr, dest.align, b_val, b_size, b_align)
self.memory.write_scalar(ptr, ptr_align.min(a_align), a_val, a_size)?;
self.memory.write_scalar(b_ptr, ptr_align.min(b_align), b_val, b_size)
}
}
}
@ -608,6 +641,9 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
) -> EvalResult<'tcx, MPlaceTy<'tcx>> {
let mplace = match place.place {
Place::Local { frame, local } => {
// FIXME: Consider not doing anything for a ZST, and just returning
// a fake pointer?
// We need the layout of the local. We can NOT use the layout we got,
// that might e.g. be a downcast variant!
let local_layout = self.layout_of_local(frame, local)?;
@ -707,6 +743,7 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
}
/// Every place can be read from, so we can turm them into an operand
#[inline(always)]
pub fn place_to_op(&self, place: PlaceTy<'tcx>) -> EvalResult<'tcx, OpTy<'tcx>> {
let op = match place.place {
Place::Ptr(mplace) => {

9
src/librustc_mir/interpret/validity.rs
View file

@ -124,7 +124,8 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
}
}
/// This function checks the memory where `ptr` points to.
/// This function checks the memory where `dest` points to. The place must be sized
/// (i.e., dest.extra == PlaceExtra::None).
/// It will error if the bits at the destination do not match the ones described by the layout.
pub fn validate_mplace(
&self,
@ -205,11 +206,11 @@ impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> EvalContext<'a, 'mir, 'tcx, M> {
// See https://github.com/rust-lang/rust/issues/32836#issuecomment-406875389
Ok(())
},
layout::FieldPlacement::Array { count, .. } => {
for i in 0..count {
layout::FieldPlacement::Array { .. } => {
for (i, field) in self.mplace_array_fields(dest)?.enumerate() {
let field = field?;
let mut path = path.clone();
self.dump_field_name(&mut path, dest.layout.ty, i as usize, variant).unwrap();
let field = self.mplace_field(dest, i)?;
self.validate_mplace(field, path, seen, todo)?;
}
Ok(())