mv compiler to compiler/
This commit is contained in:
mark
Vadim Petrochenkov
parent
db534b3ac2
commit
9e5f7d5631
1686 changed files with 941 additions and 1051 deletions
455
compiler/rustc_mir/src/interpret/intern.rs
Normal file
455
compiler/rustc_mir/src/interpret/intern.rs
Normal file
|
|
@ -0,0 +1,455 @@
|
|||
//! This module specifies the type based interner for constants.
|
||||
//!
|
||||
//! After a const evaluation has computed a value, before we destroy the const evaluator's session
|
||||
//! memory, we need to extract all memory allocations to the global memory pool so they stay around.
|
||||
|
||||
use super::validity::RefTracking;
|
||||
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
|
||||
use rustc_hir as hir;
|
||||
use rustc_middle::mir::interpret::InterpResult;
|
||||
use rustc_middle::ty::{self, layout::TyAndLayout, query::TyCtxtAt, Ty};
|
||||
use rustc_target::abi::Size;
|
||||
|
||||
use rustc_ast::Mutability;
|
||||
|
||||
use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, Scalar, ValueVisitor};
|
||||
|
||||
pub trait CompileTimeMachine<'mir, 'tcx> = Machine<
|
||||
'mir,
|
||||
'tcx,
|
||||
MemoryKind = !,
|
||||
PointerTag = (),
|
||||
ExtraFnVal = !,
|
||||
FrameExtra = (),
|
||||
AllocExtra = (),
|
||||
MemoryMap = FxHashMap<AllocId, (MemoryKind<!>, Allocation)>,
|
||||
>;
|
||||
|
||||
struct InternVisitor<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> {
|
||||
/// The ectx from which we intern.
|
||||
ecx: &'rt mut InterpCx<'mir, 'tcx, M>,
|
||||
/// Previously encountered safe references.
|
||||
ref_tracking: &'rt mut RefTracking<(MPlaceTy<'tcx>, InternMode)>,
|
||||
/// A list of all encountered allocations. After type-based interning, we traverse this list to
|
||||
/// also intern allocations that are only referenced by a raw pointer or inside a union.
|
||||
leftover_allocations: &'rt mut FxHashSet<AllocId>,
|
||||
/// The root kind of the value that we're looking at. This field is never mutated and only used
|
||||
/// for sanity assertions that will ICE when `const_qualif` screws up.
|
||||
mode: InternMode,
|
||||
/// This field stores whether we are *currently* inside an `UnsafeCell`. This can affect
|
||||
/// the intern mode of references we encounter.
|
||||
inside_unsafe_cell: bool,
|
||||
|
||||
/// This flag is to avoid triggering UnsafeCells are not allowed behind references in constants
|
||||
/// for promoteds.
|
||||
/// It's a copy of `mir::Body`'s ignore_interior_mut_in_const_validation field
|
||||
ignore_interior_mut_in_const: bool,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
|
||||
enum InternMode {
|
||||
/// A static and its current mutability. Below shared references inside a `static mut`,
|
||||
/// this is *immutable*, and below mutable references inside an `UnsafeCell`, this
|
||||
/// is *mutable*.
|
||||
Static(hir::Mutability),
|
||||
/// The "base value" of a const, which can have `UnsafeCell` (as in `const FOO: Cell<i32>`),
|
||||
/// but that interior mutability is simply ignored.
|
||||
ConstBase,
|
||||
/// The "inner values" of a const with references, where `UnsafeCell` is an error.
|
||||
ConstInner,
|
||||
}
|
||||
|
||||
/// Signalling data structure to ensure we don't recurse
|
||||
/// into the memory of other constants or statics
|
||||
struct IsStaticOrFn;
|
||||
|
||||
fn mutable_memory_in_const(tcx: TyCtxtAt<'_>, kind: &str) {
|
||||
// FIXME: show this in validation instead so we can point at where in the value the error is?
|
||||
tcx.sess.span_err(tcx.span, &format!("mutable memory ({}) is not allowed in constant", kind));
|
||||
}
|
||||
|
||||
/// Intern an allocation without looking at its children.
|
||||
/// `mode` is the mode of the environment where we found this pointer.
|
||||
/// `mutablity` is the mutability of the place to be interned; even if that says
|
||||
/// `immutable` things might become mutable if `ty` is not frozen.
|
||||
/// `ty` can be `None` if there is no potential interior mutability
|
||||
/// to account for (e.g. for vtables).
|
||||
fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>>(
|
||||
ecx: &'rt mut InterpCx<'mir, 'tcx, M>,
|
||||
leftover_allocations: &'rt mut FxHashSet<AllocId>,
|
||||
alloc_id: AllocId,
|
||||
mode: InternMode,
|
||||
ty: Option<Ty<'tcx>>,
|
||||
) -> Option<IsStaticOrFn> {
|
||||
trace!("intern_shallow {:?} with {:?}", alloc_id, mode);
|
||||
// remove allocation
|
||||
let tcx = ecx.tcx;
|
||||
let (kind, mut alloc) = match ecx.memory.alloc_map.remove(&alloc_id) {
|
||||
Some(entry) => entry,
|
||||
None => {
|
||||
// Pointer not found in local memory map. It is either a pointer to the global
|
||||
// map, or dangling.
|
||||
// If the pointer is dangling (neither in local nor global memory), we leave it
|
||||
// to validation to error -- it has the much better error messages, pointing out where
|
||||
// in the value the dangling reference lies.
|
||||
// The `delay_span_bug` ensures that we don't forget such a check in validation.
|
||||
if tcx.get_global_alloc(alloc_id).is_none() {
|
||||
tcx.sess.delay_span_bug(ecx.tcx.span, "tried to intern dangling pointer");
|
||||
}
|
||||
// treat dangling pointers like other statics
|
||||
// just to stop trying to recurse into them
|
||||
return Some(IsStaticOrFn);
|
||||
}
|
||||
};
|
||||
// This match is just a canary for future changes to `MemoryKind`, which most likely need
|
||||
// changes in this function.
|
||||
match kind {
|
||||
MemoryKind::Stack | MemoryKind::Vtable | MemoryKind::CallerLocation => {}
|
||||
}
|
||||
// Set allocation mutability as appropriate. This is used by LLVM to put things into
|
||||
// read-only memory, and also by Miri when evaluating other globals that
|
||||
// access this one.
|
||||
if let InternMode::Static(mutability) = mode {
|
||||
// For this, we need to take into account `UnsafeCell`. When `ty` is `None`, we assume
|
||||
// no interior mutability.
|
||||
let frozen = ty.map_or(true, |ty| ty.is_freeze(ecx.tcx, ecx.param_env));
|
||||
// For statics, allocation mutability is the combination of the place mutability and
|
||||
// the type mutability.
|
||||
// The entire allocation needs to be mutable if it contains an `UnsafeCell` anywhere.
|
||||
let immutable = mutability == Mutability::Not && frozen;
|
||||
if immutable {
|
||||
alloc.mutability = Mutability::Not;
|
||||
} else {
|
||||
// Just making sure we are not "upgrading" an immutable allocation to mutable.
|
||||
assert_eq!(alloc.mutability, Mutability::Mut);
|
||||
}
|
||||
} else {
|
||||
// No matter what, *constants are never mutable*. Mutating them is UB.
|
||||
// See const_eval::machine::MemoryExtra::can_access_statics for why
|
||||
// immutability is so important.
|
||||
|
||||
// There are no sensible checks we can do here; grep for `mutable_memory_in_const` to
|
||||
// find the checks we are doing elsewhere to avoid even getting here for memory
|
||||
// that "wants" to be mutable.
|
||||
alloc.mutability = Mutability::Not;
|
||||
};
|
||||
// link the alloc id to the actual allocation
|
||||
let alloc = tcx.intern_const_alloc(alloc);
|
||||
leftover_allocations.extend(alloc.relocations().iter().map(|&(_, ((), reloc))| reloc));
|
||||
tcx.set_alloc_id_memory(alloc_id, alloc);
|
||||
None
|
||||
}
|
||||
|
||||
impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> InternVisitor<'rt, 'mir, 'tcx, M> {
|
||||
fn intern_shallow(
|
||||
&mut self,
|
||||
alloc_id: AllocId,
|
||||
mode: InternMode,
|
||||
ty: Option<Ty<'tcx>>,
|
||||
) -> Option<IsStaticOrFn> {
|
||||
intern_shallow(self.ecx, self.leftover_allocations, alloc_id, mode, ty)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
|
||||
for InternVisitor<'rt, 'mir, 'tcx, M>
|
||||
{
|
||||
type V = MPlaceTy<'tcx>;
|
||||
|
||||
#[inline(always)]
|
||||
fn ecx(&self) -> &InterpCx<'mir, 'tcx, M> {
|
||||
&self.ecx
|
||||
}
|
||||
|
||||
fn visit_aggregate(
|
||||
&mut self,
|
||||
mplace: MPlaceTy<'tcx>,
|
||||
fields: impl Iterator<Item = InterpResult<'tcx, Self::V>>,
|
||||
) -> InterpResult<'tcx> {
|
||||
if let Some(def) = mplace.layout.ty.ty_adt_def() {
|
||||
if Some(def.did) == self.ecx.tcx.lang_items().unsafe_cell_type() {
|
||||
if self.mode == InternMode::ConstInner && !self.ignore_interior_mut_in_const {
|
||||
// We do not actually make this memory mutable. But in case the user
|
||||
// *expected* it to be mutable, make sure we error. This is just a
|
||||
// sanity check to prevent users from accidentally exploiting the UB
|
||||
// they caused. It also helps us to find cases where const-checking
|
||||
// failed to prevent an `UnsafeCell` (but as `ignore_interior_mut_in_const`
|
||||
// shows that part is not airtight).
|
||||
mutable_memory_in_const(self.ecx.tcx, "`UnsafeCell`");
|
||||
}
|
||||
// We are crossing over an `UnsafeCell`, we can mutate again. This means that
|
||||
// References we encounter inside here are interned as pointing to mutable
|
||||
// allocations.
|
||||
// Remember the `old` value to handle nested `UnsafeCell`.
|
||||
let old = std::mem::replace(&mut self.inside_unsafe_cell, true);
|
||||
let walked = self.walk_aggregate(mplace, fields);
|
||||
self.inside_unsafe_cell = old;
|
||||
return walked;
|
||||
}
|
||||
}
|
||||
self.walk_aggregate(mplace, fields)
|
||||
}
|
||||
|
||||
fn visit_value(&mut self, mplace: MPlaceTy<'tcx>) -> InterpResult<'tcx> {
|
||||
// Handle Reference types, as these are the only relocations supported by const eval.
|
||||
// Raw pointers (and boxes) are handled by the `leftover_relocations` logic.
|
||||
let tcx = self.ecx.tcx;
|
||||
let ty = mplace.layout.ty;
|
||||
if let ty::Ref(_, referenced_ty, ref_mutability) = ty.kind {
|
||||
let value = self.ecx.read_immediate(mplace.into())?;
|
||||
let mplace = self.ecx.ref_to_mplace(value)?;
|
||||
assert_eq!(mplace.layout.ty, referenced_ty);
|
||||
// Handle trait object vtables.
|
||||
if let ty::Dynamic(..) =
|
||||
tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind
|
||||
{
|
||||
// Validation will error (with a better message) on an invalid vtable pointer
|
||||
// so we can safely not do anything if this is not a real pointer.
|
||||
if let Scalar::Ptr(vtable) = mplace.meta.unwrap_meta() {
|
||||
// Explicitly choose const mode here, since vtables are immutable, even
|
||||
// if the reference of the fat pointer is mutable.
|
||||
self.intern_shallow(vtable.alloc_id, InternMode::ConstInner, None);
|
||||
} else {
|
||||
// Let validation show the error message, but make sure it *does* error.
|
||||
tcx.sess
|
||||
.delay_span_bug(tcx.span, "vtables pointers cannot be integer pointers");
|
||||
}
|
||||
}
|
||||
// Check if we have encountered this pointer+layout combination before.
|
||||
// Only recurse for allocation-backed pointers.
|
||||
if let Scalar::Ptr(ptr) = mplace.ptr {
|
||||
// Compute the mode with which we intern this.
|
||||
let ref_mode = match self.mode {
|
||||
InternMode::Static(mutbl) => {
|
||||
// In statics, merge outer mutability with reference mutability and
|
||||
// take into account whether we are in an `UnsafeCell`.
|
||||
|
||||
// The only way a mutable reference actually works as a mutable reference is
|
||||
// by being in a `static mut` directly or behind another mutable reference.
|
||||
// If there's an immutable reference or we are inside a `static`, then our
|
||||
// mutable reference is equivalent to an immutable one. As an example:
|
||||
// `&&mut Foo` is semantically equivalent to `&&Foo`
|
||||
match ref_mutability {
|
||||
_ if self.inside_unsafe_cell => {
|
||||
// Inside an `UnsafeCell` is like inside a `static mut`, the "outer"
|
||||
// mutability does not matter.
|
||||
InternMode::Static(ref_mutability)
|
||||
}
|
||||
Mutability::Not => {
|
||||
// A shared reference, things become immutable.
|
||||
// We do *not* consier `freeze` here -- that is done more precisely
|
||||
// when traversing the referenced data (by tracking `UnsafeCell`).
|
||||
InternMode::Static(Mutability::Not)
|
||||
}
|
||||
Mutability::Mut => {
|
||||
// Mutable reference.
|
||||
InternMode::Static(mutbl)
|
||||
}
|
||||
}
|
||||
}
|
||||
InternMode::ConstBase | InternMode::ConstInner => {
|
||||
// Ignore `UnsafeCell`, everything is immutable. Do some sanity checking
|
||||
// for mutable references that we encounter -- they must all be ZST.
|
||||
// This helps to prevent users from accidentally exploiting UB that they
|
||||
// caused (by somehow getting a mutable reference in a `const`).
|
||||
if ref_mutability == Mutability::Mut {
|
||||
match referenced_ty.kind {
|
||||
ty::Array(_, n) if n.eval_usize(*tcx, self.ecx.param_env) == 0 => {}
|
||||
ty::Slice(_)
|
||||
if mplace.meta.unwrap_meta().to_machine_usize(self.ecx)?
|
||||
== 0 => {}
|
||||
_ => mutable_memory_in_const(tcx, "`&mut`"),
|
||||
}
|
||||
} else {
|
||||
// A shared reference. We cannot check `freeze` here due to references
|
||||
// like `&dyn Trait` that are actually immutable. We do check for
|
||||
// concrete `UnsafeCell` when traversing the pointee though (if it is
|
||||
// a new allocation, not yet interned).
|
||||
}
|
||||
// Go on with the "inner" rules.
|
||||
InternMode::ConstInner
|
||||
}
|
||||
};
|
||||
match self.intern_shallow(ptr.alloc_id, ref_mode, Some(referenced_ty)) {
|
||||
// No need to recurse, these are interned already and statics may have
|
||||
// cycles, so we don't want to recurse there
|
||||
Some(IsStaticOrFn) => {}
|
||||
// intern everything referenced by this value. The mutability is taken from the
|
||||
// reference. It is checked above that mutable references only happen in
|
||||
// `static mut`
|
||||
None => self.ref_tracking.track((mplace, ref_mode), || ()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
} else {
|
||||
// Not a reference -- proceed recursively.
|
||||
self.walk_value(mplace)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Hash, Eq)]
|
||||
pub enum InternKind {
|
||||
/// The `mutability` of the static, ignoring the type which may have interior mutability.
|
||||
Static(hir::Mutability),
|
||||
Constant,
|
||||
Promoted,
|
||||
}
|
||||
|
||||
/// Intern `ret` and everything it references.
|
||||
///
|
||||
/// This *cannot raise an interpreter error*. Doing so is left to validation, which
|
||||
/// tracks where in the value we are and thus can show much better error messages.
|
||||
/// Any errors here would anyway be turned into `const_err` lints, whereas validation failures
|
||||
/// are hard errors.
|
||||
pub fn intern_const_alloc_recursive<M: CompileTimeMachine<'mir, 'tcx>>(
|
||||
ecx: &mut InterpCx<'mir, 'tcx, M>,
|
||||
intern_kind: InternKind,
|
||||
ret: MPlaceTy<'tcx>,
|
||||
ignore_interior_mut_in_const: bool,
|
||||
) where
|
||||
'tcx: 'mir,
|
||||
{
|
||||
let tcx = ecx.tcx;
|
||||
let base_intern_mode = match intern_kind {
|
||||
InternKind::Static(mutbl) => InternMode::Static(mutbl),
|
||||
// `Constant` includes array lengths.
|
||||
// `Promoted` includes non-`Copy` array initializers and `rustc_args_required_const` arguments.
|
||||
InternKind::Constant | InternKind::Promoted => InternMode::ConstBase,
|
||||
};
|
||||
|
||||
// Type based interning.
|
||||
// `ref_tracking` tracks typed references we have already interned and still need to crawl for
|
||||
// more typed information inside them.
|
||||
// `leftover_allocations` collects *all* allocations we see, because some might not
|
||||
// be available in a typed way. They get interned at the end.
|
||||
let mut ref_tracking = RefTracking::empty();
|
||||
let leftover_allocations = &mut FxHashSet::default();
|
||||
|
||||
// start with the outermost allocation
|
||||
intern_shallow(
|
||||
ecx,
|
||||
leftover_allocations,
|
||||
// The outermost allocation must exist, because we allocated it with
|
||||
// `Memory::allocate`.
|
||||
ret.ptr.assert_ptr().alloc_id,
|
||||
base_intern_mode,
|
||||
Some(ret.layout.ty),
|
||||
);
|
||||
|
||||
ref_tracking.track((ret, base_intern_mode), || ());
|
||||
|
||||
while let Some(((mplace, mode), _)) = ref_tracking.todo.pop() {
|
||||
let res = InternVisitor {
|
||||
ref_tracking: &mut ref_tracking,
|
||||
ecx,
|
||||
mode,
|
||||
leftover_allocations,
|
||||
ignore_interior_mut_in_const,
|
||||
inside_unsafe_cell: false,
|
||||
}
|
||||
.visit_value(mplace);
|
||||
// We deliberately *ignore* interpreter errors here. When there is a problem, the remaining
|
||||
// references are "leftover"-interned, and later validation will show a proper error
|
||||
// and point at the right part of the value causing the problem.
|
||||
match res {
|
||||
Ok(()) => {}
|
||||
Err(error) => {
|
||||
ecx.tcx.sess.delay_span_bug(
|
||||
ecx.tcx.span,
|
||||
&format!(
|
||||
"error during interning should later cause validation failure: {}",
|
||||
error
|
||||
),
|
||||
);
|
||||
// Some errors shouldn't come up because creating them causes
|
||||
// an allocation, which we should avoid. When that happens,
|
||||
// dedicated error variants should be introduced instead.
|
||||
assert!(
|
||||
!error.kind.allocates(),
|
||||
"interning encountered allocating error: {}",
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Intern the rest of the allocations as mutable. These might be inside unions, padding, raw
|
||||
// pointers, ... So we can't intern them according to their type rules
|
||||
|
||||
let mut todo: Vec<_> = leftover_allocations.iter().cloned().collect();
|
||||
while let Some(alloc_id) = todo.pop() {
|
||||
if let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) {
|
||||
// We can't call the `intern_shallow` method here, as its logic is tailored to safe
|
||||
// references and a `leftover_allocations` set (where we only have a todo-list here).
|
||||
// So we hand-roll the interning logic here again.
|
||||
match intern_kind {
|
||||
// Statics may contain mutable allocations even behind relocations.
|
||||
// Even for immutable statics it would be ok to have mutable allocations behind
|
||||
// raw pointers, e.g. for `static FOO: *const AtomicUsize = &AtomicUsize::new(42)`.
|
||||
InternKind::Static(_) => {}
|
||||
// Raw pointers in promoteds may only point to immutable things so we mark
|
||||
// everything as immutable.
|
||||
// It is UB to mutate through a raw pointer obtained via an immutable reference:
|
||||
// Since all references and pointers inside a promoted must by their very definition
|
||||
// be created from an immutable reference (and promotion also excludes interior
|
||||
// mutability), mutating through them would be UB.
|
||||
// There's no way we can check whether the user is using raw pointers correctly,
|
||||
// so all we can do is mark this as immutable here.
|
||||
InternKind::Promoted => {
|
||||
// See const_eval::machine::MemoryExtra::can_access_statics for why
|
||||
// immutability is so important.
|
||||
alloc.mutability = Mutability::Not;
|
||||
}
|
||||
InternKind::Constant => {
|
||||
// If it's a constant, we should not have any "leftovers" as everything
|
||||
// is tracked by const-checking.
|
||||
// FIXME: downgrade this to a warning? It rejects some legitimate consts,
|
||||
// such as `const CONST_RAW: *const Vec<i32> = &Vec::new() as *const _;`.
|
||||
ecx.tcx
|
||||
.sess
|
||||
.span_err(ecx.tcx.span, "untyped pointers are not allowed in constant");
|
||||
// For better errors later, mark the allocation as immutable.
|
||||
alloc.mutability = Mutability::Not;
|
||||
}
|
||||
}
|
||||
let alloc = tcx.intern_const_alloc(alloc);
|
||||
tcx.set_alloc_id_memory(alloc_id, alloc);
|
||||
for &(_, ((), reloc)) in alloc.relocations().iter() {
|
||||
if leftover_allocations.insert(reloc) {
|
||||
todo.push(reloc);
|
||||
}
|
||||
}
|
||||
} else if ecx.memory.dead_alloc_map.contains_key(&alloc_id) {
|
||||
// Codegen does not like dangling pointers, and generally `tcx` assumes that
|
||||
// all allocations referenced anywhere actually exist. So, make sure we error here.
|
||||
ecx.tcx.sess.span_err(ecx.tcx.span, "encountered dangling pointer in final constant");
|
||||
} else if ecx.tcx.get_global_alloc(alloc_id).is_none() {
|
||||
// We have hit an `AllocId` that is neither in local or global memory and isn't
|
||||
// marked as dangling by local memory. That should be impossible.
|
||||
span_bug!(ecx.tcx.span, "encountered unknown alloc id {:?}", alloc_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
|
||||
/// A helper function that allocates memory for the layout given and gives you access to mutate
|
||||
/// it. Once your own mutation code is done, the backing `Allocation` is removed from the
|
||||
/// current `Memory` and returned.
|
||||
pub(crate) fn intern_with_temp_alloc(
|
||||
&mut self,
|
||||
layout: TyAndLayout<'tcx>,
|
||||
f: impl FnOnce(
|
||||
&mut InterpCx<'mir, 'tcx, M>,
|
||||
MPlaceTy<'tcx, M::PointerTag>,
|
||||
) -> InterpResult<'tcx, ()>,
|
||||
) -> InterpResult<'tcx, &'tcx Allocation> {
|
||||
let dest = self.allocate(layout, MemoryKind::Stack);
|
||||
f(self, dest)?;
|
||||
let ptr = dest.ptr.assert_ptr();
|
||||
assert_eq!(ptr.offset, Size::ZERO);
|
||||
let mut alloc = self.memory.alloc_map.remove(&ptr.alloc_id).unwrap().1;
|
||||
alloc.mutability = Mutability::Not;
|
||||
Ok(self.tcx.intern_const_alloc(alloc))
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue