Auto merge of #116564 - oli-obk:evaluated_static_in_metadata, r=RalfJung,cjgillot

Store static initializers in metadata instead of the MIR of statics. This means that adding generic statics would be even more difficult, as we can't evaluate statics from other crates anymore, but the subtle issue I have encountered make me think that having this be an explicit problem is better. The issue is that ```rust static mut FOO: &mut u32 = &mut 42; static mut BAR = unsafe { FOO }; ``` gets different allocations, instead of referring to the same one. This is also true for non-static mut, but promotion makes `static FOO: &u32 = &42;` annoying to demo. Fixes https://github.com/rust-lang/rust/issues/61345 ## Why is this being done? In order to ensure all crates see the same nested allocations (which is the last issue that needs fixing before we can stabilize [`const_mut_refs`](https://github.com/rust-lang/rust/issues/57349)), I am working on creating anonymous (from the Rust side, to LLVM it's like a regular static item) static items for the nested allocations in a static. If we evaluate the static item in a downstream crate again, we will end up duplicating its nested allocations (and in some cases, like the `match` case, even duplicate the main allocation).
2024-02-15 10:28:31 +00:00 · 2024-02-15 10:28:31 +00:00 · 6a4222b511
commit 6a4222b511
parent 4ae1e79876 73b38c661d
44 changed files with 391 additions and 193 deletions
--- a/compiler/rustc_const_eval/messages.ftl
+++ b/compiler/rustc_const_eval/messages.ftl
@ -313,6 +313,8 @@ const_eval_realloc_or_alloc_with_offset =
        *[other] {""}
    } {$ptr} which does not point to the beginning of an object

+const_eval_recursive_static = encountered static that tried to initialize itself with itself
+
 const_eval_remainder_by_zero =
    calculating the remainder with a divisor of zero
 const_eval_remainder_overflow =
--- a/compiler/rustc_const_eval/src/const_eval/error.rs
+++ b/compiler/rustc_const_eval/src/const_eval/error.rs
@ -19,6 +19,7 @@ use crate::interpret::{ErrorHandled, InterpError, InterpErrorInfo, MachineStopTy
 pub enum ConstEvalErrKind {
    ConstAccessesMutGlobal,
    ModifiedGlobal,
+    RecursiveStatic,
    AssertFailure(AssertKind<ConstInt>),
    Panic { msg: Symbol, line: u32, col: u32, file: Symbol },
 }
@ -31,13 +32,14 @@ impl MachineStopType for ConstEvalErrKind {
            ConstAccessesMutGlobal => const_eval_const_accesses_mut_global,
            ModifiedGlobal => const_eval_modified_global,
            Panic { .. } => const_eval_panic,
+            RecursiveStatic => const_eval_recursive_static,
            AssertFailure(x) => x.diagnostic_message(),
        }
    }
    fn add_args(self: Box<Self>, adder: &mut dyn FnMut(DiagnosticArgName, DiagnosticArgValue)) {
        use ConstEvalErrKind::*;
        match *self {
-            ConstAccessesMutGlobal | ModifiedGlobal => {}
+            RecursiveStatic | ConstAccessesMutGlobal | ModifiedGlobal => {}
            AssertFailure(kind) => kind.add_args(adder),
            Panic { msg, line, col, file } => {
                adder("msg".into(), msg.into_diagnostic_arg());
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@ -2,12 +2,12 @@ use either::{Left, Right};

 use rustc_hir::def::DefKind;
 use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo};
-use rustc_middle::mir::pretty::write_allocation_bytes;
 use rustc_middle::mir::{self, ConstAlloc, ConstValue};
 use rustc_middle::traits::Reveal;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::def_id::LocalDefId;
 use rustc_span::Span;
 use rustc_target::abi::{self, Abi};

@ -17,8 +17,9 @@ use crate::errors;
 use crate::errors::ConstEvalError;
 use crate::interpret::eval_nullary_intrinsic;
 use crate::interpret::{
-    intern_const_alloc_recursive, CtfeValidationMode, GlobalId, Immediate, InternKind, InterpCx,
-    InterpError, InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup,
+    create_static_alloc, intern_const_alloc_recursive, take_static_root_alloc, CtfeValidationMode,
+    GlobalId, Immediate, InternKind, InterpCx, InterpError, InterpResult, MPlaceTy, MemoryKind,
+    OpTy, RefTracking, StackPopCleanup,
 };

 // Returns a pointer to where the result lives
@ -46,7 +47,21 @@ fn eval_body_using_ecx<'mir, 'tcx>(
    );
    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
    assert!(layout.is_sized());
-    let ret = ecx.allocate(layout, MemoryKind::Stack)?;
+
+    let intern_kind = if cid.promoted.is_some() {
+        InternKind::Promoted
+    } else {
+        match tcx.static_mutability(cid.instance.def_id()) {
+            Some(m) => InternKind::Static(m),
+            None => InternKind::Constant,
+        }
+    };
+
+    let ret = if let InternKind::Static(_) = intern_kind {
+        create_static_alloc(ecx, cid.instance.def_id(), layout)?
+    } else {
+        ecx.allocate(layout, MemoryKind::Stack)?
+    };

    trace!(
        "eval_body_using_ecx: pushing stack frame for global: {}{}",
@ -66,14 +81,6 @@ fn eval_body_using_ecx<'mir, 'tcx>(
    while ecx.step()? {}

    // Intern the result
-    let intern_kind = if cid.promoted.is_some() {
-        InternKind::Promoted
-    } else {
-        match tcx.static_mutability(cid.instance.def_id()) {
-            Some(m) => InternKind::Static(m),
-            None => InternKind::Constant,
-        }
-    };
    intern_const_alloc_recursive(ecx, intern_kind, &ret)?;

    Ok(ret)
@ -249,11 +256,37 @@ pub fn eval_to_const_value_raw_provider<'tcx>(
    tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
 }

+#[instrument(skip(tcx), level = "debug")]
+pub fn eval_static_initializer_provider<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def_id: LocalDefId,
+) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
+    assert!(tcx.is_static(def_id.to_def_id()));
+
+    let instance = ty::Instance::mono(tcx, def_id.to_def_id());
+    let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
+    let mut ecx = InterpCx::new(
+        tcx,
+        tcx.def_span(def_id),
+        ty::ParamEnv::reveal_all(),
+        // Statics (and promoteds inside statics) may access other statics, because unlike consts
+        // they do not have to behave "as if" they were evaluated at runtime.
+        CompileTimeInterpreter::new(CanAccessMutGlobal::Yes, CheckAlignment::Error),
+    );
+    let alloc_id = eval_in_interpreter(&mut ecx, cid, true)?.alloc_id;
+    let alloc = take_static_root_alloc(&mut ecx, alloc_id);
+    let alloc = tcx.mk_const_alloc(alloc);
+    Ok(alloc)
+}
+
 #[instrument(skip(tcx), level = "debug")]
 pub fn eval_to_allocation_raw_provider<'tcx>(
    tcx: TyCtxt<'tcx>,
    key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>,
 ) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
+    // This shouldn't be used for statics, since statics are conceptually places,
+    // not values -- so what we do here could break pointer identity.
+    assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
    // Const eval always happens in Reveal::All mode in order to be able to use the hidden types of
    // opaque types. This is needed for trivial things like `size_of`, but also for using associated
    // types that are not specified in the opaque type.
@ -273,7 +306,7 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
    let def = cid.instance.def.def_id();
    let is_static = tcx.is_static(def);

-    let ecx = InterpCx::new(
+    let mut ecx = InterpCx::new(
        tcx,
        tcx.def_span(def),
        key.param_env,
@ -283,11 +316,11 @@ pub fn eval_to_allocation_raw_provider<'tcx>(
        // so we have to reject reading mutable global memory.
        CompileTimeInterpreter::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
    );
-    eval_in_interpreter(ecx, cid, is_static)
+    eval_in_interpreter(&mut ecx, cid, is_static)
 }

 pub fn eval_in_interpreter<'mir, 'tcx>(
-    mut ecx: InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
+    ecx: &mut InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>,
    cid: GlobalId<'tcx>,
    is_static: bool,
 ) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
@ -295,7 +328,7 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
    debug_assert_eq!(is_static, ecx.tcx.static_mutability(cid.instance.def_id()).is_some());

    let res = ecx.load_mir(cid.instance.def, cid.promoted);
-    match res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body)) {
+    match res.and_then(|body| eval_body_using_ecx(ecx, cid, body)) {
        Err(error) => {
            let (error, backtrace) = error.into_parts();
            backtrace.print_backtrace();
@ -330,8 +363,11 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
        }
        Ok(mplace) => {
            // Since evaluation had no errors, validate the resulting constant.
-            // This is a separate `try` block to provide more targeted error reporting.
+
+            // Temporarily allow access to the static_root_alloc_id for the purpose of validation.
+            let static_root_alloc_id = ecx.machine.static_root_alloc_id.take();
            let validation = const_validate_mplace(&ecx, &mplace, cid);
+            ecx.machine.static_root_alloc_id = static_root_alloc_id;

            let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();

@ -383,15 +419,9 @@ pub fn const_report_error<'mir, 'tcx>(

    let ub_note = matches!(error, InterpError::UndefinedBehavior(_)).then(|| {});

-    let alloc = ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner();
-    let mut bytes = String::new();
-    if alloc.size() != abi::Size::ZERO {
-        bytes = "\n".into();
-        // FIXME(translation) there might be pieces that are translatable.
-        write_allocation_bytes(*ecx.tcx, alloc, &mut bytes, "    ").unwrap();
-    }
-    let raw_bytes =
-        errors::RawBytesNote { size: alloc.size().bytes(), align: alloc.align.bytes(), bytes };
+    let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
+    let (size, align, _) = ecx.get_alloc_info(alloc_id);
+    let raw_bytes = errors::RawBytesNote { size: size.bytes(), align: align.bytes(), bytes };

    crate::const_eval::report(
        *ecx.tcx,
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@ -58,6 +58,9 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> {

    /// Whether to check alignment during evaluation.
    pub(super) check_alignment: CheckAlignment,
+
+    /// Used to prevent reads from a static's base allocation, as that may allow for self-initialization.
+    pub(crate) static_root_alloc_id: Option<AllocId>,
 }

 #[derive(Copy, Clone)]
@ -91,6 +94,7 @@ impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
            stack: Vec::new(),
            can_access_mut_global,
            check_alignment,
+            static_root_alloc_id: None,
        }
    }
 }
@ -746,6 +750,17 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
        // Everything else is fine.
        Ok(())
    }
+
+    fn before_alloc_read(
+        ecx: &InterpCx<'mir, 'tcx, Self>,
+        alloc_id: AllocId,
+    ) -> InterpResult<'tcx> {
+        if Some(alloc_id) == ecx.machine.static_root_alloc_id {
+            Err(ConstEvalErrKind::RecursiveStatic.into())
+        } else {
+            Ok(())
+        }
+    }
 }

 // Please do not add any code below the above `Machine` trait impl. I (oli-obk) plan more cleanups
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@ -153,7 +153,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                    );
                }

-                self.copy_op(src, dest, /*allow_transmute*/ true)?;
+                self.copy_op_allow_transmute(src, dest)?;
            }
        }
        Ok(())
@ -441,7 +441,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                    if src_field.layout.is_1zst() && cast_ty_field.is_1zst() {
                        // Skip 1-ZST fields.
                    } else if src_field.layout.ty == cast_ty_field.ty {
-                        self.copy_op(&src_field, &dst_field, /*allow_transmute*/ false)?;
+                        self.copy_op(&src_field, &dst_field)?;
                    } else {
                        if found_cast_field {
                            span_bug!(self.cur_span(), "unsize_into: more than one field to cast");
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@ -899,7 +899,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                .local_to_op(self.frame(), mir::RETURN_PLACE, None)
                .expect("return place should always be live");
            let dest = self.frame().return_place.clone();
-            let err = self.copy_op(&op, &dest, /*allow_transmute*/ true);
+            let err = if self.stack().len() == 1 {
+                // The initializer of constants and statics will get validated separately
+                // after the constant has been fully evaluated. While we could fall back to the default
+                // code path, that will cause -Zenforce-validity to cycle on static initializers.
+                // Reading from a static's memory is not allowed during its evaluation, and will always
+                // trigger a cycle error. Validation must read from the memory of the current item.
+                // For Miri this means we do not validate the root frame return value,
+                // but Miri anyway calls `read_target_isize` on that so separate validation
+                // is not needed.
+                self.copy_op_no_dest_validation(&op, &dest)
+            } else {
+                self.copy_op_allow_transmute(&op, &dest)
+            };
            trace!("return value: {:?}", self.dump_place(&dest));
            // We delay actually short-circuiting on this error until *after* the stack frame is
            // popped, since we want this error to be attributed to the caller, whose type defines
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@ -85,6 +85,8 @@ pub enum InternKind {
 ///
 /// This *cannot raise an interpreter error*. Doing so is left to validation, which
 /// tracks where in the value we are and thus can show much better error messages.
+///
+/// For `InternKind::Static` the root allocation will not be interned, but must be handled by the caller.
 #[instrument(level = "debug", skip(ecx))]
 pub fn intern_const_alloc_recursive<
    'mir,
@ -97,12 +99,12 @@ pub fn intern_const_alloc_recursive<
 ) -> Result<(), ErrorGuaranteed> {
    // We are interning recursively, and for mutability we are distinguishing the "root" allocation
    // that we are starting in, and all other allocations that we are encountering recursively.
-    let (base_mutability, inner_mutability) = match intern_kind {
+    let (base_mutability, inner_mutability, is_static) = match intern_kind {
        InternKind::Constant | InternKind::Promoted => {
            // Completely immutable. Interning anything mutably here can only lead to unsoundness,
            // since all consts are conceptually independent values but share the same underlying
            // memory.
-            (Mutability::Not, Mutability::Not)
+            (Mutability::Not, Mutability::Not, false)
        }
        InternKind::Static(Mutability::Not) => {
            (
@ -115,22 +117,31 @@ pub fn intern_const_alloc_recursive<
                // Inner allocations are never mutable. They can only arise via the "tail
                // expression" / "outer scope" rule, and we treat them consistently with `const`.
                Mutability::Not,
+                true,
            )
        }
        InternKind::Static(Mutability::Mut) => {
            // Just make everything mutable. We accept code like
            // `static mut X = &mut [42]`, so even inner allocations need to be mutable.
-            (Mutability::Mut, Mutability::Mut)
+            (Mutability::Mut, Mutability::Mut, true)
        }
    };

    // Intern the base allocation, and initialize todo list for recursive interning.
    let base_alloc_id = ret.ptr().provenance.unwrap().alloc_id();
+    trace!(?base_alloc_id, ?base_mutability);
    // First we intern the base allocation, as it requires a different mutability.
    // This gives us the initial set of nested allocations, which will then all be processed
    // recursively in the loop below.
-    let mut todo: Vec<_> =
-        intern_shallow(ecx, base_alloc_id, base_mutability).unwrap().map(|prov| prov).collect();
+    let mut todo: Vec<_> = if is_static {
+        // Do not steal the root allocation, we need it later for `take_static_root_alloc`
+        // But still change its mutability to match the requested one.
+        let alloc = ecx.memory.alloc_map.get_mut(&base_alloc_id).unwrap();
+        alloc.1.mutability = base_mutability;
+        alloc.1.provenance().ptrs().iter().map(|&(_, prov)| prov).collect()
+    } else {
+        intern_shallow(ecx, base_alloc_id, base_mutability).unwrap().map(|prov| prov).collect()
+    };
    // We need to distinguish "has just been interned" from "was already in `tcx`",
    // so we track this in a separate set.
    let mut just_interned: FxHashSet<_> = std::iter::once(base_alloc_id).collect();
@ -148,7 +159,17 @@ pub fn intern_const_alloc_recursive<
    // before validation, and interning doesn't know the type of anything, this means we can't show
    // better errors. Maybe we should consider doing validation before interning in the future.
    while let Some(prov) = todo.pop() {
+        trace!(?prov);
        let alloc_id = prov.alloc_id();
+
+        if base_alloc_id == alloc_id && is_static {
+            // This is a pointer to the static itself. It's ok for a static to refer to itself,
+            // even mutably. Whether that mutable pointer is legal at all is checked in validation.
+            // See tests/ui/statics/recursive_interior_mut.rs for how such a situation can occur.
+            // We also already collected all the nested allocations, so there's no need to do that again.
+            continue;
+        }
+
        // Crucially, we check this *before* checking whether the `alloc_id`
        // has already been interned. The point of this check is to ensure that when
        // there are multiple pointers to the same allocation, they are *all* immutable.
@ -176,6 +197,7 @@ pub fn intern_const_alloc_recursive<
            // `&None::<Cell<i32>>` lead to promotion that can produce mutable pointers. We rely
            // on the promotion analysis not screwing up to ensure that it is sound to intern
            // promoteds as immutable.
+            trace!("found bad mutable pointer");
            found_bad_mutable_pointer = true;
        }
        if ecx.tcx.try_get_global_alloc(alloc_id).is_some() {
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@ -120,7 +120,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                let val = self.tcx.span_as_caller_location(span);
                let val =
                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
-                self.copy_op(&val, dest, /* allow_transmute */ false)?;
+                self.copy_op(&val, dest)?;
            }

            sym::min_align_of_val | sym::size_of_val => {
@ -157,7 +157,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                    tcx.const_eval_global_id(self.param_env, gid, Some(tcx.span))
                })?;
                let val = self.const_val_to_op(val, ty, Some(dest.layout))?;
-                self.copy_op(&val, dest, /*allow_transmute*/ false)?;
+                self.copy_op(&val, dest)?;
            }

            sym::ctpop
@ -391,7 +391,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                    } else {
                        self.project_index(&input, i)?.into()
                    };
-                    self.copy_op(&value, &place, /*allow_transmute*/ false)?;
+                    self.copy_op(&value, &place)?;
                }
            }
            sym::simd_extract => {
@ -401,15 +401,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                    index < input_len,
                    "index `{index}` must be in bounds of vector with length {input_len}"
                );
-                self.copy_op(
-                    &self.project_index(&input, index)?,
-                    dest,
-                    /*allow_transmute*/ false,
-                )?;
+                self.copy_op(&self.project_index(&input, index)?, dest)?;
            }
            sym::likely | sym::unlikely | sym::black_box => {
                // These just return their argument
-                self.copy_op(&args[0], dest, /*allow_transmute*/ false)?;
+                self.copy_op(&args[0], dest)?;
            }
            sym::raw_eq => {
                let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@ -388,6 +388,8 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
    /// Takes read-only access to the allocation so we can keep all the memory read
    /// operations take `&self`. Use a `RefCell` in `AllocExtra` if you
    /// need to mutate.
+    ///
+    /// This is not invoked for ZST accesses, as no read actually happens.
    #[inline(always)]
    fn before_memory_read(
        _tcx: TyCtxtAt<'tcx>,
@ -399,7 +401,20 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
        Ok(())
    }

+    /// Hook for performing extra checks on any memory read access,
+    /// that involves an allocation, even ZST reads.
+    ///
+    /// Used to prevent statics from self-initializing by reading from their own memory
+    /// as it is being initialized.
+    fn before_alloc_read(
+        _ecx: &InterpCx<'mir, 'tcx, Self>,
+        _alloc_id: AllocId,
+    ) -> InterpResult<'tcx> {
+        Ok(())
+    }
+
    /// Hook for performing extra checks on a memory write access.
+    /// This is not invoked for ZST accesses, as no write actually happens.
    #[inline(always)]
    fn before_memory_write(
        _tcx: TyCtxtAt<'tcx>,
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@ -624,19 +624,20 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
            size,
            CheckInAllocMsg::MemoryAccessTest,
            |alloc_id, offset, prov| {
+                // We want to call the hook on *all* accesses that involve an AllocId,
+                // including zero-sized accesses. That means we have to do it here
+                // rather than below in the `Some` branch.
+                M::before_alloc_read(self, alloc_id)?;
                let alloc = self.get_alloc_raw(alloc_id)?;
                Ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc)))
            },
        )?;
+
        if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc {
            let range = alloc_range(offset, size);
            M::before_memory_read(self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?;
            Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
        } else {
-            // Even in this branch we have to be sure that we actually access the allocation, in
-            // order to ensure that `static FOO: Type = FOO;` causes a cycle error instead of
-            // magically pulling *any* ZST value from the ether. However, the `get_raw` above is
-            // always called when `ptr` has an `AllocId`.
            Ok(None)
        }
    }
@ -855,6 +856,21 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
        DumpAllocs { ecx: self, allocs }
    }

+    /// Print the allocation's bytes, without any nested allocations.
+    pub fn print_alloc_bytes_for_diagnostics(&self, id: AllocId) -> String {
+        // Using the "raw" access to avoid the `before_alloc_read` hook, we specifically
+        // want to be able to read all memory for diagnostics, even if that is cyclic.
+        let alloc = self.get_alloc_raw(id).unwrap();
+        let mut bytes = String::new();
+        if alloc.size() != Size::ZERO {
+            bytes = "\n".into();
+            // FIXME(translation) there might be pieces that are translatable.
+            rustc_middle::mir::pretty::write_allocation_bytes(*self.tcx, alloc, &mut bytes, "    ")
+                .unwrap();
+        }
+        bytes
+    }
+
    /// Find leaked allocations. Allocations reachable from `static_roots` or a `Global` allocation
    /// are not considered leaked, as well as leaks whose kind's `may_leak()` returns true.
    pub fn find_leaked_allocations(
--- a/compiler/rustc_const_eval/src/interpret/mod.rs
+++ b/compiler/rustc_const_eval/src/interpret/mod.rs
@ -39,4 +39,5 @@ use self::{
 };

 pub(crate) use self::intrinsics::eval_nullary_intrinsic;
+pub(crate) use self::util::{create_static_alloc, take_static_root_alloc};
 use eval_context::{from_known_layout, mir_assign_valid_types};
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@ -759,14 +759,57 @@ where
    }

    /// Copies the data from an operand to a place.
-    /// `allow_transmute` indicates whether the layouts may disagree.
+    /// The layouts of the `src` and `dest` may disagree.
+    /// Does not perform validation of the destination.
+    /// The only known use case for this function is checking the return
+    /// value of a static during stack frame popping.
+    #[inline(always)]
+    pub(super) fn copy_op_no_dest_validation(
+        &mut self,
+        src: &impl Readable<'tcx, M::Provenance>,
+        dest: &impl Writeable<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        self.copy_op_inner(
+            src, dest, /* allow_transmute */ true, /* validate_dest */ false,
+        )
+    }
+
+    /// Copies the data from an operand to a place.
+    /// The layouts of the `src` and `dest` may disagree.
+    #[inline(always)]
+    pub fn copy_op_allow_transmute(
+        &mut self,
+        src: &impl Readable<'tcx, M::Provenance>,
+        dest: &impl Writeable<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        self.copy_op_inner(
+            src, dest, /* allow_transmute */ true, /* validate_dest */ true,
+        )
+    }
+
+    /// Copies the data from an operand to a place.
+    /// `src` and `dest` must have the same layout and the copied value will be validated.
    #[inline(always)]
-    #[instrument(skip(self), level = "debug")]
    pub fn copy_op(
        &mut self,
        src: &impl Readable<'tcx, M::Provenance>,
        dest: &impl Writeable<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        self.copy_op_inner(
+            src, dest, /* allow_transmute */ false, /* validate_dest */ true,
+        )
+    }
+
+    /// Copies the data from an operand to a place.
+    /// `allow_transmute` indicates whether the layouts may disagree.
+    #[inline(always)]
+    #[instrument(skip(self), level = "debug")]
+    fn copy_op_inner(
+        &mut self,
+        src: &impl Readable<'tcx, M::Provenance>,
+        dest: &impl Writeable<'tcx, M::Provenance>,
        allow_transmute: bool,
+        validate_dest: bool,
    ) -> InterpResult<'tcx> {
        // Generally for transmutation, data must be valid both at the old and new type.
        // But if the types are the same, the 2nd validation below suffices.
@ -777,7 +820,7 @@ where
        // Do the actual copy.
        self.copy_op_no_validate(src, dest, allow_transmute)?;

-        if M::enforce_validity(self, dest.layout()) {
+        if validate_dest && M::enforce_validity(self, dest.layout()) {
            // Data got changed, better make sure it matches the type!
            self.validate_operand(&dest.to_op(self)?)?;
        }
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@ -151,12 +151,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
            Use(ref operand) => {
                // Avoid recomputing the layout
                let op = self.eval_operand(operand, Some(dest.layout))?;
-                self.copy_op(&op, &dest, /*allow_transmute*/ false)?;
+                self.copy_op(&op, &dest)?;
            }

            CopyForDeref(place) => {
                let op = self.eval_place_to_op(place, Some(dest.layout))?;
-                self.copy_op(&op, &dest, /* allow_transmute*/ false)?;
+                self.copy_op(&op, &dest)?;
            }

            BinaryOp(bin_op, box (ref left, ref right)) => {
@ -316,7 +316,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
            let field_index = active_field_index.unwrap_or(field_index);
            let field_dest = self.project_field(&variant_dest, field_index.as_usize())?;
            let op = self.eval_operand(operand, Some(field_dest.layout))?;
-            self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
+            self.copy_op(&op, &field_dest)?;
        }
        self.write_discriminant(variant_index, dest)
    }
@ -339,7 +339,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
        } else {
            // Write the src to the first element.
            let first = self.project_index(&dest, 0)?;
-            self.copy_op(&src, &first, /*allow_transmute*/ false)?;
+            self.copy_op(&src, &first)?;

            // This is performance-sensitive code for big static/const arrays! So we
            // avoid writing each operand individually and instead just make many copies
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@ -481,7 +481,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
        // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This
        // is true for all `copy_op`, but there are a lot of special cases for argument passing
        // specifically.)
-        self.copy_op(&caller_arg_copy, &callee_arg, /*allow_transmute*/ true)?;
+        self.copy_op_allow_transmute(&caller_arg_copy, &callee_arg)?;
        // If this was an in-place pass, protect the place it comes from for the duration of the call.
        if let FnArg::InPlace(place) = caller_arg {
            M::protect_in_place_function_argument(self, place)?;
--- a/compiler/rustc_const_eval/src/interpret/util.rs
+++ b/compiler/rustc_const_eval/src/interpret/util.rs
@ -1,9 +1,15 @@
-use rustc_middle::mir::interpret::InterpResult;
+use crate::const_eval::CompileTimeEvalContext;
+use crate::interpret::{MemPlaceMeta, MemoryKind};
+use rustc_middle::mir::interpret::{AllocId, Allocation, InterpResult, Pointer};
+use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{
    self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
+use rustc_span::def_id::DefId;
 use std::ops::ControlFlow;

+use super::MPlaceTy;
+
 /// Checks whether a type contains generic parameters which must be instantiated.
 ///
 /// In case it does, returns a `TooGeneric` const eval error. Note that due to polymorphization
@ -73,3 +79,23 @@ where
        Ok(())
    }
 }
+
+pub(crate) fn take_static_root_alloc<'mir, 'tcx: 'mir>(
+    ecx: &mut CompileTimeEvalContext<'mir, 'tcx>,
+    alloc_id: AllocId,
+) -> Allocation {
+    ecx.memory.alloc_map.swap_remove(&alloc_id).unwrap().1
+}
+
+pub(crate) fn create_static_alloc<'mir, 'tcx: 'mir>(
+    ecx: &mut CompileTimeEvalContext<'mir, 'tcx>,
+    static_def_id: DefId,
+    layout: TyAndLayout<'tcx>,
+) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
+    let alloc = Allocation::try_uninit(layout.size, layout.align.abi)?;
+    let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id);
+    assert_eq!(ecx.machine.static_root_alloc_id, None);
+    ecx.machine.static_root_alloc_id = Some(alloc_id);
+    assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none());
+    Ok(ecx.ptr_with_meta_to_mplace(Pointer::from(alloc_id).into(), MemPlaceMeta::None, layout))
+}
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@ -27,9 +27,9 @@ use rustc_target::abi::{
 use std::hash::Hash;

 use super::{
-    format_interp_error, AllocId, CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx,
-    InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, Pointer, Projectable, Scalar,
-    ValueVisitor,
+    format_interp_error, machine::AllocMap, AllocId, CheckInAllocMsg, GlobalAlloc, ImmTy,
+    Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, Pointer, Projectable,
+    Scalar, ValueVisitor,
 };

 // for the validation errors
@ -712,11 +712,14 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
    fn in_mutable_memory(&self, op: &OpTy<'tcx, M::Provenance>) -> bool {
        if let Some(mplace) = op.as_mplace_or_imm().left() {
            if let Some(alloc_id) = mplace.ptr().provenance.and_then(|p| p.get_alloc_id()) {
-                if self.ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner().mutability
-                    == Mutability::Mut
-                {
-                    return true;
+                let mutability = match self.ecx.tcx.global_alloc(alloc_id) {
+                    GlobalAlloc::Static(_) => {
+                        self.ecx.memory.alloc_map.get(alloc_id).unwrap().1.mutability
                    }
+                    GlobalAlloc::Memory(alloc) => alloc.inner().mutability,
+                    _ => span_bug!(self.ecx.tcx.span, "not a memory allocation"),
+                };
+                return mutability == Mutability::Mut;
            }
        }
        false
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@ -40,6 +40,7 @@ pub fn provide(providers: &mut Providers) {
    const_eval::provide(providers);
    providers.eval_to_const_value_raw = const_eval::eval_to_const_value_raw_provider;
    providers.eval_to_allocation_raw = const_eval::eval_to_allocation_raw_provider;
+    providers.eval_static_initializer = const_eval::eval_static_initializer_provider;
    providers.hooks.const_caller_location = util::caller_location::const_caller_location_provider;
    providers.eval_to_valtree = |tcx, param_env_and_value| {
        let (param_env, raw) = param_env_and_value.into_parts();
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@ -250,6 +250,15 @@ provide! { tcx, def_id, other, cdata,
    fn_arg_names => { table }
    coroutine_kind => { table_direct }
    coroutine_for_closure => { table }
+    eval_static_initializer => {
+        Ok(cdata
+            .root
+            .tables
+            .eval_static_initializer
+            .get(cdata, def_id.index)
+            .map(|lazy| lazy.decode((cdata, tcx)))
+            .unwrap_or_else(|| panic!("{def_id:?} does not have eval_static_initializer")))
+    }
    trait_def => { table }
    deduced_param_attrs => { table }
    is_type_alias_impl_trait => {
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@ -1045,11 +1045,9 @@ fn should_encode_mir(
            (true, mir_opt_base)
        }
        // Constants
-        DefKind::AnonConst
-        | DefKind::InlineConst
-        | DefKind::AssocConst
-        | DefKind::Static(..)
-        | DefKind::Const => (true, false),
+        DefKind::AnonConst | DefKind::InlineConst | DefKind::AssocConst | DefKind::Const => {
+            (true, false)
+        }
        // Coroutines require optimized MIR to compute layout.
        DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => (false, true),
        // Full-fledged functions + closures
@ -1454,6 +1452,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                    .coroutine_for_closure
                    .set_some(def_id.index, self.tcx.coroutine_for_closure(def_id).into());
            }
+            if let DefKind::Static(_) = def_kind {
+                if !self.tcx.is_foreign_item(def_id) {
+                    let data = self.tcx.eval_static_initializer(def_id).unwrap();
+                    record!(self.tables.eval_static_initializer[def_id] <- data);
+                }
+            }
            if let DefKind::Enum | DefKind::Struct | DefKind::Union = def_kind {
                self.encode_info_for_adt(local_id);
            }
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@ -443,6 +443,7 @@ define_tables! {
    fn_arg_names: Table<DefIndex, LazyArray<Ident>>,
    coroutine_kind: Table<DefIndex, hir::CoroutineKind>,
    coroutine_for_closure: Table<DefIndex, RawDefId>,
+    eval_static_initializer: Table<DefIndex, LazyValue<mir::interpret::ConstAllocation<'static>>>,
    trait_def: Table<DefIndex, LazyValue<ty::TraitDef>>,
    trait_item_def_id: Table<DefIndex, RawDefId>,
    expn_that_defined: Table<DefIndex, LazyValue<ExpnId>>,
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@ -1,4 +1,4 @@
-use super::{AllocId, AllocRange, Pointer, Scalar};
+use super::{AllocId, AllocRange, ConstAllocation, Pointer, Scalar};

 use crate::error;
 use crate::mir::{ConstAlloc, ConstValue};
@ -83,6 +83,7 @@ impl Into<ErrorGuaranteed> for ReportedErrorInfo {
 TrivialTypeTraversalImpls! { ErrorHandled }

 pub type EvalToAllocationRawResult<'tcx> = Result<ConstAlloc<'tcx>, ErrorHandled>;
+pub type EvalStaticInitializerRawResult<'tcx> = Result<ConstAllocation<'tcx>, ErrorHandled>;
 pub type EvalToConstValueResult<'tcx> = Result<ConstValue<'tcx>, ErrorHandled>;
 /// `Ok(None)` indicates the constant was fine, but the valtree couldn't be constructed.
 /// This is needed in `thir::pattern::lower_inline_const`.
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@ -142,11 +142,12 @@ use crate::ty::GenericArgKind;
 use crate::ty::{self, Instance, Ty, TyCtxt};

 pub use self::error::{
-    BadBytesAccess, CheckAlignMsg, CheckInAllocMsg, ErrorHandled, EvalToAllocationRawResult,
-    EvalToConstValueResult, EvalToValTreeResult, ExpectedKind, InterpError, InterpErrorInfo,
-    InterpResult, InvalidMetaKind, InvalidProgramInfo, MachineStopType, Misalignment, PointerKind,
-    ReportedErrorInfo, ResourceExhaustionInfo, ScalarSizeMismatch, UndefinedBehaviorInfo,
-    UnsupportedOpInfo, ValidationErrorInfo, ValidationErrorKind,
+    BadBytesAccess, CheckAlignMsg, CheckInAllocMsg, ErrorHandled, EvalStaticInitializerRawResult,
+    EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult, ExpectedKind,
+    InterpError, InterpErrorInfo, InterpResult, InvalidMetaKind, InvalidProgramInfo,
+    MachineStopType, Misalignment, PointerKind, ReportedErrorInfo, ResourceExhaustionInfo,
+    ScalarSizeMismatch, UndefinedBehaviorInfo, UnsupportedOpInfo, ValidationErrorInfo,
+    ValidationErrorKind,
 };

 pub use self::value::Scalar;
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@ -1,7 +1,7 @@
 use super::{ErrorHandled, EvalToConstValueResult, EvalToValTreeResult, GlobalId};

 use crate::mir;
-use crate::query::{TyCtxtAt, TyCtxtEnsure};
+use crate::query::TyCtxtEnsure;
 use crate::ty::visit::TypeVisitableExt;
 use crate::ty::GenericArgs;
 use crate::ty::{self, TyCtxt};
@ -173,44 +173,6 @@ impl<'tcx> TyCtxt<'tcx> {
            self.eval_to_valtree(inputs)
        }
    }
-
-    /// Evaluate a static's initializer, returning the allocation of the initializer's memory.
-    #[inline(always)]
-    pub fn eval_static_initializer(
-        self,
-        def_id: DefId,
-    ) -> Result<mir::ConstAllocation<'tcx>, ErrorHandled> {
-        self.at(DUMMY_SP).eval_static_initializer(def_id)
-    }
-}
-
-impl<'tcx> TyCtxtAt<'tcx> {
-    /// Evaluate a static's initializer, returning the allocation of the initializer's memory.
-    ///
-    /// The span is entirely ignored here, but still helpful for better query cycle errors.
-    pub fn eval_static_initializer(
-        self,
-        def_id: DefId,
-    ) -> Result<mir::ConstAllocation<'tcx>, ErrorHandled> {
-        trace!("eval_static_initializer: Need to compute {:?}", def_id);
-        assert!(self.is_static(def_id));
-        let instance = ty::Instance::mono(*self, def_id);
-        let gid = GlobalId { instance, promoted: None };
-        self.eval_to_allocation(gid, ty::ParamEnv::reveal_all())
-    }
-
-    /// Evaluate anything constant-like, returning the allocation of the final memory.
-    ///
-    /// The span is entirely ignored here, but still helpful for better query cycle errors.
-    fn eval_to_allocation(
-        self,
-        gid: GlobalId<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Result<mir::ConstAllocation<'tcx>, ErrorHandled> {
-        trace!("eval_to_allocation: Need to compute {:?}", gid);
-        let raw_const = self.eval_to_allocation_raw(param_env.and(gid))?;
-        Ok(self.global_alloc(raw_const.alloc_id).unwrap_memory())
-    }
 }

 impl<'tcx> TyCtxtEnsure<'tcx> {
@ -232,15 +194,4 @@ impl<'tcx> TyCtxtEnsure<'tcx> {
        let inputs = self.tcx.erase_regions(param_env.and(cid));
        self.eval_to_const_value_raw(inputs)
    }
-
-    /// Evaluate a static's initializer, returning the allocation of the initializer's memory.
-    pub fn eval_static_initializer(self, def_id: DefId) {
-        trace!("eval_static_initializer: Need to compute {:?}", def_id);
-        assert!(self.tcx.is_static(def_id));
-        let instance = ty::Instance::mono(self.tcx, def_id);
-        let gid = GlobalId { instance, promoted: None };
-        let param_env = ty::ParamEnv::reveal_all();
-        trace!("eval_to_allocation: Need to compute {:?}", gid);
-        self.eval_to_allocation_raw(param_env.and(gid))
-    }
 }
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@ -2,7 +2,7 @@
 //!
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html

-use crate::mir::interpret::{AllocRange, ConstAllocation, Scalar};
+use crate::mir::interpret::{AllocRange, Scalar};
 use crate::mir::visit::MirVisitable;
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable};
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@ -4,6 +4,8 @@ use std::fs;
 use std::io::{self, Write as _};
 use std::path::{Path, PathBuf};

+use crate::mir::interpret::ConstAllocation;
+
 use super::graphviz::write_mir_fn_graphviz;
 use rustc_ast::InlineAsmTemplatePiece;
 use rustc_middle::mir::interpret::{
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@ -337,6 +337,7 @@ tcx_lifetime! {
    rustc_middle::mir::ConstValue,
    rustc_middle::mir::interpret::GlobalId,
    rustc_middle::mir::interpret::LitToConstInput,
+    rustc_middle::mir::interpret::EvalStaticInitializerRawResult,
    rustc_middle::traits::query::MethodAutoderefStepsResult,
    rustc_middle::traits::query::type_op::AscribeUserType,
    rustc_middle::traits::query::type_op::Eq,
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@ -20,7 +20,8 @@ use crate::middle::stability::{self, DeprecationEntry};
 use crate::mir;
 use crate::mir::interpret::GlobalId;
 use crate::mir::interpret::{
-    EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult,
+    EvalStaticInitializerRawResult, EvalToAllocationRawResult, EvalToConstValueResult,
+    EvalToValTreeResult,
 };
 use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
@ -1061,7 +1062,7 @@ rustc_queries! {

    /// Evaluates a constant and returns the computed allocation.
    ///
-    /// **Do not use this** directly, use the `tcx.eval_static_initializer` wrapper.
+    /// **Do not use this** directly, use the `eval_to_const_value` or `eval_to_valtree` instead.
    query eval_to_allocation_raw(key: ty::ParamEnvAnd<'tcx, GlobalId<'tcx>>)
        -> EvalToAllocationRawResult<'tcx> {
        desc { |tcx|
@ -1071,6 +1072,16 @@ rustc_queries! {
        cache_on_disk_if { true }
    }

+    /// Evaluate a static's initializer, returning the allocation of the initializer's memory.
+    query eval_static_initializer(key: DefId) -> EvalStaticInitializerRawResult<'tcx> {
+        desc { |tcx|
+            "evaluating initializer of static `{}`",
+            tcx.def_path_str(key)
+        }
+        cache_on_disk_if { key.is_local() }
+        separate_provide_extern
+    }
+
    /// Evaluates const items or anonymous constants
    /// (such as enum variant explicit discriminants or array lengths)
    /// into a representation suitable for the type system and const generics.
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@ -126,6 +126,7 @@ parameterized_over_tcx! {
    crate::middle::exported_symbols::ExportedSymbol,
    crate::mir::Body,
    crate::mir::CoroutineLayout,
+    crate::mir::interpret::ConstAllocation,
    ty::Ty,
    ty::FnSig,
    ty::GenericPredicates,
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@ -399,7 +399,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                    };
                    for (field_index, op) in fields.into_iter().enumerate() {
                        let field_dest = self.ecx.project_field(&variant_dest, field_index).ok()?;
-                        self.ecx.copy_op(op, &field_dest, /*allow_transmute*/ false).ok()?;
+                        self.ecx.copy_op(op, &field_dest).ok()?;
                    }
                    self.ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), &dest).ok()?;
                    self.ecx
@ -1181,8 +1181,7 @@ fn op_to_prop_const<'tcx>(
    }

    // Everything failed: create a new allocation to hold the data.
-    let alloc_id =
-        ecx.intern_with_temp_alloc(op.layout, |ecx, dest| ecx.copy_op(op, dest, false)).ok()?;
+    let alloc_id = ecx.intern_with_temp_alloc(op.layout, |ecx, dest| ecx.copy_op(op, dest)).ok()?;
    let value = ConstValue::Indirect { alloc_id, offset: Size::ZERO };

    // Check that we do not leak a pointer.
--- a/src/tools/miri/src/shims/intrinsics/mod.rs
+++ b/src/tools/miri/src/shims/intrinsics/mod.rs
@ -108,12 +108,12 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
            "volatile_load" => {
                let [place] = check_arg_count(args)?;
                let place = this.deref_pointer(place)?;
-                this.copy_op(&place, dest, /*allow_transmute*/ false)?;
+                this.copy_op(&place, dest)?;
            }
            "volatile_store" => {
                let [place, dest] = check_arg_count(args)?;
                let place = this.deref_pointer(place)?;
-                this.copy_op(dest, &place, /*allow_transmute*/ false)?;
+                this.copy_op(dest, &place)?;
            }

            "write_bytes" | "volatile_set_memory" => {
--- a/src/tools/miri/src/shims/x86/mod.rs
+++ b/src/tools/miri/src/shims/x86/mod.rs
@ -299,7 +299,6 @@ fn bin_op_simd_float_first<'tcx, F: rustc_apfloat::Float>(
        this.copy_op(
            &this.project_index(&left, i)?,
            &this.project_index(&dest, i)?,
-            /*allow_transmute*/ false,
        )?;
    }

@ -424,7 +423,6 @@ fn unary_op_ss<'tcx>(
        this.copy_op(
            &this.project_index(&op, i)?,
            &this.project_index(&dest, i)?,
-            /*allow_transmute*/ false,
        )?;
    }

@ -484,7 +482,6 @@ fn round_first<'tcx, F: rustc_apfloat::Float>(
        this.copy_op(
            &this.project_index(&left, i)?,
            &this.project_index(&dest, i)?,
-            /*allow_transmute*/ false,
        )?;
    }

--- a/src/tools/miri/src/shims/x86/sse.rs
+++ b/src/tools/miri/src/shims/x86/sse.rs
@ -211,7 +211,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                    this.copy_op(
                        &this.project_index(&left, i)?,
                        &this.project_index(&dest, i)?,
-                        /*allow_transmute*/ false,
                    )?;
                }
            }
--- a/src/tools/miri/src/shims/x86/sse2.rs
+++ b/src/tools/miri/src/shims/x86/sse2.rs
@ -443,7 +443,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                    this.copy_op(
                        &this.project_index(&op, i)?,
                        &this.project_index(&dest, i)?,
-                        /*allow_transmute*/ false,
                    )?;
                }
            }
@ -584,7 +583,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                    this.copy_op(
                        &this.project_index(&left, i)?,
                        &this.project_index(&dest, i)?,
-                        /*allow_transmute*/ false,
                    )?;
                }
            }
--- a/src/tools/miri/src/shims/x86/sse41.rs
+++ b/src/tools/miri/src/shims/x86/sse41.rs
@ -60,7 +60,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                        this.copy_op(
                            &this.project_index(&left, i)?,
                            &dest,
-                            /*allow_transmute*/ false,
                        )?;
                    }
                }
--- a/tests/ui/consts/recursive-zst-static.default.stderr
+++ b/tests/ui/consts/recursive-zst-static.default.stderr
@ -1,21 +1,35 @@
-error[E0391]: cycle detected when const-evaluating + checking `FOO`
+error[E0080]: could not evaluate static initializer
  --> $DIR/recursive-zst-static.rs:10:18
   |
 LL | static FOO: () = FOO;
-   |                  ^^^
+   |                  ^^^ encountered static that tried to initialize itself with itself
+
+error[E0391]: cycle detected when evaluating initializer of static `A`
+  --> $DIR/recursive-zst-static.rs:13:16
   |
-   = note: ...which immediately requires const-evaluating + checking `FOO` again
+LL | static A: () = B;
+   |                ^
+   |
+note: ...which requires evaluating initializer of static `B`...
+  --> $DIR/recursive-zst-static.rs:14:16
+   |
+LL | static B: () = A;
+   |                ^
+   = note: ...which again requires evaluating initializer of static `A`, completing the cycle
 note: cycle used when linting top-level module
  --> $DIR/recursive-zst-static.rs:10:1
   |
 LL | / static FOO: () = FOO;
 LL | |
-LL | | fn main() {
+LL | |
+LL | | static A: () = B;
+...  |
 LL | |     FOO
 LL | | }
   | |_^
   = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information

-error: aborting due to 1 previous error
+error: aborting due to 2 previous errors

-For more information about this error, try `rustc --explain E0391`.
+Some errors have detailed explanations: E0080, E0391.
+For more information about an error, try `rustc --explain E0080`.
--- a/tests/ui/consts/recursive-zst-static.rs
+++ b/tests/ui/consts/recursive-zst-static.rs
@ -7,7 +7,11 @@
 // can depend on this fact and will thus do unsound things when it is violated.
 // See https://github.com/rust-lang/rust/issues/71078 for more details.

-static FOO: () = FOO; //~ cycle detected when const-evaluating + checking `FOO`
+static FOO: () = FOO;
+//~^ ERROR could not evaluate static initializer
+
+static A: () = B; //~ cycle detected when evaluating initializer of static `A`
+static B: () = A;

 fn main() {
    FOO
--- a/tests/ui/consts/recursive-zst-static.unleash.stderr
+++ b/tests/ui/consts/recursive-zst-static.unleash.stderr
@ -1,21 +1,35 @@
-error[E0391]: cycle detected when const-evaluating + checking `FOO`
+error[E0080]: could not evaluate static initializer
  --> $DIR/recursive-zst-static.rs:10:18
   |
 LL | static FOO: () = FOO;
-   |                  ^^^
+   |                  ^^^ encountered static that tried to initialize itself with itself
+
+error[E0391]: cycle detected when evaluating initializer of static `A`
+  --> $DIR/recursive-zst-static.rs:13:16
   |
-   = note: ...which immediately requires const-evaluating + checking `FOO` again
+LL | static A: () = B;
+   |                ^
+   |
+note: ...which requires evaluating initializer of static `B`...
+  --> $DIR/recursive-zst-static.rs:14:16
+   |
+LL | static B: () = A;
+   |                ^
+   = note: ...which again requires evaluating initializer of static `A`, completing the cycle
 note: cycle used when linting top-level module
  --> $DIR/recursive-zst-static.rs:10:1
   |
 LL | / static FOO: () = FOO;
 LL | |
-LL | | fn main() {
+LL | |
+LL | | static A: () = B;
+...  |
 LL | |     FOO
 LL | | }
   | |_^
   = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information

-error: aborting due to 1 previous error
+error: aborting due to 2 previous errors

-For more information about this error, try `rustc --explain E0391`.
+Some errors have detailed explanations: E0080, E0391.
+For more information about an error, try `rustc --explain E0080`.
--- a/tests/ui/consts/write-to-static-mut-in-static.rs
+++ b/tests/ui/consts/write-to-static-mut-in-static.rs
@ -3,8 +3,8 @@ pub static mut B: () = unsafe { A = 1; };
 //~^ ERROR could not evaluate static initializer

 pub static mut C: u32 = unsafe { C = 1; 0 };
-//~^ ERROR cycle detected

 pub static D: u32 = D;
+//~^ ERROR could not evaluate static initializer

 fn main() {}
--- a/tests/ui/consts/write-to-static-mut-in-static.stderr
+++ b/tests/ui/consts/write-to-static-mut-in-static.stderr
@ -4,27 +4,12 @@ error[E0080]: could not evaluate static initializer
 LL | pub static mut B: () = unsafe { A = 1; };
   |                                 ^^^^^ modifying a static's initial value from another static's initializer

-error[E0391]: cycle detected when const-evaluating + checking `C`
-  --> $DIR/write-to-static-mut-in-static.rs:5:34
+error[E0080]: could not evaluate static initializer
+  --> $DIR/write-to-static-mut-in-static.rs:7:21
   |
-LL | pub static mut C: u32 = unsafe { C = 1; 0 };
-   |                                  ^^^^^
-   |
-   = note: ...which immediately requires const-evaluating + checking `C` again
-note: cycle used when linting top-level module
-  --> $DIR/write-to-static-mut-in-static.rs:1:1
-   |
-LL | / pub static mut A: u32 = 0;
-LL | | pub static mut B: () = unsafe { A = 1; };
-LL | |
-LL | |
-...  |
-LL | |
-LL | | fn main() {}
-   | |____________^
-   = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
+LL | pub static D: u32 = D;
+   |                     ^ encountered static that tried to initialize itself with itself

 error: aborting due to 2 previous errors

-Some errors have detailed explanations: E0080, E0391.
-For more information about an error, try `rustc --explain E0080`.
+For more information about this error, try `rustc --explain E0080`.
--- a/tests/ui/recursion/recursive-static-definition.rs
+++ b/tests/ui/recursion/recursive-static-definition.rs
@ -1,4 +1,12 @@
 pub static FOO: u32 = FOO;
-//~^ ERROR cycle detected when const-evaluating + checking `FOO`
+//~^ ERROR could not evaluate static initializer
+
+#[derive(Copy, Clone)]
+pub union Foo {
+    x: u32,
+}
+
+pub static BAR: Foo = BAR;
+//~^ ERROR could not evaluate static initializer

 fn main() {}
--- a/tests/ui/recursion/recursive-static-definition.stderr
+++ b/tests/ui/recursion/recursive-static-definition.stderr
@ -1,20 +1,15 @@
-error[E0391]: cycle detected when const-evaluating + checking `FOO`
+error[E0080]: could not evaluate static initializer
  --> $DIR/recursive-static-definition.rs:1:23
   |
 LL | pub static FOO: u32 = FOO;
-   |                       ^^^
-   |
-   = note: ...which immediately requires const-evaluating + checking `FOO` again
-note: cycle used when linting top-level module
-  --> $DIR/recursive-static-definition.rs:1:1
-   |
-LL | / pub static FOO: u32 = FOO;
-LL | |
-LL | |
-LL | | fn main() {}
-   | |____________^
-   = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
+   |                       ^^^ encountered static that tried to initialize itself with itself

-error: aborting due to 1 previous error
+error[E0080]: could not evaluate static initializer
+  --> $DIR/recursive-static-definition.rs:9:23
+   |
+LL | pub static BAR: Foo = BAR;
+   |                       ^^^ encountered static that tried to initialize itself with itself

-For more information about this error, try `rustc --explain E0391`.
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0080`.
--- a/tests/ui/statics/recursive_interior_mut.rs
+++ b/tests/ui/statics/recursive_interior_mut.rs
@ -0,0 +1,20 @@
+// check-pass
+
+use std::cell::Cell;
+use std::ptr::NonNull;
+
+struct ChunkFooter {
+    prev: Cell<NonNull<ChunkFooter>>,
+}
+
+struct EmptyChunkFooter(ChunkFooter);
+
+unsafe impl Sync for EmptyChunkFooter {}
+
+static EMPTY_CHUNK: EmptyChunkFooter = EmptyChunkFooter(ChunkFooter {
+    prev: Cell::new(unsafe {
+        NonNull::new_unchecked(&EMPTY_CHUNK as *const EmptyChunkFooter as *mut ChunkFooter)
+    }),
+});
+
+fn main() {}
--- a/tests/ui/treat-err-as-bug/err.rs
+++ b/tests/ui/treat-err-as-bug/err.rs
@ -1,7 +1,7 @@
 // compile-flags: -Ztreat-err-as-bug
 // failure-status: 101
 // error-pattern: aborting due to `-Z treat-err-as-bug=1`
-// error-pattern: [eval_to_allocation_raw] const-evaluating + checking `C`
+// error-pattern: [eval_static_initializer] evaluating initializer of static `C`
 // normalize-stderr-test "note: .*\n\n" -> ""
 // normalize-stderr-test "thread 'rustc' panicked.*:\n.*\n" -> ""
 // rustc-env:RUST_BACKTRACE=0
--- a/tests/ui/treat-err-as-bug/err.stderr
+++ b/tests/ui/treat-err-as-bug/err.stderr
@ -7,6 +7,6 @@ LL | pub static C: u32 = 0 - 1;
 error: the compiler unexpectedly panicked. this is a bug.

 query stack during panic:
-#0 [eval_to_allocation_raw] const-evaluating + checking `C`
+#0 [eval_static_initializer] evaluating initializer of static `C`
 #1 [lint_mod] linting top-level module
 end of query stack