Support allocation failures when interperting MIR

Note that this breaks Miri.

Closes #79601

compiler/rustc_middle/src/lib.rs

@@ -48,6 +48,7 @@
 #![feature(associated_type_defaults)]
 #![feature(iter_zip)]
 #![feature(thread_local_const_init)]
+#![feature(try_reserve)]
 #![recursion_limit = "512"]
 
 #[macro_use]

compiler/rustc_middle/src/mir/interpret/allocation.rs

@@ -11,8 +11,9 @@ use rustc_data_structures::sorted_map::SortedMap;
 use rustc_target::abi::{Align, HasDataLayout, Size};
 
 use super::{
-    read_target_uint, write_target_uint, AllocId, InterpError, Pointer, Scalar, ScalarMaybeUninit,
-    UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo,
+    read_target_uint, write_target_uint, AllocId, InterpError, InterpResult, Pointer,
+    ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, UndefinedBehaviorInfo, UninitBytesAccess,
+    UnsupportedOpInfo,
 };
 
 /// This type represents an Allocation in the Miri/CTFE core engine.
@@ -121,15 +122,23 @@ impl<Tag> Allocation<Tag> {
         Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
     }
 
-    pub fn uninit(size: Size, align: Align) -> Self {
-        Allocation {
-            bytes: vec![0; size.bytes_usize()],
+    /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
+    /// available to the compiler to do so.
+    pub fn uninit(size: Size, align: Align) -> InterpResult<'static, Self> {
+        let mut bytes = Vec::new();
+        bytes.try_reserve(size.bytes_usize()).map_err(|_| {
+            InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
+        })?;
+        bytes.resize(size.bytes_usize(), 0);
+        bytes.fill(0);
+        Ok(Allocation {
+            bytes: bytes,
             relocations: Relocations::new(),
             init_mask: InitMask::new(size, false),
             align,
             mutability: Mutability::Mut,
             extra: (),
-        }
+        })
     }
 }
 

compiler/rustc_middle/src/mir/interpret/error.rs

@@ -423,6 +423,8 @@ pub enum ResourceExhaustionInfo {
     ///
     /// The exact limit is set by the `const_eval_limit` attribute.
     StepLimitReached,
+    /// There is not enough memory to perform an allocation.
+    MemoryExhausted,
 }
 
 impl fmt::Display for ResourceExhaustionInfo {
@@ -435,6 +437,9 @@ impl fmt::Display for ResourceExhaustionInfo {
             StepLimitReached => {
                 write!(f, "exceeded interpreter step limit (see `#[const_eval_limit]`)")
             }
+            MemoryExhausted => {
+                write!(f, "tried to allocate more memory than available to compiler")
+            }
         }
     }
 }

compiler/rustc_middle/src/ty/vtable.rs

@@ -1,6 +1,6 @@
 use std::convert::TryFrom;
 
-use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar};
+use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar, InterpResult};
 use crate::ty::fold::TypeFoldable;
 use crate::ty::{self, DefId, SubstsRef, Ty, TyCtxt};
 use rustc_ast::Mutability;
@@ -28,11 +28,11 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         ty: Ty<'tcx>,
         poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
-    ) -> AllocId {
+    ) -> InterpResult<'tcx, AllocId> {
         let tcx = self;
         let vtables_cache = tcx.vtables_cache.lock();
         if let Some(alloc_id) = vtables_cache.get(&(ty, poly_trait_ref)).cloned() {
-            return alloc_id;
+            return Ok(alloc_id);
         }
         drop(vtables_cache);
 
@@ -60,7 +60,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let ptr_align = tcx.data_layout.pointer_align.abi;
 
         let vtable_size = ptr_size * u64::try_from(vtable_entries.len()).unwrap();
-        let mut vtable = Allocation::uninit(vtable_size, ptr_align);
+        let mut vtable = Allocation::uninit(vtable_size, ptr_align)?;
 
         // No need to do any alignment checks on the memory accesses below, because we know the
         // allocation is correctly aligned as we created it above. Also we're only offsetting by
@@ -101,6 +101,6 @@ impl<'tcx> TyCtxt<'tcx> {
         let alloc_id = tcx.create_memory_alloc(tcx.intern_const_alloc(vtable));
         let mut vtables_cache = self.vtables_cache.lock();
         vtables_cache.insert((ty, poly_trait_ref), alloc_id);
-        alloc_id
+        Ok(alloc_id)
     }
 }