diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 6976999c0e4..aee385ab050 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -453,7 +453,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
         for elem in place_ref.projection[base..].iter() {
             cg_base = match elem.clone() {
-                mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()),
+                mir::ProjectionElem::Deref => {
+                    // custom allocators can change box's abi, making it unable to be derefed directly
+                    if cg_base.layout.ty.is_box()
+                        && matches!(cg_base.layout.abi, Abi::Aggregate { .. })
+                    {
+                        let ptr = cg_base.project_field(bx, 0).project_field(bx, 0);
+
+                        bx.load_operand(ptr).deref(bx.cx())
+                    } else {
+                        bx.load_operand(cg_base).deref(bx.cx())
+                    }
+                }
                 mir::ProjectionElem::Field(ref field, _) => {
                     cg_base.project_field(bx, field.index())
                 }
diff --git a/src/test/ui/box/issue-81270-ice.rs b/src/test/ui/box/issue-81270-ice.rs
new file mode 100644
index 00000000000..fb42aed7aae
--- /dev/null
+++ b/src/test/ui/box/issue-81270-ice.rs
@@ -0,0 +1,22 @@
+// check-pass
+#![feature(allocator_api)]
+
+use std::alloc::Allocator;
+
+struct BigAllocator([usize; 2]);
+
+unsafe impl Allocator for BigAllocator {
+    fn allocate(
+        &self,
+        _: std::alloc::Layout,
+    ) -> Result<std::ptr::NonNull<[u8]>, std::alloc::AllocError> {
+        todo!()
+    }
+    unsafe fn deallocate(&self, _: std::ptr::NonNull<u8>, _: std::alloc::Layout) {
+        todo!()
+    }
+}
+
+fn main() {
+    Box::new_in((), BigAllocator([0; 2]));
+}