Erase query cache values

compiler/rustc_middle/src/query/erase.rs

@@ -0,0 +1,225 @@
+use crate::ty;
+use std::intrinsics::type_name;
+use std::{
+    fmt,
+    mem::{size_of, transmute_copy, MaybeUninit},
+};
+
+#[derive(Copy, Clone)]
+pub struct Erased<T: Copy> {
+    data: MaybeUninit<T>,
+}
+
+impl<T: Copy> fmt::Debug for Erased<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "Erased")
+    }
+}
+
+pub trait EraseType: Copy {
+    type Result: Copy;
+}
+
+// Allow `type_alias_bounds` since compilation will fail without `EraseType`.
+#[allow(type_alias_bounds)]
+pub type Erase<T: Copy + EraseType> = Erased<impl Copy>;
+
+#[inline(always)]
+pub fn erase<T: EraseType>(src: T) -> Erase<T> {
+    assert_eq!(
+        size_of::<T>(),
+        size_of::<T::Result>(),
+        "size of {} must match erased type {}",
+        type_name::<T>(),
+        type_name::<T::Result>()
+    );
+    Erased::<<T as EraseType>::Result> {
+        // SAFETY: Is it safe to transmute to MaybeUninit for types with the same sizes.
+        data: unsafe { transmute_copy(&src) },
+    }
+}
+
+/// Restores an erased value.
+#[inline(always)]
+pub fn restore<T: EraseType>(value: Erase<T>) -> T {
+    let value: Erased<<T as EraseType>::Result> = value;
+    // SAFETY: Due to the use of impl Trait in `Erase` the only way to safetly create an instance
+    // of `Erase` is to call `erase`, so we know that `value.data` is a valid instance of `T` of
+    // the right size.
+    unsafe { transmute_copy(&value.data) }
+}
+
+impl<T> EraseType for &'_ T {
+    type Result = [u8; size_of::<*const ()>()];
+}
+
+impl<T> EraseType for &'_ [T] {
+    type Result = [u8; size_of::<*const [()]>()];
+}
+
+impl<T> EraseType for &'_ ty::List<T> {
+    type Result = [u8; size_of::<*const ()>()];
+}
+
+impl<T: Copy, E: Copy> EraseType for Result<T, E> {
+    type Result = Self;
+}
+
+impl<T: Copy> EraseType for Option<T> {
+    type Result = Self;
+}
+
+impl<T: Copy> EraseType for rustc_hir::MaybeOwner<T> {
+    type Result = Self;
+}
+
+impl<T: Copy> EraseType for ty::Visibility<T> {
+    type Result = Self;
+}
+
+impl<T: Copy> EraseType for ty::Binder<'_, T> {
+    type Result = Self;
+}
+
+impl<T: Copy> EraseType for ty::EarlyBinder<T> {
+    type Result = Self;
+}
+
+impl<T0: Copy, T1: Copy> EraseType for (T0, T1) {
+    type Result = Self;
+}
+
+macro_rules! trivial {
+    ($($ty:ty),+ $(,)?) => {
+        $(
+            impl EraseType for $ty {
+                type Result = [u8; size_of::<$ty>()];
+            }
+        )*
+    }
+}
+
+trivial! {
+    (),
+    bool,
+    rustc_ast::expand::allocator::AllocatorKind,
+    rustc_attr::ConstStability,
+    rustc_attr::DefaultBodyStability,
+    rustc_attr::Deprecation,
+    rustc_attr::Stability,
+    rustc_data_structures::svh::Svh,
+    rustc_errors::ErrorGuaranteed,
+    rustc_hir::Constness,
+    rustc_hir::def_id::DefId,
+    rustc_hir::def_id::DefIndex,
+    rustc_hir::def_id::LocalDefId,
+    rustc_hir::def::DefKind,
+    rustc_hir::Defaultness,
+    rustc_hir::definitions::DefKey,
+    rustc_hir::GeneratorKind,
+    rustc_hir::HirId,
+    rustc_hir::IsAsync,
+    rustc_hir::ItemLocalId,
+    rustc_hir::LangItem,
+    rustc_hir::OwnerId,
+    rustc_hir::Upvar,
+    rustc_index::bit_set::FiniteBitSet<u32>,
+    rustc_middle::metadata::ModChild,
+    rustc_middle::middle::dependency_format::Linkage,
+    rustc_middle::middle::exported_symbols::SymbolExportInfo,
+    rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
+    rustc_middle::middle::resolve_bound_vars::ResolvedArg,
+    rustc_middle::middle::stability::DeprecationEntry,
+    rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::interpret::AllocId,
+    rustc_middle::mir::interpret::ErrorHandled,
+    rustc_middle::mir::interpret::LitToConstError,
+    rustc_middle::thir::ExprId,
+    rustc_middle::traits::CodegenObligationError,
+    rustc_middle::traits::EvaluationResult,
+    rustc_middle::traits::OverflowError,
+    rustc_middle::traits::query::NoSolution,
+    rustc_middle::traits::WellFormedLoc,
+    rustc_middle::ty::adjustment::CoerceUnsizedInfo,
+    rustc_middle::ty::AssocItem,
+    rustc_middle::ty::AssocItemContainer,
+    rustc_middle::ty::BoundVariableKind,
+    rustc_middle::ty::DeducedParamAttrs,
+    rustc_middle::ty::Destructor,
+    rustc_middle::ty::fast_reject::SimplifiedType,
+    rustc_middle::ty::ImplPolarity,
+    rustc_middle::ty::Representability,
+    rustc_middle::ty::ReprOptions,
+    rustc_middle::ty::UnusedGenericParams,
+    rustc_middle::ty::util::AlwaysRequiresDrop,
+    rustc_session::config::CrateType,
+    rustc_session::config::EntryFnType,
+    rustc_session::config::OptLevel,
+    rustc_session::config::SymbolManglingVersion,
+    rustc_session::cstore::CrateDepKind,
+    rustc_session::cstore::ExternCrate,
+    rustc_session::cstore::LinkagePreference,
+    rustc_session::Limits,
+    rustc_session::lint::LintExpectationId,
+    rustc_span::def_id::CrateNum,
+    rustc_span::def_id::DefPathHash,
+    rustc_span::ExpnHash,
+    rustc_span::ExpnId,
+    rustc_span::Span,
+    rustc_span::Symbol,
+    rustc_span::symbol::Ident,
+    rustc_target::spec::PanicStrategy,
+    rustc_type_ir::Variance,
+    u32,
+    usize,
+}
+
+macro_rules! tcx_lifetime {
+    ($($($fake_path:ident)::+),+ $(,)?) => {
+        $(
+            impl<'tcx> EraseType for $($fake_path)::+<'tcx> {
+                type Result = [u8; size_of::<$($fake_path)::+<'static>>()];
+            }
+        )*
+    }
+}
+
+tcx_lifetime! {
+    rustc_middle::hir::Owner,
+    rustc_middle::middle::exported_symbols::ExportedSymbol,
+    rustc_middle::mir::ConstantKind,
+    rustc_middle::mir::DestructuredConstant,
+    rustc_middle::mir::interpret::ConstAlloc,
+    rustc_middle::mir::interpret::ConstValue,
+    rustc_middle::mir::interpret::GlobalId,
+    rustc_middle::mir::interpret::LitToConstInput,
+    rustc_middle::traits::ChalkEnvironmentAndGoal,
+    rustc_middle::traits::query::MethodAutoderefStepsResult,
+    rustc_middle::traits::query::type_op::AscribeUserType,
+    rustc_middle::traits::query::type_op::Eq,
+    rustc_middle::traits::query::type_op::ProvePredicate,
+    rustc_middle::traits::query::type_op::Subtype,
+    rustc_middle::ty::AdtDef,
+    rustc_middle::ty::AliasTy,
+    rustc_middle::ty::Clause,
+    rustc_middle::ty::ClosureTypeInfo,
+    rustc_middle::ty::Const,
+    rustc_middle::ty::DestructuredConst,
+    rustc_middle::ty::ExistentialTraitRef,
+    rustc_middle::ty::FnSig,
+    rustc_middle::ty::GenericArg,
+    rustc_middle::ty::GenericPredicates,
+    rustc_middle::ty::inhabitedness::InhabitedPredicate,
+    rustc_middle::ty::Instance,
+    rustc_middle::ty::InstanceDef,
+    rustc_middle::ty::layout::FnAbiError,
+    rustc_middle::ty::layout::LayoutError,
+    rustc_middle::ty::ParamEnv,
+    rustc_middle::ty::Predicate,
+    rustc_middle::ty::SymbolName,
+    rustc_middle::ty::TraitRef,
+    rustc_middle::ty::Ty,
+    rustc_middle::ty::UnevaluatedConst,
+    rustc_middle::ty::ValTree,
+    rustc_middle::ty::VtblEntry,
+}

compiler/rustc_middle/src/query/mod.rs

@@ -7,6 +7,7 @@
 use crate::ty::{self, print::describe_as_module, TyCtxt};
 use rustc_span::def_id::LOCAL_CRATE;
 
+pub mod erase;
 mod keys;
 pub use keys::{AsLocalKey, Key, LocalCrate};
 

compiler/rustc_middle/src/ty/query.rs

@@ -17,6 +17,7 @@ use crate::mir::interpret::{
 };
 use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
+use crate::query::erase::{erase, restore, Erase};
 use crate::query::{AsLocalKey, Key};
 use crate::thir;
 use crate::traits::query::{
@@ -57,6 +58,8 @@ use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
 use rustc_index::vec::IndexVec;
+pub(crate) use rustc_query_system::query::QueryJobId;
+use rustc_query_system::query::*;
 use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
 use rustc_session::cstore::{CrateDepKind, CrateSource};
 use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
@@ -66,18 +69,19 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi;
 use rustc_target::spec::PanicStrategy;
+
+use std::marker::PhantomData;
 use std::mem;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-pub(crate) use rustc_query_system::query::QueryJobId;
-use rustc_query_system::query::*;
-
 #[derive(Default)]
 pub struct QuerySystem<'tcx> {
     pub arenas: QueryArenas<'tcx>,
     pub caches: QueryCaches<'tcx>,
+    // Since we erase query value types we tell the typesystem about them with `PhantomData`.
+    _phantom_values: QueryPhantomValues<'tcx>,
 }
 
 #[derive(Copy, Clone)]
@@ -263,8 +267,8 @@ macro_rules! define_callbacks {
                 pub fn $name<'tcx>(
                     _tcx: TyCtxt<'tcx>,
                     value: query_provided::$name<'tcx>,
-                ) -> query_values::$name<'tcx> {
-                    query_if_arena!([$($modifiers)*]
+                ) -> Erase<query_values::$name<'tcx>> {
+                    erase(query_if_arena!([$($modifiers)*]
                         {
                             if mem::needs_drop::<query_provided::$name<'tcx>>() {
                                 &*_tcx.query_system.arenas.$name.alloc(value)
@@ -273,7 +277,7 @@ macro_rules! define_callbacks {
                             }
                         }
                         (value)
-                    )
+                    ))
                 }
             )*
         }
@@ -282,7 +286,7 @@ macro_rules! define_callbacks {
             use super::*;
 
             $(
-                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
+                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache;
             )*
         }
 
@@ -334,6 +338,11 @@ macro_rules! define_callbacks {
             }
         }
 
+        #[derive(Default)]
+        pub struct QueryPhantomValues<'tcx> {
+            $($(#[$attr])* pub $name: PhantomData<query_values::$name<'tcx>>,)*
+        }
+
         #[derive(Default)]
         pub struct QueryCaches<'tcx> {
             $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
@@ -395,10 +404,10 @@ macro_rules! define_callbacks {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
-                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
+                restore::<$V>(match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
                     Some(value) => value,
                     None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
-                }
+                })
             })*
         }
 
@@ -459,7 +468,7 @@ macro_rules! define_callbacks {
                 span: Span,
                 key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<$V>;)*
+            ) -> Option<Erase<$V>>;)*
         }
     };
 }
@@ -486,7 +495,8 @@ macro_rules! define_feedable {
                 opt_remap_env_constness!([$($modifiers)*][key]);
 
                 let tcx = self.tcx;
-                let value = query_provided_to_value::$name(tcx, value);
+                let erased = query_provided_to_value::$name(tcx, value);
+                let value = restore::<$V>(erased);
                 let cache = &tcx.query_system.caches.$name;
 
                 match try_get_cached(tcx, cache, &key) {
@@ -505,7 +515,7 @@ macro_rules! define_feedable {
                             &value,
                             hash_result!([$($modifiers)*]),
                         );
-                        cache.complete(key, value, dep_node_index);
+                        cache.complete(key, erased, dep_node_index);
                         value
                     }
                 }