Auto merge of #70674 - cjgillot:query-arena-all, r=matthewjasper

Have the per-query caches store the results on arenas

This PR leverages the cache for each query to serve as storage area for the query results.

It introduces a new cache `ArenaCache`, which moves the result to an arena,
and only stores the reference in the hash map.
This allows to remove a sizeable part of the usage of the global `TyCtxt` arena.

I only migrated queries that already used arenas before.

Cargo.lock

@@ -4126,6 +4126,7 @@ dependencies = [
 name = "rustc_query_system"
 version = "0.0.0"
 dependencies = [
+ "arena",
  "log",
  "parking_lot 0.10.2",
  "rustc-rayon-core",

src/librustc_codegen_llvm/attributes.rs

@@ -253,7 +253,7 @@ pub fn from_fn_attrs(cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value, instance: ty::
         inline(cx, llfn, attributes::InlineAttr::Hint);
     }
 
-    inline(cx, llfn, codegen_fn_attrs.inline);
+    inline(cx, llfn, codegen_fn_attrs.inline.clone());
 
     // The `uwtable` attribute according to LLVM is:
     //
@@ -350,15 +350,12 @@ pub fn provide(providers: &mut Providers<'_>) {
         if tcx.sess.opts.actually_rustdoc {
             // rustdoc needs to be able to document functions that use all the features, so
             // whitelist them all
-            tcx.arena
+            llvm_util::all_known_features().map(|(a, b)| (a.to_string(), b)).collect()
-                .alloc(llvm_util::all_known_features().map(|(a, b)| (a.to_string(), b)).collect())
         } else {
-            tcx.arena.alloc(
+            llvm_util::target_feature_whitelist(tcx.sess)
-                llvm_util::target_feature_whitelist(tcx.sess)
+                .iter()
-                    .iter()
+                .map(|&(a, b)| (a.to_string(), b))
-                    .map(|&(a, b)| (a.to_string(), b))
+                .collect()
-                    .collect(),
-            )
         }
     };
 
@@ -390,7 +387,7 @@ pub fn provide_extern(providers: &mut Providers<'_>) {
             }));
         }
 
-        tcx.arena.alloc(ret)
+        ret
     };
 }
 

src/librustc_codegen_ssa/back/symbol_export.rs

@@ -42,14 +42,11 @@ pub fn crates_export_threshold(crate_types: &[config::CrateType]) -> SymbolExpor
     }
 }
 
-fn reachable_non_generics_provider(
+fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<SymbolExportLevel> {
-    tcx: TyCtxt<'_>,
-    cnum: CrateNum,
-) -> &DefIdMap<SymbolExportLevel> {
     assert_eq!(cnum, LOCAL_CRATE);
 
     if !tcx.sess.opts.output_types.should_codegen() {
-        return tcx.arena.alloc(Default::default());
+        return Default::default();
     }
 
     // Check to see if this crate is a "special runtime crate". These
@@ -145,7 +142,7 @@ fn reachable_non_generics_provider(
         reachable_non_generics.insert(id, SymbolExportLevel::C);
     }
 
-    tcx.arena.alloc(reachable_non_generics)
+    reachable_non_generics
 }
 
 fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
@@ -281,7 +278,7 @@ fn exported_symbols_provider_local(
 fn upstream_monomorphizations_provider(
     tcx: TyCtxt<'_>,
     cnum: CrateNum,
-) -> &DefIdMap<FxHashMap<SubstsRef<'_>, CrateNum>> {
+) -> DefIdMap<FxHashMap<SubstsRef<'_>, CrateNum>> {
     debug_assert!(cnum == LOCAL_CRATE);
 
     let cnums = tcx.all_crate_nums(LOCAL_CRATE);
@@ -338,7 +335,7 @@ fn upstream_monomorphizations_provider(
         }
     }
 
-    tcx.arena.alloc(instances)
+    instances
 }
 
 fn upstream_monomorphizations_for_provider(

src/librustc_codegen_ssa/base.rs

@@ -908,7 +908,7 @@ pub fn provide_both(providers: &mut Providers<'_>) {
             .map(|id| &module_map[&id])
             .flat_map(|module| module.foreign_items.iter().cloned())
             .collect();
-        tcx.arena.alloc(dllimports)
+        dllimports
     };
 
     providers.is_dllimport_foreign_item =

src/librustc_lint/levels.rs

@@ -22,7 +22,7 @@ use rustc_span::symbol::{sym, Symbol};
 
 use std::cmp;
 
-fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> &LintLevelMap {
+fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> LintLevelMap {
     assert_eq!(cnum, LOCAL_CRATE);
     let store = unerased_lint_store(tcx);
     let levels = LintLevelsBuilder::new(tcx.sess, false, &store);
@@ -37,7 +37,7 @@ fn lint_levels(tcx: TyCtxt<'_>, cnum: CrateNum) -> &LintLevelMap {
     intravisit::walk_crate(&mut builder, krate);
     builder.levels.pop(push);
 
-    tcx.arena.alloc(builder.levels.build_map())
+    builder.levels.build_map()
 }
 
 pub struct LintLevelsBuilder<'s> {

src/librustc_macros/src/query.rs

@@ -107,7 +107,9 @@ impl Parse for QueryModifier {
             let block = input.parse()?;
             Ok(QueryModifier::LoadCached(tcx, id, block))
         } else if modifier == "storage" {
-            let ty = input.parse()?;
+            let args;
+            parenthesized!(args in input);
+            let ty = args.parse()?;
             Ok(QueryModifier::Storage(ty))
         } else if modifier == "fatal_cycle" {
             Ok(QueryModifier::FatalCycle)

src/librustc_metadata/rmeta/decoder.rs

@@ -939,8 +939,8 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     /// Iterates over the diagnostic items in the given crate.
-    fn get_diagnostic_items(&self, tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap<Symbol, DefId> {
+    fn get_diagnostic_items(&self) -> FxHashMap<Symbol, DefId> {
-        tcx.arena.alloc(if self.root.is_proc_macro_crate() {
+        if self.root.is_proc_macro_crate() {
             // Proc macro crates do not export any diagnostic-items to the target.
             Default::default()
         } else {
@@ -949,7 +949,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
                 .decode(self)
                 .map(|(name, def_index)| (name, self.local_def_id(def_index)))
                 .collect()
-        })
+        }
     }
 
     /// Iterates over each child of the given item.

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -88,15 +88,11 @@ impl IntoArgs for (CrateNum, DefId) {
 
 provide! { <'tcx> tcx, def_id, other, cdata,
     type_of => { cdata.get_type(def_id.index, tcx) }
-    generics_of => {
+    generics_of => { cdata.get_generics(def_id.index, tcx.sess) }
-        tcx.arena.alloc(cdata.get_generics(def_id.index, tcx.sess))
-    }
     explicit_predicates_of => { cdata.get_explicit_predicates(def_id.index, tcx) }
     inferred_outlives_of => { cdata.get_inferred_outlives(def_id.index, tcx) }
     super_predicates_of => { cdata.get_super_predicates(def_id.index, tcx) }
-    trait_def => {
+    trait_def => { cdata.get_trait_def(def_id.index, tcx.sess) }
-        tcx.arena.alloc(cdata.get_trait_def(def_id.index, tcx.sess))
-    }
     adt_def => { cdata.get_adt_def(def_id.index, tcx) }
     adt_destructor => {
         let _ = cdata;
@@ -117,8 +113,8 @@ provide! { <'tcx> tcx, def_id, other, cdata,
             bug!("coerce_unsized_info: `{:?}` is missing its info", def_id);
         })
     }
-    optimized_mir => { tcx.arena.alloc(cdata.get_optimized_mir(tcx, def_id.index)) }
+    optimized_mir => { cdata.get_optimized_mir(tcx, def_id.index) }
-    promoted_mir => { tcx.arena.alloc(cdata.get_promoted_mir(tcx, def_id.index)) }
+    promoted_mir => { cdata.get_promoted_mir(tcx, def_id.index) }
     mir_const_qualif => { cdata.mir_const_qualif(def_id.index) }
     fn_sig => { cdata.fn_sig(def_id.index, tcx) }
     inherent_impls => { cdata.get_inherent_implementations_for_type(tcx, def_id.index) }
@@ -178,7 +174,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
             })
             .collect();
 
-        tcx.arena.alloc(reachable_non_generics)
+        reachable_non_generics
     }
     native_libraries => { Lrc::new(cdata.get_native_libraries(tcx.sess)) }
     foreign_modules => { cdata.get_foreign_modules(tcx) }
@@ -220,7 +216,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
     }
     defined_lib_features => { cdata.get_lib_features(tcx) }
     defined_lang_items => { cdata.get_lang_items(tcx) }
-    diagnostic_items => { cdata.get_diagnostic_items(tcx) }
+    diagnostic_items => { cdata.get_diagnostic_items() }
     missing_lang_items => { cdata.get_missing_lang_items(tcx) }
 
     missing_extern_crate_item => {
@@ -363,7 +359,7 @@ pub fn provide(providers: &mut Providers<'_>) {
                 }
             }
 
-            tcx.arena.alloc(visible_parent_map)
+            visible_parent_map
         },
 
         dependency_formats: |tcx, cnum| {

src/librustc_middle/arena.rs

@@ -12,36 +12,13 @@ macro_rules! arena_types {
     ($macro:path, $args:tt, $tcx:lifetime) => (
         $macro!($args, [
             [] layouts: rustc_target::abi::Layout,
-            [] generics: rustc_middle::ty::Generics,
+            // AdtDef are interned and compared by address
-            [] trait_def: rustc_middle::ty::TraitDef,
             [] adt_def: rustc_middle::ty::AdtDef,
-            [] steal_mir: rustc_middle::ty::steal::Steal<rustc_middle::mir::Body<$tcx>>,
-            [] mir: rustc_middle::mir::Body<$tcx>,
-            [] steal_promoted: rustc_middle::ty::steal::Steal<
-                rustc_index::vec::IndexVec<
-                    rustc_middle::mir::Promoted,
-                    rustc_middle::mir::Body<$tcx>
-                >
-            >,
-            [] promoted: rustc_index::vec::IndexVec<
-                rustc_middle::mir::Promoted,
-                rustc_middle::mir::Body<$tcx>
-            >,
             [decode] tables: rustc_middle::ty::TypeckTables<$tcx>,
-            [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<$tcx>,
             [] const_allocs: rustc_middle::mir::interpret::Allocation,
-            [] vtable_method: Option<(
+            // Required for the incremental on-disk cache
-                rustc_hir::def_id::DefId,
+            [few, decode] mir_keys: rustc_hir::def_id::DefIdSet,
-                rustc_middle::ty::subst::SubstsRef<$tcx>
-            )>,
-            [few, decode] collect_and_partition_mono_items: rustc_hir::def_id::DefIdSet,
-            [few, decode] mir_keys: rustc_data_structures::fx::FxHashSet<rustc_hir::def_id::LocalDefId>,
-            [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph,
             [] region_scope_tree: rustc_middle::middle::region::ScopeTree,
-            [] item_local_set: rustc_hir::ItemLocalSet,
-            [decode] mir_const_qualif: rustc_index::bit_set::BitSet<rustc_middle::mir::Local>,
-            [] trait_impls_of: rustc_middle::ty::trait_def::TraitImpls,
-            [] associated_items: rustc_middle::ty::AssociatedItems,
             [] dropck_outlives:
                 rustc_middle::infer::canonical::Canonical<'tcx,
                     rustc_middle::infer::canonical::QueryResponse<'tcx,
@@ -80,42 +57,10 @@ macro_rules! arena_types {
                 rustc_middle::infer::canonical::Canonical<'tcx,
                     rustc_middle::infer::canonical::QueryResponse<'tcx, rustc_middle::ty::Ty<'tcx>>
                 >,
-            [few] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls,
-            [few] upstream_monomorphizations:
-                rustc_hir::def_id::DefIdMap<
-                    rustc_data_structures::fx::FxHashMap<
-                        rustc_middle::ty::subst::SubstsRef<'tcx>,
-                        rustc_hir::def_id::CrateNum
-                    >
-                >,
-            [few] diagnostic_items: rustc_data_structures::fx::FxHashMap<
-                rustc_span::symbol::Symbol,
-                rustc_hir::def_id::DefId,
-            >,
-            [few] resolve_lifetimes: rustc_middle::middle::resolve_lifetime::ResolveLifetimes,
-            [few] lint_levels: rustc_middle::lint::LintLevelMap,
-            [few] stability_index: rustc_middle::middle::stability::Index<'tcx>,
-            [few] features: rustc_feature::Features,
             [few] all_traits: Vec<rustc_hir::def_id::DefId>,
             [few] privacy_access_levels: rustc_middle::middle::privacy::AccessLevels,
-            [few] target_features_whitelist: rustc_data_structures::fx::FxHashMap<
-                String,
-                Option<rustc_span::symbol::Symbol>
-            >,
-            [few] wasm_import_module_map: rustc_data_structures::fx::FxHashMap<
-                rustc_hir::def_id::DefId,
-                String
-            >,
-            [few] get_lib_features: rustc_middle::middle::lib_features::LibFeatures,
-            [few] defined_lib_features: rustc_hir::lang_items::LanguageItems,
-            [few] visible_parent_map: rustc_hir::def_id::DefIdMap<rustc_hir::def_id::DefId>,
             [few] foreign_module: rustc_middle::middle::cstore::ForeignModule,
             [few] foreign_modules: Vec<rustc_middle::middle::cstore::ForeignModule>,
-            [few] reachable_non_generics: rustc_hir::def_id::DefIdMap<
-                rustc_middle::middle::exported_symbols::SymbolExportLevel
-            >,
-            [few] crate_variances: rustc_middle::ty::CrateVariancesMap<'tcx>,
-            [few] inferred_outlives_crate: rustc_middle::ty::CratePredicatesMap<'tcx>,
             [] upvars: rustc_data_structures::fx::FxIndexMap<rustc_hir::HirId, rustc_hir::Upvar>,
             [] object_safety_violations: rustc_middle::traits::ObjectSafetyViolation,
             [] codegen_unit: rustc_middle::mir::mono::CodegenUnit<$tcx>,

src/librustc_middle/query/mod.rs

@@ -102,12 +102,13 @@ rustc_queries! {
 
         /// Maps from the `DefId` of an item (trait/struct/enum/fn) to its
         /// associated generics.
-        query generics_of(key: DefId) -> &'tcx ty::Generics {
+        query generics_of(key: DefId) -> ty::Generics {
+            storage(ArenaCacheSelector<'tcx>)
             cache_on_disk_if { key.is_local() }
             load_cached(tcx, id) {
                 let generics: Option<ty::Generics> = tcx.queries.on_disk_cache
                                                         .try_load_query_result(tcx, id);
-                generics.map(|x| &*tcx.arena.alloc(x))
+                generics
             }
         }
 
@@ -134,7 +135,8 @@ rustc_queries! {
             desc { "looking up the native libraries of a linked crate" }
         }
 
-        query lint_levels(_: CrateNum) -> &'tcx LintLevelMap {
+        query lint_levels(_: CrateNum) -> LintLevelMap {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "computing the lint levels for items in this crate" }
         }
@@ -156,7 +158,8 @@ rustc_queries! {
         /// Set of all the `DefId`s in this crate that have MIR associated with
         /// them. This includes all the body owners, but also things like struct
         /// constructors.
-        query mir_keys(_: CrateNum) -> &'tcx FxHashSet<LocalDefId> {
+        query mir_keys(_: CrateNum) -> FxHashSet<LocalDefId> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "getting a list of all mir_keys" }
         }
 
@@ -170,7 +173,8 @@ rustc_queries! {
 
         /// Fetch the MIR for a given `DefId` right after it's built - this includes
         /// unreachable code.
-        query mir_built(_: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
+        query mir_built(_: LocalDefId) -> Steal<mir::Body<'tcx>> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "building MIR for" }
         }
 
@@ -178,40 +182,31 @@ rustc_queries! {
         /// ready for const evaluation.
         ///
         /// See the README for the `mir` module for details.
-        query mir_const(_: DefId) -> &'tcx Steal<mir::Body<'tcx>> {
+        query mir_const(_: DefId) -> Steal<mir::Body<'tcx>> {
+            storage(ArenaCacheSelector<'tcx>)
             no_hash
         }
 
         query mir_validated(key: LocalDefId) ->
             (
-                &'tcx Steal<mir::Body<'tcx>>,
+                Steal<mir::Body<'tcx>>,
-                &'tcx Steal<IndexVec<mir::Promoted, mir::Body<'tcx>>>
+                Steal<IndexVec<mir::Promoted, mir::Body<'tcx>>>
             ) {
+            storage(ArenaCacheSelector<'tcx>)
             no_hash
             desc { |tcx| "processing `{}`", tcx.def_path_str(key.to_def_id()) }
         }
 
         /// MIR after our optimization passes have run. This is MIR that is ready
         /// for codegen. This is also the only query that can fetch non-local MIR, at present.
-        query optimized_mir(key: DefId) -> &'tcx mir::Body<'tcx> {
+        query optimized_mir(key: DefId) -> mir::Body<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             cache_on_disk_if { key.is_local() }
-            load_cached(tcx, id) {
-                let mir: Option<crate::mir::Body<'tcx>>
-                    = tcx.queries.on_disk_cache.try_load_query_result(tcx, id);
-                mir.map(|x| &*tcx.arena.alloc(x))
-            }
         }
 
-        query promoted_mir(key: DefId) -> &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>> {
+        query promoted_mir(key: DefId) -> IndexVec<mir::Promoted, mir::Body<'tcx>> {
+            storage(ArenaCacheSelector<'tcx>)
             cache_on_disk_if { key.is_local() }
-            load_cached(tcx, id) {
-                let promoted: Option<
-                    rustc_index::vec::IndexVec<
-                        crate::mir::Promoted,
-                        crate::mir::Body<'tcx>
-                    >> = tcx.queries.on_disk_cache.try_load_query_result(tcx, id);
-                promoted.map(|p| &*tcx.arena.alloc(p))
-            }
         }
     }
 
@@ -245,7 +240,8 @@ rustc_queries! {
     }
 
     Linking {
-        query wasm_import_module_map(_: CrateNum) -> &'tcx FxHashMap<DefId, String> {
+        query wasm_import_module_map(_: CrateNum) -> FxHashMap<DefId, String> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "wasm import module map" }
         }
     }
@@ -283,8 +279,11 @@ rustc_queries! {
             }}
         }
 
-        query trait_def(_: DefId) -> &'tcx ty::TraitDef {}
+        query trait_def(_: DefId) -> ty::TraitDef {
-        query adt_def(_: DefId) -> &'tcx ty::AdtDef {}
+            storage(ArenaCacheSelector<'tcx>)
+        }
+        query adt_def(_: DefId) -> &'tcx ty::AdtDef {
+        }
         query adt_destructor(_: DefId) -> Option<ty::Destructor> {}
 
         // The cycle error here should be reported as an error by `check_representable`.
@@ -345,7 +344,8 @@ rustc_queries! {
         query generator_kind(_: DefId) -> Option<hir::GeneratorKind> {}
 
         /// Gets a map with the variance of every item; use `item_variance` instead.
-        query crate_variances(_: CrateNum) -> &'tcx ty::CrateVariancesMap<'tcx> {
+        query crate_variances(_: CrateNum) -> ty::CrateVariancesMap<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "computing the variances for items in this crate" }
         }
 
@@ -356,7 +356,8 @@ rustc_queries! {
     TypeChecking {
         /// Maps from thee `DefId` of a type to its (inferred) outlives.
         query inferred_outlives_crate(_: CrateNum)
-            -> &'tcx ty::CratePredicatesMap<'tcx> {
+            -> ty::CratePredicatesMap<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "computing the inferred outlives predicates for items in this crate" }
         }
     }
@@ -366,10 +367,13 @@ rustc_queries! {
         query associated_item_def_ids(_: DefId) -> &'tcx [DefId] {}
 
         /// Maps from a trait item to the trait item "descriptor".
-        query associated_item(_: DefId) -> ty::AssocItem {}
+        query associated_item(_: DefId) -> ty::AssocItem {
+            storage(ArenaCacheSelector<'tcx>)
+        }
 
         /// Collects the associated items defined on a trait or impl.
-        query associated_items(key: DefId) -> &'tcx ty::AssociatedItems {
+        query associated_items(key: DefId) -> ty::AssociatedItems<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { |tcx| "collecting associated items of {}", tcx.def_path_str(key) }
         }
 
@@ -393,6 +397,7 @@ rustc_queries! {
         query unsafety_check_result(key: LocalDefId) -> mir::UnsafetyCheckResult {
             desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key.to_def_id()) }
             cache_on_disk_if { true }
+            storage(ArenaCacheSelector<'tcx>)
         }
 
         /// HACK: when evaluated, this reports a "unsafe derive on repr(packed)" error
@@ -472,7 +477,7 @@ rustc_queries! {
                     .queries.on_disk_cache
                     .try_load_query_result(tcx, id);
 
-                typeck_tables.map(|tables| &*tcx.arena.alloc(tables))
+                typeck_tables.map(|x| &*tcx.arena.alloc(x))
             }
         }
     }
@@ -495,7 +500,8 @@ rustc_queries! {
     BorrowChecking {
         /// Borrow-checks the function body. If this is a closure, returns
         /// additional requirements that the closure's creator must verify.
-        query mir_borrowck(key: LocalDefId) -> &'tcx mir::BorrowCheckResult<'tcx> {
+        query mir_borrowck(key: LocalDefId) -> mir::BorrowCheckResult<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { |tcx| "borrow-checking `{}`", tcx.def_path_str(key.to_def_id()) }
             cache_on_disk_if(tcx, opt_result) {
                 tcx.is_closure(key.to_def_id())
@@ -509,7 +515,8 @@ rustc_queries! {
         /// Not meant to be used directly outside of coherence.
         /// (Defined only for `LOCAL_CRATE`.)
         query crate_inherent_impls(k: CrateNum)
-            -> &'tcx CrateInherentImpls {
+            -> CrateInherentImpls {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "all inherent impls defined in crate `{:?}`", k }
         }
@@ -610,7 +617,8 @@ rustc_queries! {
         /// in the case of closures, this will be redirected to the enclosing function.
         query region_scope_tree(_: DefId) -> &'tcx region::ScopeTree {}
 
-        query mir_shims(key: ty::InstanceDef<'tcx>) -> &'tcx mir::Body<'tcx> {
+        query mir_shims(key: ty::InstanceDef<'tcx>) -> mir::Body<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { |tcx| "generating MIR shim for `{}`", tcx.def_path_str(key.def_id()) }
         }
 
@@ -639,6 +647,7 @@ rustc_queries! {
 
     Codegen {
         query codegen_fn_attrs(_: DefId) -> CodegenFnAttrs {
+            storage(ArenaCacheSelector<'tcx>)
             cache_on_disk_if { true }
         }
     }
@@ -684,10 +693,12 @@ rustc_queries! {
         query all_local_trait_impls(key: CrateNum) -> &'tcx BTreeMap<DefId, Vec<hir::HirId>> {
             desc { "local trait impls" }
         }
-        query trait_impls_of(key: DefId) -> &'tcx ty::trait_def::TraitImpls {
+        query trait_impls_of(key: DefId) -> ty::trait_def::TraitImpls {
+            storage(ArenaCacheSelector<'tcx>)
             desc { |tcx| "trait impls of `{}`", tcx.def_path_str(key) }
         }
-        query specialization_graph_of(key: DefId) -> &'tcx specialization_graph::Graph {
+        query specialization_graph_of(key: DefId) -> specialization_graph::Graph {
+            storage(ArenaCacheSelector<'tcx>)
             desc { |tcx| "building specialization graph of trait `{}`", tcx.def_path_str(key) }
             cache_on_disk_if { true }
         }
@@ -829,7 +840,8 @@ rustc_queries! {
         // Does not include external symbols that don't have a corresponding DefId,
         // like the compiler-generated `main` function and so on.
         query reachable_non_generics(_: CrateNum)
-            -> &'tcx DefIdMap<SymbolExportLevel> {
+            -> DefIdMap<SymbolExportLevel> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "looking up the exported symbols of a crate" }
         }
         query is_reachable_non_generic(_: DefId) -> bool {}
@@ -845,7 +857,8 @@ rustc_queries! {
         /// better, `Instance::upstream_monomorphization()`.
         query upstream_monomorphizations(
             k: CrateNum
-        ) -> &'tcx DefIdMap<FxHashMap<SubstsRef<'tcx>, CrateNum>> {
+        ) -> DefIdMap<FxHashMap<SubstsRef<'tcx>, CrateNum>> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "collecting available upstream monomorphizations `{:?}`", k }
         }
 
@@ -930,7 +943,8 @@ rustc_queries! {
 
     Other {
         query dllimport_foreign_items(_: CrateNum)
-            -> &'tcx FxHashSet<DefId> {
+            -> FxHashSet<DefId> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "dllimport_foreign_items" }
         }
         query is_dllimport_foreign_item(_: DefId) -> bool {}
@@ -948,7 +962,8 @@ rustc_queries! {
 
     BorrowChecking {
         /// Lifetime resolution. See `middle::resolve_lifetimes`.
-        query resolve_lifetimes(_: CrateNum) -> &'tcx ResolveLifetimes {
+        query resolve_lifetimes(_: CrateNum) -> ResolveLifetimes {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "resolving lifetimes" }
         }
         query named_region_map(_: LocalDefId) ->
@@ -981,7 +996,8 @@ rustc_queries! {
         query item_children(_: DefId) -> &'tcx [Export<hir::HirId>] {}
         query extern_mod_stmt_cnum(_: DefId) -> Option<CrateNum> {}
 
-        query get_lib_features(_: CrateNum) -> &'tcx LibFeatures {
+        query get_lib_features(_: CrateNum) -> LibFeatures {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "calculating the lib features map" }
         }
@@ -992,13 +1008,15 @@ rustc_queries! {
         /// Returns the lang items defined in another crate by loading it from metadata.
         // FIXME: It is illegal to pass a `CrateNum` other than `LOCAL_CRATE` here, just get rid
         // of that argument?
-        query get_lang_items(_: CrateNum) -> &'tcx LanguageItems {
+        query get_lang_items(_: CrateNum) -> LanguageItems {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "calculating the lang items map" }
         }
 
         /// Returns all diagnostic items defined in all crates.
-        query all_diagnostic_items(_: CrateNum) -> &'tcx FxHashMap<Symbol, DefId> {
+        query all_diagnostic_items(_: CrateNum) -> FxHashMap<Symbol, DefId> {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "calculating the diagnostic items map" }
         }
@@ -1009,7 +1027,8 @@ rustc_queries! {
         }
 
         /// Returns the diagnostic items defined in a crate.
-        query diagnostic_items(_: CrateNum) -> &'tcx FxHashMap<Symbol, DefId> {
+        query diagnostic_items(_: CrateNum) -> FxHashMap<Symbol, DefId> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "calculating the diagnostic items map in a crate" }
         }
 
@@ -1017,7 +1036,8 @@ rustc_queries! {
             desc { "calculating the missing lang items in a crate" }
         }
         query visible_parent_map(_: CrateNum)
-            -> &'tcx DefIdMap<DefId> {
+            -> DefIdMap<DefId> {
+            storage(ArenaCacheSelector<'tcx>)
             desc { "calculating the visible parent map" }
         }
         query missing_extern_crate_item(_: CrateNum) -> bool {
@@ -1051,7 +1071,8 @@ rustc_queries! {
             desc { |tcx| "names_imported_by_glob_use for `{}`", tcx.def_path_str(def_id.to_def_id()) }
         }
 
-        query stability_index(_: CrateNum) -> &'tcx stability::Index<'tcx> {
+        query stability_index(_: CrateNum) -> stability::Index<'tcx> {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "calculating the stability index for the local crate" }
         }
@@ -1242,7 +1263,8 @@ rustc_queries! {
     }
 
     Other {
-        query target_features_whitelist(_: CrateNum) -> &'tcx FxHashMap<String, Option<Symbol>> {
+        query target_features_whitelist(_: CrateNum) -> FxHashMap<String, Option<Symbol>> {
+            storage(ArenaCacheSelector<'tcx>)
             eval_always
             desc { "looking up the whitelist of target features" }
         }

src/librustc_middle/ty/context.rs

@@ -991,22 +991,15 @@ pub struct GlobalCtxt<'tcx> {
 }
 
 impl<'tcx> TyCtxt<'tcx> {
-    pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> &'tcx Steal<Body<'tcx>> {
+    pub fn alloc_steal_mir(self, mir: Body<'tcx>) -> Steal<Body<'tcx>> {
-        self.arena.alloc(Steal::new(mir))
+        Steal::new(mir)
     }
 
     pub fn alloc_steal_promoted(
         self,
         promoted: IndexVec<Promoted, Body<'tcx>>,
-    ) -> &'tcx Steal<IndexVec<Promoted, Body<'tcx>>> {
+    ) -> Steal<IndexVec<Promoted, Body<'tcx>>> {
-        self.arena.alloc(Steal::new(promoted))
+        Steal::new(promoted)
-    }
-
-    pub fn intern_promoted(
-        self,
-        promoted: IndexVec<Promoted, Body<'tcx>>,
-    ) -> &'tcx IndexVec<Promoted, Body<'tcx>> {
-        self.arena.alloc(promoted)
     }
 
     pub fn alloc_adt_def(
@@ -1016,8 +1009,7 @@ impl<'tcx> TyCtxt<'tcx> {
         variants: IndexVec<VariantIdx, ty::VariantDef>,
         repr: ReprOptions,
     ) -> &'tcx ty::AdtDef {
-        let def = ty::AdtDef::new(self, did, kind, variants, repr);
+        self.arena.alloc(ty::AdtDef::new(self, did, kind, variants, repr))
-        self.arena.alloc(def)
     }
 
     pub fn intern_const_alloc(self, alloc: Allocation) -> &'tcx Allocation {
@@ -2745,7 +2737,7 @@ pub fn provide(providers: &mut ty::query::Providers<'_>) {
     };
     providers.features_query = |tcx, cnum| {
         assert_eq!(cnum, LOCAL_CRATE);
-        tcx.arena.alloc(tcx.sess.features_untracked().clone())
+        tcx.sess.features_untracked()
     };
     providers.is_panic_runtime = |tcx, cnum| {
         assert_eq!(cnum, LOCAL_CRATE);

src/librustc_middle/ty/mod.rs

@@ -256,13 +256,13 @@ impl AssocItem {
 /// it is relatively expensive. Instead, items are indexed by `Symbol` and hygienic comparison is
 /// done only on items with the same name.
 #[derive(Debug, Clone, PartialEq, HashStable)]
-pub struct AssociatedItems {
+pub struct AssociatedItems<'tcx> {
-    items: SortedIndexMultiMap<u32, Symbol, ty::AssocItem>,
+    items: SortedIndexMultiMap<u32, Symbol, &'tcx ty::AssocItem>,
 }
 
-impl AssociatedItems {
+impl<'tcx> AssociatedItems<'tcx> {
     /// Constructs an `AssociatedItems` map from a series of `ty::AssocItem`s in definition order.
-    pub fn new(items_in_def_order: impl IntoIterator<Item = ty::AssocItem>) -> Self {
+    pub fn new(items_in_def_order: impl IntoIterator<Item = &'tcx ty::AssocItem>) -> Self {
         let items = items_in_def_order.into_iter().map(|item| (item.ident.name, item)).collect();
         AssociatedItems { items }
     }
@@ -272,7 +272,7 @@ impl AssociatedItems {
     /// New code should avoid relying on definition order. If you need a particular associated item
     /// for a known trait, make that trait a lang item instead of indexing this array.
     pub fn in_definition_order(&self) -> impl '_ + Iterator<Item = &ty::AssocItem> {
-        self.items.iter().map(|(_, v)| v)
+        self.items.iter().map(|(_, v)| *v)
     }
 
     /// Returns an iterator over all associated items with the given name, ignoring hygiene.
@@ -280,7 +280,7 @@ impl AssociatedItems {
         &self,
         name: Symbol,
     ) -> impl '_ + Iterator<Item = &ty::AssocItem> {
-        self.items.get_by_key(&name)
+        self.items.get_by_key(&name).map(|v| *v)
     }
 
     /// Returns an iterator over all associated items with the given name.
@@ -2671,7 +2671,7 @@ impl<'tcx> TyCtxt<'tcx> {
             .and_then(|def_id| self.hir().get(self.hir().as_local_hir_id(def_id)).ident())
     }
 
-    pub fn opt_associated_item(self, def_id: DefId) -> Option<AssocItem> {
+    pub fn opt_associated_item(self, def_id: DefId) -> Option<&'tcx AssocItem> {
         let is_associated_item = if let Some(def_id) = def_id.as_local() {
             match self.hir().get(self.hir().as_local_hir_id(def_id)) {
                 Node::TraitItem(_) | Node::ImplItem(_) => true,

src/librustc_middle/ty/query/plumbing.rs

@@ -213,7 +213,7 @@ macro_rules! query_storage {
         <<$K as Key>::CacheSelector as CacheSelector<$K, $V>>::Cache
     };
     ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
-        $ty
+        <$ty as CacheSelector<$K, $V>>::Cache
     };
     ([$other:ident $(($($other_args:tt)*))* $(, $($modifiers:tt)*)*][$($args:tt)*]) => {
         query_storage!([$($($modifiers)*)*][$($args)*])
@@ -328,6 +328,10 @@ macro_rules! define_queries_inner {
         $(impl<$tcx> QueryConfig<TyCtxt<$tcx>> for queries::$name<$tcx> {
             type Key = $($K)*;
             type Value = $V;
+            type Stored = <
+                query_storage!([$($modifiers)*][$($K)*, $V])
+                as QueryStorage
+            >::Stored;
             const NAME: &'static str = stringify!($name);
             const CATEGORY: ProfileCategory = $category;
         }
@@ -426,8 +430,10 @@ macro_rules! define_queries_inner {
 
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V {
+            pub fn $name(self, key: query_helper_param_ty!($($K)*))
-                self.at(DUMMY_SP).$name(key)
+                -> <queries::$name<$tcx> as QueryConfig<TyCtxt<$tcx>>>::Stored
+            {
+                self.at(DUMMY_SP).$name(key.into_query_param())
             })*
 
             /// All self-profiling events generated by the query engine use
@@ -463,7 +469,9 @@ macro_rules! define_queries_inner {
         impl TyCtxtAt<$tcx> {
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V {
+            pub fn $name(self, key: query_helper_param_ty!($($K)*))
+                -> <queries::$name<$tcx> as QueryConfig<TyCtxt<$tcx>>>::Stored
+            {
                 get_query::<queries::$name<'_>, _>(self.tcx, self.span, key.into_query_param())
             })*
         }

src/librustc_middle/ty/trait_def.rs

@@ -187,7 +187,7 @@ pub(super) fn all_local_trait_impls<'tcx>(
 }
 
 // Query provider for `trait_impls_of`.
-pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> &TraitImpls {
+pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> TraitImpls {
     let mut impls = TraitImpls::default();
 
     {
@@ -219,7 +219,7 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> &Trai
         }
     }
 
-    tcx.arena.alloc(impls)
+    impls
 }
 
 impl<'a> HashStable<StableHashingContext<'a>> for TraitImpls {

src/librustc_mir/borrow_check/mod.rs

@@ -92,7 +92,7 @@ pub fn provide(providers: &mut Providers<'_>) {
     *providers = Providers { mir_borrowck, ..*providers };
 }
 
-fn mir_borrowck(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &BorrowCheckResult<'_> {
+fn mir_borrowck(tcx: TyCtxt<'_>, def_id: LocalDefId) -> BorrowCheckResult<'_> {
     let (input_body, promoted) = tcx.mir_validated(def_id);
     debug!("run query mir_borrowck: {}", tcx.def_path_str(def_id.to_def_id()));
 
@@ -103,7 +103,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &BorrowCheckResult<'_> {
     });
     debug!("mir_borrowck done");
 
-    tcx.arena.alloc(opt_closure_req)
+    opt_closure_req
 }
 
 fn do_mir_borrowck<'a, 'tcx>(

src/librustc_mir/shim.rs

@@ -26,7 +26,7 @@ pub fn provide(providers: &mut Providers<'_>) {
     providers.mir_shims = make_shim;
 }
 
-fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx Body<'tcx> {
+fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'tcx> {
     debug!("make_shim({:?})", instance);
 
     let mut result = match instance {
@@ -128,7 +128,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> &'tcx
 
     debug!("make_shim({:?}) = {:?}", instance, result);
 
-    tcx.arena.alloc(result)
+    result
 }
 
 #[derive(Copy, Clone, Debug, PartialEq)]
@@ -828,7 +828,7 @@ fn build_call_shim<'tcx>(
     body
 }
 
-pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &Body<'_> {
+pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
     debug_assert!(tcx.is_constructor(ctor_id));
 
     let span =
@@ -895,5 +895,5 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> &Body<'_> {
         |_, _| Ok(()),
     );
 
-    tcx.arena.alloc(body)
+    body
 }

src/librustc_mir/transform/mod.rs

@@ -60,7 +60,7 @@ fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 
 /// Finds the full set of `DefId`s within the current crate that have
 /// MIR associated with them.
-fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> &FxHashSet<LocalDefId> {
+fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> FxHashSet<LocalDefId> {
     assert_eq!(krate, LOCAL_CRATE);
 
     let mut set = FxHashSet::default();
@@ -97,7 +97,7 @@ fn mir_keys(tcx: TyCtxt<'_>, krate: CrateNum) -> &FxHashSet<LocalDefId> {
         .krate()
         .visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }.as_deep_visitor());
 
-    tcx.arena.alloc(set)
+    set
 }
 
 /// Where a specific `mir::Body` comes from.
@@ -211,7 +211,7 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def_id: DefId) -> ConstQualifs {
     validator.qualifs_in_return_place()
 }
 
-fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<Body<'_>> {
+fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> Steal<Body<'_>> {
     let def_id = def_id.expect_local();
 
     // Unsafety check uses the raw mir, so make sure it is run
@@ -241,7 +241,7 @@ fn mir_const(tcx: TyCtxt<'_>, def_id: DefId) -> &Steal<Body<'_>> {
 fn mir_validated(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
-) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) {
+) -> (Steal<Body<'tcx>>, Steal<IndexVec<Promoted, Body<'tcx>>>) {
     // Ensure that we compute the `mir_const_qualif` for constants at
     // this point, before we steal the mir-const result.
     let _ = tcx.mir_const_qualif(def_id.to_def_id());
@@ -360,7 +360,7 @@ fn run_optimization_passes<'tcx>(
     );
 }
 
-fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> {
+fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> Body<'_> {
     if tcx.is_constructor(def_id) {
         // There's no reason to run all of the MIR passes on constructors when
         // we can just output the MIR we want directly. This also saves const
@@ -381,12 +381,12 @@ fn optimized_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &Body<'_> {
 
     debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR");
 
-    tcx.arena.alloc(body)
+    body
 }
 
-fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &IndexVec<Promoted, Body<'_>> {
+fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> IndexVec<Promoted, Body<'_>> {
     if tcx.is_constructor(def_id) {
-        return tcx.intern_promoted(IndexVec::new());
+        return IndexVec::new();
     }
 
     let def_id = def_id.expect_local();
@@ -401,5 +401,5 @@ fn promoted_mir(tcx: TyCtxt<'_>, def_id: DefId) -> &IndexVec<Promoted, Body<'_>>
 
     debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR");
 
-    tcx.intern_promoted(promoted)
+    promoted
 }

src/librustc_mir_build/build/mod.rs

@@ -21,7 +21,7 @@ use rustc_target::spec::PanicStrategy;
 
 use super::lints;
 
-crate fn mir_built(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &ty::steal::Steal<Body<'_>> {
+crate fn mir_built(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::steal::Steal<Body<'_>> {
     tcx.alloc_steal_mir(mir_build(tcx, def_id))
 }
 

src/librustc_mir_build/lints.rs

@@ -24,7 +24,7 @@ crate fn check<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, def_id: LocalDefId) {
             Some(AssocItem {
                 container: AssocItemContainer::TraitContainer(trait_def_id), ..
             }) => {
-                let trait_substs_count = tcx.generics_of(trait_def_id).count();
+                let trait_substs_count = tcx.generics_of(*trait_def_id).count();
                 &InternalSubsts::identity_for_item(tcx, def_id.to_def_id())[..trait_substs_count]
             }
             _ => &[],

src/librustc_passes/diagnostic_items.rs

@@ -93,18 +93,18 @@ fn extract(attrs: &[ast::Attribute]) -> Option<Symbol> {
 }
 
 /// Traverse and collect the diagnostic items in the current
-fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap<Symbol, DefId> {
+fn collect<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap<Symbol, DefId> {
     // Initialize the collector.
     let mut collector = DiagnosticItemCollector::new(tcx);
 
     // Collect diagnostic items in this crate.
     tcx.hir().krate().visit_all_item_likes(&mut collector);
 
-    tcx.arena.alloc(collector.items)
+    collector.items
 }
 
 /// Traverse and collect all the diagnostic items in all crates.
-fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap<Symbol, DefId> {
+fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> FxHashMap<Symbol, DefId> {
     // Initialize the collector.
     let mut collector = FxHashMap::default();
 
@@ -115,7 +115,7 @@ fn collect_all<'tcx>(tcx: TyCtxt<'tcx>) -> &'tcx FxHashMap<Symbol, DefId> {
         }
     }
 
-    tcx.arena.alloc(collector)
+    collector
 }
 
 pub fn provide(providers: &mut Providers<'_>) {

src/librustc_passes/lang_items.rs

@@ -169,6 +169,6 @@ fn collect(tcx: TyCtxt<'_>) -> LanguageItems {
 pub fn provide(providers: &mut Providers<'_>) {
     providers.get_lang_items = |tcx, id| {
         assert_eq!(id, LOCAL_CRATE);
-        tcx.arena.alloc(collect(tcx))
+        collect(tcx)
     };
 }

src/librustc_passes/lib_features.rs

@@ -138,6 +138,6 @@ fn collect(tcx: TyCtxt<'_>) -> LibFeatures {
 pub fn provide(providers: &mut Providers<'_>) {
     providers.get_lib_features = |tcx, id| {
         assert_eq!(id, LOCAL_CRATE);
-        tcx.arena.alloc(collect(tcx))
+        collect(tcx)
     };
 }

src/librustc_passes/reachable.rs

@@ -23,7 +23,7 @@ use rustc_target::spec::abi::Abi;
 // Returns true if the given item must be inlined because it may be
 // monomorphized or it was marked with `#[inline]`. This will only return
 // true for functions.
-fn item_might_be_inlined(tcx: TyCtxt<'tcx>, item: &hir::Item<'_>, attrs: CodegenFnAttrs) -> bool {
+fn item_might_be_inlined(tcx: TyCtxt<'tcx>, item: &hir::Item<'_>, attrs: &CodegenFnAttrs) -> bool {
     if attrs.requests_inline() {
         return true;
     }

src/librustc_passes/stability.rs

@@ -480,7 +480,7 @@ pub(crate) fn provide(providers: &mut Providers<'_>) {
     *providers = Providers { check_mod_unstable_api_usage, ..*providers };
     providers.stability_index = |tcx, cnum| {
         assert_eq!(cnum, LOCAL_CRATE);
-        tcx.arena.alloc(new_index(tcx))
+        new_index(tcx)
     };
 }
 

src/librustc_query_system/Cargo.toml

@@ -10,6 +10,7 @@ path = "lib.rs"
 doctest = false
 
 [dependencies]
+arena = { path = "../libarena" }
 log = { version = "0.4", features = ["release_max_level_info", "std"] }
 rustc-rayon-core = "0.3.0"
 rustc_data_structures = { path = "../librustc_data_structures" }

src/librustc_query_system/query/caches.rs

@@ -2,19 +2,29 @@ use crate::dep_graph::DepNodeIndex;
 use crate::query::plumbing::{QueryLookup, QueryState};
 use crate::query::QueryContext;
 
+use arena::TypedArena;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sharded::Sharded;
+use rustc_data_structures::sync::WorkerLocal;
 use std::default::Default;
 use std::hash::Hash;
 use std::marker::PhantomData;
 
-pub trait CacheSelector<K: Hash, V> {
+pub trait CacheSelector<K, V> {
-    type Cache: QueryCache<Key = K, Value = V>;
+    type Cache;
 }
 
-pub trait QueryCache: Default {
+pub trait QueryStorage: Default {
-    type Key: Hash;
     type Value;
+    type Stored: Clone;
+
+    /// Store a value without putting it in the cache.
+    /// This is meant to be used with cycle errors.
+    fn store_nocache(&self, value: Self::Value) -> Self::Stored;
+}
+
+pub trait QueryCache: QueryStorage {
+    type Key: Hash;
     type Sharded: Default;
 
     /// Checks if the query is already computed and in the cache.
@@ -30,7 +40,7 @@ pub trait QueryCache: Default {
         on_miss: OnMiss,
     ) -> R
     where
-        OnHit: FnOnce(&Self::Value, DepNodeIndex) -> R,
+        OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R,
         OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R;
 
     fn complete<CTX: QueryContext>(
@@ -40,7 +50,7 @@ pub trait QueryCache: Default {
         key: Self::Key,
         value: Self::Value,
         index: DepNodeIndex,
-    );
+    ) -> Self::Stored;
 
     fn iter<R, L>(
         &self,
@@ -66,9 +76,19 @@ impl<K, V> Default for DefaultCache<K, V> {
     }
 }
 
+impl<K: Eq + Hash, V: Clone> QueryStorage for DefaultCache<K, V> {
+    type Value = V;
+    type Stored = V;
+
+    #[inline]
+    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
+        // We have no dedicated storage
+        value
+    }
+}
+
 impl<K: Eq + Hash, V: Clone> QueryCache for DefaultCache<K, V> {
     type Key = K;
-    type Value = V;
     type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
 
     #[inline(always)]
@@ -99,8 +119,94 @@ impl<K: Eq + Hash, V: Clone> QueryCache for DefaultCache<K, V> {
         key: K,
         value: V,
         index: DepNodeIndex,
-    ) {
+    ) -> Self::Stored {
-        lock_sharded_storage.insert(key, (value, index));
+        lock_sharded_storage.insert(key, (value.clone(), index));
+        value
+    }
+
+    fn iter<R, L>(
+        &self,
+        shards: &Sharded<L>,
+        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+    ) -> R {
+        let mut shards = shards.lock_shards();
+        let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
+        let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+        f(Box::new(results))
+    }
+}
+
+pub struct ArenaCacheSelector<'tcx>(PhantomData<&'tcx ()>);
+
+impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<K, V> for ArenaCacheSelector<'tcx> {
+    type Cache = ArenaCache<'tcx, K, V>;
+}
+
+pub struct ArenaCache<'tcx, K, V> {
+    arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
+    phantom: PhantomData<(K, &'tcx V)>,
+}
+
+impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
+    fn default() -> Self {
+        ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), phantom: PhantomData }
+    }
+}
+
+impl<'tcx, K: Eq + Hash, V: 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
+    type Value = V;
+    type Stored = &'tcx V;
+
+    #[inline]
+    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
+        let value = self.arena.alloc((value, DepNodeIndex::INVALID));
+        let value = unsafe { &*(&value.0 as *const _) };
+        &value
+    }
+}
+
+impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> {
+    type Key = K;
+    type Sharded = FxHashMap<K, &'tcx (V, DepNodeIndex)>;
+
+    #[inline(always)]
+    fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
+        &self,
+        state: &QueryState<CTX, Self>,
+        key: K,
+        on_hit: OnHit,
+        on_miss: OnMiss,
+    ) -> R
+    where
+        OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
+        OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
+    {
+        let mut lookup = state.get_lookup(&key);
+        let lock = &mut *lookup.lock;
+
+        let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
+
+        if let Some((_, value)) = result {
+            on_hit(&&value.0, value.1)
+        } else {
+            on_miss(key, lookup)
+        }
+    }
+
+    #[inline]
+    fn complete<CTX: QueryContext>(
+        &self,
+        _: CTX,
+        lock_sharded_storage: &mut Self::Sharded,
+        key: K,
+        value: V,
+        index: DepNodeIndex,
+    ) -> Self::Stored {
+        let value = self.arena.alloc((value, index));
+        let value = unsafe { &*(value as *const _) };
+        lock_sharded_storage.insert(key, value);
+        &value.0
     }
 
     fn iter<R, L>(

src/librustc_query_system/query/config.rs

@@ -20,7 +20,8 @@ pub trait QueryConfig<CTX> {
     const CATEGORY: ProfileCategory;
 
     type Key: Eq + Hash + Clone + Debug;
-    type Value: Clone;
+    type Value;
+    type Stored: Clone;
 }
 
 pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
@@ -28,7 +29,7 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
     const EVAL_ALWAYS: bool;
     const DEP_KIND: CTX::DepKind;
 
-    type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
+    type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
 
     // Don't use this method to access query results, instead use the methods on TyCtxt
     fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;

src/librustc_query_system/query/mod.rs

@@ -7,7 +7,9 @@ pub use self::job::deadlock;
 pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
 
 mod caches;
-pub use self::caches::{CacheSelector, DefaultCacheSelector, QueryCache};
+pub use self::caches::{
+    ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage,
+};
 
 mod config;
 pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};

src/librustc_query_system/query/plumbing.rs

@@ -148,7 +148,6 @@ struct JobOwner<'tcx, CTX: QueryContext, C>
 where
     C: QueryCache,
     C::Key: Eq + Hash + Clone + Debug,
-    C::Value: Clone,
 {
     state: &'tcx QueryState<CTX, C>,
     key: C::Key,
@@ -159,7 +158,6 @@ impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C>
 where
     C: QueryCache,
     C::Key: Eq + Hash + Clone + Debug,
-    C::Value: Clone,
 {
     /// Either gets a `JobOwner` corresponding the query, allowing us to
     /// start executing the query, or returns with the result of the query.
@@ -177,7 +175,7 @@ where
         mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
     ) -> TryGetJob<'b, CTX, C>
     where
-        Q: QueryDescription<CTX, Key = C::Key, Value = C::Value, Cache = C>,
+        Q: QueryDescription<CTX, Key = C::Key, Stored = C::Stored, Value = C::Value, Cache = C>,
         CTX: QueryContext,
     {
         let lock = &mut *lookup.lock;
@@ -229,7 +227,8 @@ where
         // so we just return the error.
         #[cfg(not(parallel_compiler))]
         return TryGetJob::Cycle(cold_path(|| {
-            Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span))
+            let value = Q::handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
+            Q::query_state(tcx).cache.store_nocache(value)
         }));
 
         // With parallel queries we might just have to wait on some other
@@ -239,7 +238,9 @@ where
             let result = latch.wait_on(tcx, span);
 
             if let Err(cycle) = result {
-                return TryGetJob::Cycle(Q::handle_cycle_error(tcx, cycle));
+                let value = Q::handle_cycle_error(tcx, cycle);
+                let value = Q::query_state(tcx).cache.store_nocache(value);
+                return TryGetJob::Cycle(value);
             }
 
             let cached = try_get_cached(
@@ -261,7 +262,7 @@ where
     /// Completes the query by updating the query cache with the `result`,
     /// signals the waiter and forgets the JobOwner, so it won't poison the query
     #[inline(always)]
-    fn complete(self, tcx: CTX, result: &C::Value, dep_node_index: DepNodeIndex) {
+    fn complete(self, tcx: CTX, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored {
         // We can move out of `self` here because we `mem::forget` it below
         let key = unsafe { ptr::read(&self.key) };
         let state = self.state;
@@ -269,18 +270,18 @@ where
         // Forget ourself so our destructor won't poison the query
         mem::forget(self);
 
-        let job = {
+        let (job, result) = {
-            let result = result.clone();
             let mut lock = state.shards.get_shard_by_value(&key).lock();
             let job = match lock.active.remove(&key).unwrap() {
                 QueryResult::Started(job) => job,
                 QueryResult::Poisoned => panic!(),
             };
-            state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index);
+            let result = state.cache.complete(tcx, &mut lock.cache, key, result, dep_node_index);
-            job
+            (job, result)
         };
 
         job.signal_complete();
+        result
     }
 }
 
@@ -297,7 +298,6 @@ where
 impl<'tcx, CTX: QueryContext, C: QueryCache> Drop for JobOwner<'tcx, CTX, C>
 where
     C::Key: Eq + Hash + Clone + Debug,
-    C::Value: Clone,
 {
     #[inline(never)]
     #[cold]
@@ -331,7 +331,6 @@ pub struct CycleError<Q> {
 enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache>
 where
     C::Key: Eq + Hash + Clone + Debug,
-    C::Value: Clone,
 {
     /// The query is not yet started. Contains a guard to the cache eventually used to start it.
     NotYetStarted(JobOwner<'tcx, CTX, C>),
@@ -340,10 +339,10 @@ where
     /// Returns the result of the query and its dep-node index
     /// if it succeeded or a cycle error if it failed.
     #[cfg(parallel_compiler)]
-    JobCompleted((C::Value, DepNodeIndex)),
+    JobCompleted((C::Stored, DepNodeIndex)),
 
     /// Trying to execute the query resulted in a cycle.
-    Cycle(C::Value),
+    Cycle(C::Stored),
 }
 
 /// Checks if the query is already computed and in the cache.
@@ -362,7 +361,7 @@ fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
 where
     C: QueryCache,
     CTX: QueryContext,
-    OnHit: FnOnce(&C::Value, DepNodeIndex) -> R,
+    OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R,
     OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R,
 {
     state.cache.lookup(
@@ -388,7 +387,7 @@ fn try_execute_query<Q, CTX>(
     span: Span,
     key: Q::Key,
     lookup: QueryLookup<'_, CTX, Q::Key, <Q::Cache as QueryCache>::Sharded>,
-) -> Q::Value
+) -> Q::Stored
 where
     Q: QueryDescription<CTX>,
     CTX: QueryContext,
@@ -427,9 +426,7 @@ where
             tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
         }
 
-        job.complete(tcx, &result, dep_node_index);
+        return job.complete(tcx, result, dep_node_index);
-
-        return result;
     }
 
     let dep_node = Q::to_dep_node(tcx, &key);
@@ -454,8 +451,7 @@ where
             })
         });
         if let Some((result, dep_node_index)) = loaded {
-            job.complete(tcx, &result, dep_node_index);
+            return job.complete(tcx, result, dep_node_index);
-            return result;
         }
     }
 
@@ -558,7 +554,7 @@ fn force_query_with_job<Q, CTX>(
     key: Q::Key,
     job: JobOwner<'_, CTX, Q::Cache>,
     dep_node: DepNode<CTX::DepKind>,
-) -> (Q::Value, DepNodeIndex)
+) -> (Q::Stored, DepNodeIndex)
 where
     Q: QueryDescription<CTX>,
     CTX: QueryContext,
@@ -603,13 +599,13 @@ where
         }
     }
 
-    job.complete(tcx, &result, dep_node_index);
+    let result = job.complete(tcx, result, dep_node_index);
 
     (result, dep_node_index)
 }
 
 #[inline(never)]
-pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Value
+pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
 where
     Q: QueryDescription<CTX>,
     CTX: QueryContext,

src/librustc_resolve/late/lifetimes.rs

@@ -294,7 +294,7 @@ pub fn provide(providers: &mut ty::query::Providers<'_>) {
 /// entire crate. You should not read the result of this query
 /// directly, but rather use `named_region_map`, `is_late_bound_map`,
 /// etc.
-fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> &ResolveLifetimes {
+fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> ResolveLifetimes {
     assert_eq!(for_krate, LOCAL_CRATE);
 
     let named_region_map = krate(tcx);
@@ -314,7 +314,7 @@ fn resolve_lifetimes(tcx: TyCtxt<'_>, for_krate: CrateNum) -> &ResolveLifetimes
         map.insert(hir_id.local_id, v);
     }
 
-    tcx.arena.alloc(rl)
+    rl
 }
 
 fn krate(tcx: TyCtxt<'_>) -> NamedRegionMap {

src/librustc_trait_selection/traits/specialize/mod.rs

@@ -254,7 +254,7 @@ fn fulfill_implication<'a, 'tcx>(
 pub(super) fn specialization_graph_provider(
     tcx: TyCtxt<'_>,
     trait_id: DefId,
-) -> &specialization_graph::Graph {
+) -> specialization_graph::Graph {
     let mut sg = specialization_graph::Graph::new();
 
     let mut trait_impls: Vec<_> = tcx.all_impls(trait_id).collect();
@@ -286,7 +286,7 @@ pub(super) fn specialization_graph_provider(
         }
     }
 
-    tcx.arena.alloc(sg)
+    sg
 }
 
 fn report_overlap_conflict(

src/librustc_ty/ty.rs

@@ -220,9 +220,9 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: DefId) -> &[DefId] {
     }
 }
 
-fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::AssociatedItems {
+fn associated_items(tcx: TyCtxt<'_>, def_id: DefId) -> ty::AssociatedItems<'_> {
     let items = tcx.associated_item_def_ids(def_id).iter().map(|did| tcx.associated_item(*did));
-    tcx.arena.alloc(ty::AssociatedItems::new(items))
+    ty::AssociatedItems::new(items)
 }
 
 fn def_span(tcx: TyCtxt<'_>, def_id: DefId) -> Span {

src/librustc_typeck/coherence/inherent_impls.rs

@@ -17,13 +17,13 @@ use rustc_ast::ast;
 use rustc_span::Span;
 
 /// On-demand query: yields a map containing all types mapped to their inherent impls.
-pub fn crate_inherent_impls(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CrateInherentImpls {
+pub fn crate_inherent_impls(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateInherentImpls {
     assert_eq!(crate_num, LOCAL_CRATE);
 
     let krate = tcx.hir().krate();
     let mut collect = InherentCollect { tcx, impls_map: Default::default() };
     krate.visit_all_item_likes(&mut collect);
-    tcx.arena.alloc(collect.impls_map)
+    collect.impls_map
 }
 
 /// On-demand query: yields a vector of the inherent impls for a specific type.

src/librustc_typeck/collect.rs

@@ -1002,7 +1002,7 @@ fn super_predicates_of(tcx: TyCtxt<'_>, trait_def_id: DefId) -> ty::GenericPredi
     ty::GenericPredicates { parent: None, predicates: superbounds }
 }
 
-fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::TraitDef {
+fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> ty::TraitDef {
     let hir_id = tcx.hir().as_local_hir_id(def_id.expect_local());
     let item = tcx.hir().expect_item(hir_id);
 
@@ -1033,16 +1033,7 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::TraitDef {
         ty::trait_def::TraitSpecializationKind::None
     };
     let def_path_hash = tcx.def_path_hash(def_id);
-    let def = ty::TraitDef::new(
+    ty::TraitDef::new(def_id, unsafety, paren_sugar, is_auto, is_marker, spec_kind, def_path_hash)
-        def_id,
-        unsafety,
-        paren_sugar,
-        is_auto,
-        is_marker,
-        spec_kind,
-        def_path_hash,
-    );
-    tcx.arena.alloc(def)
 }
 
 fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option<Span> {
@@ -1158,7 +1149,7 @@ fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option<S
     }
 }
 
-fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::Generics {
+fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> ty::Generics {
     use rustc_hir::*;
 
     let hir_id = tcx.hir().as_local_hir_id(def_id.expect_local());
@@ -1403,14 +1394,14 @@ fn generics_of(tcx: TyCtxt<'_>, def_id: DefId) -> &ty::Generics {
 
     let param_def_id_to_index = params.iter().map(|param| (param.def_id, param.index)).collect();
 
-    tcx.arena.alloc(ty::Generics {
+    ty::Generics {
         parent: parent_def_id,
         parent_count,
         params,
         param_def_id_to_index,
         has_self: has_self || parent_has_self,
         has_late_bound_regions: has_late_bound_regions(tcx, node),
-    })
+    }
 }
 
 fn are_suggestable_generic_args(generic_args: &[hir::GenericArg<'_>]) -> bool {

src/librustc_typeck/outlives/mod.rs

@@ -58,7 +58,7 @@ fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[(ty::Predicate
     }
 }
 
-fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CratePredicatesMap<'_> {
+fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CratePredicatesMap<'_> {
     assert_eq!(crate_num, LOCAL_CRATE);
 
     // Compute a map from each struct/enum/union S to the **explicit**
@@ -105,5 +105,5 @@ fn inferred_outlives_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CratePredic
         })
         .collect();
 
-    tcx.arena.alloc(ty::CratePredicatesMap { predicates })
+    ty::CratePredicatesMap { predicates }
 }

src/librustc_typeck/variance/mod.rs

@@ -29,12 +29,12 @@ pub fn provide(providers: &mut Providers<'_>) {
     *providers = Providers { variances_of, crate_variances, ..*providers };
 }
 
-fn crate_variances(tcx: TyCtxt<'_>, crate_num: CrateNum) -> &CrateVariancesMap<'_> {
+fn crate_variances(tcx: TyCtxt<'_>, crate_num: CrateNum) -> CrateVariancesMap<'_> {
     assert_eq!(crate_num, LOCAL_CRATE);
     let mut arena = arena::TypedArena::default();
     let terms_cx = terms::determine_parameters_to_be_inferred(tcx, &mut arena);
     let constraints_cx = constraints::add_constraints_from_crate(terms_cx);
-    tcx.arena.alloc(solve::solve_constraints(constraints_cx))
+    solve::solve_constraints(constraints_cx)
 }
 
 fn variances_of(tcx: TyCtxt<'_>, item_def_id: DefId) -> &[ty::Variance] {

src/test/run-make-fulldeps/hotplug_codegen_backend/the_backend.rs

@@ -53,7 +53,7 @@ impl CodegenBackend for TheBackend {
         rustc_symbol_mangling::provide(providers);
 
         providers.target_features_whitelist = |tcx, _cnum| {
-            tcx.arena.alloc(Default::default()) // Just a dummy
+            Default::default() // Just a dummy
         };
         providers.is_reachable_non_generic = |_tcx, _defid| true;
         providers.exported_symbols = |_tcx, _crate| &[];