Auto merge of #82183 - michaelwoerister:lazier-defpathhash-loading2, r=wesleywiser

Simplify lazy DefPathHash decoding by using an on-disk hash table.

This PR simplifies the logic around mapping `DefPathHash` values encountered during incremental compilation to valid `DefId`s in the current session. It is able to do so by using an on-disk hash table encoding that allows for looking up values directly, i.e. without deserializing the entire table.

The main simplification comes from not having to keep track of `DefPathHashes` being used during the compilation session.

Cargo.lock

@@ -2321,6 +2321,15 @@ dependencies = [
  "rustc-std-workspace-core",
 ]
 
+[[package]]
+name = "odht"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b18a8d1c919d3e7b5c49708d08ef7d60bc2150a7c3a8244257c54ca3f625010"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.7.2"
@@ -3859,6 +3868,7 @@ version = "0.0.0"
 name = "rustc_hir"
 version = "0.0.0"
 dependencies = [
+ "odht",
  "rustc_ast",
  "rustc_data_structures",
  "rustc_feature",
@@ -4045,6 +4055,7 @@ name = "rustc_metadata"
 version = "0.0.0"
 dependencies = [
  "libc",
+ "odht",
  "rustc_ast",
  "rustc_attr",
  "rustc_data_structures",

compiler/rustc_hir/Cargo.toml

@@ -17,3 +17,4 @@ rustc_serialize = { path = "../rustc_serialize" }
 rustc_ast = { path = "../rustc_ast" }
 tracing = "0.1"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
+odht = { version = "0.2.1", features = ["nightly"] }

compiler/rustc_hir/src/def_path_hash_map.rs

@@ -0,0 +1,37 @@
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_span::def_id::{DefIndex, DefPathHash};
+
+#[derive(Clone, Default)]
+pub struct Config;
+
+impl odht::Config for Config {
+    type Key = DefPathHash;
+    type Value = DefIndex;
+
+    type EncodedKey = [u8; 16];
+    type EncodedValue = [u8; 4];
+
+    type H = odht::UnHashFn;
+
+    #[inline]
+    fn encode_key(k: &DefPathHash) -> [u8; 16] {
+        k.0.to_le_bytes()
+    }
+
+    #[inline]
+    fn encode_value(v: &DefIndex) -> [u8; 4] {
+        v.as_u32().to_le_bytes()
+    }
+
+    #[inline]
+    fn decode_key(k: &[u8; 16]) -> DefPathHash {
+        DefPathHash(Fingerprint::from_le_bytes(*k))
+    }
+
+    #[inline]
+    fn decode_value(v: &[u8; 4]) -> DefIndex {
+        DefIndex::from_u32(u32::from_le_bytes(*v))
+    }
+}
+
+pub type DefPathHashMap = odht::HashTableOwned<Config>;

compiler/rustc_hir/src/definitions.rs

@@ -6,11 +6,11 @@
 
 pub use crate::def_id::DefPathHash;
 use crate::def_id::{CrateNum, DefIndex, LocalDefId, StableCrateId, CRATE_DEF_INDEX, LOCAL_CRATE};
+use crate::def_path_hash_map::DefPathHashMap;
 use crate::hir;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::StableHasher;
-use rustc_data_structures::unhash::UnhashMap;
 use rustc_index::vec::IndexVec;
 use rustc_span::hygiene::ExpnId;
 use rustc_span::symbol::{kw, sym, Symbol};
@@ -28,7 +28,7 @@ use tracing::debug;
 pub struct DefPathTable {
     index_to_key: IndexVec<DefIndex, DefKey>,
     def_path_hashes: IndexVec<DefIndex, DefPathHash>,
-    def_path_hash_to_index: UnhashMap<DefPathHash, DefIndex>,
+    def_path_hash_to_index: DefPathHashMap,
 }
 
 impl DefPathTable {
@@ -44,7 +44,7 @@ impl DefPathTable {
 
         // Check for hash collisions of DefPathHashes. These should be
         // exceedingly rare.
-        if let Some(existing) = self.def_path_hash_to_index.insert(def_path_hash, index) {
+        if let Some(existing) = self.def_path_hash_to_index.insert(&def_path_hash, &index) {
             let def_path1 = DefPath::make(LOCAL_CRATE, existing, |idx| self.def_key(idx));
             let def_path2 = DefPath::make(LOCAL_CRATE, index, |idx| self.def_key(idx));
 
@@ -87,7 +87,7 @@ impl DefPathTable {
 
     pub fn enumerated_keys_and_path_hashes(
         &self,
-    ) -> impl Iterator<Item = (DefIndex, &DefKey, &DefPathHash)> + '_ {
+    ) -> impl Iterator<Item = (DefIndex, &DefKey, &DefPathHash)> + ExactSizeIterator + '_ {
         self.index_to_key
             .iter_enumerated()
             .map(move |(index, key)| (index, key, &self.def_path_hashes[index]))
@@ -110,6 +110,9 @@ pub struct Definitions {
     expansions_that_defined: FxHashMap<LocalDefId, ExpnId>,
 
     def_id_to_span: IndexVec<LocalDefId, Span>,
+
+    /// The [StableCrateId] of the local crate.
+    stable_crate_id: StableCrateId,
 }
 
 /// A unique identifier that we can use to lookup a definition
@@ -356,6 +359,7 @@ impl Definitions {
             hir_id_to_def_id: Default::default(),
             expansions_that_defined: Default::default(),
             def_id_to_span,
+            stable_crate_id,
         }
     }
 
@@ -439,11 +443,17 @@ impl Definitions {
     }
 
     #[inline(always)]
-    pub fn local_def_path_hash_to_def_id(&self, hash: DefPathHash) -> Option<LocalDefId> {
+    pub fn local_def_path_hash_to_def_id(&self, hash: DefPathHash) -> LocalDefId {
+        debug_assert!(hash.stable_crate_id() == self.stable_crate_id);
         self.table
             .def_path_hash_to_index
             .get(&hash)
-            .map(|&local_def_index| LocalDefId { local_def_index })
+            .map(|local_def_index| LocalDefId { local_def_index })
+            .unwrap()
+    }
+
+    pub fn def_path_hash_to_def_index_map(&self) -> &DefPathHashMap {
+        &self.table.def_path_hash_to_index
     }
 }
 

compiler/rustc_hir/src/lib.rs

@@ -16,6 +16,7 @@ extern crate rustc_data_structures;
 
 mod arena;
 pub mod def;
+pub mod def_path_hash_map;
 pub mod definitions;
 pub use rustc_span::def_id;
 mod hir;

compiler/rustc_metadata/Cargo.toml

@@ -8,6 +8,7 @@ doctest = false
 
 [dependencies]
 libc = "0.2"
+odht = { version = "0.2.1", features = ["nightly"] }
 snap = "1"
 tracing = "0.1"
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }

compiler/rustc_metadata/src/creader.rs

@@ -45,7 +45,7 @@ pub struct CStore {
 
     /// This map is used to verify we get no hash conflicts between
     /// `StableCrateId` values.
-    stable_crate_ids: FxHashMap<StableCrateId, CrateNum>,
+    pub(crate) stable_crate_ids: FxHashMap<StableCrateId, CrateNum>,
 
     /// Unused externs of the crate
     unused_externs: Vec<Symbol>,

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -48,7 +48,26 @@ use rustc_span::hygiene::HygieneDecodeContext;
 
 mod cstore_impl;
 
-crate struct MetadataBlob(MetadataRef);
+/// A reference to the raw binary version of crate metadata.
+/// A `MetadataBlob` internally is just a reference counted pointer to
+/// the actual data, so cloning it is cheap.
+#[derive(Clone)]
+crate struct MetadataBlob(Lrc<MetadataRef>);
+
+// This is needed so we can create an OwningRef into the blob.
+// The data behind a `MetadataBlob` has a stable address because it is
+// contained within an Rc/Arc.
+unsafe impl rustc_data_structures::owning_ref::StableAddress for MetadataBlob {}
+
+// This is needed so we can create an OwningRef into the blob.
+impl std::ops::Deref for MetadataBlob {
+    type Target = [u8];
+
+    #[inline]
+    fn deref(&self) -> &[u8] {
+        &self.0[..]
+    }
+}
 
 // A map from external crate numbers (as decoded from some crate file) to
 // local crate numbers (as generated during this session). Each external
@@ -76,10 +95,8 @@ crate struct CrateMetadata {
     raw_proc_macros: Option<&'static [ProcMacro]>,
     /// Source maps for code from the crate.
     source_map_import_info: OnceCell<Vec<ImportedSourceFile>>,
-    /// For every definition in this crate, maps its `DefPathHash` to its
+    /// For every definition in this crate, maps its `DefPathHash` to its `DefIndex`.
-    /// `DefIndex`. See `raw_def_id_to_def_id` for more details about how
+    def_path_hash_map: DefPathHashMapRef<'static>,
-    /// this is used.
-    def_path_hash_map: OnceCell<UnhashMap<DefPathHash, DefIndex>>,
     /// Likewise for ExpnHash.
     expn_hash_map: OnceCell<UnhashMap<ExpnHash, ExpnIndex>>,
     /// Used for decoding interpret::AllocIds in a cached & thread-safe manner.
@@ -134,6 +151,7 @@ struct ImportedSourceFile {
 pub(super) struct DecodeContext<'a, 'tcx> {
     opaque: opaque::Decoder<'a>,
     cdata: Option<CrateMetadataRef<'a>>,
+    blob: &'a MetadataBlob,
     sess: Option<&'tcx Session>,
     tcx: Option<TyCtxt<'tcx>>,
 
@@ -148,7 +166,8 @@ pub(super) struct DecodeContext<'a, 'tcx> {
 
 /// Abstract over the various ways one can create metadata decoders.
 pub(super) trait Metadata<'a, 'tcx>: Copy {
-    fn raw_bytes(self) -> &'a [u8];
+    fn blob(self) -> &'a MetadataBlob;
+
     fn cdata(self) -> Option<CrateMetadataRef<'a>> {
         None
     }
@@ -162,8 +181,9 @@ pub(super) trait Metadata<'a, 'tcx>: Copy {
     fn decoder(self, pos: usize) -> DecodeContext<'a, 'tcx> {
         let tcx = self.tcx();
         DecodeContext {
-            opaque: opaque::Decoder::new(self.raw_bytes(), pos),
+            opaque: opaque::Decoder::new(self.blob(), pos),
             cdata: self.cdata(),
+            blob: self.blob(),
             sess: self.sess().or(tcx.map(|tcx| tcx.sess)),
             tcx,
             last_source_file_index: 0,
@@ -176,17 +196,19 @@ pub(super) trait Metadata<'a, 'tcx>: Copy {
 }
 
 impl<'a, 'tcx> Metadata<'a, 'tcx> for &'a MetadataBlob {
-    fn raw_bytes(self) -> &'a [u8] {
+    #[inline]
-        &self.0
+    fn blob(self) -> &'a MetadataBlob {
+        self
     }
 }
 
 impl<'a, 'tcx> Metadata<'a, 'tcx> for (&'a MetadataBlob, &'tcx Session) {
-    fn raw_bytes(self) -> &'a [u8] {
+    #[inline]
-        let (blob, _) = self;
+    fn blob(self) -> &'a MetadataBlob {
-        &blob.0
+        self.0
     }
 
+    #[inline]
     fn sess(self) -> Option<&'tcx Session> {
         let (_, sess) = self;
         Some(sess)
@@ -194,33 +216,41 @@ impl<'a, 'tcx> Metadata<'a, 'tcx> for (&'a MetadataBlob, &'tcx Session) {
 }
 
 impl<'a, 'tcx> Metadata<'a, 'tcx> for &'a CrateMetadataRef<'a> {
-    fn raw_bytes(self) -> &'a [u8] {
+    #[inline]
-        self.blob.raw_bytes()
+    fn blob(self) -> &'a MetadataBlob {
+        &self.blob
     }
+    #[inline]
     fn cdata(self) -> Option<CrateMetadataRef<'a>> {
         Some(*self)
     }
 }
 
 impl<'a, 'tcx> Metadata<'a, 'tcx> for (&'a CrateMetadataRef<'a>, &'tcx Session) {
-    fn raw_bytes(self) -> &'a [u8] {
+    #[inline]
-        self.0.raw_bytes()
+    fn blob(self) -> &'a MetadataBlob {
+        &self.0.blob
     }
+    #[inline]
     fn cdata(self) -> Option<CrateMetadataRef<'a>> {
         Some(*self.0)
     }
+    #[inline]
     fn sess(self) -> Option<&'tcx Session> {
         Some(&self.1)
     }
 }
 
 impl<'a, 'tcx> Metadata<'a, 'tcx> for (&'a CrateMetadataRef<'a>, TyCtxt<'tcx>) {
-    fn raw_bytes(self) -> &'a [u8] {
+    #[inline]
-        self.0.raw_bytes()
+    fn blob(self) -> &'a MetadataBlob {
+        &self.0.blob
     }
+    #[inline]
     fn cdata(self) -> Option<CrateMetadataRef<'a>> {
         Some(*self.0)
     }
+    #[inline]
     fn tcx(self) -> Option<TyCtxt<'tcx>> {
         Some(self.1)
     }
@@ -246,12 +276,21 @@ impl<'a: 'x, 'tcx: 'x, 'x, T: Decodable<DecodeContext<'a, 'tcx>>> Lazy<[T]> {
 }
 
 impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
+    #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
-        self.tcx.expect("missing TyCtxt in DecodeContext")
+        debug_assert!(self.tcx.is_some(), "missing TyCtxt in DecodeContext");
+        self.tcx.unwrap()
     }
 
-    fn cdata(&self) -> CrateMetadataRef<'a> {
+    #[inline]
-        self.cdata.expect("missing CrateMetadata in DecodeContext")
+    pub fn blob(&self) -> &'a MetadataBlob {
+        self.blob
+    }
+
+    #[inline]
+    pub fn cdata(&self) -> CrateMetadataRef<'a> {
+        debug_assert!(self.cdata.is_some(), "missing CrateMetadata in DecodeContext");
+        self.cdata.unwrap()
     }
 
     fn map_encoded_cnum_to_current(&self, cnum: CrateNum) -> CrateNum {
@@ -276,6 +315,11 @@ impl<'a, 'tcx> DecodeContext<'a, 'tcx> {
         self.lazy_state = LazyState::Previous(NonZeroUsize::new(position + min_size).unwrap());
         Ok(Lazy::from_position_and_meta(NonZeroUsize::new(position).unwrap(), meta))
     }
+
+    #[inline]
+    pub fn read_raw_bytes(&mut self, len: usize) -> &'a [u8] {
+        self.opaque.read_raw_bytes(len)
+    }
 }
 
 impl<'a, 'tcx> TyDecoder<'tcx> for DecodeContext<'a, 'tcx> {
@@ -586,11 +630,11 @@ implement_ty_decoder!(DecodeContext<'a, 'tcx>);
 
 impl MetadataBlob {
     crate fn new(metadata_ref: MetadataRef) -> MetadataBlob {
-        MetadataBlob(metadata_ref)
+        MetadataBlob(Lrc::new(metadata_ref))
     }
 
     crate fn is_compatible(&self) -> bool {
-        self.raw_bytes().starts_with(METADATA_HEADER)
+        self.blob().starts_with(METADATA_HEADER)
     }
 
     crate fn get_rustc_version(&self) -> String {
@@ -599,7 +643,7 @@ impl MetadataBlob {
     }
 
     crate fn get_root(&self) -> CrateRoot<'tcx> {
-        let slice = self.raw_bytes();
+        let slice = &self.blob()[..];
         let offset = METADATA_HEADER.len();
         let pos = (((slice[offset + 0] as u32) << 24)
             | ((slice[offset + 1] as u32) << 16)
@@ -1552,58 +1596,6 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
             .or_insert_with(|| self.root.tables.def_keys.get(self, index).unwrap().decode(self))
     }
 
-    /// Finds the corresponding `DefId` for the provided `DefPathHash`, if it exists.
-    /// This is used by incremental compilation to map a serialized `DefPathHash` to
-    /// its `DefId` in the current session.
-    /// Normally, only one 'main' crate will change between incremental compilation sessions:
-    /// all dependencies will be completely unchanged. In this case, we can avoid
-    /// decoding every `DefPathHash` in the crate, since the `DefIndex` from the previous
-    /// session will still be valid. If our 'guess' is wrong (the `DefIndex` no longer exists,
-    /// or has a different `DefPathHash`, then we need to decode all `DefPathHashes` to determine
-    /// the correct mapping).
-    fn def_path_hash_to_def_id(
-        &self,
-        krate: CrateNum,
-        index_guess: u32,
-        hash: DefPathHash,
-    ) -> Option<DefId> {
-        let def_index_guess = DefIndex::from_u32(index_guess);
-        let old_hash = self
-            .root
-            .tables
-            .def_path_hashes
-            .get(self, def_index_guess)
-            .map(|lazy| lazy.decode(self));
-
-        // Fast path: the definition and its index is unchanged from the
-        // previous compilation session. There is no need to decode anything
-        // else
-        if old_hash == Some(hash) {
-            return Some(DefId { krate, index: def_index_guess });
-        }
-
-        let is_proc_macro = self.is_proc_macro_crate();
-
-        // Slow path: We need to find out the new `DefIndex` of the provided
-        // `DefPathHash`, if its still exists. This requires decoding every `DefPathHash`
-        // stored in this crate.
-        let map = self.cdata.def_path_hash_map.get_or_init(|| {
-            let end_id = self.root.tables.def_path_hashes.size() as u32;
-            let mut map = UnhashMap::with_capacity_and_hasher(end_id as usize, Default::default());
-            for i in 0..end_id {
-                let def_index = DefIndex::from_u32(i);
-                // There may be gaps in the encoded table if we're decoding a proc-macro crate
-                if let Some(hash) = self.root.tables.def_path_hashes.get(self, def_index) {
-                    map.insert(hash.decode(self), def_index);
-                } else if !is_proc_macro {
-                    panic!("Missing def_path_hashes entry for {:?}", def_index);
-                }
-            }
-            map
-        });
-        map.get(&hash).map(|index| DefId { krate, index: *index })
-    }
-
     // Returns the path leading to the thing with this `id`.
     fn def_path(&self, id: DefIndex) -> DefPath {
         debug!("def_path(cnum={:?}, id={:?})", self.cnum, id);
@@ -1626,6 +1618,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
         self.def_path_hash_unlocked(index, &mut def_path_hashes)
     }
 
+    #[inline]
+    fn def_path_hash_to_def_index(&self, hash: DefPathHash) -> DefIndex {
+        self.def_path_hash_map.def_path_hash_to_def_index(&hash)
+    }
+
     fn expn_hash_to_expn_id(&self, index_guess: u32, hash: ExpnHash) -> ExpnId {
         debug_assert_eq!(ExpnId::from_hash(hash), None);
         let index_guess = ExpnIndex::from_u32(index_guess);
@@ -1892,13 +1889,18 @@ impl CrateMetadata {
         let alloc_decoding_state =
             AllocDecodingState::new(root.interpret_alloc_index.decode(&blob).collect());
         let dependencies = Lock::new(cnum_map.iter().cloned().collect());
+
+        // Pre-decode the DefPathHash->DefIndex table. This is a cheap operation
+        // that does not copy any data. It just does some data verification.
+        let def_path_hash_map = root.def_path_hash_map.decode(&blob);
+
         CrateMetadata {
             blob,
             root,
             trait_impls,
             raw_proc_macros,
             source_map_import_info: OnceCell::new(),
-            def_path_hash_map: Default::default(),
+            def_path_hash_map,
             expn_hash_map: Default::default(),
             alloc_decoding_state,
             cnum,

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -498,6 +498,10 @@ impl CrateStore for CStore {
         self.get_crate_data(cnum).root.stable_crate_id
     }
 
+    fn stable_crate_id_to_crate_num(&self, stable_crate_id: StableCrateId) -> CrateNum {
+        self.stable_crate_ids[&stable_crate_id]
+    }
+
     /// Returns the `DefKey` for a given `DefId`. This indicates the
     /// parent `DefId` as well as some idea of what kind of data the
     /// `DefId` refers to.
@@ -513,14 +517,9 @@ impl CrateStore for CStore {
         self.get_crate_data(def.krate).def_path_hash(def.index)
     }
 
-    // See `CrateMetadataRef::def_path_hash_to_def_id` for more details
+    fn def_path_hash_to_def_id(&self, cnum: CrateNum, hash: DefPathHash) -> DefId {
-    fn def_path_hash_to_def_id(
+        let def_index = self.get_crate_data(cnum).def_path_hash_to_def_index(hash);
-        &self,
+        DefId { krate: cnum, index: def_index }
-        cnum: CrateNum,
-        index_guess: u32,
-        hash: DefPathHash,
-    ) -> Option<DefId> {
-        self.get_crate_data(cnum).def_path_hash_to_def_id(cnum, index_guess, hash)
     }
 
     fn expn_hash_to_expn_id(&self, cnum: CrateNum, index_guess: u32, hash: ExpnHash) -> ExpnId {

compiler/rustc_metadata/src/rmeta/def_path_hash_map.rs

@@ -0,0 +1,58 @@
+use crate::rmeta::DecodeContext;
+use crate::rmeta::EncodeContext;
+use crate::rmeta::MetadataBlob;
+use rustc_data_structures::owning_ref::OwningRef;
+use rustc_hir::def_path_hash_map::{Config as HashMapConfig, DefPathHashMap};
+use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder};
+use rustc_span::def_id::{DefIndex, DefPathHash};
+
+crate enum DefPathHashMapRef<'tcx> {
+    OwnedFromMetadata(odht::HashTable<HashMapConfig, OwningRef<MetadataBlob, [u8]>>),
+    BorrowedFromTcx(&'tcx DefPathHashMap),
+}
+
+impl DefPathHashMapRef<'tcx> {
+    #[inline]
+    pub fn def_path_hash_to_def_index(&self, def_path_hash: &DefPathHash) -> DefIndex {
+        match *self {
+            DefPathHashMapRef::OwnedFromMetadata(ref map) => map.get(def_path_hash).unwrap(),
+            DefPathHashMapRef::BorrowedFromTcx(_) => {
+                panic!("DefPathHashMap::BorrowedFromTcx variant only exists for serialization")
+            }
+        }
+    }
+}
+
+impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for DefPathHashMapRef<'tcx> {
+    fn encode(&self, e: &mut EncodeContext<'a, 'tcx>) -> opaque::EncodeResult {
+        match *self {
+            DefPathHashMapRef::BorrowedFromTcx(def_path_hash_map) => {
+                let bytes = def_path_hash_map.raw_bytes();
+                e.emit_usize(bytes.len())?;
+                e.emit_raw_bytes(bytes)
+            }
+            DefPathHashMapRef::OwnedFromMetadata(_) => {
+                panic!("DefPathHashMap::OwnedFromMetadata variant only exists for deserialization")
+            }
+        }
+    }
+}
+
+impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for DefPathHashMapRef<'static> {
+    fn decode(d: &mut DecodeContext<'a, 'tcx>) -> Result<DefPathHashMapRef<'static>, String> {
+        // Import TyDecoder so we can access the DecodeContext::position() method
+        use crate::rustc_middle::ty::codec::TyDecoder;
+
+        let len = d.read_usize()?;
+        let pos = d.position();
+        let o = OwningRef::new(d.blob().clone()).map(|x| &x[pos..pos + len]);
+
+        // Although we already have the data we need via the OwningRef, we still need
+        // to advance the DecodeContext's position so it's in a valid state after
+        // the method. We use read_raw_bytes() for that.
+        let _ = d.read_raw_bytes(len);
+
+        let inner = odht::HashTable::from_raw_bytes(o).map_err(|e| format!("{}", e))?;
+        Ok(DefPathHashMapRef::OwnedFromMetadata(inner))
+    }
+}

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1,3 +1,4 @@
+use crate::rmeta::def_path_hash_map::DefPathHashMapRef;
 use crate::rmeta::table::{FixedSizeEncoding, TableBuilder};
 use crate::rmeta::*;
 
@@ -472,6 +473,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         }
     }
 
+    fn encode_def_path_hash_map(&mut self) -> Lazy<DefPathHashMapRef<'tcx>> {
+        self.lazy(DefPathHashMapRef::BorrowedFromTcx(
+            self.tcx.resolutions(()).definitions.def_path_hash_to_def_index_map(),
+        ))
+    }
+
     fn encode_source_map(&mut self) -> Lazy<[rustc_span::SourceFile]> {
         let source_map = self.tcx.sess.source_map();
         let all_source_files = source_map.files();
@@ -675,6 +682,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let (syntax_contexts, expn_data, expn_hashes) = self.encode_hygiene();
         let hygiene_bytes = self.position() - i;
 
+        i = self.position();
+        let def_path_hash_map = self.encode_def_path_hash_map();
+        let def_path_hash_map_bytes = self.position() - i;
+
         // Encode source_map. This needs to be done last,
         // since encoding `Span`s tells us which `SourceFiles` we actually
         // need to encode.
@@ -722,6 +733,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             syntax_contexts,
             expn_data,
             expn_hashes,
+            def_path_hash_map,
         });
 
         let total_bytes = self.position();
@@ -744,6 +756,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             eprintln!("            impl bytes: {}", impl_bytes);
             eprintln!("    exp. symbols bytes: {}", exported_symbols_bytes);
             eprintln!("  def-path table bytes: {}", def_path_table_bytes);
+            eprintln!(" def-path hashes bytes: {}", def_path_hash_map_bytes);
             eprintln!(" proc-macro-data-bytes: {}", proc_macro_data_bytes);
             eprintln!("             mir bytes: {}", mir_bytes);
             eprintln!("            item bytes: {}", item_bytes);

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -1,4 +1,5 @@
 use decoder::Metadata;
+use def_path_hash_map::DefPathHashMapRef;
 use table::{Table, TableBuilder};
 
 use rustc_ast::{self as ast, MacroDef};
@@ -35,6 +36,7 @@ use encoder::EncodeContext;
 use rustc_span::hygiene::SyntaxContextData;
 
 mod decoder;
+mod def_path_hash_map;
 mod encoder;
 mod table;
 
@@ -231,6 +233,8 @@ crate struct CrateRoot<'tcx> {
     expn_data: ExpnDataTable,
     expn_hashes: ExpnHashTable,
 
+    def_path_hash_map: Lazy<DefPathHashMapRef<'tcx>>,
+
     source_map: Lazy<[rustc_span::SourceFile]>,
 
     compiler_builtins: bool,

compiler/rustc_metadata/src/rmeta/table.rs

@@ -199,7 +199,7 @@ where
         debug!("Table::lookup: index={:?} len={:?}", i, self.meta);
 
         let start = self.position.get();
-        let bytes = &metadata.raw_bytes()[start..start + self.meta];
+        let bytes = &metadata.blob()[start..start + self.meta];
         <Option<T>>::maybe_read_from_bytes_at(bytes, i.index())?
     }
 

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -336,7 +336,11 @@ impl DepNodeExt for DepNode {
     /// has been removed.
     fn extract_def_id(&self, tcx: TyCtxt<'tcx>) -> Option<DefId> {
         if self.kind.can_reconstruct_query_key() {
-            tcx.on_disk_cache.as_ref()?.def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into()))
+            Some(
+                tcx.on_disk_cache
+                    .as_ref()?
+                    .def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into())),
+            )
         } else {
             None
         }
@@ -385,17 +389,7 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for DefId {
     }
 
     fn to_fingerprint(&self, tcx: TyCtxt<'tcx>) -> Fingerprint {
-        let hash = tcx.def_path_hash(*self);
+        tcx.def_path_hash(*self).0
-        // If this is a foreign `DefId`, store its current value
-        // in the incremental cache. When we decode the cache,
-        // we will use the old DefIndex as an initial guess for
-        // a lookup into the crate metadata.
-        if !self.is_local() {
-            if let Some(cache) = &tcx.on_disk_cache {
-                cache.store_foreign_def_id_hash(*self, hash);
-            }
-        }
-        hash.0
     }
 
     fn to_debug_str(&self, tcx: TyCtxt<'tcx>) -> String {

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -92,12 +92,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     type DepKind = DepKind;
     type StableHashingContext = StableHashingContext<'tcx>;
 
-    fn register_reused_dep_node(&self, dep_node: &DepNode) {
+    #[inline]
-        if let Some(cache) = self.on_disk_cache.as_ref() {
-            cache.register_reused_dep_node(*self, dep_node)
-        }
-    }
-
     fn create_stable_hashing_context(&self) -> Self::StableHashingContext {
         TyCtxt::create_stable_hashing_context(*self)
     }

compiler/rustc_middle/src/middle/cstore.rs

@@ -199,14 +199,10 @@ pub trait CrateStore: std::fmt::Debug {
     // incr.  comp. uses to identify a CrateNum.
     fn crate_name(&self, cnum: CrateNum) -> Symbol;
     fn stable_crate_id(&self, cnum: CrateNum) -> StableCrateId;
+    fn stable_crate_id_to_crate_num(&self, stable_crate_id: StableCrateId) -> CrateNum;
 
     /// Fetch a DefId from a DefPathHash for a foreign crate.
-    fn def_path_hash_to_def_id(
+    fn def_path_hash_to_def_id(&self, cnum: CrateNum, hash: DefPathHash) -> DefId;
-        &self,
-        cnum: CrateNum,
-        index_guess: u32,
-        hash: DefPathHash,
-    ) -> Option<DefId>;
     fn expn_hash_to_expn_id(&self, cnum: CrateNum, index_guess: u32, hash: ExpnHash) -> ExpnId;
 
     // utility functions

compiler/rustc_middle/src/ty/context.rs

@@ -1,7 +1,7 @@
 //! Type context book-keeping.
 
 use crate::arena::Arena;
-use crate::dep_graph::{DepGraph, DepNode};
+use crate::dep_graph::DepGraph;
 use crate::hir::place::Place as HirPlace;
 use crate::ich::{NodeIdHashingMode, StableHashingContext};
 use crate::infer::canonical::{Canonical, CanonicalVarInfo, CanonicalVarInfos};
@@ -83,23 +83,7 @@ pub trait OnDiskCache<'tcx>: rustc_data_structures::sync::Sync {
     /// Converts a `DefPathHash` to its corresponding `DefId` in the current compilation
     /// session, if it still exists. This is used during incremental compilation to
     /// turn a deserialized `DefPathHash` into its current `DefId`.
-    fn def_path_hash_to_def_id(
+    fn def_path_hash_to_def_id(&self, tcx: TyCtxt<'tcx>, def_path_hash: DefPathHash) -> DefId;
-        &self,
-        tcx: TyCtxt<'tcx>,
-        def_path_hash: DefPathHash,
-    ) -> Option<DefId>;
-
-    /// If the given `dep_node`'s hash still exists in the current compilation,
-    /// and its current `DefId` is foreign, calls `store_foreign_def_id` with it.
-    ///
-    /// Normally, `store_foreign_def_id_hash` can be called directly by
-    /// the dependency graph when we construct a `DepNode`. However,
-    /// when we re-use a deserialized `DepNode` from the previous compilation
-    /// session, we only have the `DefPathHash` available. This method is used
-    /// to that any `DepNode` that we re-use has a `DefPathHash` -> `RawId` written
-    /// out for usage in the next compilation session.
-    fn register_reused_dep_node(&self, tcx: TyCtxt<'tcx>, dep_node: &DepNode);
-    fn store_foreign_def_id_hash(&self, def_id: DefId, hash: DefPathHash);
 
     fn drop_serialized_data(&self, tcx: TyCtxt<'tcx>);
 
@@ -1316,6 +1300,17 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Maps a StableCrateId to the corresponding CrateNum. This method assumes
+    /// that the crate in question has already been loaded by the CrateStore.
+    #[inline]
+    pub fn stable_crate_id_to_crate_num(self, stable_crate_id: StableCrateId) -> CrateNum {
+        if stable_crate_id == self.sess.local_stable_crate_id() {
+            LOCAL_CRATE
+        } else {
+            self.untracked_resolutions.cstore.stable_crate_id_to_crate_num(stable_crate_id)
+        }
+    }
+
     pub fn def_path_debug_str(self, def_id: DefId) -> String {
         // We are explicitly not going through queries here in order to get
         // crate name and stable crate id since this code is called from debug!()

compiler/rustc_query_impl/src/on_disk_cache.rs

@@ -1,12 +1,12 @@
 use crate::QueryCtxt;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_data_structures::memmap::Mmap;
-use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, OnceCell, RwLock};
+use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, RwLock};
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathHash;
 use rustc_index::vec::{Idx, IndexVec};
-use rustc_middle::dep_graph::{DepNode, DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
 use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use rustc_middle::mir::{self, interpret};
 use rustc_middle::thir;
@@ -25,7 +25,6 @@ use rustc_span::hygiene::{
 use rustc_span::source_map::{SourceMap, StableSourceFileId};
 use rustc_span::CachingSourceMapView;
 use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span};
-use std::collections::hash_map::Entry;
 use std::mem;
 
 const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE;
@@ -51,8 +50,6 @@ pub struct OnDiskCache<'sess> {
     // session.
     current_side_effects: Lock<FxHashMap<DepNodeIndex, QuerySideEffects>>,
 
-    cnum_map: OnceCell<UnhashMap<StableCrateId, CrateNum>>,
-
     source_map: &'sess SourceMap,
     file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
 
@@ -87,27 +84,11 @@ pub struct OnDiskCache<'sess> {
     expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
     // Additional information used when decoding hygiene data.
     hygiene_context: HygieneDecodeContext,
-    // Maps `DefPathHash`es to their `RawDefId`s from the *previous*
+    // Maps `ExpnHash`es to their raw value from the *previous*
     // compilation session. This is used as an initial 'guess' when
-    // we try to map a `DefPathHash` to its `DefId` in the current compilation
+    // we try to map an `ExpnHash` to its value in the current
-    // session.
+    // compilation session.
-    foreign_def_path_hashes: UnhashMap<DefPathHash, RawDefId>,
-    // Likewise for ExpnId.
     foreign_expn_data: UnhashMap<ExpnHash, u32>,
-
-    // The *next* compilation sessison's `foreign_def_path_hashes` - at
-    // the end of our current compilation session, this will get written
-    // out to the `foreign_def_path_hashes` field of the `Footer`, which
-    // will become `foreign_def_path_hashes` of the next compilation session.
-    // This stores any `DefPathHash` that we may need to map to a `DefId`
-    // during the next compilation session.
-    latest_foreign_def_path_hashes: Lock<UnhashMap<DefPathHash, RawDefId>>,
-
-    // Caches all lookups of `DefPathHashes`, both for local and foreign
-    // definitions. A definition from the previous compilation session
-    // may no longer exist in the current compilation session, so
-    // we use `Option<DefId>` so that we can cache a lookup failure.
-    def_path_hash_to_def_id_cache: Lock<UnhashMap<DefPathHash, Option<DefId>>>,
 }
 
 // This type is used only for serialization and deserialization.
@@ -122,7 +103,6 @@ struct Footer {
     syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
     // See `OnDiskCache.expn_data`
     expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
-    foreign_def_path_hashes: UnhashMap<DefPathHash, RawDefId>,
     foreign_expn_data: UnhashMap<ExpnHash, u32>,
 }
 
@@ -145,19 +125,6 @@ impl AbsoluteBytePos {
     }
 }
 
-/// Represents a potentially invalid `DefId`. This is used during incremental
-/// compilation to represent a `DefId` from the *previous* compilation session,
-/// which may no longer be valid. This is used to help map a `DefPathHash`
-/// to a `DefId` in the current compilation session.
-#[derive(Encodable, Decodable, Copy, Clone, Debug)]
-crate struct RawDefId {
-    // We deliberately do not use `CrateNum` and `DefIndex`
-    // here, since a crate/index from the previous compilation
-    // session may no longer exist.
-    pub krate: u32,
-    pub index: u32,
-}
-
 /// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that
 /// the source crate is represented as a [StableCrateId] instead of as a
 /// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded
@@ -170,8 +137,8 @@ struct EncodedSourceFileId {
 }
 
 impl EncodedSourceFileId {
-    fn translate(&self, cnum_map: &UnhashMap<StableCrateId, CrateNum>) -> StableSourceFileId {
+    fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId {
-        let cnum = cnum_map[&self.stable_crate_id];
+        let cnum = tcx.stable_crate_id_to_crate_num(self.stable_crate_id);
         StableSourceFileId { file_name_hash: self.file_name_hash, cnum }
     }
 
@@ -211,7 +178,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
             serialized_data: RwLock::new(Some(data)),
             file_index_to_stable_id: footer.file_index_to_stable_id,
             file_index_to_file: Default::default(),
-            cnum_map: OnceCell::new(),
             source_map: sess.source_map(),
             current_side_effects: Default::default(),
             query_result_index: footer.query_result_index.into_iter().collect(),
@@ -221,9 +187,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
             expn_data: footer.expn_data,
             foreign_expn_data: footer.foreign_expn_data,
             hygiene_context: Default::default(),
-            foreign_def_path_hashes: footer.foreign_def_path_hashes,
-            latest_foreign_def_path_hashes: Default::default(),
-            def_path_hash_to_def_id_cache: Default::default(),
         }
     }
 
@@ -232,7 +195,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
             serialized_data: RwLock::new(None),
             file_index_to_stable_id: Default::default(),
             file_index_to_file: Default::default(),
-            cnum_map: OnceCell::new(),
             source_map,
             current_side_effects: Default::default(),
             query_result_index: Default::default(),
@@ -242,9 +204,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
             expn_data: UnhashMap::default(),
             foreign_expn_data: UnhashMap::default(),
             hygiene_context: Default::default(),
-            foreign_def_path_hashes: Default::default(),
-            latest_foreign_def_path_hashes: Default::default(),
-            def_path_hash_to_def_id_cache: Default::default(),
         }
     }
 
@@ -254,13 +213,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
     /// In order to serialize the new on-disk cache, the former on-disk cache file needs to be
     /// deleted, hence we won't be able to refer to its memmapped data.
     fn drop_serialized_data(&self, tcx: TyCtxt<'tcx>) {
-        // Register any dep nodes that we reused from the previous session,
-        // but didn't `DepNode::construct` in this session. This ensures
-        // that their `DefPathHash` to `RawDefId` mappings are registered
-        // in 'latest_foreign_def_path_hashes' if necessary, since that
-        // normally happens in `DepNode::construct`.
-        tcx.dep_graph.register_reused_dep_nodes(tcx);
-
         // Load everything into memory so we can write it out to the on-disk
         // cache. The vast majority of cacheable query results should already
         // be in memory, so this should be a cheap operation.
@@ -294,7 +246,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
                 (file_to_file_index, file_index_to_stable_id)
             };
 
-            let latest_foreign_def_path_hashes = self.latest_foreign_def_path_hashes.lock().clone();
             let hygiene_encode_context = HygieneEncodeContext::default();
 
             let mut encoder = CacheEncoder {
@@ -306,7 +257,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
                 source_map: CachingSourceMapView::new(tcx.sess.source_map()),
                 file_to_file_index,
                 hygiene_context: &hygiene_encode_context,
-                latest_foreign_def_path_hashes,
             };
 
             // Encode query results.
@@ -383,9 +333,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
                 },
             )?;
 
-            let foreign_def_path_hashes =
-                std::mem::take(&mut encoder.latest_foreign_def_path_hashes);
-
             // `Encode the file footer.
             let footer_pos = encoder.position() as u64;
             encoder.encode_tagged(
@@ -398,7 +345,6 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
                     syntax_contexts,
                     expn_data,
                     foreign_expn_data,
-                    foreign_def_path_hashes,
                 },
             )?;
 
@@ -413,81 +359,22 @@ impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
         })
     }
 
-    fn def_path_hash_to_def_id(&self, tcx: TyCtxt<'tcx>, hash: DefPathHash) -> Option<DefId> {
+    fn def_path_hash_to_def_id(&self, tcx: TyCtxt<'tcx>, hash: DefPathHash) -> DefId {
-        let mut cache = self.def_path_hash_to_def_id_cache.lock();
-        match cache.entry(hash) {
-            Entry::Occupied(e) => *e.get(),
-            Entry::Vacant(e) => {
         debug!("def_path_hash_to_def_id({:?})", hash);
-                // Check if the `DefPathHash` corresponds to a definition in the current
-                // crate
-                if let Some(def_id) =
-                    tcx.definitions_untracked().local_def_path_hash_to_def_id(hash)
-                {
-                    let def_id = def_id.to_def_id();
-                    e.insert(Some(def_id));
-                    return Some(def_id);
-                }
-                // This `raw_def_id` represents the `DefId` of this `DefPathHash` in
-                // the *previous* compliation session. The `DefPathHash` includes the
-                // owning crate, so if the corresponding definition still exists in the
-                // current compilation session, the crate is guaranteed to be the same
-                // (otherwise, we would compute a different `DefPathHash`).
-                let raw_def_id = self.get_raw_def_id(&hash)?;
-                debug!("def_path_hash_to_def_id({:?}): raw_def_id = {:?}", hash, raw_def_id);
-                // If the owning crate no longer exists, the corresponding definition definitely
-                // no longer exists.
-                let krate = self.try_remap_cnum(tcx, hash.stable_crate_id())?;
-                debug!("def_path_hash_to_def_id({:?}): krate = {:?}", hash, krate);
-                // If our `DefPathHash` corresponded to a definition in the local crate,
-                // we should have either found it in `local_def_path_hash_to_def_id`, or
-                // never attempted to load it in the first place. Any query result or `DepNode`
-                // that references a local `DefId` should depend on some HIR-related `DepNode`.
-                // If a local definition is removed/modified such that its old `DefPathHash`
-                // no longer has a corresponding definition, that HIR-related `DepNode` should
-                // end up red. This should prevent us from ever calling
-                // `tcx.def_path_hash_to_def_id`, since we'll end up recomputing any
-                // queries involved.
-                debug_assert_ne!(krate, LOCAL_CRATE);
-                // Try to find a definition in the current session, using the previous `DefIndex`
-                // as an initial guess.
-                let opt_def_id =
-                    tcx.cstore_untracked().def_path_hash_to_def_id(krate, raw_def_id.index, hash);
-                debug!("def_path_to_def_id({:?}): opt_def_id = {:?}", hash, opt_def_id);
-                e.insert(opt_def_id);
-                opt_def_id
-            }
-        }
-    }
 
-    fn register_reused_dep_node(&self, tcx: TyCtxt<'sess>, dep_node: &DepNode) {
+        let stable_crate_id = hash.stable_crate_id();
-        // For reused dep nodes, we only need to store the mapping if the node
-        // is one whose query key we can reconstruct from the hash. We use the
-        // mapping to aid that reconstruction in the next session. While we also
-        // use it to decode `DefId`s we encoded in the cache as `DefPathHashes`,
-        // they're already registered during `DefId` encoding.
-        if dep_node.kind.can_reconstruct_query_key() {
-            let hash = DefPathHash(dep_node.hash.into());
 
-            // We can't simply copy the `RawDefId` from `foreign_def_path_hashes` to
+        // If this is a DefPathHash from the local crate, we can look up the
-            // `latest_foreign_def_path_hashes`, since the `RawDefId` might have
+        // DefId in the tcx's `Definitions`.
-            // changed in the current compilation session (e.g. we've added/removed crates,
+        if stable_crate_id == tcx.sess.local_stable_crate_id() {
-            // or added/removed definitions before/after the target definition).
+            tcx.definitions_untracked().local_def_path_hash_to_def_id(hash).to_def_id()
-            if let Some(def_id) = self.def_path_hash_to_def_id(tcx, hash) {
+        } else {
-                if !def_id.is_local() {
+            // If this is a DefPathHash from an upstream crate, let the CrateStore map
-                    self.store_foreign_def_id_hash(def_id, hash);
+            // it to a DefId.
+            let cnum = tcx.cstore_untracked().stable_crate_id_to_crate_num(stable_crate_id);
+            tcx.cstore_untracked().def_path_hash_to_def_id(cnum, hash)
         }
     }
-        }
-    }
-
-    fn store_foreign_def_id_hash(&self, def_id: DefId, hash: DefPathHash) {
-        // We may overwrite an existing entry, but it will have the same value,
-        // so it's fine
-        self.latest_foreign_def_path_hashes
-            .lock()
-            .insert(hash, RawDefId { krate: def_id.krate.as_u32(), index: def_id.index.as_u32() });
-    }
 }
 
 impl<'sess> OnDiskCache<'sess> {
@@ -518,17 +405,6 @@ impl<'sess> OnDiskCache<'sess> {
         debug_assert!(prev.is_none());
     }
 
-    fn get_raw_def_id(&self, hash: &DefPathHash) -> Option<RawDefId> {
-        self.foreign_def_path_hashes.get(hash).copied()
-    }
-
-    fn try_remap_cnum(&self, tcx: TyCtxt<'_>, stable_crate_id: StableCrateId) -> Option<CrateNum> {
-        let cnum_map = self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx));
-        debug!("try_remap_cnum({:?}): cnum_map={:?}", stable_crate_id, cnum_map);
-
-        cnum_map.get(&stable_crate_id).copied()
-    }
-
     /// Returns the cached query result if there is something in the cache for
     /// the given `SerializedDepNodeIndex`; otherwise returns `None`.
     pub fn try_load_query_result<'tcx, T>(
@@ -586,14 +462,11 @@ impl<'sess> OnDiskCache<'sess> {
     where
         T: Decodable<CacheDecoder<'a, 'tcx>>,
     {
-        let cnum_map = self.cnum_map.get_or_init(|| Self::compute_cnum_map(tcx));
-
         let serialized_data = self.serialized_data.read();
         let mut decoder = CacheDecoder {
             tcx,
             opaque: opaque::Decoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()),
             source_map: self.source_map,
-            cnum_map,
             file_index_to_file: &self.file_index_to_file,
             file_index_to_stable_id: &self.file_index_to_stable_id,
             alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(),
@@ -604,23 +477,6 @@ impl<'sess> OnDiskCache<'sess> {
         };
         f(&mut decoder)
     }
-
-    // This function builds mapping from previous-session-`CrateNum` to
-    // current-session-`CrateNum`. There might be `CrateNum`s from the previous
-    // `Session` that don't occur in the current one. For these, the mapping
-    // maps to None.
-    fn compute_cnum_map(tcx: TyCtxt<'_>) -> UnhashMap<StableCrateId, CrateNum> {
-        tcx.dep_graph.with_ignore(|| {
-            tcx.crates(())
-                .iter()
-                .chain(std::iter::once(&LOCAL_CRATE))
-                .map(|&cnum| {
-                    let hash = tcx.def_path_hash(cnum.as_def_id()).stable_crate_id();
-                    (hash, cnum)
-                })
-                .collect()
-        })
-    }
 }
 
 //- DECODING -------------------------------------------------------------------
@@ -632,7 +488,6 @@ pub struct CacheDecoder<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     opaque: opaque::Decoder<'a>,
     source_map: &'a SourceMap,
-    cnum_map: &'a UnhashMap<StableCrateId, CrateNum>,
     file_index_to_file: &'a Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
     file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, EncodedSourceFileId>,
     alloc_decoding_session: AllocDecodingSession<'a>,
@@ -645,10 +500,10 @@ pub struct CacheDecoder<'a, 'tcx> {
 impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
     fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc<SourceFile> {
         let CacheDecoder {
+            tcx,
             ref file_index_to_file,
             ref file_index_to_stable_id,
             ref source_map,
-            ref cnum_map,
             ..
         } = *self;
 
@@ -656,7 +511,7 @@ impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
             .borrow_mut()
             .entry(index)
             .or_insert_with(|| {
-                let stable_id = file_index_to_stable_id[&index].translate(cnum_map);
+                let stable_id = file_index_to_stable_id[&index].translate(tcx);
                 source_map
                     .source_file_by_stable_id(stable_id)
                     .expect("failed to lookup `SourceFile` in new context")
@@ -798,7 +653,7 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for ExpnId {
             return Ok(expn_id);
         }
 
-        let krate = decoder.cnum_map[&hash.stable_crate_id()];
+        let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id());
 
         let expn_id = if krate == LOCAL_CRATE {
             // We look up the position of the associated `ExpnData` and decode it.
@@ -871,7 +726,7 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span {
 impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for CrateNum {
     fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Result<Self, String> {
         let stable_id = StableCrateId::decode(d)?;
-        let cnum = d.cnum_map[&stable_id];
+        let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id);
         Ok(cnum)
     }
 }
@@ -899,12 +754,7 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
         // If we get to this point, then all of the query inputs were green,
         // which means that the definition with this hash is guaranteed to
         // still exist in the current compilation session.
-        Ok(d.tcx()
+        Ok(d.tcx().on_disk_cache.as_ref().unwrap().def_path_hash_to_def_id(d.tcx(), def_path_hash))
-            .on_disk_cache
-            .as_ref()
-            .unwrap()
-            .def_path_hash_to_def_id(d.tcx(), def_path_hash)
-            .unwrap())
     }
 }
 
@@ -969,7 +819,6 @@ pub struct CacheEncoder<'a, 'tcx, E: OpaqueEncoder> {
     source_map: CachingSourceMapView<'tcx>,
     file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>,
     hygiene_context: &'a HygieneEncodeContext,
-    latest_foreign_def_path_hashes: UnhashMap<DefPathHash, RawDefId>,
 }
 
 impl<'a, 'tcx, E> CacheEncoder<'a, 'tcx, E>
@@ -1102,17 +951,7 @@ where
     E: 'a + OpaqueEncoder,
 {
     fn encode(&self, s: &mut CacheEncoder<'a, 'tcx, E>) -> Result<(), E::Error> {
-        let def_path_hash = s.tcx.def_path_hash(*self);
+        s.tcx.def_path_hash(*self).encode(s)
-        // Store additional information when we encode a foreign `DefId`,
-        // so that we can map its `DefPathHash` back to a `DefId` in the next
-        // compilation session.
-        if !self.is_local() {
-            s.latest_foreign_def_path_hashes.insert(
-                def_path_hash,
-                RawDefId { krate: self.krate.as_u32(), index: self.index.as_u32() },
-            );
-        }
-        def_path_hash.encode(s)
     }
 }
 

compiler/rustc_query_system/src/dep_graph/dep_node.rs

@@ -53,18 +53,6 @@ use std::hash::Hash;
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)]
 pub struct DepNode<K> {
     pub kind: K,
-    // Important - whenever a `DepNode` is constructed, we need to make
-    // sure to register a `DefPathHash -> DefId` mapping if needed.
-    // This is currently done in two places:
-    //
-    // * When a `DepNode::construct` is called, `arg.to_fingerprint()`
-    //   is responsible for calling `OnDiskCache::store_foreign_def_id_hash`
-    //   if needed
-    // * When we serialize the on-disk cache, `OnDiskCache::serialize` is
-    //   responsible for calling `DepGraph::register_reused_dep_nodes`.
-    //
-    // FIXME: Enforce this by preventing manual construction of `DefNode`
-    // (e.g. add a `_priv: ()` field)
     pub hash: PackedFingerprint,
 }
 

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -760,20 +760,6 @@ impl<K: DepKind> DepGraph<K> {
         }
     }
 
-    // Register reused dep nodes (i.e. nodes we've marked red or green) with the context.
-    pub fn register_reused_dep_nodes<Ctxt: DepContext<DepKind = K>>(&self, tcx: Ctxt) {
-        let data = self.data.as_ref().unwrap();
-        for prev_index in data.colors.values.indices() {
-            match data.colors.get(prev_index) {
-                Some(DepNodeColor::Red) | Some(DepNodeColor::Green(_)) => {
-                    let dep_node = data.previous.index_to_node(prev_index);
-                    tcx.register_reused_dep_node(&dep_node);
-                }
-                None => {}
-            }
-        }
-    }
-
     pub fn print_incremental_info(&self) {
         if let Some(data) = &self.data {
             data.current.encoder.borrow().print_incremental_info(

compiler/rustc_query_system/src/dep_graph/mod.rs

@@ -27,8 +27,6 @@ pub trait DepContext: Copy {
     /// Access the DepGraph.
     fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
 
-    fn register_reused_dep_node(&self, dep_node: &DepNode<Self::DepKind>);
-
     /// Access the profiler.
     fn profiler(&self) -> &SelfProfilerRef;
 

src/tools/tidy/src/deps.rs

@@ -144,6 +144,7 @@ const PERMITTED_DEPENDENCIES: &[&str] = &[
     "num-integer",
     "num-traits",
     "object",
+    "odht",
     "once_cell",
     "opaque-debug",
     "parking_lot",