Optimize incremental_verify_ich

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -71,6 +71,7 @@ impl rustc_query_system::dep_graph::DepKind for DepKind {
 }
 
 impl<'tcx> DepContext for TyCtxt<'tcx> {
+    type Implicit<'a> = TyCtxt<'a>;
     type DepKind = DepKind;
 
     #[inline]
@@ -78,6 +79,11 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
         TyCtxt::with_stable_hashing_context(self, f)
     }
 
+    #[inline]
+    fn with_context<R>(f: impl FnOnce(TyCtxt<'_>) -> R) -> R {
+        ty::tls::with(|tcx| f(tcx))
+    }
+
     #[inline]
     fn dep_graph(&self) -> &DepGraph {
         &self.dep_graph

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -535,13 +535,14 @@ impl<K: DepKind> DepGraph<K> {
             // value to an existing node.
             //
             // For sanity, we still check that the loaded stable hash and the new one match.
-            if let Some(dep_node_index) = data.dep_node_index_of_opt(&node) {
+            if let Some(prev_index) = data.previous.node_to_index_opt(&node)
-                let _current_fingerprint =
+                && let Some(dep_node_index) = { data.current.prev_index_to_index.lock()[prev_index] }
-                    crate::query::incremental_verify_ich(cx, data, result, &node, hash_result);
+            {
+                crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result);
 
                 #[cfg(debug_assertions)]
                 if hash_result.is_some() {
-                    data.current.record_edge(dep_node_index, node, _current_fingerprint);
+                    data.current.record_edge(dep_node_index, node, data.prev_fingerprint_of(prev_index));
                 }
 
                 return dep_node_index;
@@ -626,13 +627,19 @@ impl<K: DepKind> DepGraphData<K> {
 
     /// Returns true if the given node has been marked as green during the
     /// current compilation session. Used in various assertions
-    pub fn is_green(&self, dep_node: &DepNode<K>) -> bool {
+    #[inline]
-        self.node_color(dep_node).map_or(false, |c| c.is_green())
+    pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
+        self.colors.get(prev_index).map_or(false, |c| c.is_green())
     }
 
     #[inline]
-    pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
+    pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
-        self.previous.fingerprint_of(dep_node)
+        self.previous.fingerprint_by_index(prev_index)
+    }
+
+    #[inline]
+    pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode<K> {
+        self.previous.index_to_node(prev_index)
     }
 
     pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
@@ -643,7 +650,7 @@ impl<K: DepKind> DepGraphData<K> {
 impl<K: DepKind> DepGraph<K> {
     #[inline]
     pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
-        self.data.as_ref().and_then(|data| data.dep_node_index_of_opt(dep_node)).is_some()
+        self.data.as_ref().map_or(false, |data| data.dep_node_exists(dep_node))
     }
 
     /// Checks whether a previous work product exists for `v` and, if

compiler/rustc_query_system/src/dep_graph/mod.rs

@@ -23,11 +23,15 @@ use std::{fmt, panic};
 use self::graph::{print_markframe_trace, MarkFrame};
 
 pub trait DepContext: Copy {
+    type Implicit<'a>: DepContext;
     type DepKind: self::DepKind;
 
     /// Create a hashing context for hashing new results.
     fn with_stable_hashing_context<R>(self, f: impl FnOnce(StableHashingContext<'_>) -> R) -> R;
 
+    /// Access the implicit context.
+    fn with_context<R>(f: impl FnOnce(Self::Implicit<'_>) -> R) -> R;
+
     /// Access the DepGraph.
     fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
 

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -79,11 +79,6 @@ impl<K: DepKind> SerializedDepGraph<K> {
         self.index.get(dep_node).cloned()
     }
 
-    #[inline]
-    pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
-        self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index])
-    }
-
     #[inline]
     pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
         self.fingerprints[dep_node_index]

compiler/rustc_query_system/src/query/plumbing.rs

@@ -7,6 +7,7 @@ use crate::dep_graph::{DepGraphData, HasDepContext};
 use crate::ich::StableHashingContext;
 use crate::query::caches::QueryCache;
 use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
+use crate::query::SerializedDepNodeIndex;
 use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
 use crate::values::Value;
 use crate::HandleCycleError;
@@ -19,7 +20,6 @@ use rustc_data_structures::sharded::Sharded;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_data_structures::sync::{Lock, LockGuard};
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
-use rustc_session::Session;
 use rustc_span::{Span, DUMMY_SP};
 use std::cell::Cell;
 use std::collections::hash_map::Entry;
@@ -537,7 +537,7 @@ where
 
     let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
 
-    debug_assert!(dep_graph_data.is_green(dep_node));
+    debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
 
     // First we try to load the result from the on-disk cache.
     // Some things are never cached on disk.
@@ -561,8 +561,7 @@ where
                 dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
             }
 
-            let prev_fingerprint =
+            let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
-                dep_graph_data.prev_fingerprint_of(dep_node).unwrap_or(Fingerprint::ZERO);
             // If `-Zincremental-verify-ich` is specified, re-hash results from
             // the cache and make sure that they have the expected fingerprint.
             //
@@ -578,7 +577,7 @@ where
                     *qcx.dep_context(),
                     dep_graph_data,
                     &result,
-                    dep_node,
+                    prev_dep_node_index,
                     query.hash_result(),
                 );
             }
@@ -623,7 +622,7 @@ where
         *qcx.dep_context(),
         dep_graph_data,
         &result,
-        dep_node,
+        prev_dep_node_index,
         query.hash_result(),
     );
 
@@ -636,32 +635,38 @@ pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
     tcx: Tcx,
     dep_graph_data: &DepGraphData<Tcx::DepKind>,
     result: &V,
-    dep_node: &DepNode<Tcx::DepKind>,
+    prev_index: SerializedDepNodeIndex,
     hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
-) -> Fingerprint
+) where
-where
     Tcx: DepContext,
 {
-    assert!(
+    if !dep_graph_data.is_index_green(prev_index) {
-        dep_graph_data.is_green(dep_node),
+        incremental_verify_ich_not_green::<Tcx>(prev_index)
-        "fingerprint for green query instance not loaded from cache: {dep_node:?}",
+    }
-    );
 
     let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
         tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
     });
 
-    let old_hash = dep_graph_data.prev_fingerprint_of(dep_node);
+    let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
 
-    if Some(new_hash) != old_hash {
+    if new_hash != old_hash {
-        incremental_verify_ich_failed(
+        incremental_verify_ich_failed::<Tcx>(prev_index, DebugArg::from(&result));
-            tcx.sess(),
+    }
-            DebugArg::from(&dep_node),
-            DebugArg::from(&result),
-        );
 }
 
-    new_hash
+#[cold]
+#[inline(never)]
+fn incremental_verify_ich_not_green<Tcx>(prev_index: SerializedDepNodeIndex)
+where
+    Tcx: DepContext,
+{
+    Tcx::with_context(|tcx| {
+        panic!(
+            "fingerprint for green query instance not loaded from cache: {:?}",
+            tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
+        )
+    })
 }
 
 // This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is
@@ -706,7 +711,11 @@ impl std::fmt::Debug for DebugArg<'_> {
 // different implementations for LLVM to chew on (and filling up the final
 // binary, too).
 #[cold]
-fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) {
+#[inline(never)]
+fn incremental_verify_ich_failed<Tcx>(prev_index: SerializedDepNodeIndex, result: DebugArg<'_>)
+where
+    Tcx: DepContext,
+{
     // When we emit an error message and panic, we try to debug-print the `DepNode`
     // and query result. Unfortunately, this can cause us to run additional queries,
     // which may result in another fingerprint mismatch while we're in the middle
@@ -719,21 +728,25 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result:
 
     let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
 
+    Tcx::with_context(|tcx| {
         if old_in_panic {
-        sess.emit_err(crate::error::Reentrant);
+            tcx.sess().emit_err(crate::error::Reentrant);
         } else {
-        let run_cmd = if let Some(crate_name) = &sess.opts.crate_name {
+            let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
                 format!("`cargo clean -p {crate_name}` or `cargo clean`")
             } else {
                 "`cargo clean`".to_string()
             };
 
-        sess.emit_err(crate::error::IncrementCompilation {
+            let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
+
+            let dep_node = tcx.sess().emit_err(crate::error::IncrementCompilation {
                 run_cmd,
                 dep_node: format!("{dep_node:?}"),
             });
             panic!("Found unstable fingerprints for {dep_node:?}: {result:?}");
         }
+    });
 
     INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic));
 }