1
Fork 0

Auto merge of #109371 - Zoxc:verify-hash-opt, r=cjgillot

Optimize `incremental_verify_ich`

This optimizes `incremental_verify_ich` by operating on `SerializedDepNodeIndex`, saving 2 hashmap lookups. The panic paths are also changed to get a `TyCtxt` reference using TLS.
This commit is contained in:
bors 2023-03-25 09:22:15 +00:00
commit e10eab5956
3 changed files with 66 additions and 85 deletions

View file

@ -535,16 +535,22 @@ impl<K: DepKind> DepGraph<K> {
// value to an existing node. // value to an existing node.
// //
// For sanity, we still check that the loaded stable hash and the new one match. // For sanity, we still check that the loaded stable hash and the new one match.
if let Some(dep_node_index) = data.dep_node_index_of_opt(&node) { if let Some(prev_index) = data.previous.node_to_index_opt(&node) {
let _current_fingerprint = let dep_node_index = data.current.prev_index_to_index.lock()[prev_index];
crate::query::incremental_verify_ich(cx, data, result, &node, hash_result); if let Some(dep_node_index) = dep_node_index {
crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result);
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
if hash_result.is_some() { if hash_result.is_some() {
data.current.record_edge(dep_node_index, node, _current_fingerprint); data.current.record_edge(
dep_node_index,
node,
data.prev_fingerprint_of(prev_index),
);
}
return dep_node_index;
} }
return dep_node_index;
} }
let mut edges = SmallVec::new(); let mut edges = SmallVec::new();
@ -626,13 +632,19 @@ impl<K: DepKind> DepGraphData<K> {
/// Returns true if the given node has been marked as green during the /// Returns true if the given node has been marked as green during the
/// current compilation session. Used in various assertions /// current compilation session. Used in various assertions
pub fn is_green(&self, dep_node: &DepNode<K>) -> bool { #[inline]
self.node_color(dep_node).map_or(false, |c| c.is_green()) pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
self.colors.get(prev_index).map_or(false, |c| c.is_green())
} }
#[inline] #[inline]
pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> { pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
self.previous.fingerprint_of(dep_node) self.previous.fingerprint_by_index(prev_index)
}
#[inline]
pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode<K> {
self.previous.index_to_node(prev_index)
} }
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) { pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
@ -643,7 +655,7 @@ impl<K: DepKind> DepGraphData<K> {
impl<K: DepKind> DepGraph<K> { impl<K: DepKind> DepGraph<K> {
#[inline] #[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool { pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
self.data.as_ref().and_then(|data| data.dep_node_index_of_opt(dep_node)).is_some() self.data.as_ref().map_or(false, |data| data.dep_node_exists(dep_node))
} }
/// Checks whether a previous work product exists for `v` and, if /// Checks whether a previous work product exists for `v` and, if

View file

@ -79,11 +79,6 @@ impl<K: DepKind> SerializedDepGraph<K> {
self.index.get(dep_node).cloned() self.index.get(dep_node).cloned()
} }
#[inline]
pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index])
}
#[inline] #[inline]
pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint { pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
self.fingerprints[dep_node_index] self.fingerprints[dep_node_index]

View file

@ -7,6 +7,7 @@ use crate::dep_graph::{DepGraphData, HasDepContext};
use crate::ich::StableHashingContext; use crate::ich::StableHashingContext;
use crate::query::caches::QueryCache; use crate::query::caches::QueryCache;
use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
use crate::query::SerializedDepNodeIndex;
use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame}; use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
use crate::values::Value; use crate::values::Value;
use crate::HandleCycleError; use crate::HandleCycleError;
@ -19,7 +20,6 @@ use rustc_data_structures::sharded::Sharded;
use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::sync::{Lock, LockGuard};
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
use rustc_session::Session;
use rustc_span::{Span, DUMMY_SP}; use rustc_span::{Span, DUMMY_SP};
use std::cell::Cell; use std::cell::Cell;
use std::collections::hash_map::Entry; use std::collections::hash_map::Entry;
@ -537,7 +537,7 @@ where
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?; let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
debug_assert!(dep_graph_data.is_green(dep_node)); debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
// First we try to load the result from the on-disk cache. // First we try to load the result from the on-disk cache.
// Some things are never cached on disk. // Some things are never cached on disk.
@ -561,8 +561,7 @@ where
dep_graph_data.mark_debug_loaded_from_disk(*dep_node) dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
} }
let prev_fingerprint = let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
dep_graph_data.prev_fingerprint_of(dep_node).unwrap_or(Fingerprint::ZERO);
// If `-Zincremental-verify-ich` is specified, re-hash results from // If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint. // the cache and make sure that they have the expected fingerprint.
// //
@ -578,7 +577,7 @@ where
*qcx.dep_context(), *qcx.dep_context(),
dep_graph_data, dep_graph_data,
&result, &result,
dep_node, prev_dep_node_index,
query.hash_result(), query.hash_result(),
); );
} }
@ -623,7 +622,7 @@ where
*qcx.dep_context(), *qcx.dep_context(),
dep_graph_data, dep_graph_data,
&result, &result,
dep_node, prev_dep_node_index,
query.hash_result(), query.hash_result(),
); );
@ -636,77 +635,50 @@ pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
tcx: Tcx, tcx: Tcx,
dep_graph_data: &DepGraphData<Tcx::DepKind>, dep_graph_data: &DepGraphData<Tcx::DepKind>,
result: &V, result: &V,
dep_node: &DepNode<Tcx::DepKind>, prev_index: SerializedDepNodeIndex,
hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>, hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
) -> Fingerprint ) where
where
Tcx: DepContext, Tcx: DepContext,
{ {
assert!( if !dep_graph_data.is_index_green(prev_index) {
dep_graph_data.is_green(dep_node), incremental_verify_ich_not_green(tcx, prev_index)
"fingerprint for green query instance not loaded from cache: {dep_node:?}", }
);
let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| { let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result)) tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
}); });
let old_hash = dep_graph_data.prev_fingerprint_of(dep_node); let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
if Some(new_hash) != old_hash { if new_hash != old_hash {
incremental_verify_ich_failed( incremental_verify_ich_failed(tcx, prev_index, result);
tcx.sess(),
DebugArg::from(&dep_node),
DebugArg::from(&result),
);
}
new_hash
}
// This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is
// currently not exposed publicly.
//
// The PR which added this attempted to use `&dyn Debug` instead, but that
// showed statistically significant worse compiler performance. It's not
// actually clear what the cause there was -- the code should be cold. If this
// can be replaced with `&dyn Debug` with on perf impact, then it probably
// should be.
extern "C" {
type Opaque;
}
struct DebugArg<'a> {
value: &'a Opaque,
fmt: fn(&Opaque, &mut std::fmt::Formatter<'_>) -> std::fmt::Result,
}
impl<'a, T> From<&'a T> for DebugArg<'a>
where
T: std::fmt::Debug,
{
fn from(value: &'a T) -> DebugArg<'a> {
DebugArg {
value: unsafe { std::mem::transmute(value) },
fmt: unsafe {
std::mem::transmute(<T as std::fmt::Debug>::fmt as fn(_, _) -> std::fmt::Result)
},
}
} }
} }
impl std::fmt::Debug for DebugArg<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
(self.fmt)(self.value, f)
}
}
// Note that this is marked #[cold] and intentionally takes the equivalent of
// `dyn Debug` for its arguments, as we want to avoid generating a bunch of
// different implementations for LLVM to chew on (and filling up the final
// binary, too).
#[cold] #[cold]
fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) { #[inline(never)]
fn incremental_verify_ich_not_green<Tcx>(tcx: Tcx, prev_index: SerializedDepNodeIndex)
where
Tcx: DepContext,
{
panic!(
"fingerprint for green query instance not loaded from cache: {:?}",
tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
)
}
// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`,
// as we want to avoid generating a bunch of different implementations for LLVM to
// chew on (and filling up the final binary, too).
#[cold]
#[inline(never)]
fn incremental_verify_ich_failed<Tcx>(
tcx: Tcx,
prev_index: SerializedDepNodeIndex,
result: &dyn Debug,
) where
Tcx: DepContext,
{
// When we emit an error message and panic, we try to debug-print the `DepNode` // When we emit an error message and panic, we try to debug-print the `DepNode`
// and query result. Unfortunately, this can cause us to run additional queries, // and query result. Unfortunately, this can cause us to run additional queries,
// which may result in another fingerprint mismatch while we're in the middle // which may result in another fingerprint mismatch while we're in the middle
@ -720,15 +692,17 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result:
let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true)); let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
if old_in_panic { if old_in_panic {
sess.emit_err(crate::error::Reentrant); tcx.sess().emit_err(crate::error::Reentrant);
} else { } else {
let run_cmd = if let Some(crate_name) = &sess.opts.crate_name { let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
format!("`cargo clean -p {crate_name}` or `cargo clean`") format!("`cargo clean -p {crate_name}` or `cargo clean`")
} else { } else {
"`cargo clean`".to_string() "`cargo clean`".to_string()
}; };
sess.emit_err(crate::error::IncrementCompilation { let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
let dep_node = tcx.sess().emit_err(crate::error::IncrementCompilation {
run_cmd, run_cmd,
dep_node: format!("{dep_node:?}"), dep_node: format!("{dep_node:?}"),
}); });