Auto merge of #109371 - Zoxc:verify-hash-opt, r=cjgillot
Optimize `incremental_verify_ich` This optimizes `incremental_verify_ich` by operating on `SerializedDepNodeIndex`, saving 2 hashmap lookups. The panic paths are also changed to get a `TyCtxt` reference using TLS.
This commit is contained in:
commit
e10eab5956
3 changed files with 66 additions and 85 deletions
|
@ -535,16 +535,22 @@ impl<K: DepKind> DepGraph<K> {
|
||||||
// value to an existing node.
|
// value to an existing node.
|
||||||
//
|
//
|
||||||
// For sanity, we still check that the loaded stable hash and the new one match.
|
// For sanity, we still check that the loaded stable hash and the new one match.
|
||||||
if let Some(dep_node_index) = data.dep_node_index_of_opt(&node) {
|
if let Some(prev_index) = data.previous.node_to_index_opt(&node) {
|
||||||
let _current_fingerprint =
|
let dep_node_index = data.current.prev_index_to_index.lock()[prev_index];
|
||||||
crate::query::incremental_verify_ich(cx, data, result, &node, hash_result);
|
if let Some(dep_node_index) = dep_node_index {
|
||||||
|
crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result);
|
||||||
|
|
||||||
#[cfg(debug_assertions)]
|
#[cfg(debug_assertions)]
|
||||||
if hash_result.is_some() {
|
if hash_result.is_some() {
|
||||||
data.current.record_edge(dep_node_index, node, _current_fingerprint);
|
data.current.record_edge(
|
||||||
|
dep_node_index,
|
||||||
|
node,
|
||||||
|
data.prev_fingerprint_of(prev_index),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return dep_node_index;
|
||||||
}
|
}
|
||||||
|
|
||||||
return dep_node_index;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut edges = SmallVec::new();
|
let mut edges = SmallVec::new();
|
||||||
|
@ -626,13 +632,19 @@ impl<K: DepKind> DepGraphData<K> {
|
||||||
|
|
||||||
/// Returns true if the given node has been marked as green during the
|
/// Returns true if the given node has been marked as green during the
|
||||||
/// current compilation session. Used in various assertions
|
/// current compilation session. Used in various assertions
|
||||||
pub fn is_green(&self, dep_node: &DepNode<K>) -> bool {
|
#[inline]
|
||||||
self.node_color(dep_node).map_or(false, |c| c.is_green())
|
pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
|
||||||
|
self.colors.get(prev_index).map_or(false, |c| c.is_green())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn prev_fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
|
pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
|
||||||
self.previous.fingerprint_of(dep_node)
|
self.previous.fingerprint_by_index(prev_index)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode<K> {
|
||||||
|
self.previous.index_to_node(prev_index)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
|
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode<K>) {
|
||||||
|
@ -643,7 +655,7 @@ impl<K: DepKind> DepGraphData<K> {
|
||||||
impl<K: DepKind> DepGraph<K> {
|
impl<K: DepKind> DepGraph<K> {
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
|
pub fn dep_node_exists(&self, dep_node: &DepNode<K>) -> bool {
|
||||||
self.data.as_ref().and_then(|data| data.dep_node_index_of_opt(dep_node)).is_some()
|
self.data.as_ref().map_or(false, |data| data.dep_node_exists(dep_node))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Checks whether a previous work product exists for `v` and, if
|
/// Checks whether a previous work product exists for `v` and, if
|
||||||
|
|
|
@ -79,11 +79,6 @@ impl<K: DepKind> SerializedDepGraph<K> {
|
||||||
self.index.get(dep_node).cloned()
|
self.index.get(dep_node).cloned()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[inline]
|
|
||||||
pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
|
|
||||||
self.index.get(dep_node).map(|&node_index| self.fingerprints[node_index])
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
|
pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
|
||||||
self.fingerprints[dep_node_index]
|
self.fingerprints[dep_node_index]
|
||||||
|
|
|
@ -7,6 +7,7 @@ use crate::dep_graph::{DepGraphData, HasDepContext};
|
||||||
use crate::ich::StableHashingContext;
|
use crate::ich::StableHashingContext;
|
||||||
use crate::query::caches::QueryCache;
|
use crate::query::caches::QueryCache;
|
||||||
use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
|
use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
|
||||||
|
use crate::query::SerializedDepNodeIndex;
|
||||||
use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
|
use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
|
||||||
use crate::values::Value;
|
use crate::values::Value;
|
||||||
use crate::HandleCycleError;
|
use crate::HandleCycleError;
|
||||||
|
@ -19,7 +20,6 @@ use rustc_data_structures::sharded::Sharded;
|
||||||
use rustc_data_structures::stack::ensure_sufficient_stack;
|
use rustc_data_structures::stack::ensure_sufficient_stack;
|
||||||
use rustc_data_structures::sync::{Lock, LockGuard};
|
use rustc_data_structures::sync::{Lock, LockGuard};
|
||||||
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
|
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError};
|
||||||
use rustc_session::Session;
|
|
||||||
use rustc_span::{Span, DUMMY_SP};
|
use rustc_span::{Span, DUMMY_SP};
|
||||||
use std::cell::Cell;
|
use std::cell::Cell;
|
||||||
use std::collections::hash_map::Entry;
|
use std::collections::hash_map::Entry;
|
||||||
|
@ -537,7 +537,7 @@ where
|
||||||
|
|
||||||
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
|
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
|
||||||
|
|
||||||
debug_assert!(dep_graph_data.is_green(dep_node));
|
debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
|
||||||
|
|
||||||
// First we try to load the result from the on-disk cache.
|
// First we try to load the result from the on-disk cache.
|
||||||
// Some things are never cached on disk.
|
// Some things are never cached on disk.
|
||||||
|
@ -561,8 +561,7 @@ where
|
||||||
dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
|
dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
|
||||||
}
|
}
|
||||||
|
|
||||||
let prev_fingerprint =
|
let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
|
||||||
dep_graph_data.prev_fingerprint_of(dep_node).unwrap_or(Fingerprint::ZERO);
|
|
||||||
// If `-Zincremental-verify-ich` is specified, re-hash results from
|
// If `-Zincremental-verify-ich` is specified, re-hash results from
|
||||||
// the cache and make sure that they have the expected fingerprint.
|
// the cache and make sure that they have the expected fingerprint.
|
||||||
//
|
//
|
||||||
|
@ -578,7 +577,7 @@ where
|
||||||
*qcx.dep_context(),
|
*qcx.dep_context(),
|
||||||
dep_graph_data,
|
dep_graph_data,
|
||||||
&result,
|
&result,
|
||||||
dep_node,
|
prev_dep_node_index,
|
||||||
query.hash_result(),
|
query.hash_result(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
@ -623,7 +622,7 @@ where
|
||||||
*qcx.dep_context(),
|
*qcx.dep_context(),
|
||||||
dep_graph_data,
|
dep_graph_data,
|
||||||
&result,
|
&result,
|
||||||
dep_node,
|
prev_dep_node_index,
|
||||||
query.hash_result(),
|
query.hash_result(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
@ -636,77 +635,50 @@ pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
|
||||||
tcx: Tcx,
|
tcx: Tcx,
|
||||||
dep_graph_data: &DepGraphData<Tcx::DepKind>,
|
dep_graph_data: &DepGraphData<Tcx::DepKind>,
|
||||||
result: &V,
|
result: &V,
|
||||||
dep_node: &DepNode<Tcx::DepKind>,
|
prev_index: SerializedDepNodeIndex,
|
||||||
hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
|
hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
|
||||||
) -> Fingerprint
|
) where
|
||||||
where
|
|
||||||
Tcx: DepContext,
|
Tcx: DepContext,
|
||||||
{
|
{
|
||||||
assert!(
|
if !dep_graph_data.is_index_green(prev_index) {
|
||||||
dep_graph_data.is_green(dep_node),
|
incremental_verify_ich_not_green(tcx, prev_index)
|
||||||
"fingerprint for green query instance not loaded from cache: {dep_node:?}",
|
}
|
||||||
);
|
|
||||||
|
|
||||||
let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
|
let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| {
|
||||||
tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
|
tcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, result))
|
||||||
});
|
});
|
||||||
|
|
||||||
let old_hash = dep_graph_data.prev_fingerprint_of(dep_node);
|
let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
|
||||||
|
|
||||||
if Some(new_hash) != old_hash {
|
if new_hash != old_hash {
|
||||||
incremental_verify_ich_failed(
|
incremental_verify_ich_failed(tcx, prev_index, result);
|
||||||
tcx.sess(),
|
|
||||||
DebugArg::from(&dep_node),
|
|
||||||
DebugArg::from(&result),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
new_hash
|
|
||||||
}
|
|
||||||
|
|
||||||
// This DebugArg business is largely a mirror of std::fmt::ArgumentV1, which is
|
|
||||||
// currently not exposed publicly.
|
|
||||||
//
|
|
||||||
// The PR which added this attempted to use `&dyn Debug` instead, but that
|
|
||||||
// showed statistically significant worse compiler performance. It's not
|
|
||||||
// actually clear what the cause there was -- the code should be cold. If this
|
|
||||||
// can be replaced with `&dyn Debug` with on perf impact, then it probably
|
|
||||||
// should be.
|
|
||||||
extern "C" {
|
|
||||||
type Opaque;
|
|
||||||
}
|
|
||||||
|
|
||||||
struct DebugArg<'a> {
|
|
||||||
value: &'a Opaque,
|
|
||||||
fmt: fn(&Opaque, &mut std::fmt::Formatter<'_>) -> std::fmt::Result,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl<'a, T> From<&'a T> for DebugArg<'a>
|
|
||||||
where
|
|
||||||
T: std::fmt::Debug,
|
|
||||||
{
|
|
||||||
fn from(value: &'a T) -> DebugArg<'a> {
|
|
||||||
DebugArg {
|
|
||||||
value: unsafe { std::mem::transmute(value) },
|
|
||||||
fmt: unsafe {
|
|
||||||
std::mem::transmute(<T as std::fmt::Debug>::fmt as fn(_, _) -> std::fmt::Result)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for DebugArg<'_> {
|
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
||||||
(self.fmt)(self.value, f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Note that this is marked #[cold] and intentionally takes the equivalent of
|
|
||||||
// `dyn Debug` for its arguments, as we want to avoid generating a bunch of
|
|
||||||
// different implementations for LLVM to chew on (and filling up the final
|
|
||||||
// binary, too).
|
|
||||||
#[cold]
|
#[cold]
|
||||||
fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: DebugArg<'_>) {
|
#[inline(never)]
|
||||||
|
fn incremental_verify_ich_not_green<Tcx>(tcx: Tcx, prev_index: SerializedDepNodeIndex)
|
||||||
|
where
|
||||||
|
Tcx: DepContext,
|
||||||
|
{
|
||||||
|
panic!(
|
||||||
|
"fingerprint for green query instance not loaded from cache: {:?}",
|
||||||
|
tcx.dep_graph().data().unwrap().prev_node_of(prev_index)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note that this is marked #[cold] and intentionally takes `dyn Debug` for `result`,
|
||||||
|
// as we want to avoid generating a bunch of different implementations for LLVM to
|
||||||
|
// chew on (and filling up the final binary, too).
|
||||||
|
#[cold]
|
||||||
|
#[inline(never)]
|
||||||
|
fn incremental_verify_ich_failed<Tcx>(
|
||||||
|
tcx: Tcx,
|
||||||
|
prev_index: SerializedDepNodeIndex,
|
||||||
|
result: &dyn Debug,
|
||||||
|
) where
|
||||||
|
Tcx: DepContext,
|
||||||
|
{
|
||||||
// When we emit an error message and panic, we try to debug-print the `DepNode`
|
// When we emit an error message and panic, we try to debug-print the `DepNode`
|
||||||
// and query result. Unfortunately, this can cause us to run additional queries,
|
// and query result. Unfortunately, this can cause us to run additional queries,
|
||||||
// which may result in another fingerprint mismatch while we're in the middle
|
// which may result in another fingerprint mismatch while we're in the middle
|
||||||
|
@ -720,15 +692,17 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result:
|
||||||
let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
|
let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
|
||||||
|
|
||||||
if old_in_panic {
|
if old_in_panic {
|
||||||
sess.emit_err(crate::error::Reentrant);
|
tcx.sess().emit_err(crate::error::Reentrant);
|
||||||
} else {
|
} else {
|
||||||
let run_cmd = if let Some(crate_name) = &sess.opts.crate_name {
|
let run_cmd = if let Some(crate_name) = &tcx.sess().opts.crate_name {
|
||||||
format!("`cargo clean -p {crate_name}` or `cargo clean`")
|
format!("`cargo clean -p {crate_name}` or `cargo clean`")
|
||||||
} else {
|
} else {
|
||||||
"`cargo clean`".to_string()
|
"`cargo clean`".to_string()
|
||||||
};
|
};
|
||||||
|
|
||||||
sess.emit_err(crate::error::IncrementCompilation {
|
let dep_node = tcx.dep_graph().data().unwrap().prev_node_of(prev_index);
|
||||||
|
|
||||||
|
let dep_node = tcx.sess().emit_err(crate::error::IncrementCompilation {
|
||||||
run_cmd,
|
run_cmd,
|
||||||
dep_node: format!("{dep_node:?}"),
|
dep_node: format!("{dep_node:?}"),
|
||||||
});
|
});
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue