bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

mv compiler to compiler/

Browse source
This commit is contained in:
mark 2020-08-27 22:58:48 -05:00 • committed by Vadim Petrochenkov
parent db534b3ac2
commit 9e5f7d5631
1686 changed files with 941 additions and 1051 deletions
Download patch file Download diff file Expand all files Collapse all files

3
compiler/rustc_query_system/src/query/README.md Normal file
View file

@ -0,0 +1,3 @@
For more information about how the query system works, see the [rustc dev guide].
[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html

220
compiler/rustc_query_system/src/query/caches.rs Normal file
View file

@ -0,0 +1,220 @@
use crate::dep_graph::DepNodeIndex;
use crate::query::plumbing::{QueryLookup, QueryState};
use crate::query::QueryContext;
use rustc_arena::TypedArena;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sharded::Sharded;
use rustc_data_structures::sync::WorkerLocal;
use std::default::Default;
use std::hash::Hash;
use std::marker::PhantomData;
pub trait CacheSelector<K, V> {
type Cache;
}
pub trait QueryStorage: Default {
type Value;
type Stored: Clone;
/// Store a value without putting it in the cache.
/// This is meant to be used with cycle errors.
fn store_nocache(&self, value: Self::Value) -> Self::Stored;
}
pub trait QueryCache: QueryStorage {
type Key: Hash;
type Sharded: Default;
/// Checks if the query is already computed and in the cache.
/// It returns the shard index and a lock guard to the shard,
/// which will be used if the query is not in the cache and we need
/// to compute it.
fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
&self,
state: &QueryState<CTX, Self>,
key: Self::Key,
// `on_hit` can be called while holding a lock to the query state shard.
on_hit: OnHit,
on_miss: OnMiss,
) -> R
where
OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R,
OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R;
fn complete(
&self,
lock_sharded_storage: &mut Self::Sharded,
key: Self::Key,
value: Self::Value,
index: DepNodeIndex,
) -> Self::Stored;
fn iter<R, L>(
&self,
shards: &Sharded<L>,
get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
f: impl for<'a> FnOnce(
Box<dyn Iterator<Item = (&'a Self::Key, &'a Self::Value, DepNodeIndex)> + 'a>,
) -> R,
) -> R;
}
pub struct DefaultCacheSelector;
impl<K: Eq + Hash, V: Clone> CacheSelector<K, V> for DefaultCacheSelector {
type Cache = DefaultCache<K, V>;
}
pub struct DefaultCache<K, V>(PhantomData<(K, V)>);
impl<K, V> Default for DefaultCache<K, V> {
fn default() -> Self {
DefaultCache(PhantomData)
}
}
impl<K: Eq + Hash, V: Clone> QueryStorage for DefaultCache<K, V> {
type Value = V;
type Stored = V;
#[inline]
fn store_nocache(&self, value: Self::Value) -> Self::Stored {
// We have no dedicated storage
value
}
}
impl<K: Eq + Hash, V: Clone> QueryCache for DefaultCache<K, V> {
type Key = K;
type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
#[inline(always)]
fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
&self,
state: &QueryState<CTX, Self>,
key: K,
on_hit: OnHit,
on_miss: OnMiss,
) -> R
where
OnHit: FnOnce(&V, DepNodeIndex) -> R,
OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
{
let mut lookup = state.get_lookup(&key);
let lock = &mut *lookup.lock;
let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
}
#[inline]
fn complete(
&self,
lock_sharded_storage: &mut Self::Sharded,
key: K,
value: V,
index: DepNodeIndex,
) -> Self::Stored {
lock_sharded_storage.insert(key, (value.clone(), index));
value
}
fn iter<R, L>(
&self,
shards: &Sharded<L>,
get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
) -> R {
let mut shards = shards.lock_shards();
let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
f(Box::new(results))
}
}
pub struct ArenaCacheSelector<'tcx>(PhantomData<&'tcx ()>);
impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<K, V> for ArenaCacheSelector<'tcx> {
type Cache = ArenaCache<'tcx, K, V>;
}
pub struct ArenaCache<'tcx, K, V> {
arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
phantom: PhantomData<(K, &'tcx V)>,
}
impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
fn default() -> Self {
ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), phantom: PhantomData }
}
}
impl<'tcx, K: Eq + Hash, V: 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
type Value = V;
type Stored = &'tcx V;
#[inline]
fn store_nocache(&self, value: Self::Value) -> Self::Stored {
let value = self.arena.alloc((value, DepNodeIndex::INVALID));
let value = unsafe { &*(&value.0 as *const _) };
&value
}
}
impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> {
type Key = K;
type Sharded = FxHashMap<K, &'tcx (V, DepNodeIndex)>;
#[inline(always)]
fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
&self,
state: &QueryState<CTX, Self>,
key: K,
on_hit: OnHit,
on_miss: OnMiss,
) -> R
where
OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
{
let mut lookup = state.get_lookup(&key);
let lock = &mut *lookup.lock;
let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
if let Some((_, value)) = result {
on_hit(&&value.0, value.1)
} else {
on_miss(key, lookup)
}
}
#[inline]
fn complete(
&self,
lock_sharded_storage: &mut Self::Sharded,
key: K,
value: V,
index: DepNodeIndex,
) -> Self::Stored {
let value = self.arena.alloc((value, index));
let value = unsafe { &*(value as *const _) };
lock_sharded_storage.insert(key, value);
&value.0
}
fn iter<R, L>(
&self,
shards: &Sharded<L>,
get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
) -> R {
let mut shards = shards.lock_shards();
let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
f(Box::new(results))
}
}

133
compiler/rustc_query_system/src/query/config.rs Normal file
View file

@ -0,0 +1,133 @@
//! Query configuration and description traits.
use crate::dep_graph::DepNode;
use crate::dep_graph::SerializedDepNodeIndex;
use crate::query::caches::QueryCache;
use crate::query::plumbing::CycleError;
use crate::query::{QueryContext, QueryState};
use rustc_data_structures::profiling::ProfileCategory;
use rustc_data_structures::fingerprint::Fingerprint;
use std::borrow::Cow;
use std::fmt::Debug;
use std::hash::Hash;
// The parameter `CTX` is required in librustc_middle:
// implementations may need to access the `'tcx` lifetime in `CTX = TyCtxt<'tcx>`.
pub trait QueryConfig<CTX> {
const NAME: &'static str;
const CATEGORY: ProfileCategory;
type Key: Eq + Hash + Clone + Debug;
type Value;
type Stored: Clone;
}
pub(crate) struct QueryVtable<CTX: QueryContext, K, V> {
pub anon: bool,
pub dep_kind: CTX::DepKind,
pub eval_always: bool,
// Don't use this method to compute query results, instead use the methods on TyCtxt
pub compute: fn(CTX, K) -> V,
pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option<Fingerprint>,
pub handle_cycle_error: fn(CTX, CycleError<CTX::Query>) -> V,
pub cache_on_disk: fn(CTX, &K, Option<&V>) -> bool,
pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option<V>,
}
impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode<CTX::DepKind>
where
K: crate::dep_graph::DepNodeParams<CTX>,
{
DepNode::construct(tcx, self.dep_kind, key)
}
pub(crate) fn compute(&self, tcx: CTX, key: K) -> V {
(self.compute)(tcx, key)
}
pub(crate) fn hash_result(
&self,
hcx: &mut CTX::StableHashingContext,
value: &V,
) -> Option<Fingerprint> {
(self.hash_result)(hcx, value)
}
pub(crate) fn handle_cycle_error(&self, tcx: CTX, error: CycleError<CTX::Query>) -> V {
(self.handle_cycle_error)(tcx, error)
}
pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool {
(self.cache_on_disk)(tcx, key, value)
}
pub(crate) fn try_load_from_disk(&self, tcx: CTX, index: SerializedDepNodeIndex) -> Option<V> {
(self.try_load_from_disk)(tcx, index)
}
}
pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
const ANON: bool;
const EVAL_ALWAYS: bool;
const DEP_KIND: CTX::DepKind;
type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
// Don't use this method to access query results, instead use the methods on TyCtxt
fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;
fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>
where
Self::Key: crate::dep_graph::DepNodeParams<CTX>,
{
DepNode::construct(tcx, Self::DEP_KIND, key)
}
// Don't use this method to compute query results, instead use the methods on TyCtxt
fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
fn hash_result(
hcx: &mut CTX::StableHashingContext,
result: &Self::Value,
) -> Option<Fingerprint>;
fn handle_cycle_error(tcx: CTX, error: CycleError<CTX::Query>) -> Self::Value;
}
pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>;
#[inline]
fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool {
false
}
fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
panic!("QueryDescription::load_from_disk() called for an unsupported query.")
}
}
pub(crate) trait QueryVtableExt<CTX: QueryContext, K, V> {
const VTABLE: QueryVtable<CTX, K, V>;
}
impl<CTX, Q> QueryVtableExt<CTX, Q::Key, Q::Value> for Q
where
CTX: QueryContext,
Q: QueryDescription<CTX>,
{
const VTABLE: QueryVtable<CTX, Q::Key, Q::Value> = QueryVtable {
anon: Q::ANON,
dep_kind: Q::DEP_KIND,
eval_always: Q::EVAL_ALWAYS,
compute: Q::compute,
hash_result: Q::hash_result,
handle_cycle_error: Q::handle_cycle_error,
cache_on_disk: Q::cache_on_disk,
try_load_from_disk: Q::try_load_from_disk,
};
}

570
compiler/rustc_query_system/src/query/job.rs Normal file
View file

@ -0,0 +1,570 @@
use crate::dep_graph::{DepContext, DepKind};
use crate::query::plumbing::CycleError;
use crate::query::QueryContext;
use rustc_data_structures::fx::FxHashMap;
use rustc_span::Span;
use std::convert::TryFrom;
use std::marker::PhantomData;
use std::num::NonZeroU32;
#[cfg(parallel_compiler)]
use {
parking_lot::{Condvar, Mutex},
rustc_data_structures::fx::FxHashSet,
rustc_data_structures::stable_hasher::{HashStable, StableHasher},
rustc_data_structures::sync::Lock,
rustc_data_structures::sync::Lrc,
rustc_data_structures::{jobserver, OnDrop},
rustc_rayon_core as rayon_core,
rustc_span::DUMMY_SP,
std::iter::FromIterator,
std::{mem, process},
};
/// Represents a span and a query key.
#[derive(Clone, Debug)]
pub struct QueryInfo<Q> {
/// The span corresponding to the reason for which this query was required.
pub span: Span,
pub query: Q,
}
type QueryMap<CTX> = FxHashMap<QueryJobId<<CTX as DepContext>::DepKind>, QueryJobInfo<CTX>>;
/// A value uniquely identifiying an active query job within a shard in the query cache.
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
pub struct QueryShardJobId(pub NonZeroU32);
/// A value uniquely identifiying an active query job.
#[derive(Copy, Clone, Eq, PartialEq, Hash)]
pub struct QueryJobId<K> {
/// Which job within a shard is this
pub job: QueryShardJobId,
/// In which shard is this job
pub shard: u16,
/// What kind of query this job is
pub kind: K,
}
impl<K: DepKind> QueryJobId<K> {
pub fn new(job: QueryShardJobId, shard: usize, kind: K) -> Self {
QueryJobId { job, shard: u16::try_from(shard).unwrap(), kind }
}
fn query<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> CTX::Query {
map.get(&self).unwrap().info.query.clone()
}
#[cfg(parallel_compiler)]
fn span<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Span {
map.get(&self).unwrap().job.span
}
#[cfg(parallel_compiler)]
fn parent<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Option<QueryJobId<K>> {
map.get(&self).unwrap().job.parent
}
#[cfg(parallel_compiler)]
fn latch<'a, CTX: QueryContext<DepKind = K>>(
self,
map: &'a QueryMap<CTX>,
) -> Option<&'a QueryLatch<CTX>> {
map.get(&self).unwrap().job.latch.as_ref()
}
}
pub struct QueryJobInfo<CTX: QueryContext> {
pub info: QueryInfo<CTX::Query>,
pub job: QueryJob<CTX>,
}
/// Represents an active query job.
#[derive(Clone)]
pub struct QueryJob<CTX: QueryContext> {
pub id: QueryShardJobId,
/// The span corresponding to the reason for which this query was required.
pub span: Span,
/// The parent query job which created this job and is implicitly waiting on it.
pub parent: Option<QueryJobId<CTX::DepKind>>,
/// The latch that is used to wait on this job.
#[cfg(parallel_compiler)]
latch: Option<QueryLatch<CTX>>,
dummy: PhantomData<QueryLatch<CTX>>,
}
impl<CTX: QueryContext> QueryJob<CTX> {
/// Creates a new query job.
pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId<CTX::DepKind>>) -> Self {
QueryJob {
id,
span,
parent,
#[cfg(parallel_compiler)]
latch: None,
dummy: PhantomData,
}
}
#[cfg(parallel_compiler)]
pub(super) fn latch(&mut self, _id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
if self.latch.is_none() {
self.latch = Some(QueryLatch::new());
}
self.latch.as_ref().unwrap().clone()
}
#[cfg(not(parallel_compiler))]
pub(super) fn latch(&mut self, id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
QueryLatch { id, dummy: PhantomData }
}
/// Signals to waiters that the query is complete.
///
/// This does nothing for single threaded rustc,
/// as there are no concurrent jobs which could be waiting on us
pub fn signal_complete(self) {
#[cfg(parallel_compiler)]
{
if let Some(latch) = self.latch {
latch.set();
}
}
}
}
#[cfg(not(parallel_compiler))]
#[derive(Clone)]
pub(super) struct QueryLatch<CTX: QueryContext> {
id: QueryJobId<CTX::DepKind>,
dummy: PhantomData<CTX>,
}
#[cfg(not(parallel_compiler))]
impl<CTX: QueryContext> QueryLatch<CTX> {
pub(super) fn find_cycle_in_stack(&self, tcx: CTX, span: Span) -> CycleError<CTX::Query> {
let query_map = tcx.try_collect_active_jobs().unwrap();
// Get the current executing query (waiter) and find the waitee amongst its parents
let mut current_job = tcx.current_query_job();
let mut cycle = Vec::new();
while let Some(job) = current_job {
let info = query_map.get(&job).unwrap();
cycle.push(info.info.clone());
if job == self.id {
cycle.reverse();
// This is the end of the cycle
// The span entry we included was for the usage
// of the cycle itself, and not part of the cycle
// Replace it with the span which caused the cycle to form
cycle[0].span = span;
// Find out why the cycle itself was used
let usage = info
.job
.parent
.as_ref()
.map(|parent| (info.info.span, parent.query(&query_map)));
return CycleError { usage, cycle };
}
current_job = info.job.parent;
}
panic!("did not find a cycle")
}
}
#[cfg(parallel_compiler)]
struct QueryWaiter<CTX: QueryContext> {
query: Option<QueryJobId<CTX::DepKind>>,
condvar: Condvar,
span: Span,
cycle: Lock<Option<CycleError<CTX::Query>>>,
}
#[cfg(parallel_compiler)]
impl<CTX: QueryContext> QueryWaiter<CTX> {
fn notify(&self, registry: &rayon_core::Registry) {
rayon_core::mark_unblocked(registry);
self.condvar.notify_one();
}
}
#[cfg(parallel_compiler)]
struct QueryLatchInfo<CTX: QueryContext> {
complete: bool,
waiters: Vec<Lrc<QueryWaiter<CTX>>>,
}
#[cfg(parallel_compiler)]
#[derive(Clone)]
pub(super) struct QueryLatch<CTX: QueryContext> {
info: Lrc<Mutex<QueryLatchInfo<CTX>>>,
}
#[cfg(parallel_compiler)]
impl<CTX: QueryContext> QueryLatch<CTX> {
fn new() -> Self {
QueryLatch {
info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })),
}
}
}
#[cfg(parallel_compiler)]
impl<CTX: QueryContext> QueryLatch<CTX> {
/// Awaits for the query job to complete.
pub(super) fn wait_on(&self, tcx: CTX, span: Span) -> Result<(), CycleError<CTX::Query>> {
let query = tcx.current_query_job();
let waiter =
Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() });
self.wait_on_inner(&waiter);
// FIXME: Get rid of this lock. We have ownership of the QueryWaiter
// although another thread may still have a Lrc reference so we cannot
// use Lrc::get_mut
let mut cycle = waiter.cycle.lock();
match cycle.take() {
None => Ok(()),
Some(cycle) => Err(cycle),
}
}
}
#[cfg(parallel_compiler)]
impl<CTX: QueryContext> QueryLatch<CTX> {
/// Awaits the caller on this latch by blocking the current thread.
fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter<CTX>>) {
let mut info = self.info.lock();
if !info.complete {
// We push the waiter on to the `waiters` list. It can be accessed inside
// the `wait` call below, by 1) the `set` method or 2) by deadlock detection.
// Both of these will remove it from the `waiters` list before resuming
// this thread.
info.waiters.push(waiter.clone());
// If this detects a deadlock and the deadlock handler wants to resume this thread
// we have to be in the `wait` call. This is ensured by the deadlock handler
// getting the self.info lock.
rayon_core::mark_blocked();
jobserver::release_thread();
waiter.condvar.wait(&mut info);
// Release the lock before we potentially block in `acquire_thread`
mem::drop(info);
jobserver::acquire_thread();
}
}
/// Sets the latch and resumes all waiters on it
fn set(&self) {
let mut info = self.info.lock();
debug_assert!(!info.complete);
info.complete = true;
let registry = rayon_core::Registry::current();
for waiter in info.waiters.drain(..) {
waiter.notify(&registry);
}
}
/// Removes a single waiter from the list of waiters.
/// This is used to break query cycles.
fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter<CTX>> {
let mut info = self.info.lock();
debug_assert!(!info.complete);
// Remove the waiter from the list of waiters
info.waiters.remove(waiter)
}
}
/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
#[cfg(parallel_compiler)]
type Waiter<K> = (QueryJobId<K>, usize);
/// Visits all the non-resumable and resumable waiters of a query.
/// Only waiters in a query are visited.
/// `visit` is called for every waiter and is passed a query waiting on `query_ref`
/// and a span indicating the reason the query waited on `query_ref`.
/// If `visit` returns Some, this function returns.
/// For visits of non-resumable waiters it returns the return value of `visit`.
/// For visits of resumable waiters it returns Some(Some(Waiter)) which has the
/// required information to resume the waiter.
/// If all `visit` calls returns None, this function also returns None.
#[cfg(parallel_compiler)]
fn visit_waiters<CTX: QueryContext, F>(
query_map: &QueryMap<CTX>,
query: QueryJobId<CTX::DepKind>,
mut visit: F,
) -> Option<Option<Waiter<CTX::DepKind>>>
where
F: FnMut(Span, QueryJobId<CTX::DepKind>) -> Option<Option<Waiter<CTX::DepKind>>>,
{
// Visit the parent query which is a non-resumable waiter since it's on the same stack
if let Some(parent) = query.parent(query_map) {
if let Some(cycle) = visit(query.span(query_map), parent) {
return Some(cycle);
}
}
// Visit the explicit waiters which use condvars and are resumable
if let Some(latch) = query.latch(query_map) {
for (i, waiter) in latch.info.lock().waiters.iter().enumerate() {
if let Some(waiter_query) = waiter.query {
if visit(waiter.span, waiter_query).is_some() {
// Return a value which indicates that this waiter can be resumed
return Some(Some((query, i)));
}
}
}
}
None
}
/// Look for query cycles by doing a depth first search starting at `query`.
/// `span` is the reason for the `query` to execute. This is initially DUMMY_SP.
/// If a cycle is detected, this initial value is replaced with the span causing
/// the cycle.
#[cfg(parallel_compiler)]
fn cycle_check<CTX: QueryContext>(
query_map: &QueryMap<CTX>,
query: QueryJobId<CTX::DepKind>,
span: Span,
stack: &mut Vec<(Span, QueryJobId<CTX::DepKind>)>,
visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
) -> Option<Option<Waiter<CTX::DepKind>>> {
if !visited.insert(query) {
return if let Some(p) = stack.iter().position(|q| q.1 == query) {
// We detected a query cycle, fix up the initial span and return Some
// Remove previous stack entries
stack.drain(0..p);
// Replace the span for the first query with the cycle cause
stack[0].0 = span;
Some(None)
} else {
None
};
}
// Query marked as visited is added it to the stack
stack.push((span, query));
// Visit all the waiters
let r = visit_waiters(query_map, query, |span, successor| {
cycle_check(query_map, successor, span, stack, visited)
});
// Remove the entry in our stack if we didn't find a cycle
if r.is_none() {
stack.pop();
}
r
}
/// Finds out if there's a path to the compiler root (aka. code which isn't in a query)
/// from `query` without going through any of the queries in `visited`.
/// This is achieved with a depth first search.
#[cfg(parallel_compiler)]
fn connected_to_root<CTX: QueryContext>(
query_map: &QueryMap<CTX>,
query: QueryJobId<CTX::DepKind>,
visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
) -> bool {
// We already visited this or we're deliberately ignoring it
if !visited.insert(query) {
return false;
}
// This query is connected to the root (it has no query parent), return true
if query.parent(query_map).is_none() {
return true;
}
visit_waiters(query_map, query, |_, successor| {
connected_to_root(query_map, successor, visited).then_some(None)
})
.is_some()
}
// Deterministically pick an query from a list
#[cfg(parallel_compiler)]
fn pick_query<'a, CTX, T, F>(query_map: &QueryMap<CTX>, tcx: CTX, queries: &'a [T], f: F) -> &'a T
where
CTX: QueryContext,
F: Fn(&T) -> (Span, QueryJobId<CTX::DepKind>),
{
// Deterministically pick an entry point
// FIXME: Sort this instead
let mut hcx = tcx.create_stable_hashing_context();
queries
.iter()
.min_by_key(|v| {
let (span, query) = f(v);
let mut stable_hasher = StableHasher::new();
query.query(query_map).hash_stable(&mut hcx, &mut stable_hasher);
// Prefer entry points which have valid spans for nicer error messages
// We add an integer to the tuple ensuring that entry points
// with valid spans are picked first
let span_cmp = if span == DUMMY_SP { 1 } else { 0 };
(span_cmp, stable_hasher.finish::<u64>())
})
.unwrap()
}
/// Looks for query cycles starting from the last query in `jobs`.
/// If a cycle is found, all queries in the cycle is removed from `jobs` and
/// the function return true.
/// If a cycle was not found, the starting query is removed from `jobs` and
/// the function returns false.
#[cfg(parallel_compiler)]
fn remove_cycle<CTX: QueryContext>(
query_map: &QueryMap<CTX>,
jobs: &mut Vec<QueryJobId<CTX::DepKind>>,
wakelist: &mut Vec<Lrc<QueryWaiter<CTX>>>,
tcx: CTX,
) -> bool {
let mut visited = FxHashSet::default();
let mut stack = Vec::new();
// Look for a cycle starting with the last query in `jobs`
if let Some(waiter) =
cycle_check(query_map, jobs.pop().unwrap(), DUMMY_SP, &mut stack, &mut visited)
{
// The stack is a vector of pairs of spans and queries; reverse it so that
// the earlier entries require later entries
let (mut spans, queries): (Vec<_>, Vec<_>) = stack.into_iter().rev().unzip();
// Shift the spans so that queries are matched with the span for their waitee
spans.rotate_right(1);
// Zip them back together
let mut stack: Vec<_> = spans.into_iter().zip(queries).collect();
// Remove the queries in our cycle from the list of jobs to look at
for r in &stack {
if let Some(pos) = jobs.iter().position(|j| j == &r.1) {
jobs.remove(pos);
}
}
// Find the queries in the cycle which are
// connected to queries outside the cycle
let entry_points = stack
.iter()
.filter_map(|&(span, query)| {
if query.parent(query_map).is_none() {
// This query is connected to the root (it has no query parent)
Some((span, query, None))
} else {
let mut waiters = Vec::new();
// Find all the direct waiters who lead to the root
visit_waiters(query_map, query, |span, waiter| {
// Mark all the other queries in the cycle as already visited
let mut visited = FxHashSet::from_iter(stack.iter().map(|q| q.1));
if connected_to_root(query_map, waiter, &mut visited) {
waiters.push((span, waiter));
}
None
});
if waiters.is_empty() {
None
} else {
// Deterministically pick one of the waiters to show to the user
let waiter = *pick_query(query_map, tcx, &waiters, |s| *s);
Some((span, query, Some(waiter)))
}
}
})
.collect::<Vec<(Span, QueryJobId<CTX::DepKind>, Option<(Span, QueryJobId<CTX::DepKind>)>)>>();
// Deterministically pick an entry point
let (_, entry_point, usage) = pick_query(query_map, tcx, &entry_points, |e| (e.0, e.1));
// Shift the stack so that our entry point is first
let entry_point_pos = stack.iter().position(|(_, query)| query == entry_point);
if let Some(pos) = entry_point_pos {
stack.rotate_left(pos);
}
let usage = usage.as_ref().map(|(span, query)| (*span, query.query(query_map)));
// Create the cycle error
let error = CycleError {
usage,
cycle: stack
.iter()
.map(|&(s, ref q)| QueryInfo { span: s, query: q.query(query_map) })
.collect(),
};
// We unwrap `waiter` here since there must always be one
// edge which is resumeable / waited using a query latch
let (waitee_query, waiter_idx) = waiter.unwrap();
// Extract the waiter we want to resume
let waiter = waitee_query.latch(query_map).unwrap().extract_waiter(waiter_idx);
// Set the cycle error so it will be picked up when resumed
*waiter.cycle.lock() = Some(error);
// Put the waiter on the list of things to resume
wakelist.push(waiter);
true
} else {
false
}
}
/// Detects query cycles by using depth first search over all active query jobs.
/// If a query cycle is found it will break the cycle by finding an edge which
/// uses a query latch and then resuming that waiter.
/// There may be multiple cycles involved in a deadlock, so this searches
/// all active queries for cycles before finally resuming all the waiters at once.
#[cfg(parallel_compiler)]
pub fn deadlock<CTX: QueryContext>(tcx: CTX, registry: &rayon_core::Registry) {
let on_panic = OnDrop(|| {
eprintln!("deadlock handler panicked, aborting process");
process::abort();
});
let mut wakelist = Vec::new();
let query_map = tcx.try_collect_active_jobs().unwrap();
let mut jobs: Vec<QueryJobId<CTX::DepKind>> = query_map.keys().cloned().collect();
let mut found_cycle = false;
while jobs.len() > 0 {
if remove_cycle(&query_map, &mut jobs, &mut wakelist, tcx) {
found_cycle = true;
}
}
// Check that a cycle was found. It is possible for a deadlock to occur without
// a query cycle if a query which can be waited on uses Rayon to do multithreading
// internally. Such a query (X) may be executing on 2 threads (A and B) and A may
// wait using Rayon on B. Rayon may then switch to executing another query (Y)
// which in turn will wait on X causing a deadlock. We have a false dependency from
// X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
// only considers the true dependency and won't detect a cycle.
assert!(found_cycle);
// FIXME: Ensure this won't cause a deadlock before we return
for waiter in wakelist.into_iter() {
waiter.notify(registry);
}
on_panic.disable();
}

54
compiler/rustc_query_system/src/query/mod.rs Normal file
View file

@ -0,0 +1,54 @@
mod plumbing;
pub use self::plumbing::*;
mod job;
#[cfg(parallel_compiler)]
pub use self::job::deadlock;
pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
mod caches;
pub use self::caches::{
ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage,
};
mod config;
pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
use crate::dep_graph::{DepContext, DepGraph};
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::stable_hasher::HashStable;
use rustc_data_structures::sync::Lock;
use rustc_data_structures::thin_vec::ThinVec;
use rustc_errors::Diagnostic;
use rustc_span::def_id::DefId;
pub trait QueryContext: DepContext {
type Query: Clone + HashStable<Self::StableHashingContext>;
fn incremental_verify_ich(&self) -> bool;
fn verbose(&self) -> bool;
/// Get string representation from DefPath.
fn def_path_str(&self, def_id: DefId) -> String;
/// Access the DepGraph.
fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
/// Get the query information from the TLS context.
fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>>;
fn try_collect_active_jobs(
&self,
) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>>;
/// Executes a job by changing the `ImplicitCtxt` to point to the
/// new query job while it executes. It returns the diagnostics
/// captured during execution and the actual result.
fn start_query<R>(
&self,
token: QueryJobId<Self::DepKind>,
diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
compute: impl FnOnce(Self) -> R,
) -> R;
}

752
compiler/rustc_query_system/src/query/plumbing.rs Normal file
View file

@ -0,0 +1,752 @@
//! The implementation of the query system itself. This defines the macros that
//! generate the actual methods on tcx which find and execute the provider,
//! manage the caches, and so forth.
use crate::dep_graph::{DepKind, DepNode};
use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
use crate::query::caches::QueryCache;
use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt};
use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
use crate::query::QueryContext;
#[cfg(not(parallel_compiler))]
use rustc_data_structures::cold_path;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::{FxHashMap, FxHasher};
use rustc_data_structures::sharded::Sharded;
use rustc_data_structures::sync::{Lock, LockGuard};
use rustc_data_structures::thin_vec::ThinVec;
use rustc_errors::{Diagnostic, FatalError};
use rustc_span::source_map::DUMMY_SP;
use rustc_span::Span;
use std::collections::hash_map::Entry;
use std::convert::TryFrom;
use std::fmt::Debug;
use std::hash::{Hash, Hasher};
use std::mem;
use std::num::NonZeroU32;
use std::ptr;
#[cfg(debug_assertions)]
use std::sync::atomic::{AtomicUsize, Ordering};
pub(super) struct QueryStateShard<CTX: QueryContext, K, C> {
pub(super) cache: C,
active: FxHashMap<K, QueryResult<CTX>>,
/// Used to generate unique ids for active jobs.
jobs: u32,
}
impl<CTX: QueryContext, K, C: Default> Default for QueryStateShard<CTX, K, C> {
fn default() -> QueryStateShard<CTX, K, C> {
QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 }
}
}
pub struct QueryState<CTX: QueryContext, C: QueryCache> {
cache: C,
shards: Sharded<QueryStateShard<CTX, C::Key, C::Sharded>>,
#[cfg(debug_assertions)]
pub cache_hits: AtomicUsize,
}
impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
#[inline]
pub(super) fn get_lookup<'tcx>(
&'tcx self,
key: &C::Key,
) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> {
// We compute the key's hash once and then use it for both the
// shard lookup and the hashmap lookup. This relies on the fact
// that both of them use `FxHasher`.
let mut hasher = FxHasher::default();
key.hash(&mut hasher);
let key_hash = hasher.finish();
let shard = self.shards.get_shard_index_by_hash(key_hash);
let lock = self.shards.get_shard_by_index(shard).lock();
QueryLookup { key_hash, shard, lock }
}
}
/// Indicates the state of a query for a given key in a query map.
enum QueryResult<CTX: QueryContext> {
/// An already executing query. The query job can be used to await for its completion.
Started(QueryJob<CTX>),
/// The query panicked. Queries trying to wait on this will raise a fatal error which will
/// silently panic.
Poisoned,
}
impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
#[inline(always)]
pub fn iter_results<R>(
&self,
f: impl for<'a> FnOnce(
Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
) -> R,
) -> R {
self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
}
#[inline(always)]
pub fn all_inactive(&self) -> bool {
let shards = self.shards.lock_shards();
shards.iter().all(|shard| shard.active.is_empty())
}
pub fn try_collect_active_jobs(
&self,
kind: CTX::DepKind,
make_query: fn(C::Key) -> CTX::Query,
jobs: &mut FxHashMap<QueryJobId<CTX::DepKind>, QueryJobInfo<CTX>>,
) -> Option<()>
where
C::Key: Clone,
{
// We use try_lock_shards here since we are called from the
// deadlock handler, and this shouldn't be locked.
let shards = self.shards.try_lock_shards()?;
let shards = shards.iter().enumerate();
jobs.extend(shards.flat_map(|(shard_id, shard)| {
shard.active.iter().filter_map(move |(k, v)| {
if let QueryResult::Started(ref job) = *v {
let id =
QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), kind };
let info = QueryInfo { span: job.span, query: make_query(k.clone()) };
Some((id, QueryJobInfo { info, job: job.clone() }))
} else {
None
}
})
}));
Some(())
}
}
impl<CTX: QueryContext, C: QueryCache> Default for QueryState<CTX, C> {
fn default() -> QueryState<CTX, C> {
QueryState {
cache: C::default(),
shards: Default::default(),
#[cfg(debug_assertions)]
cache_hits: AtomicUsize::new(0),
}
}
}
/// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
pub struct QueryLookup<'tcx, CTX: QueryContext, K, C> {
pub(super) key_hash: u64,
shard: usize,
pub(super) lock: LockGuard<'tcx, QueryStateShard<CTX, K, C>>,
}
/// A type representing the responsibility to execute the job in the `job` field.
/// This will poison the relevant query if dropped.
struct JobOwner<'tcx, CTX: QueryContext, C>
where
C: QueryCache,
C::Key: Eq + Hash + Clone + Debug,
{
state: &'tcx QueryState<CTX, C>,
key: C::Key,
id: QueryJobId<CTX::DepKind>,
}
impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C>
where
C: QueryCache,
C::Key: Eq + Hash + Clone + Debug,
{
/// Either gets a `JobOwner` corresponding the query, allowing us to
/// start executing the query, or returns with the result of the query.
/// This function assumes that `try_get_cached` is already called and returned `lookup`.
/// If the query is executing elsewhere, this will wait for it and return the result.
/// If the query panicked, this will silently panic.
///
/// This function is inlined because that results in a noticeable speed-up
/// for some compile-time benchmarks.
#[inline(always)]
fn try_start<'a, 'b>(
tcx: CTX,
state: &'b QueryState<CTX, C>,
span: Span,
key: &C::Key,
mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> TryGetJob<'b, CTX, C>
where
CTX: QueryContext,
{
let lock = &mut *lookup.lock;
let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) {
Entry::Occupied(mut entry) => {
match entry.get_mut() {
QueryResult::Started(job) => {
// For parallel queries, we'll block and wait until the query running
// in another thread has completed. Record how long we wait in the
// self-profiler.
let _query_blocked_prof_timer = if cfg!(parallel_compiler) {
Some(tcx.profiler().query_blocked())
} else {
None
};
// Create the id of the job we're waiting for
let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind);
(job.latch(id), _query_blocked_prof_timer)
}
QueryResult::Poisoned => FatalError.raise(),
}
}
Entry::Vacant(entry) => {
// No job entry for this query. Return a new one to be started later.
// Generate an id unique within this shard.
let id = lock.jobs.checked_add(1).unwrap();
lock.jobs = id;
let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind);
let job = tcx.current_query_job();
let job = QueryJob::new(id, span, job);
entry.insert(QueryResult::Started(job));
let owner = JobOwner { state, id: global_id, key: (*key).clone() };
return TryGetJob::NotYetStarted(owner);
}
};
mem::drop(lookup.lock);
// If we are single-threaded we know that we have cycle error,
// so we just return the error.
#[cfg(not(parallel_compiler))]
return TryGetJob::Cycle(cold_path(|| {
let value = query.handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
state.cache.store_nocache(value)
}));
// With parallel queries we might just have to wait on some other
// thread.
#[cfg(parallel_compiler)]
{
let result = latch.wait_on(tcx, span);
if let Err(cycle) = result {
let value = query.handle_cycle_error(tcx, cycle);
let value = state.cache.store_nocache(value);
return TryGetJob::Cycle(value);
}
let cached = try_get_cached(
tcx,
state,
(*key).clone(),
|value, index| (value.clone(), index),
|_, _| panic!("value must be in cache after waiting"),
);
if let Some(prof_timer) = _query_blocked_prof_timer.take() {
prof_timer.finish_with_query_invocation_id(cached.1.into());
}
return TryGetJob::JobCompleted(cached);
}
}
/// Completes the query by updating the query cache with the `result`,
/// signals the waiter and forgets the JobOwner, so it won't poison the query
#[inline(always)]
fn complete(self, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored {
// We can move out of `self` here because we `mem::forget` it below
let key = unsafe { ptr::read(&self.key) };
let state = self.state;
// Forget ourself so our destructor won't poison the query
mem::forget(self);
let (job, result) = {
let mut lock = state.shards.get_shard_by_value(&key).lock();
let job = match lock.active.remove(&key).unwrap() {
QueryResult::Started(job) => job,
QueryResult::Poisoned => panic!(),
};
let result = state.cache.complete(&mut lock.cache, key, result, dep_node_index);
(job, result)
};
job.signal_complete();
result
}
}
#[inline(always)]
fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
where
F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R,
{
let diagnostics = Lock::new(ThinVec::new());
let result = f(Some(&diagnostics));
(result, diagnostics.into_inner())
}
impl<'tcx, CTX: QueryContext, C: QueryCache> Drop for JobOwner<'tcx, CTX, C>
where
C::Key: Eq + Hash + Clone + Debug,
{
#[inline(never)]
#[cold]
fn drop(&mut self) {
// Poison the query so jobs waiting on it panic.
let state = self.state;
let shard = state.shards.get_shard_by_value(&self.key);
let job = {
let mut shard = shard.lock();
let job = match shard.active.remove(&self.key).unwrap() {
QueryResult::Started(job) => job,
QueryResult::Poisoned => panic!(),
};
shard.active.insert(self.key.clone(), QueryResult::Poisoned);
job
};
// Also signal the completion of the job, so waiters
// will continue execution.
job.signal_complete();
}
}
#[derive(Clone)]
pub struct CycleError<Q> {
/// The query and related span that uses the cycle.
pub usage: Option<(Span, Q)>,
pub cycle: Vec<QueryInfo<Q>>,
}
/// The result of `try_start`.
enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache>
where
C::Key: Eq + Hash + Clone + Debug,
{
/// The query is not yet started. Contains a guard to the cache eventually used to start it.
NotYetStarted(JobOwner<'tcx, CTX, C>),
/// The query was already completed.
/// Returns the result of the query and its dep-node index
/// if it succeeded or a cycle error if it failed.
#[cfg(parallel_compiler)]
JobCompleted((C::Stored, DepNodeIndex)),
/// Trying to execute the query resulted in a cycle.
Cycle(C::Stored),
}
/// Checks if the query is already computed and in the cache.
/// It returns the shard index and a lock guard to the shard,
/// which will be used if the query is not in the cache and we need
/// to compute it.
#[inline(always)]
fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
tcx: CTX,
state: &QueryState<CTX, C>,
key: C::Key,
// `on_hit` can be called while holding a lock to the query cache
on_hit: OnHit,
on_miss: OnMiss,
) -> R
where
C: QueryCache,
CTX: QueryContext,
OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R,
OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R,
{
state.cache.lookup(
state,
key,
|value, index| {
if unlikely!(tcx.profiler().enabled()) {
tcx.profiler().query_cache_hit(index.into());
}
#[cfg(debug_assertions)]
{
state.cache_hits.fetch_add(1, Ordering::Relaxed);
}
on_hit(value, index)
},
on_miss,
)
}
#[inline(always)]
fn try_execute_query<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
span: Span,
key: C::Key,
lookup: QueryLookup<'_, CTX, C::Key, C::Sharded>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> C::Stored
where
C: QueryCache,
C::Key: Eq + Clone + Debug + crate::dep_graph::DepNodeParams<CTX>,
C::Stored: Clone,
CTX: QueryContext,
{
let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
TryGetJob::NotYetStarted(job) => job,
TryGetJob::Cycle(result) => return result,
#[cfg(parallel_compiler)]
TryGetJob::JobCompleted((v, index)) => {
tcx.dep_graph().read_index(index);
return v;
}
};
// Fast path for when incr. comp. is off. `to_dep_node` is
// expensive for some `DepKind`s.
if !tcx.dep_graph().is_fully_enabled() {
let null_dep_node = DepNode::new_no_params(DepKind::NULL);
return force_query_with_job(tcx, key, job, null_dep_node, query).0;
}
if query.anon {
let prof_timer = tcx.profiler().query_provider();
let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
tcx.start_query(job.id, diagnostics, |tcx| {
tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key))
})
});
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
tcx.dep_graph().read_index(dep_node_index);
if unlikely!(!diagnostics.is_empty()) {
tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
}
return job.complete(result, dep_node_index);
}
let dep_node = query.to_dep_node(tcx, &key);
if !query.eval_always {
// The diagnostics for this query will be
// promoted to the current session during
// `try_mark_green()`, so we can ignore them here.
let loaded = tcx.start_query(job.id, None, |tcx| {
let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
marked.map(|(prev_dep_node_index, dep_node_index)| {
(
load_from_disk_and_cache_in_memory(
tcx,
key.clone(),
prev_dep_node_index,
dep_node_index,
&dep_node,
query,
),
dep_node_index,
)
})
});
if let Some((result, dep_node_index)) = loaded {
return job.complete(result, dep_node_index);
}
}
let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query);
tcx.dep_graph().read_index(dep_node_index);
result
}
fn load_from_disk_and_cache_in_memory<CTX, K, V>(
tcx: CTX,
key: K,
prev_dep_node_index: SerializedDepNodeIndex,
dep_node_index: DepNodeIndex,
dep_node: &DepNode<CTX::DepKind>,
query: &QueryVtable<CTX, K, V>,
) -> V
where
CTX: QueryContext,
{
// Note this function can be called concurrently from the same query
// We must ensure that this is handled correctly.
debug_assert!(tcx.dep_graph().is_green(dep_node));
// First we try to load the result from the on-disk cache.
let result = if query.cache_on_disk(tcx, &key, None) {
let prof_timer = tcx.profiler().incr_cache_loading();
let result = query.try_load_from_disk(tcx, prev_dep_node_index);
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// We always expect to find a cached result for things that
// can be forced from `DepNode`.
debug_assert!(
!dep_node.kind.can_reconstruct_query_key() || result.is_some(),
"missing on-disk cache entry for {:?}",
dep_node
);
result
} else {
// Some things are never cached on disk.
None
};
let result = if let Some(result) = result {
result
} else {
// We could not load a result from the on-disk cache, so
// recompute.
let prof_timer = tcx.profiler().query_provider();
// The dep-graph for this computation is already in-place.
let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key));
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
result
};
// If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint.
if unlikely!(tcx.incremental_verify_ich()) {
incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query);
}
result
}
#[inline(never)]
#[cold]
fn incremental_verify_ich<CTX, K, V>(
tcx: CTX,
result: &V,
dep_node: &DepNode<CTX::DepKind>,
dep_node_index: DepNodeIndex,
query: &QueryVtable<CTX, K, V>,
) where
CTX: QueryContext,
{
assert!(
Some(tcx.dep_graph().fingerprint_of(dep_node_index))
== tcx.dep_graph().prev_fingerprint_of(dep_node),
"fingerprint for green query instance not loaded from cache: {:?}",
dep_node,
);
debug!("BEGIN verify_ich({:?})", dep_node);
let mut hcx = tcx.create_stable_hashing_context();
let new_hash = query.hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
debug!("END verify_ich({:?})", dep_node);
let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
}
#[inline(always)]
fn force_query_with_job<C, CTX>(
tcx: CTX,
key: C::Key,
job: JobOwner<'_, CTX, C>,
dep_node: DepNode<CTX::DepKind>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> (C::Stored, DepNodeIndex)
where
C: QueryCache,
C::Key: Eq + Clone + Debug,
C::Stored: Clone,
CTX: QueryContext,
{
// If the following assertion triggers, it can have two reasons:
// 1. Something is wrong with DepNode creation, either here or
// in `DepGraph::try_mark_green()`.
// 2. Two distinct query keys get mapped to the same `DepNode`
// (see for example #48923).
assert!(
!tcx.dep_graph().dep_node_exists(&dep_node),
"forcing query with already existing `DepNode`\n\
- query-key: {:?}\n\
- dep-node: {:?}",
key,
dep_node
);
let prof_timer = tcx.profiler().query_provider();
let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
tcx.start_query(job.id, diagnostics, |tcx| {
if query.eval_always {
tcx.dep_graph().with_eval_always_task(
dep_node,
tcx,
key,
query.compute,
query.hash_result,
)
} else {
tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result)
}
})
});
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
if unlikely!(!diagnostics.is_empty()) {
if dep_node.kind != DepKind::NULL {
tcx.store_diagnostics(dep_node_index, diagnostics);
}
}
let result = job.complete(result, dep_node_index);
(result, dep_node_index)
}
#[inline(never)]
fn get_query_impl<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
span: Span,
key: C::Key,
query: &QueryVtable<CTX, C::Key, C::Value>,
) -> C::Stored
where
CTX: QueryContext,
C: QueryCache,
C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
C::Stored: Clone,
{
try_get_cached(
tcx,
state,
key,
|value, index| {
tcx.dep_graph().read_index(index);
value.clone()
},
|key, lookup| try_execute_query(tcx, state, span, key, lookup, query),
)
}
/// Ensure that either this query has all green inputs or been executed.
/// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
///
/// This function is particularly useful when executing passes for their
/// side-effects -- e.g., in order to report errors for erroneous programs.
///
/// Note: The optimization is only available during incr. comp.
#[inline(never)]
fn ensure_query_impl<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
key: C::Key,
query: &QueryVtable<CTX, C::Key, C::Value>,
) where
C: QueryCache,
C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
if query.eval_always {
let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
return;
}
// Ensuring an anonymous query makes no sense
assert!(!query.anon);
let dep_node = query.to_dep_node(tcx, &key);
match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) {
None => {
// A None return from `try_mark_green_and_read` means that this is either
// a new dep node or that the dep node has already been marked red.
// Either way, we can't call `dep_graph.read()` as we don't have the
// DepNodeIndex. We must invoke the query itself. The performance cost
// this introduces should be negligible as we'll immediately hit the
// in-memory cache, or another query down the line will.
let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
}
Some((_, dep_node_index)) => {
tcx.profiler().query_cache_hit(dep_node_index.into());
}
}
}
#[inline(never)]
fn force_query_impl<CTX, C>(
tcx: CTX,
state: &QueryState<CTX, C>,
key: C::Key,
span: Span,
dep_node: DepNode<CTX::DepKind>,
query: &QueryVtable<CTX, C::Key, C::Value>,
) where
C: QueryCache,
C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
// We may be concurrently trying both execute and force a query.
// Ensure that only one of them runs the query.
try_get_cached(
tcx,
state,
key,
|_, _| {
// Cache hit, do nothing
},
|key, lookup| {
let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
TryGetJob::NotYetStarted(job) => job,
TryGetJob::Cycle(_) => return,
#[cfg(parallel_compiler)]
TryGetJob::JobCompleted(_) => return,
};
force_query_with_job(tcx, key, job, dep_node, query);
},
);
}
#[inline(always)]
pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
where
Q: QueryDescription<CTX>,
Q::Key: crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE)
}
#[inline(always)]
pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
where
Q: QueryDescription<CTX>,
Q::Key: crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE)
}
#[inline(always)]
pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
where
Q: QueryDescription<CTX>,
Q::Key: crate::dep_graph::DepNodeParams<CTX>,
CTX: QueryContext,
{
force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE)
}