mv compiler to compiler/

compiler/rustc_query_system/src/cache.rs

@@ -0,0 +1,62 @@
+//! Cache for candidate selection.
+
+use crate::dep_graph::DepNodeIndex;
+use crate::query::QueryContext;
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::HashMapExt;
+use rustc_data_structures::sync::Lock;
+
+use std::hash::Hash;
+
+#[derive(Clone)]
+pub struct Cache<Key, Value> {
+    hashmap: Lock<FxHashMap<Key, WithDepNode<Value>>>,
+}
+
+impl<Key, Value> Default for Cache<Key, Value> {
+    fn default() -> Self {
+        Self { hashmap: Default::default() }
+    }
+}
+
+impl<Key, Value> Cache<Key, Value> {
+    /// Actually frees the underlying memory in contrast to what stdlib containers do on `clear`
+    pub fn clear(&self) {
+        *self.hashmap.borrow_mut() = Default::default();
+    }
+}
+
+impl<Key: Eq + Hash, Value: Clone> Cache<Key, Value> {
+    pub fn get<CTX: QueryContext>(&self, key: &Key, tcx: CTX) -> Option<Value> {
+        Some(self.hashmap.borrow().get(key)?.get(tcx))
+    }
+
+    pub fn insert(&self, key: Key, dep_node: DepNodeIndex, value: Value) {
+        self.hashmap.borrow_mut().insert(key, WithDepNode::new(dep_node, value));
+    }
+
+    pub fn insert_same(&self, key: Key, dep_node: DepNodeIndex, value: Value)
+    where
+        Value: Eq,
+    {
+        self.hashmap.borrow_mut().insert_same(key, WithDepNode::new(dep_node, value));
+    }
+}
+
+#[derive(Clone, Eq, PartialEq)]
+pub struct WithDepNode<T> {
+    dep_node: DepNodeIndex,
+    cached_value: T,
+}
+
+impl<T: Clone> WithDepNode<T> {
+    pub fn new(dep_node: DepNodeIndex, cached_value: T) -> Self {
+        WithDepNode { dep_node, cached_value }
+    }
+
+    pub fn get<CTX: QueryContext>(&self, tcx: CTX) -> T {
+        tcx.dep_graph().read_index(self.dep_node);
+        self.cached_value.clone()
+    }
+}

compiler/rustc_query_system/src/dep_graph/README.md

@@ -0,0 +1,4 @@
+To learn more about how dependency tracking works in rustc, see the [rustc
+guide].
+
+[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html

compiler/rustc_query_system/src/dep_graph/debug.rs

@@ -0,0 +1,58 @@
+//! Code for debugging the dep-graph.
+
+use super::{DepKind, DepNode};
+use std::error::Error;
+
+/// A dep-node filter goes from a user-defined string to a query over
+/// nodes. Right now the format is like this:
+///
+///     x & y & z
+///
+/// where the format-string of the dep-node must contain `x`, `y`, and
+/// `z`.
+#[derive(Debug)]
+pub struct DepNodeFilter {
+    text: String,
+}
+
+impl DepNodeFilter {
+    pub fn new(text: &str) -> Self {
+        DepNodeFilter { text: text.trim().to_string() }
+    }
+
+    /// Returns `true` if all nodes always pass the filter.
+    pub fn accepts_all(&self) -> bool {
+        self.text.is_empty()
+    }
+
+    /// Tests whether `node` meets the filter, returning true if so.
+    pub fn test<K: DepKind>(&self, node: &DepNode<K>) -> bool {
+        let debug_str = format!("{:?}", node);
+        self.text.split('&').map(|s| s.trim()).all(|f| debug_str.contains(f))
+    }
+}
+
+/// A filter like `F -> G` where `F` and `G` are valid dep-node
+/// filters. This can be used to test the source/target independently.
+pub struct EdgeFilter {
+    pub source: DepNodeFilter,
+    pub target: DepNodeFilter,
+}
+
+impl EdgeFilter {
+    pub fn new(test: &str) -> Result<EdgeFilter, Box<dyn Error>> {
+        let parts: Vec<_> = test.split("->").collect();
+        if parts.len() != 2 {
+            Err(format!("expected a filter like `a&b -> c&d`, not `{}`", test).into())
+        } else {
+            Ok(EdgeFilter {
+                source: DepNodeFilter::new(parts[0]),
+                target: DepNodeFilter::new(parts[1]),
+            })
+        }
+    }
+
+    pub fn test<K: DepKind>(&self, source: &DepNode<K>, target: &DepNode<K>) -> bool {
+        self.source.test(source) && self.target.test(target)
+    }
+}

compiler/rustc_query_system/src/dep_graph/dep_node.rs

@@ -0,0 +1,179 @@
+//! This module defines the `DepNode` type which the compiler uses to represent
+//! nodes in the dependency graph. A `DepNode` consists of a `DepKind` (which
+//! specifies the kind of thing it represents, like a piece of HIR, MIR, etc)
+//! and a `Fingerprint`, a 128 bit hash value the exact meaning of which
+//! depends on the node's `DepKind`. Together, the kind and the fingerprint
+//! fully identify a dependency node, even across multiple compilation sessions.
+//! In other words, the value of the fingerprint does not depend on anything
+//! that is specific to a given compilation session, like an unpredictable
+//! interning key (e.g., NodeId, DefId, Symbol) or the numeric value of a
+//! pointer. The concept behind this could be compared to how git commit hashes
+//! uniquely identify a given commit and has a few advantages:
+//!
+//! * A `DepNode` can simply be serialized to disk and loaded in another session
+//!   without the need to do any "rebasing (like we have to do for Spans and
+//!   NodeIds) or "retracing" like we had to do for `DefId` in earlier
+//!   implementations of the dependency graph.
+//! * A `Fingerprint` is just a bunch of bits, which allows `DepNode` to
+//!   implement `Copy`, `Sync`, `Send`, `Freeze`, etc.
+//! * Since we just have a bit pattern, `DepNode` can be mapped from disk into
+//!   memory without any post-processing (e.g., "abomination-style" pointer
+//!   reconstruction).
+//! * Because a `DepNode` is self-contained, we can instantiate `DepNodes` that
+//!   refer to things that do not exist anymore. In previous implementations
+//!   `DepNode` contained a `DefId`. A `DepNode` referring to something that
+//!   had been removed between the previous and the current compilation session
+//!   could not be instantiated because the current compilation session
+//!   contained no `DefId` for thing that had been removed.
+//!
+//! `DepNode` definition happens in `librustc_middle` with the `define_dep_nodes!()` macro.
+//! This macro defines the `DepKind` enum and a corresponding `DepConstructor` enum. The
+//! `DepConstructor` enum links a `DepKind` to the parameters that are needed at runtime in order
+//! to construct a valid `DepNode` fingerprint.
+//!
+//! Because the macro sees what parameters a given `DepKind` requires, it can
+//! "infer" some properties for each kind of `DepNode`:
+//!
+//! * Whether a `DepNode` of a given kind has any parameters at all. Some
+//!   `DepNode`s could represent global concepts with only one value.
+//! * Whether it is possible, in principle, to reconstruct a query key from a
+//!   given `DepNode`. Many `DepKind`s only require a single `DefId` parameter,
+//!   in which case it is possible to map the node's fingerprint back to the
+//!   `DefId` it was computed from. In other cases, too much information gets
+//!   lost during fingerprint computation.
+
+use super::{DepContext, DepKind};
+
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+
+use std::fmt;
+use std::hash::Hash;
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)]
+pub struct DepNode<K> {
+    pub kind: K,
+    pub hash: Fingerprint,
+}
+
+impl<K: DepKind> DepNode<K> {
+    /// Creates a new, parameterless DepNode. This method will assert
+    /// that the DepNode corresponding to the given DepKind actually
+    /// does not require any parameters.
+    pub fn new_no_params(kind: K) -> DepNode<K> {
+        debug_assert!(!kind.has_params());
+        DepNode { kind, hash: Fingerprint::ZERO }
+    }
+
+    pub fn construct<Ctxt, Key>(tcx: Ctxt, kind: K, arg: &Key) -> DepNode<K>
+    where
+        Ctxt: crate::query::QueryContext<DepKind = K>,
+        Key: DepNodeParams<Ctxt>,
+    {
+        let hash = arg.to_fingerprint(tcx);
+        let dep_node = DepNode { kind, hash };
+
+        #[cfg(debug_assertions)]
+        {
+            if !kind.can_reconstruct_query_key() && tcx.debug_dep_node() {
+                tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx));
+            }
+        }
+
+        dep_node
+    }
+}
+
+impl<K: DepKind> fmt::Debug for DepNode<K> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        K::debug_node(self, f)
+    }
+}
+
+pub trait DepNodeParams<Ctxt: DepContext>: fmt::Debug + Sized {
+    fn can_reconstruct_query_key() -> bool;
+
+    /// This method turns the parameters of a DepNodeConstructor into an opaque
+    /// Fingerprint to be used in DepNode.
+    /// Not all DepNodeParams support being turned into a Fingerprint (they
+    /// don't need to if the corresponding DepNode is anonymous).
+    fn to_fingerprint(&self, _: Ctxt) -> Fingerprint {
+        panic!("Not implemented. Accidentally called on anonymous node?")
+    }
+
+    fn to_debug_str(&self, _: Ctxt) -> String {
+        format!("{:?}", self)
+    }
+
+    /// This method tries to recover the query key from the given `DepNode`,
+    /// something which is needed when forcing `DepNode`s during red-green
+    /// evaluation. The query system will only call this method if
+    /// `can_reconstruct_query_key()` is `true`.
+    /// It is always valid to return `None` here, in which case incremental
+    /// compilation will treat the query as having changed instead of forcing it.
+    fn recover(tcx: Ctxt, dep_node: &DepNode<Ctxt::DepKind>) -> Option<Self>;
+}
+
+impl<Ctxt: DepContext, T> DepNodeParams<Ctxt> for T
+where
+    T: HashStable<Ctxt::StableHashingContext> + fmt::Debug,
+{
+    #[inline]
+    default fn can_reconstruct_query_key() -> bool {
+        false
+    }
+
+    default fn to_fingerprint(&self, tcx: Ctxt) -> Fingerprint {
+        let mut hcx = tcx.create_stable_hashing_context();
+        let mut hasher = StableHasher::new();
+
+        self.hash_stable(&mut hcx, &mut hasher);
+
+        hasher.finish()
+    }
+
+    default fn to_debug_str(&self, _: Ctxt) -> String {
+        format!("{:?}", *self)
+    }
+
+    default fn recover(_: Ctxt, _: &DepNode<Ctxt::DepKind>) -> Option<Self> {
+        None
+    }
+}
+
+impl<Ctxt: DepContext> DepNodeParams<Ctxt> for () {
+    fn to_fingerprint(&self, _: Ctxt) -> Fingerprint {
+        Fingerprint::ZERO
+    }
+}
+
+/// A "work product" corresponds to a `.o` (or other) file that we
+/// save in between runs. These IDs do not have a `DefId` but rather
+/// some independent path or string that persists between runs without
+/// the need to be mapped or unmapped. (This ensures we can serialize
+/// them even in the absence of a tcx.)
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Encodable, Decodable)]
+pub struct WorkProductId {
+    hash: Fingerprint,
+}
+
+impl WorkProductId {
+    pub fn from_cgu_name(cgu_name: &str) -> WorkProductId {
+        let mut hasher = StableHasher::new();
+        cgu_name.len().hash(&mut hasher);
+        cgu_name.hash(&mut hasher);
+        WorkProductId { hash: hasher.finish() }
+    }
+
+    pub fn from_fingerprint(fingerprint: Fingerprint) -> WorkProductId {
+        WorkProductId { hash: fingerprint }
+    }
+}
+
+impl<HCX> HashStable<HCX> for WorkProductId {
+    #[inline]
+    fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
+        self.hash.hash_stable(hcx, hasher)
+    }
+}

compiler/rustc_query_system/src/dep_graph/mod.rs

@@ -0,0 +1,85 @@
+pub mod debug;
+mod dep_node;
+mod graph;
+mod prev;
+mod query;
+mod serialized;
+
+pub use dep_node::{DepNode, DepNodeParams, WorkProductId};
+pub use graph::{hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, WorkProduct};
+pub use prev::PreviousDepGraph;
+pub use query::DepGraphQuery;
+pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
+
+use rustc_data_structures::profiling::SelfProfilerRef;
+use rustc_data_structures::sync::Lock;
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::Diagnostic;
+
+use std::fmt;
+use std::hash::Hash;
+
+pub trait DepContext: Copy {
+    type DepKind: self::DepKind;
+    type StableHashingContext;
+
+    /// Create a hashing context for hashing new results.
+    fn create_stable_hashing_context(&self) -> Self::StableHashingContext;
+
+    fn debug_dep_tasks(&self) -> bool;
+    fn debug_dep_node(&self) -> bool;
+
+    /// Try to force a dep node to execute and see if it's green.
+    fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
+
+    /// Return whether the current session is tainted by errors.
+    fn has_errors_or_delayed_span_bugs(&self) -> bool;
+
+    /// Return the diagnostic handler.
+    fn diagnostic(&self) -> &rustc_errors::Handler;
+
+    /// Load data from the on-disk cache.
+    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
+
+    /// Load diagnostics associated to the node in the previous session.
+    fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
+
+    /// Register diagnostics for the given node, for use in next session.
+    fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>);
+
+    /// Register diagnostics for the given node, for use in next session.
+    fn store_diagnostics_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        diagnostics: ThinVec<Diagnostic>,
+    );
+
+    /// Access the profiler.
+    fn profiler(&self) -> &SelfProfilerRef;
+}
+
+/// Describe the different families of dependency nodes.
+pub trait DepKind: Copy + fmt::Debug + Eq + Ord + Hash {
+    const NULL: Self;
+
+    /// Return whether this kind always require evaluation.
+    fn is_eval_always(&self) -> bool;
+
+    /// Return whether this kind requires additional parameters to be executed.
+    fn has_params(&self) -> bool;
+
+    /// Implementation of `std::fmt::Debug` for `DepNode`.
+    fn debug_node(node: &DepNode<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result;
+
+    /// Execute the operation with provided dependencies.
+    fn with_deps<OP, R>(deps: Option<&Lock<TaskDeps<Self>>>, op: OP) -> R
+    where
+        OP: FnOnce() -> R;
+
+    /// Access dependencies from current implicit context.
+    fn read_deps<OP>(op: OP)
+    where
+        OP: for<'a> FnOnce(Option<&'a Lock<TaskDeps<Self>>>);
+
+    fn can_reconstruct_query_key(&self) -> bool;
+}

compiler/rustc_query_system/src/dep_graph/prev.rs

@@ -0,0 +1,61 @@
+use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex};
+use super::{DepKind, DepNode};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::FxHashMap;
+
+#[derive(Debug, Encodable, Decodable)]
+pub struct PreviousDepGraph<K: DepKind> {
+    data: SerializedDepGraph<K>,
+    index: FxHashMap<DepNode<K>, SerializedDepNodeIndex>,
+}
+
+impl<K: DepKind> Default for PreviousDepGraph<K> {
+    fn default() -> Self {
+        PreviousDepGraph { data: Default::default(), index: Default::default() }
+    }
+}
+
+impl<K: DepKind> PreviousDepGraph<K> {
+    pub fn new(data: SerializedDepGraph<K>) -> PreviousDepGraph<K> {
+        let index: FxHashMap<_, _> =
+            data.nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect();
+        PreviousDepGraph { data, index }
+    }
+
+    #[inline]
+    pub fn edge_targets_from(
+        &self,
+        dep_node_index: SerializedDepNodeIndex,
+    ) -> &[SerializedDepNodeIndex] {
+        self.data.edge_targets_from(dep_node_index)
+    }
+
+    #[inline]
+    pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode<K> {
+        self.data.nodes[dep_node_index]
+    }
+
+    #[inline]
+    pub fn node_to_index(&self, dep_node: &DepNode<K>) -> SerializedDepNodeIndex {
+        self.index[dep_node]
+    }
+
+    #[inline]
+    pub fn node_to_index_opt(&self, dep_node: &DepNode<K>) -> Option<SerializedDepNodeIndex> {
+        self.index.get(dep_node).cloned()
+    }
+
+    #[inline]
+    pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
+        self.index.get(dep_node).map(|&node_index| self.data.fingerprints[node_index])
+    }
+
+    #[inline]
+    pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
+        self.data.fingerprints[dep_node_index]
+    }
+
+    pub fn node_count(&self) -> usize {
+        self.index.len()
+    }
+}

compiler/rustc_query_system/src/dep_graph/query.rs

@@ -0,0 +1,74 @@
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::graph::implementation::{
+    Direction, Graph, NodeIndex, INCOMING, OUTGOING,
+};
+
+use super::{DepKind, DepNode};
+
+pub struct DepGraphQuery<K> {
+    pub graph: Graph<DepNode<K>, ()>,
+    pub indices: FxHashMap<DepNode<K>, NodeIndex>,
+}
+
+impl<K: DepKind> DepGraphQuery<K> {
+    pub fn new(nodes: &[DepNode<K>], edges: &[(DepNode<K>, DepNode<K>)]) -> DepGraphQuery<K> {
+        let mut graph = Graph::with_capacity(nodes.len(), edges.len());
+        let mut indices = FxHashMap::default();
+        for node in nodes {
+            indices.insert(*node, graph.add_node(*node));
+        }
+
+        for &(ref source, ref target) in edges {
+            let source = indices[source];
+            let target = indices[target];
+            graph.add_edge(source, target, ());
+        }
+
+        DepGraphQuery { graph, indices }
+    }
+
+    pub fn contains_node(&self, node: &DepNode<K>) -> bool {
+        self.indices.contains_key(&node)
+    }
+
+    pub fn nodes(&self) -> Vec<&DepNode<K>> {
+        self.graph.all_nodes().iter().map(|n| &n.data).collect()
+    }
+
+    pub fn edges(&self) -> Vec<(&DepNode<K>, &DepNode<K>)> {
+        self.graph
+            .all_edges()
+            .iter()
+            .map(|edge| (edge.source(), edge.target()))
+            .map(|(s, t)| (self.graph.node_data(s), self.graph.node_data(t)))
+            .collect()
+    }
+
+    fn reachable_nodes(&self, node: &DepNode<K>, direction: Direction) -> Vec<&DepNode<K>> {
+        if let Some(&index) = self.indices.get(node) {
+            self.graph.depth_traverse(index, direction).map(|s| self.graph.node_data(s)).collect()
+        } else {
+            vec![]
+        }
+    }
+
+    /// All nodes reachable from `node`. In other words, things that
+    /// will have to be recomputed if `node` changes.
+    pub fn transitive_successors(&self, node: &DepNode<K>) -> Vec<&DepNode<K>> {
+        self.reachable_nodes(node, OUTGOING)
+    }
+
+    /// All nodes that can reach `node`.
+    pub fn transitive_predecessors(&self, node: &DepNode<K>) -> Vec<&DepNode<K>> {
+        self.reachable_nodes(node, INCOMING)
+    }
+
+    /// Just the outgoing edges from `node`.
+    pub fn immediate_successors(&self, node: &DepNode<K>) -> Vec<&DepNode<K>> {
+        if let Some(&index) = self.indices.get(&node) {
+            self.graph.successor_nodes(index).map(|s| self.graph.node_data(s)).collect()
+        } else {
+            vec![]
+        }
+    }
+}

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -0,0 +1,45 @@
+//! The data that we will serialize and deserialize.
+
+use super::{DepKind, DepNode};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_index::vec::IndexVec;
+
+rustc_index::newtype_index! {
+    pub struct SerializedDepNodeIndex { .. }
+}
+
+/// Data for use when recompiling the **current crate**.
+#[derive(Debug, Encodable, Decodable)]
+pub struct SerializedDepGraph<K: DepKind> {
+    /// The set of all DepNodes in the graph
+    pub nodes: IndexVec<SerializedDepNodeIndex, DepNode<K>>,
+    /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to
+    /// the DepNode at the same index in the nodes vector.
+    pub fingerprints: IndexVec<SerializedDepNodeIndex, Fingerprint>,
+    /// For each DepNode, stores the list of edges originating from that
+    /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
+    /// which holds the actual DepNodeIndices of the target nodes.
+    pub edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>,
+    /// A flattened list of all edge targets in the graph. Edge sources are
+    /// implicit in edge_list_indices.
+    pub edge_list_data: Vec<SerializedDepNodeIndex>,
+}
+
+impl<K: DepKind> Default for SerializedDepGraph<K> {
+    fn default() -> Self {
+        SerializedDepGraph {
+            nodes: Default::default(),
+            fingerprints: Default::default(),
+            edge_list_indices: Default::default(),
+            edge_list_data: Default::default(),
+        }
+    }
+}
+
+impl<K: DepKind> SerializedDepGraph<K> {
+    #[inline]
+    pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] {
+        let targets = self.edge_list_indices[source];
+        &self.edge_list_data[targets.0 as usize..targets.1 as usize]
+    }
+}

compiler/rustc_query_system/src/lib.rs

@@ -0,0 +1,18 @@
+#![feature(bool_to_option)]
+#![feature(const_fn)]
+#![feature(const_panic)]
+#![feature(core_intrinsics)]
+#![feature(hash_raw_entry)]
+#![feature(min_specialization)]
+#![feature(stmt_expr_attributes)]
+
+#[macro_use]
+extern crate tracing;
+#[macro_use]
+extern crate rustc_data_structures;
+#[macro_use]
+extern crate rustc_macros;
+
+pub mod cache;
+pub mod dep_graph;
+pub mod query;

compiler/rustc_query_system/src/query/README.md

@@ -0,0 +1,3 @@
+For more information about how the query system works, see the [rustc dev guide].
+
+[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html

compiler/rustc_query_system/src/query/caches.rs

@@ -0,0 +1,220 @@
+use crate::dep_graph::DepNodeIndex;
+use crate::query::plumbing::{QueryLookup, QueryState};
+use crate::query::QueryContext;
+
+use rustc_arena::TypedArena;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sharded::Sharded;
+use rustc_data_structures::sync::WorkerLocal;
+use std::default::Default;
+use std::hash::Hash;
+use std::marker::PhantomData;
+
+pub trait CacheSelector<K, V> {
+    type Cache;
+}
+
+pub trait QueryStorage: Default {
+    type Value;
+    type Stored: Clone;
+
+    /// Store a value without putting it in the cache.
+    /// This is meant to be used with cycle errors.
+    fn store_nocache(&self, value: Self::Value) -> Self::Stored;
+}
+
+pub trait QueryCache: QueryStorage {
+    type Key: Hash;
+    type Sharded: Default;
+
+    /// Checks if the query is already computed and in the cache.
+    /// It returns the shard index and a lock guard to the shard,
+    /// which will be used if the query is not in the cache and we need
+    /// to compute it.
+    fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
+        &self,
+        state: &QueryState<CTX, Self>,
+        key: Self::Key,
+        // `on_hit` can be called while holding a lock to the query state shard.
+        on_hit: OnHit,
+        on_miss: OnMiss,
+    ) -> R
+    where
+        OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R,
+        OnMiss: FnOnce(Self::Key, QueryLookup<'_, CTX, Self::Key, Self::Sharded>) -> R;
+
+    fn complete(
+        &self,
+        lock_sharded_storage: &mut Self::Sharded,
+        key: Self::Key,
+        value: Self::Value,
+        index: DepNodeIndex,
+    ) -> Self::Stored;
+
+    fn iter<R, L>(
+        &self,
+        shards: &Sharded<L>,
+        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        f: impl for<'a> FnOnce(
+            Box<dyn Iterator<Item = (&'a Self::Key, &'a Self::Value, DepNodeIndex)> + 'a>,
+        ) -> R,
+    ) -> R;
+}
+
+pub struct DefaultCacheSelector;
+
+impl<K: Eq + Hash, V: Clone> CacheSelector<K, V> for DefaultCacheSelector {
+    type Cache = DefaultCache<K, V>;
+}
+
+pub struct DefaultCache<K, V>(PhantomData<(K, V)>);
+
+impl<K, V> Default for DefaultCache<K, V> {
+    fn default() -> Self {
+        DefaultCache(PhantomData)
+    }
+}
+
+impl<K: Eq + Hash, V: Clone> QueryStorage for DefaultCache<K, V> {
+    type Value = V;
+    type Stored = V;
+
+    #[inline]
+    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
+        // We have no dedicated storage
+        value
+    }
+}
+
+impl<K: Eq + Hash, V: Clone> QueryCache for DefaultCache<K, V> {
+    type Key = K;
+    type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
+
+    #[inline(always)]
+    fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
+        &self,
+        state: &QueryState<CTX, Self>,
+        key: K,
+        on_hit: OnHit,
+        on_miss: OnMiss,
+    ) -> R
+    where
+        OnHit: FnOnce(&V, DepNodeIndex) -> R,
+        OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
+    {
+        let mut lookup = state.get_lookup(&key);
+        let lock = &mut *lookup.lock;
+
+        let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
+
+        if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
+    }
+
+    #[inline]
+    fn complete(
+        &self,
+        lock_sharded_storage: &mut Self::Sharded,
+        key: K,
+        value: V,
+        index: DepNodeIndex,
+    ) -> Self::Stored {
+        lock_sharded_storage.insert(key, (value.clone(), index));
+        value
+    }
+
+    fn iter<R, L>(
+        &self,
+        shards: &Sharded<L>,
+        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+    ) -> R {
+        let mut shards = shards.lock_shards();
+        let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
+        let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+        f(Box::new(results))
+    }
+}
+
+pub struct ArenaCacheSelector<'tcx>(PhantomData<&'tcx ()>);
+
+impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<K, V> for ArenaCacheSelector<'tcx> {
+    type Cache = ArenaCache<'tcx, K, V>;
+}
+
+pub struct ArenaCache<'tcx, K, V> {
+    arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
+    phantom: PhantomData<(K, &'tcx V)>,
+}
+
+impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
+    fn default() -> Self {
+        ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), phantom: PhantomData }
+    }
+}
+
+impl<'tcx, K: Eq + Hash, V: 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
+    type Value = V;
+    type Stored = &'tcx V;
+
+    #[inline]
+    fn store_nocache(&self, value: Self::Value) -> Self::Stored {
+        let value = self.arena.alloc((value, DepNodeIndex::INVALID));
+        let value = unsafe { &*(&value.0 as *const _) };
+        &value
+    }
+}
+
+impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> {
+    type Key = K;
+    type Sharded = FxHashMap<K, &'tcx (V, DepNodeIndex)>;
+
+    #[inline(always)]
+    fn lookup<CTX: QueryContext, R, OnHit, OnMiss>(
+        &self,
+        state: &QueryState<CTX, Self>,
+        key: K,
+        on_hit: OnHit,
+        on_miss: OnMiss,
+    ) -> R
+    where
+        OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
+        OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R,
+    {
+        let mut lookup = state.get_lookup(&key);
+        let lock = &mut *lookup.lock;
+
+        let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
+
+        if let Some((_, value)) = result {
+            on_hit(&&value.0, value.1)
+        } else {
+            on_miss(key, lookup)
+        }
+    }
+
+    #[inline]
+    fn complete(
+        &self,
+        lock_sharded_storage: &mut Self::Sharded,
+        key: K,
+        value: V,
+        index: DepNodeIndex,
+    ) -> Self::Stored {
+        let value = self.arena.alloc((value, index));
+        let value = unsafe { &*(value as *const _) };
+        lock_sharded_storage.insert(key, value);
+        &value.0
+    }
+
+    fn iter<R, L>(
+        &self,
+        shards: &Sharded<L>,
+        get_shard: impl Fn(&mut L) -> &mut Self::Sharded,
+        f: impl for<'a> FnOnce(Box<dyn Iterator<Item = (&'a K, &'a V, DepNodeIndex)> + 'a>) -> R,
+    ) -> R {
+        let mut shards = shards.lock_shards();
+        let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect();
+        let results = shards.iter_mut().flat_map(|shard| shard.iter()).map(|(k, v)| (k, &v.0, v.1));
+        f(Box::new(results))
+    }
+}

compiler/rustc_query_system/src/query/config.rs

@@ -0,0 +1,133 @@
+//! Query configuration and description traits.
+
+use crate::dep_graph::DepNode;
+use crate::dep_graph::SerializedDepNodeIndex;
+use crate::query::caches::QueryCache;
+use crate::query::plumbing::CycleError;
+use crate::query::{QueryContext, QueryState};
+use rustc_data_structures::profiling::ProfileCategory;
+
+use rustc_data_structures::fingerprint::Fingerprint;
+use std::borrow::Cow;
+use std::fmt::Debug;
+use std::hash::Hash;
+
+// The parameter `CTX` is required in librustc_middle:
+// implementations may need to access the `'tcx` lifetime in `CTX = TyCtxt<'tcx>`.
+pub trait QueryConfig<CTX> {
+    const NAME: &'static str;
+    const CATEGORY: ProfileCategory;
+
+    type Key: Eq + Hash + Clone + Debug;
+    type Value;
+    type Stored: Clone;
+}
+
+pub(crate) struct QueryVtable<CTX: QueryContext, K, V> {
+    pub anon: bool,
+    pub dep_kind: CTX::DepKind,
+    pub eval_always: bool,
+
+    // Don't use this method to compute query results, instead use the methods on TyCtxt
+    pub compute: fn(CTX, K) -> V,
+
+    pub hash_result: fn(&mut CTX::StableHashingContext, &V) -> Option<Fingerprint>,
+    pub handle_cycle_error: fn(CTX, CycleError<CTX::Query>) -> V,
+    pub cache_on_disk: fn(CTX, &K, Option<&V>) -> bool,
+    pub try_load_from_disk: fn(CTX, SerializedDepNodeIndex) -> Option<V>,
+}
+
+impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
+    pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode<CTX::DepKind>
+    where
+        K: crate::dep_graph::DepNodeParams<CTX>,
+    {
+        DepNode::construct(tcx, self.dep_kind, key)
+    }
+
+    pub(crate) fn compute(&self, tcx: CTX, key: K) -> V {
+        (self.compute)(tcx, key)
+    }
+
+    pub(crate) fn hash_result(
+        &self,
+        hcx: &mut CTX::StableHashingContext,
+        value: &V,
+    ) -> Option<Fingerprint> {
+        (self.hash_result)(hcx, value)
+    }
+
+    pub(crate) fn handle_cycle_error(&self, tcx: CTX, error: CycleError<CTX::Query>) -> V {
+        (self.handle_cycle_error)(tcx, error)
+    }
+
+    pub(crate) fn cache_on_disk(&self, tcx: CTX, key: &K, value: Option<&V>) -> bool {
+        (self.cache_on_disk)(tcx, key, value)
+    }
+
+    pub(crate) fn try_load_from_disk(&self, tcx: CTX, index: SerializedDepNodeIndex) -> Option<V> {
+        (self.try_load_from_disk)(tcx, index)
+    }
+}
+
+pub trait QueryAccessors<CTX: QueryContext>: QueryConfig<CTX> {
+    const ANON: bool;
+    const EVAL_ALWAYS: bool;
+    const DEP_KIND: CTX::DepKind;
+
+    type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
+
+    // Don't use this method to access query results, instead use the methods on TyCtxt
+    fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX, Self::Cache>;
+
+    fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>
+    where
+        Self::Key: crate::dep_graph::DepNodeParams<CTX>,
+    {
+        DepNode::construct(tcx, Self::DEP_KIND, key)
+    }
+
+    // Don't use this method to compute query results, instead use the methods on TyCtxt
+    fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
+
+    fn hash_result(
+        hcx: &mut CTX::StableHashingContext,
+        result: &Self::Value,
+    ) -> Option<Fingerprint>;
+
+    fn handle_cycle_error(tcx: CTX, error: CycleError<CTX::Query>) -> Self::Value;
+}
+
+pub trait QueryDescription<CTX: QueryContext>: QueryAccessors<CTX> {
+    fn describe(tcx: CTX, key: Self::Key) -> Cow<'static, str>;
+
+    #[inline]
+    fn cache_on_disk(_: CTX, _: &Self::Key, _: Option<&Self::Value>) -> bool {
+        false
+    }
+
+    fn try_load_from_disk(_: CTX, _: SerializedDepNodeIndex) -> Option<Self::Value> {
+        panic!("QueryDescription::load_from_disk() called for an unsupported query.")
+    }
+}
+
+pub(crate) trait QueryVtableExt<CTX: QueryContext, K, V> {
+    const VTABLE: QueryVtable<CTX, K, V>;
+}
+
+impl<CTX, Q> QueryVtableExt<CTX, Q::Key, Q::Value> for Q
+where
+    CTX: QueryContext,
+    Q: QueryDescription<CTX>,
+{
+    const VTABLE: QueryVtable<CTX, Q::Key, Q::Value> = QueryVtable {
+        anon: Q::ANON,
+        dep_kind: Q::DEP_KIND,
+        eval_always: Q::EVAL_ALWAYS,
+        compute: Q::compute,
+        hash_result: Q::hash_result,
+        handle_cycle_error: Q::handle_cycle_error,
+        cache_on_disk: Q::cache_on_disk,
+        try_load_from_disk: Q::try_load_from_disk,
+    };
+}

compiler/rustc_query_system/src/query/job.rs

@@ -0,0 +1,570 @@
+use crate::dep_graph::{DepContext, DepKind};
+use crate::query::plumbing::CycleError;
+use crate::query::QueryContext;
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_span::Span;
+
+use std::convert::TryFrom;
+use std::marker::PhantomData;
+use std::num::NonZeroU32;
+
+#[cfg(parallel_compiler)]
+use {
+    parking_lot::{Condvar, Mutex},
+    rustc_data_structures::fx::FxHashSet,
+    rustc_data_structures::stable_hasher::{HashStable, StableHasher},
+    rustc_data_structures::sync::Lock,
+    rustc_data_structures::sync::Lrc,
+    rustc_data_structures::{jobserver, OnDrop},
+    rustc_rayon_core as rayon_core,
+    rustc_span::DUMMY_SP,
+    std::iter::FromIterator,
+    std::{mem, process},
+};
+
+/// Represents a span and a query key.
+#[derive(Clone, Debug)]
+pub struct QueryInfo<Q> {
+    /// The span corresponding to the reason for which this query was required.
+    pub span: Span,
+    pub query: Q,
+}
+
+type QueryMap<CTX> = FxHashMap<QueryJobId<<CTX as DepContext>::DepKind>, QueryJobInfo<CTX>>;
+
+/// A value uniquely identifiying an active query job within a shard in the query cache.
+#[derive(Copy, Clone, Eq, PartialEq, Hash)]
+pub struct QueryShardJobId(pub NonZeroU32);
+
+/// A value uniquely identifiying an active query job.
+#[derive(Copy, Clone, Eq, PartialEq, Hash)]
+pub struct QueryJobId<K> {
+    /// Which job within a shard is this
+    pub job: QueryShardJobId,
+
+    /// In which shard is this job
+    pub shard: u16,
+
+    /// What kind of query this job is
+    pub kind: K,
+}
+
+impl<K: DepKind> QueryJobId<K> {
+    pub fn new(job: QueryShardJobId, shard: usize, kind: K) -> Self {
+        QueryJobId { job, shard: u16::try_from(shard).unwrap(), kind }
+    }
+
+    fn query<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> CTX::Query {
+        map.get(&self).unwrap().info.query.clone()
+    }
+
+    #[cfg(parallel_compiler)]
+    fn span<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Span {
+        map.get(&self).unwrap().job.span
+    }
+
+    #[cfg(parallel_compiler)]
+    fn parent<CTX: QueryContext<DepKind = K>>(self, map: &QueryMap<CTX>) -> Option<QueryJobId<K>> {
+        map.get(&self).unwrap().job.parent
+    }
+
+    #[cfg(parallel_compiler)]
+    fn latch<'a, CTX: QueryContext<DepKind = K>>(
+        self,
+        map: &'a QueryMap<CTX>,
+    ) -> Option<&'a QueryLatch<CTX>> {
+        map.get(&self).unwrap().job.latch.as_ref()
+    }
+}
+
+pub struct QueryJobInfo<CTX: QueryContext> {
+    pub info: QueryInfo<CTX::Query>,
+    pub job: QueryJob<CTX>,
+}
+
+/// Represents an active query job.
+#[derive(Clone)]
+pub struct QueryJob<CTX: QueryContext> {
+    pub id: QueryShardJobId,
+
+    /// The span corresponding to the reason for which this query was required.
+    pub span: Span,
+
+    /// The parent query job which created this job and is implicitly waiting on it.
+    pub parent: Option<QueryJobId<CTX::DepKind>>,
+
+    /// The latch that is used to wait on this job.
+    #[cfg(parallel_compiler)]
+    latch: Option<QueryLatch<CTX>>,
+
+    dummy: PhantomData<QueryLatch<CTX>>,
+}
+
+impl<CTX: QueryContext> QueryJob<CTX> {
+    /// Creates a new query job.
+    pub fn new(id: QueryShardJobId, span: Span, parent: Option<QueryJobId<CTX::DepKind>>) -> Self {
+        QueryJob {
+            id,
+            span,
+            parent,
+            #[cfg(parallel_compiler)]
+            latch: None,
+            dummy: PhantomData,
+        }
+    }
+
+    #[cfg(parallel_compiler)]
+    pub(super) fn latch(&mut self, _id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
+        if self.latch.is_none() {
+            self.latch = Some(QueryLatch::new());
+        }
+        self.latch.as_ref().unwrap().clone()
+    }
+
+    #[cfg(not(parallel_compiler))]
+    pub(super) fn latch(&mut self, id: QueryJobId<CTX::DepKind>) -> QueryLatch<CTX> {
+        QueryLatch { id, dummy: PhantomData }
+    }
+
+    /// Signals to waiters that the query is complete.
+    ///
+    /// This does nothing for single threaded rustc,
+    /// as there are no concurrent jobs which could be waiting on us
+    pub fn signal_complete(self) {
+        #[cfg(parallel_compiler)]
+        {
+            if let Some(latch) = self.latch {
+                latch.set();
+            }
+        }
+    }
+}
+
+#[cfg(not(parallel_compiler))]
+#[derive(Clone)]
+pub(super) struct QueryLatch<CTX: QueryContext> {
+    id: QueryJobId<CTX::DepKind>,
+    dummy: PhantomData<CTX>,
+}
+
+#[cfg(not(parallel_compiler))]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+    pub(super) fn find_cycle_in_stack(&self, tcx: CTX, span: Span) -> CycleError<CTX::Query> {
+        let query_map = tcx.try_collect_active_jobs().unwrap();
+
+        // Get the current executing query (waiter) and find the waitee amongst its parents
+        let mut current_job = tcx.current_query_job();
+        let mut cycle = Vec::new();
+
+        while let Some(job) = current_job {
+            let info = query_map.get(&job).unwrap();
+            cycle.push(info.info.clone());
+
+            if job == self.id {
+                cycle.reverse();
+
+                // This is the end of the cycle
+                // The span entry we included was for the usage
+                // of the cycle itself, and not part of the cycle
+                // Replace it with the span which caused the cycle to form
+                cycle[0].span = span;
+                // Find out why the cycle itself was used
+                let usage = info
+                    .job
+                    .parent
+                    .as_ref()
+                    .map(|parent| (info.info.span, parent.query(&query_map)));
+                return CycleError { usage, cycle };
+            }
+
+            current_job = info.job.parent;
+        }
+
+        panic!("did not find a cycle")
+    }
+}
+
+#[cfg(parallel_compiler)]
+struct QueryWaiter<CTX: QueryContext> {
+    query: Option<QueryJobId<CTX::DepKind>>,
+    condvar: Condvar,
+    span: Span,
+    cycle: Lock<Option<CycleError<CTX::Query>>>,
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryWaiter<CTX> {
+    fn notify(&self, registry: &rayon_core::Registry) {
+        rayon_core::mark_unblocked(registry);
+        self.condvar.notify_one();
+    }
+}
+
+#[cfg(parallel_compiler)]
+struct QueryLatchInfo<CTX: QueryContext> {
+    complete: bool,
+    waiters: Vec<Lrc<QueryWaiter<CTX>>>,
+}
+
+#[cfg(parallel_compiler)]
+#[derive(Clone)]
+pub(super) struct QueryLatch<CTX: QueryContext> {
+    info: Lrc<Mutex<QueryLatchInfo<CTX>>>,
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+    fn new() -> Self {
+        QueryLatch {
+            info: Lrc::new(Mutex::new(QueryLatchInfo { complete: false, waiters: Vec::new() })),
+        }
+    }
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+    /// Awaits for the query job to complete.
+    pub(super) fn wait_on(&self, tcx: CTX, span: Span) -> Result<(), CycleError<CTX::Query>> {
+        let query = tcx.current_query_job();
+        let waiter =
+            Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() });
+        self.wait_on_inner(&waiter);
+        // FIXME: Get rid of this lock. We have ownership of the QueryWaiter
+        // although another thread may still have a Lrc reference so we cannot
+        // use Lrc::get_mut
+        let mut cycle = waiter.cycle.lock();
+        match cycle.take() {
+            None => Ok(()),
+            Some(cycle) => Err(cycle),
+        }
+    }
+}
+
+#[cfg(parallel_compiler)]
+impl<CTX: QueryContext> QueryLatch<CTX> {
+    /// Awaits the caller on this latch by blocking the current thread.
+    fn wait_on_inner(&self, waiter: &Lrc<QueryWaiter<CTX>>) {
+        let mut info = self.info.lock();
+        if !info.complete {
+            // We push the waiter on to the `waiters` list. It can be accessed inside
+            // the `wait` call below, by 1) the `set` method or 2) by deadlock detection.
+            // Both of these will remove it from the `waiters` list before resuming
+            // this thread.
+            info.waiters.push(waiter.clone());
+
+            // If this detects a deadlock and the deadlock handler wants to resume this thread
+            // we have to be in the `wait` call. This is ensured by the deadlock handler
+            // getting the self.info lock.
+            rayon_core::mark_blocked();
+            jobserver::release_thread();
+            waiter.condvar.wait(&mut info);
+            // Release the lock before we potentially block in `acquire_thread`
+            mem::drop(info);
+            jobserver::acquire_thread();
+        }
+    }
+
+    /// Sets the latch and resumes all waiters on it
+    fn set(&self) {
+        let mut info = self.info.lock();
+        debug_assert!(!info.complete);
+        info.complete = true;
+        let registry = rayon_core::Registry::current();
+        for waiter in info.waiters.drain(..) {
+            waiter.notify(&registry);
+        }
+    }
+
+    /// Removes a single waiter from the list of waiters.
+    /// This is used to break query cycles.
+    fn extract_waiter(&self, waiter: usize) -> Lrc<QueryWaiter<CTX>> {
+        let mut info = self.info.lock();
+        debug_assert!(!info.complete);
+        // Remove the waiter from the list of waiters
+        info.waiters.remove(waiter)
+    }
+}
+
+/// A resumable waiter of a query. The usize is the index into waiters in the query's latch
+#[cfg(parallel_compiler)]
+type Waiter<K> = (QueryJobId<K>, usize);
+
+/// Visits all the non-resumable and resumable waiters of a query.
+/// Only waiters in a query are visited.
+/// `visit` is called for every waiter and is passed a query waiting on `query_ref`
+/// and a span indicating the reason the query waited on `query_ref`.
+/// If `visit` returns Some, this function returns.
+/// For visits of non-resumable waiters it returns the return value of `visit`.
+/// For visits of resumable waiters it returns Some(Some(Waiter)) which has the
+/// required information to resume the waiter.
+/// If all `visit` calls returns None, this function also returns None.
+#[cfg(parallel_compiler)]
+fn visit_waiters<CTX: QueryContext, F>(
+    query_map: &QueryMap<CTX>,
+    query: QueryJobId<CTX::DepKind>,
+    mut visit: F,
+) -> Option<Option<Waiter<CTX::DepKind>>>
+where
+    F: FnMut(Span, QueryJobId<CTX::DepKind>) -> Option<Option<Waiter<CTX::DepKind>>>,
+{
+    // Visit the parent query which is a non-resumable waiter since it's on the same stack
+    if let Some(parent) = query.parent(query_map) {
+        if let Some(cycle) = visit(query.span(query_map), parent) {
+            return Some(cycle);
+        }
+    }
+
+    // Visit the explicit waiters which use condvars and are resumable
+    if let Some(latch) = query.latch(query_map) {
+        for (i, waiter) in latch.info.lock().waiters.iter().enumerate() {
+            if let Some(waiter_query) = waiter.query {
+                if visit(waiter.span, waiter_query).is_some() {
+                    // Return a value which indicates that this waiter can be resumed
+                    return Some(Some((query, i)));
+                }
+            }
+        }
+    }
+
+    None
+}
+
+/// Look for query cycles by doing a depth first search starting at `query`.
+/// `span` is the reason for the `query` to execute. This is initially DUMMY_SP.
+/// If a cycle is detected, this initial value is replaced with the span causing
+/// the cycle.
+#[cfg(parallel_compiler)]
+fn cycle_check<CTX: QueryContext>(
+    query_map: &QueryMap<CTX>,
+    query: QueryJobId<CTX::DepKind>,
+    span: Span,
+    stack: &mut Vec<(Span, QueryJobId<CTX::DepKind>)>,
+    visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
+) -> Option<Option<Waiter<CTX::DepKind>>> {
+    if !visited.insert(query) {
+        return if let Some(p) = stack.iter().position(|q| q.1 == query) {
+            // We detected a query cycle, fix up the initial span and return Some
+
+            // Remove previous stack entries
+            stack.drain(0..p);
+            // Replace the span for the first query with the cycle cause
+            stack[0].0 = span;
+            Some(None)
+        } else {
+            None
+        };
+    }
+
+    // Query marked as visited is added it to the stack
+    stack.push((span, query));
+
+    // Visit all the waiters
+    let r = visit_waiters(query_map, query, |span, successor| {
+        cycle_check(query_map, successor, span, stack, visited)
+    });
+
+    // Remove the entry in our stack if we didn't find a cycle
+    if r.is_none() {
+        stack.pop();
+    }
+
+    r
+}
+
+/// Finds out if there's a path to the compiler root (aka. code which isn't in a query)
+/// from `query` without going through any of the queries in `visited`.
+/// This is achieved with a depth first search.
+#[cfg(parallel_compiler)]
+fn connected_to_root<CTX: QueryContext>(
+    query_map: &QueryMap<CTX>,
+    query: QueryJobId<CTX::DepKind>,
+    visited: &mut FxHashSet<QueryJobId<CTX::DepKind>>,
+) -> bool {
+    // We already visited this or we're deliberately ignoring it
+    if !visited.insert(query) {
+        return false;
+    }
+
+    // This query is connected to the root (it has no query parent), return true
+    if query.parent(query_map).is_none() {
+        return true;
+    }
+
+    visit_waiters(query_map, query, |_, successor| {
+        connected_to_root(query_map, successor, visited).then_some(None)
+    })
+    .is_some()
+}
+
+// Deterministically pick an query from a list
+#[cfg(parallel_compiler)]
+fn pick_query<'a, CTX, T, F>(query_map: &QueryMap<CTX>, tcx: CTX, queries: &'a [T], f: F) -> &'a T
+where
+    CTX: QueryContext,
+    F: Fn(&T) -> (Span, QueryJobId<CTX::DepKind>),
+{
+    // Deterministically pick an entry point
+    // FIXME: Sort this instead
+    let mut hcx = tcx.create_stable_hashing_context();
+    queries
+        .iter()
+        .min_by_key(|v| {
+            let (span, query) = f(v);
+            let mut stable_hasher = StableHasher::new();
+            query.query(query_map).hash_stable(&mut hcx, &mut stable_hasher);
+            // Prefer entry points which have valid spans for nicer error messages
+            // We add an integer to the tuple ensuring that entry points
+            // with valid spans are picked first
+            let span_cmp = if span == DUMMY_SP { 1 } else { 0 };
+            (span_cmp, stable_hasher.finish::<u64>())
+        })
+        .unwrap()
+}
+
+/// Looks for query cycles starting from the last query in `jobs`.
+/// If a cycle is found, all queries in the cycle is removed from `jobs` and
+/// the function return true.
+/// If a cycle was not found, the starting query is removed from `jobs` and
+/// the function returns false.
+#[cfg(parallel_compiler)]
+fn remove_cycle<CTX: QueryContext>(
+    query_map: &QueryMap<CTX>,
+    jobs: &mut Vec<QueryJobId<CTX::DepKind>>,
+    wakelist: &mut Vec<Lrc<QueryWaiter<CTX>>>,
+    tcx: CTX,
+) -> bool {
+    let mut visited = FxHashSet::default();
+    let mut stack = Vec::new();
+    // Look for a cycle starting with the last query in `jobs`
+    if let Some(waiter) =
+        cycle_check(query_map, jobs.pop().unwrap(), DUMMY_SP, &mut stack, &mut visited)
+    {
+        // The stack is a vector of pairs of spans and queries; reverse it so that
+        // the earlier entries require later entries
+        let (mut spans, queries): (Vec<_>, Vec<_>) = stack.into_iter().rev().unzip();
+
+        // Shift the spans so that queries are matched with the span for their waitee
+        spans.rotate_right(1);
+
+        // Zip them back together
+        let mut stack: Vec<_> = spans.into_iter().zip(queries).collect();
+
+        // Remove the queries in our cycle from the list of jobs to look at
+        for r in &stack {
+            if let Some(pos) = jobs.iter().position(|j| j == &r.1) {
+                jobs.remove(pos);
+            }
+        }
+
+        // Find the queries in the cycle which are
+        // connected to queries outside the cycle
+        let entry_points = stack
+            .iter()
+            .filter_map(|&(span, query)| {
+                if query.parent(query_map).is_none() {
+                    // This query is connected to the root (it has no query parent)
+                    Some((span, query, None))
+                } else {
+                    let mut waiters = Vec::new();
+                    // Find all the direct waiters who lead to the root
+                    visit_waiters(query_map, query, |span, waiter| {
+                        // Mark all the other queries in the cycle as already visited
+                        let mut visited = FxHashSet::from_iter(stack.iter().map(|q| q.1));
+
+                        if connected_to_root(query_map, waiter, &mut visited) {
+                            waiters.push((span, waiter));
+                        }
+
+                        None
+                    });
+                    if waiters.is_empty() {
+                        None
+                    } else {
+                        // Deterministically pick one of the waiters to show to the user
+                        let waiter = *pick_query(query_map, tcx, &waiters, |s| *s);
+                        Some((span, query, Some(waiter)))
+                    }
+                }
+            })
+            .collect::<Vec<(Span, QueryJobId<CTX::DepKind>, Option<(Span, QueryJobId<CTX::DepKind>)>)>>();
+
+        // Deterministically pick an entry point
+        let (_, entry_point, usage) = pick_query(query_map, tcx, &entry_points, |e| (e.0, e.1));
+
+        // Shift the stack so that our entry point is first
+        let entry_point_pos = stack.iter().position(|(_, query)| query == entry_point);
+        if let Some(pos) = entry_point_pos {
+            stack.rotate_left(pos);
+        }
+
+        let usage = usage.as_ref().map(|(span, query)| (*span, query.query(query_map)));
+
+        // Create the cycle error
+        let error = CycleError {
+            usage,
+            cycle: stack
+                .iter()
+                .map(|&(s, ref q)| QueryInfo { span: s, query: q.query(query_map) })
+                .collect(),
+        };
+
+        // We unwrap `waiter` here since there must always be one
+        // edge which is resumeable / waited using a query latch
+        let (waitee_query, waiter_idx) = waiter.unwrap();
+
+        // Extract the waiter we want to resume
+        let waiter = waitee_query.latch(query_map).unwrap().extract_waiter(waiter_idx);
+
+        // Set the cycle error so it will be picked up when resumed
+        *waiter.cycle.lock() = Some(error);
+
+        // Put the waiter on the list of things to resume
+        wakelist.push(waiter);
+
+        true
+    } else {
+        false
+    }
+}
+
+/// Detects query cycles by using depth first search over all active query jobs.
+/// If a query cycle is found it will break the cycle by finding an edge which
+/// uses a query latch and then resuming that waiter.
+/// There may be multiple cycles involved in a deadlock, so this searches
+/// all active queries for cycles before finally resuming all the waiters at once.
+#[cfg(parallel_compiler)]
+pub fn deadlock<CTX: QueryContext>(tcx: CTX, registry: &rayon_core::Registry) {
+    let on_panic = OnDrop(|| {
+        eprintln!("deadlock handler panicked, aborting process");
+        process::abort();
+    });
+
+    let mut wakelist = Vec::new();
+    let query_map = tcx.try_collect_active_jobs().unwrap();
+    let mut jobs: Vec<QueryJobId<CTX::DepKind>> = query_map.keys().cloned().collect();
+
+    let mut found_cycle = false;
+
+    while jobs.len() > 0 {
+        if remove_cycle(&query_map, &mut jobs, &mut wakelist, tcx) {
+            found_cycle = true;
+        }
+    }
+
+    // Check that a cycle was found. It is possible for a deadlock to occur without
+    // a query cycle if a query which can be waited on uses Rayon to do multithreading
+    // internally. Such a query (X) may be executing on 2 threads (A and B) and A may
+    // wait using Rayon on B. Rayon may then switch to executing another query (Y)
+    // which in turn will wait on X causing a deadlock. We have a false dependency from
+    // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
+    // only considers the true dependency and won't detect a cycle.
+    assert!(found_cycle);
+
+    // FIXME: Ensure this won't cause a deadlock before we return
+    for waiter in wakelist.into_iter() {
+        waiter.notify(registry);
+    }
+
+    on_panic.disable();
+}

compiler/rustc_query_system/src/query/mod.rs

@@ -0,0 +1,54 @@
+mod plumbing;
+pub use self::plumbing::*;
+
+mod job;
+#[cfg(parallel_compiler)]
+pub use self::job::deadlock;
+pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
+
+mod caches;
+pub use self::caches::{
+    ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage,
+};
+
+mod config;
+pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
+
+use crate::dep_graph::{DepContext, DepGraph};
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::stable_hasher::HashStable;
+use rustc_data_structures::sync::Lock;
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::Diagnostic;
+use rustc_span::def_id::DefId;
+
+pub trait QueryContext: DepContext {
+    type Query: Clone + HashStable<Self::StableHashingContext>;
+
+    fn incremental_verify_ich(&self) -> bool;
+    fn verbose(&self) -> bool;
+
+    /// Get string representation from DefPath.
+    fn def_path_str(&self, def_id: DefId) -> String;
+
+    /// Access the DepGraph.
+    fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
+
+    /// Get the query information from the TLS context.
+    fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>>;
+
+    fn try_collect_active_jobs(
+        &self,
+    ) -> Option<FxHashMap<QueryJobId<Self::DepKind>, QueryJobInfo<Self>>>;
+
+    /// Executes a job by changing the `ImplicitCtxt` to point to the
+    /// new query job while it executes. It returns the diagnostics
+    /// captured during execution and the actual result.
+    fn start_query<R>(
+        &self,
+        token: QueryJobId<Self::DepKind>,
+        diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
+        compute: impl FnOnce(Self) -> R,
+    ) -> R;
+}

compiler/rustc_query_system/src/query/plumbing.rs

@@ -0,0 +1,752 @@
+//! The implementation of the query system itself. This defines the macros that
+//! generate the actual methods on tcx which find and execute the provider,
+//! manage the caches, and so forth.
+
+use crate::dep_graph::{DepKind, DepNode};
+use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use crate::query::caches::QueryCache;
+use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt};
+use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId};
+use crate::query::QueryContext;
+
+#[cfg(not(parallel_compiler))]
+use rustc_data_structures::cold_path;
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::{FxHashMap, FxHasher};
+use rustc_data_structures::sharded::Sharded;
+use rustc_data_structures::sync::{Lock, LockGuard};
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::{Diagnostic, FatalError};
+use rustc_span::source_map::DUMMY_SP;
+use rustc_span::Span;
+use std::collections::hash_map::Entry;
+use std::convert::TryFrom;
+use std::fmt::Debug;
+use std::hash::{Hash, Hasher};
+use std::mem;
+use std::num::NonZeroU32;
+use std::ptr;
+#[cfg(debug_assertions)]
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+pub(super) struct QueryStateShard<CTX: QueryContext, K, C> {
+    pub(super) cache: C,
+    active: FxHashMap<K, QueryResult<CTX>>,
+
+    /// Used to generate unique ids for active jobs.
+    jobs: u32,
+}
+
+impl<CTX: QueryContext, K, C: Default> Default for QueryStateShard<CTX, K, C> {
+    fn default() -> QueryStateShard<CTX, K, C> {
+        QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 }
+    }
+}
+
+pub struct QueryState<CTX: QueryContext, C: QueryCache> {
+    cache: C,
+    shards: Sharded<QueryStateShard<CTX, C::Key, C::Sharded>>,
+    #[cfg(debug_assertions)]
+    pub cache_hits: AtomicUsize,
+}
+
+impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
+    #[inline]
+    pub(super) fn get_lookup<'tcx>(
+        &'tcx self,
+        key: &C::Key,
+    ) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> {
+        // We compute the key's hash once and then use it for both the
+        // shard lookup and the hashmap lookup. This relies on the fact
+        // that both of them use `FxHasher`.
+        let mut hasher = FxHasher::default();
+        key.hash(&mut hasher);
+        let key_hash = hasher.finish();
+
+        let shard = self.shards.get_shard_index_by_hash(key_hash);
+        let lock = self.shards.get_shard_by_index(shard).lock();
+        QueryLookup { key_hash, shard, lock }
+    }
+}
+
+/// Indicates the state of a query for a given key in a query map.
+enum QueryResult<CTX: QueryContext> {
+    /// An already executing query. The query job can be used to await for its completion.
+    Started(QueryJob<CTX>),
+
+    /// The query panicked. Queries trying to wait on this will raise a fatal error which will
+    /// silently panic.
+    Poisoned,
+}
+
+impl<CTX: QueryContext, C: QueryCache> QueryState<CTX, C> {
+    #[inline(always)]
+    pub fn iter_results<R>(
+        &self,
+        f: impl for<'a> FnOnce(
+            Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
+        ) -> R,
+    ) -> R {
+        self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
+    }
+
+    #[inline(always)]
+    pub fn all_inactive(&self) -> bool {
+        let shards = self.shards.lock_shards();
+        shards.iter().all(|shard| shard.active.is_empty())
+    }
+
+    pub fn try_collect_active_jobs(
+        &self,
+        kind: CTX::DepKind,
+        make_query: fn(C::Key) -> CTX::Query,
+        jobs: &mut FxHashMap<QueryJobId<CTX::DepKind>, QueryJobInfo<CTX>>,
+    ) -> Option<()>
+    where
+        C::Key: Clone,
+    {
+        // We use try_lock_shards here since we are called from the
+        // deadlock handler, and this shouldn't be locked.
+        let shards = self.shards.try_lock_shards()?;
+        let shards = shards.iter().enumerate();
+        jobs.extend(shards.flat_map(|(shard_id, shard)| {
+            shard.active.iter().filter_map(move |(k, v)| {
+                if let QueryResult::Started(ref job) = *v {
+                    let id =
+                        QueryJobId { job: job.id, shard: u16::try_from(shard_id).unwrap(), kind };
+                    let info = QueryInfo { span: job.span, query: make_query(k.clone()) };
+                    Some((id, QueryJobInfo { info, job: job.clone() }))
+                } else {
+                    None
+                }
+            })
+        }));
+
+        Some(())
+    }
+}
+
+impl<CTX: QueryContext, C: QueryCache> Default for QueryState<CTX, C> {
+    fn default() -> QueryState<CTX, C> {
+        QueryState {
+            cache: C::default(),
+            shards: Default::default(),
+            #[cfg(debug_assertions)]
+            cache_hits: AtomicUsize::new(0),
+        }
+    }
+}
+
+/// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
+pub struct QueryLookup<'tcx, CTX: QueryContext, K, C> {
+    pub(super) key_hash: u64,
+    shard: usize,
+    pub(super) lock: LockGuard<'tcx, QueryStateShard<CTX, K, C>>,
+}
+
+/// A type representing the responsibility to execute the job in the `job` field.
+/// This will poison the relevant query if dropped.
+struct JobOwner<'tcx, CTX: QueryContext, C>
+where
+    C: QueryCache,
+    C::Key: Eq + Hash + Clone + Debug,
+{
+    state: &'tcx QueryState<CTX, C>,
+    key: C::Key,
+    id: QueryJobId<CTX::DepKind>,
+}
+
+impl<'tcx, CTX: QueryContext, C> JobOwner<'tcx, CTX, C>
+where
+    C: QueryCache,
+    C::Key: Eq + Hash + Clone + Debug,
+{
+    /// Either gets a `JobOwner` corresponding the query, allowing us to
+    /// start executing the query, or returns with the result of the query.
+    /// This function assumes that `try_get_cached` is already called and returned `lookup`.
+    /// If the query is executing elsewhere, this will wait for it and return the result.
+    /// If the query panicked, this will silently panic.
+    ///
+    /// This function is inlined because that results in a noticeable speed-up
+    /// for some compile-time benchmarks.
+    #[inline(always)]
+    fn try_start<'a, 'b>(
+        tcx: CTX,
+        state: &'b QueryState<CTX, C>,
+        span: Span,
+        key: &C::Key,
+        mut lookup: QueryLookup<'a, CTX, C::Key, C::Sharded>,
+        query: &QueryVtable<CTX, C::Key, C::Value>,
+    ) -> TryGetJob<'b, CTX, C>
+    where
+        CTX: QueryContext,
+    {
+        let lock = &mut *lookup.lock;
+
+        let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) {
+            Entry::Occupied(mut entry) => {
+                match entry.get_mut() {
+                    QueryResult::Started(job) => {
+                        // For parallel queries, we'll block and wait until the query running
+                        // in another thread has completed. Record how long we wait in the
+                        // self-profiler.
+                        let _query_blocked_prof_timer = if cfg!(parallel_compiler) {
+                            Some(tcx.profiler().query_blocked())
+                        } else {
+                            None
+                        };
+
+                        // Create the id of the job we're waiting for
+                        let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind);
+
+                        (job.latch(id), _query_blocked_prof_timer)
+                    }
+                    QueryResult::Poisoned => FatalError.raise(),
+                }
+            }
+            Entry::Vacant(entry) => {
+                // No job entry for this query. Return a new one to be started later.
+
+                // Generate an id unique within this shard.
+                let id = lock.jobs.checked_add(1).unwrap();
+                lock.jobs = id;
+                let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
+
+                let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind);
+
+                let job = tcx.current_query_job();
+                let job = QueryJob::new(id, span, job);
+
+                entry.insert(QueryResult::Started(job));
+
+                let owner = JobOwner { state, id: global_id, key: (*key).clone() };
+                return TryGetJob::NotYetStarted(owner);
+            }
+        };
+        mem::drop(lookup.lock);
+
+        // If we are single-threaded we know that we have cycle error,
+        // so we just return the error.
+        #[cfg(not(parallel_compiler))]
+        return TryGetJob::Cycle(cold_path(|| {
+            let value = query.handle_cycle_error(tcx, latch.find_cycle_in_stack(tcx, span));
+            state.cache.store_nocache(value)
+        }));
+
+        // With parallel queries we might just have to wait on some other
+        // thread.
+        #[cfg(parallel_compiler)]
+        {
+            let result = latch.wait_on(tcx, span);
+
+            if let Err(cycle) = result {
+                let value = query.handle_cycle_error(tcx, cycle);
+                let value = state.cache.store_nocache(value);
+                return TryGetJob::Cycle(value);
+            }
+
+            let cached = try_get_cached(
+                tcx,
+                state,
+                (*key).clone(),
+                |value, index| (value.clone(), index),
+                |_, _| panic!("value must be in cache after waiting"),
+            );
+
+            if let Some(prof_timer) = _query_blocked_prof_timer.take() {
+                prof_timer.finish_with_query_invocation_id(cached.1.into());
+            }
+
+            return TryGetJob::JobCompleted(cached);
+        }
+    }
+
+    /// Completes the query by updating the query cache with the `result`,
+    /// signals the waiter and forgets the JobOwner, so it won't poison the query
+    #[inline(always)]
+    fn complete(self, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored {
+        // We can move out of `self` here because we `mem::forget` it below
+        let key = unsafe { ptr::read(&self.key) };
+        let state = self.state;
+
+        // Forget ourself so our destructor won't poison the query
+        mem::forget(self);
+
+        let (job, result) = {
+            let mut lock = state.shards.get_shard_by_value(&key).lock();
+            let job = match lock.active.remove(&key).unwrap() {
+                QueryResult::Started(job) => job,
+                QueryResult::Poisoned => panic!(),
+            };
+            let result = state.cache.complete(&mut lock.cache, key, result, dep_node_index);
+            (job, result)
+        };
+
+        job.signal_complete();
+        result
+    }
+}
+
+#[inline(always)]
+fn with_diagnostics<F, R>(f: F) -> (R, ThinVec<Diagnostic>)
+where
+    F: FnOnce(Option<&Lock<ThinVec<Diagnostic>>>) -> R,
+{
+    let diagnostics = Lock::new(ThinVec::new());
+    let result = f(Some(&diagnostics));
+    (result, diagnostics.into_inner())
+}
+
+impl<'tcx, CTX: QueryContext, C: QueryCache> Drop for JobOwner<'tcx, CTX, C>
+where
+    C::Key: Eq + Hash + Clone + Debug,
+{
+    #[inline(never)]
+    #[cold]
+    fn drop(&mut self) {
+        // Poison the query so jobs waiting on it panic.
+        let state = self.state;
+        let shard = state.shards.get_shard_by_value(&self.key);
+        let job = {
+            let mut shard = shard.lock();
+            let job = match shard.active.remove(&self.key).unwrap() {
+                QueryResult::Started(job) => job,
+                QueryResult::Poisoned => panic!(),
+            };
+            shard.active.insert(self.key.clone(), QueryResult::Poisoned);
+            job
+        };
+        // Also signal the completion of the job, so waiters
+        // will continue execution.
+        job.signal_complete();
+    }
+}
+
+#[derive(Clone)]
+pub struct CycleError<Q> {
+    /// The query and related span that uses the cycle.
+    pub usage: Option<(Span, Q)>,
+    pub cycle: Vec<QueryInfo<Q>>,
+}
+
+/// The result of `try_start`.
+enum TryGetJob<'tcx, CTX: QueryContext, C: QueryCache>
+where
+    C::Key: Eq + Hash + Clone + Debug,
+{
+    /// The query is not yet started. Contains a guard to the cache eventually used to start it.
+    NotYetStarted(JobOwner<'tcx, CTX, C>),
+
+    /// The query was already completed.
+    /// Returns the result of the query and its dep-node index
+    /// if it succeeded or a cycle error if it failed.
+    #[cfg(parallel_compiler)]
+    JobCompleted((C::Stored, DepNodeIndex)),
+
+    /// Trying to execute the query resulted in a cycle.
+    Cycle(C::Stored),
+}
+
+/// Checks if the query is already computed and in the cache.
+/// It returns the shard index and a lock guard to the shard,
+/// which will be used if the query is not in the cache and we need
+/// to compute it.
+#[inline(always)]
+fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    key: C::Key,
+    // `on_hit` can be called while holding a lock to the query cache
+    on_hit: OnHit,
+    on_miss: OnMiss,
+) -> R
+where
+    C: QueryCache,
+    CTX: QueryContext,
+    OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R,
+    OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX, C::Key, C::Sharded>) -> R,
+{
+    state.cache.lookup(
+        state,
+        key,
+        |value, index| {
+            if unlikely!(tcx.profiler().enabled()) {
+                tcx.profiler().query_cache_hit(index.into());
+            }
+            #[cfg(debug_assertions)]
+            {
+                state.cache_hits.fetch_add(1, Ordering::Relaxed);
+            }
+            on_hit(value, index)
+        },
+        on_miss,
+    )
+}
+
+#[inline(always)]
+fn try_execute_query<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    span: Span,
+    key: C::Key,
+    lookup: QueryLookup<'_, CTX, C::Key, C::Sharded>,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) -> C::Stored
+where
+    C: QueryCache,
+    C::Key: Eq + Clone + Debug + crate::dep_graph::DepNodeParams<CTX>,
+    C::Stored: Clone,
+    CTX: QueryContext,
+{
+    let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
+        TryGetJob::NotYetStarted(job) => job,
+        TryGetJob::Cycle(result) => return result,
+        #[cfg(parallel_compiler)]
+        TryGetJob::JobCompleted((v, index)) => {
+            tcx.dep_graph().read_index(index);
+            return v;
+        }
+    };
+
+    // Fast path for when incr. comp. is off. `to_dep_node` is
+    // expensive for some `DepKind`s.
+    if !tcx.dep_graph().is_fully_enabled() {
+        let null_dep_node = DepNode::new_no_params(DepKind::NULL);
+        return force_query_with_job(tcx, key, job, null_dep_node, query).0;
+    }
+
+    if query.anon {
+        let prof_timer = tcx.profiler().query_provider();
+
+        let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
+            tcx.start_query(job.id, diagnostics, |tcx| {
+                tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key))
+            })
+        });
+
+        prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+        tcx.dep_graph().read_index(dep_node_index);
+
+        if unlikely!(!diagnostics.is_empty()) {
+            tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
+        }
+
+        return job.complete(result, dep_node_index);
+    }
+
+    let dep_node = query.to_dep_node(tcx, &key);
+
+    if !query.eval_always {
+        // The diagnostics for this query will be
+        // promoted to the current session during
+        // `try_mark_green()`, so we can ignore them here.
+        let loaded = tcx.start_query(job.id, None, |tcx| {
+            let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
+            marked.map(|(prev_dep_node_index, dep_node_index)| {
+                (
+                    load_from_disk_and_cache_in_memory(
+                        tcx,
+                        key.clone(),
+                        prev_dep_node_index,
+                        dep_node_index,
+                        &dep_node,
+                        query,
+                    ),
+                    dep_node_index,
+                )
+            })
+        });
+        if let Some((result, dep_node_index)) = loaded {
+            return job.complete(result, dep_node_index);
+        }
+    }
+
+    let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query);
+    tcx.dep_graph().read_index(dep_node_index);
+    result
+}
+
+fn load_from_disk_and_cache_in_memory<CTX, K, V>(
+    tcx: CTX,
+    key: K,
+    prev_dep_node_index: SerializedDepNodeIndex,
+    dep_node_index: DepNodeIndex,
+    dep_node: &DepNode<CTX::DepKind>,
+    query: &QueryVtable<CTX, K, V>,
+) -> V
+where
+    CTX: QueryContext,
+{
+    // Note this function can be called concurrently from the same query
+    // We must ensure that this is handled correctly.
+
+    debug_assert!(tcx.dep_graph().is_green(dep_node));
+
+    // First we try to load the result from the on-disk cache.
+    let result = if query.cache_on_disk(tcx, &key, None) {
+        let prof_timer = tcx.profiler().incr_cache_loading();
+        let result = query.try_load_from_disk(tcx, prev_dep_node_index);
+        prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+        // We always expect to find a cached result for things that
+        // can be forced from `DepNode`.
+        debug_assert!(
+            !dep_node.kind.can_reconstruct_query_key() || result.is_some(),
+            "missing on-disk cache entry for {:?}",
+            dep_node
+        );
+        result
+    } else {
+        // Some things are never cached on disk.
+        None
+    };
+
+    let result = if let Some(result) = result {
+        result
+    } else {
+        // We could not load a result from the on-disk cache, so
+        // recompute.
+        let prof_timer = tcx.profiler().query_provider();
+
+        // The dep-graph for this computation is already in-place.
+        let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key));
+
+        prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+        result
+    };
+
+    // If `-Zincremental-verify-ich` is specified, re-hash results from
+    // the cache and make sure that they have the expected fingerprint.
+    if unlikely!(tcx.incremental_verify_ich()) {
+        incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query);
+    }
+
+    result
+}
+
+#[inline(never)]
+#[cold]
+fn incremental_verify_ich<CTX, K, V>(
+    tcx: CTX,
+    result: &V,
+    dep_node: &DepNode<CTX::DepKind>,
+    dep_node_index: DepNodeIndex,
+    query: &QueryVtable<CTX, K, V>,
+) where
+    CTX: QueryContext,
+{
+    assert!(
+        Some(tcx.dep_graph().fingerprint_of(dep_node_index))
+            == tcx.dep_graph().prev_fingerprint_of(dep_node),
+        "fingerprint for green query instance not loaded from cache: {:?}",
+        dep_node,
+    );
+
+    debug!("BEGIN verify_ich({:?})", dep_node);
+    let mut hcx = tcx.create_stable_hashing_context();
+
+    let new_hash = query.hash_result(&mut hcx, result).unwrap_or(Fingerprint::ZERO);
+    debug!("END verify_ich({:?})", dep_node);
+
+    let old_hash = tcx.dep_graph().fingerprint_of(dep_node_index);
+
+    assert!(new_hash == old_hash, "found unstable fingerprints for {:?}", dep_node,);
+}
+
+#[inline(always)]
+fn force_query_with_job<C, CTX>(
+    tcx: CTX,
+    key: C::Key,
+    job: JobOwner<'_, CTX, C>,
+    dep_node: DepNode<CTX::DepKind>,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) -> (C::Stored, DepNodeIndex)
+where
+    C: QueryCache,
+    C::Key: Eq + Clone + Debug,
+    C::Stored: Clone,
+    CTX: QueryContext,
+{
+    // If the following assertion triggers, it can have two reasons:
+    // 1. Something is wrong with DepNode creation, either here or
+    //    in `DepGraph::try_mark_green()`.
+    // 2. Two distinct query keys get mapped to the same `DepNode`
+    //    (see for example #48923).
+    assert!(
+        !tcx.dep_graph().dep_node_exists(&dep_node),
+        "forcing query with already existing `DepNode`\n\
+                 - query-key: {:?}\n\
+                 - dep-node: {:?}",
+        key,
+        dep_node
+    );
+
+    let prof_timer = tcx.profiler().query_provider();
+
+    let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
+        tcx.start_query(job.id, diagnostics, |tcx| {
+            if query.eval_always {
+                tcx.dep_graph().with_eval_always_task(
+                    dep_node,
+                    tcx,
+                    key,
+                    query.compute,
+                    query.hash_result,
+                )
+            } else {
+                tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result)
+            }
+        })
+    });
+
+    prof_timer.finish_with_query_invocation_id(dep_node_index.into());
+
+    if unlikely!(!diagnostics.is_empty()) {
+        if dep_node.kind != DepKind::NULL {
+            tcx.store_diagnostics(dep_node_index, diagnostics);
+        }
+    }
+
+    let result = job.complete(result, dep_node_index);
+
+    (result, dep_node_index)
+}
+
+#[inline(never)]
+fn get_query_impl<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    span: Span,
+    key: C::Key,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) -> C::Stored
+where
+    CTX: QueryContext,
+    C: QueryCache,
+    C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
+    C::Stored: Clone,
+{
+    try_get_cached(
+        tcx,
+        state,
+        key,
+        |value, index| {
+            tcx.dep_graph().read_index(index);
+            value.clone()
+        },
+        |key, lookup| try_execute_query(tcx, state, span, key, lookup, query),
+    )
+}
+
+/// Ensure that either this query has all green inputs or been executed.
+/// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
+///
+/// This function is particularly useful when executing passes for their
+/// side-effects -- e.g., in order to report errors for erroneous programs.
+///
+/// Note: The optimization is only available during incr. comp.
+#[inline(never)]
+fn ensure_query_impl<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    key: C::Key,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) where
+    C: QueryCache,
+    C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    if query.eval_always {
+        let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
+        return;
+    }
+
+    // Ensuring an anonymous query makes no sense
+    assert!(!query.anon);
+
+    let dep_node = query.to_dep_node(tcx, &key);
+
+    match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) {
+        None => {
+            // A None return from `try_mark_green_and_read` means that this is either
+            // a new dep node or that the dep node has already been marked red.
+            // Either way, we can't call `dep_graph.read()` as we don't have the
+            // DepNodeIndex. We must invoke the query itself. The performance cost
+            // this introduces should be negligible as we'll immediately hit the
+            // in-memory cache, or another query down the line will.
+            let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
+        }
+        Some((_, dep_node_index)) => {
+            tcx.profiler().query_cache_hit(dep_node_index.into());
+        }
+    }
+}
+
+#[inline(never)]
+fn force_query_impl<CTX, C>(
+    tcx: CTX,
+    state: &QueryState<CTX, C>,
+    key: C::Key,
+    span: Span,
+    dep_node: DepNode<CTX::DepKind>,
+    query: &QueryVtable<CTX, C::Key, C::Value>,
+) where
+    C: QueryCache,
+    C::Key: Eq + Clone + crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    // We may be concurrently trying both execute and force a query.
+    // Ensure that only one of them runs the query.
+
+    try_get_cached(
+        tcx,
+        state,
+        key,
+        |_, _| {
+            // Cache hit, do nothing
+        },
+        |key, lookup| {
+            let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) {
+                TryGetJob::NotYetStarted(job) => job,
+                TryGetJob::Cycle(_) => return,
+                #[cfg(parallel_compiler)]
+                TryGetJob::JobCompleted(_) => return,
+            };
+            force_query_with_job(tcx, key, job, dep_node, query);
+        },
+    );
+}
+
+#[inline(always)]
+pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
+where
+    Q: QueryDescription<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
+
+    get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE)
+}
+
+#[inline(always)]
+pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
+where
+    Q: QueryDescription<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE)
+}
+
+#[inline(always)]
+pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
+where
+    Q: QueryDescription<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    CTX: QueryContext,
+{
+    force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE)
+}