mv compiler to compiler/

compiler/rustc_query_system/src/dep_graph/README.md

@@ -0,0 +1,4 @@
+To learn more about how dependency tracking works in rustc, see the [rustc
+guide].
+
+[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html

compiler/rustc_query_system/src/dep_graph/debug.rs

@@ -0,0 +1,58 @@
+//! Code for debugging the dep-graph.
+
+use super::{DepKind, DepNode};
+use std::error::Error;
+
+/// A dep-node filter goes from a user-defined string to a query over
+/// nodes. Right now the format is like this:
+///
+///     x & y & z
+///
+/// where the format-string of the dep-node must contain `x`, `y`, and
+/// `z`.
+#[derive(Debug)]
+pub struct DepNodeFilter {
+    text: String,
+}
+
+impl DepNodeFilter {
+    pub fn new(text: &str) -> Self {
+        DepNodeFilter { text: text.trim().to_string() }
+    }
+
+    /// Returns `true` if all nodes always pass the filter.
+    pub fn accepts_all(&self) -> bool {
+        self.text.is_empty()
+    }
+
+    /// Tests whether `node` meets the filter, returning true if so.
+    pub fn test<K: DepKind>(&self, node: &DepNode<K>) -> bool {
+        let debug_str = format!("{:?}", node);
+        self.text.split('&').map(|s| s.trim()).all(|f| debug_str.contains(f))
+    }
+}
+
+/// A filter like `F -> G` where `F` and `G` are valid dep-node
+/// filters. This can be used to test the source/target independently.
+pub struct EdgeFilter {
+    pub source: DepNodeFilter,
+    pub target: DepNodeFilter,
+}
+
+impl EdgeFilter {
+    pub fn new(test: &str) -> Result<EdgeFilter, Box<dyn Error>> {
+        let parts: Vec<_> = test.split("->").collect();
+        if parts.len() != 2 {
+            Err(format!("expected a filter like `a&b -> c&d`, not `{}`", test).into())
+        } else {
+            Ok(EdgeFilter {
+                source: DepNodeFilter::new(parts[0]),
+                target: DepNodeFilter::new(parts[1]),
+            })
+        }
+    }
+
+    pub fn test<K: DepKind>(&self, source: &DepNode<K>, target: &DepNode<K>) -> bool {
+        self.source.test(source) && self.target.test(target)
+    }
+}

compiler/rustc_query_system/src/dep_graph/dep_node.rs

@@ -0,0 +1,179 @@
+//! This module defines the `DepNode` type which the compiler uses to represent
+//! nodes in the dependency graph. A `DepNode` consists of a `DepKind` (which
+//! specifies the kind of thing it represents, like a piece of HIR, MIR, etc)
+//! and a `Fingerprint`, a 128 bit hash value the exact meaning of which
+//! depends on the node's `DepKind`. Together, the kind and the fingerprint
+//! fully identify a dependency node, even across multiple compilation sessions.
+//! In other words, the value of the fingerprint does not depend on anything
+//! that is specific to a given compilation session, like an unpredictable
+//! interning key (e.g., NodeId, DefId, Symbol) or the numeric value of a
+//! pointer. The concept behind this could be compared to how git commit hashes
+//! uniquely identify a given commit and has a few advantages:
+//!
+//! * A `DepNode` can simply be serialized to disk and loaded in another session
+//!   without the need to do any "rebasing (like we have to do for Spans and
+//!   NodeIds) or "retracing" like we had to do for `DefId` in earlier
+//!   implementations of the dependency graph.
+//! * A `Fingerprint` is just a bunch of bits, which allows `DepNode` to
+//!   implement `Copy`, `Sync`, `Send`, `Freeze`, etc.
+//! * Since we just have a bit pattern, `DepNode` can be mapped from disk into
+//!   memory without any post-processing (e.g., "abomination-style" pointer
+//!   reconstruction).
+//! * Because a `DepNode` is self-contained, we can instantiate `DepNodes` that
+//!   refer to things that do not exist anymore. In previous implementations
+//!   `DepNode` contained a `DefId`. A `DepNode` referring to something that
+//!   had been removed between the previous and the current compilation session
+//!   could not be instantiated because the current compilation session
+//!   contained no `DefId` for thing that had been removed.
+//!
+//! `DepNode` definition happens in `librustc_middle` with the `define_dep_nodes!()` macro.
+//! This macro defines the `DepKind` enum and a corresponding `DepConstructor` enum. The
+//! `DepConstructor` enum links a `DepKind` to the parameters that are needed at runtime in order
+//! to construct a valid `DepNode` fingerprint.
+//!
+//! Because the macro sees what parameters a given `DepKind` requires, it can
+//! "infer" some properties for each kind of `DepNode`:
+//!
+//! * Whether a `DepNode` of a given kind has any parameters at all. Some
+//!   `DepNode`s could represent global concepts with only one value.
+//! * Whether it is possible, in principle, to reconstruct a query key from a
+//!   given `DepNode`. Many `DepKind`s only require a single `DefId` parameter,
+//!   in which case it is possible to map the node's fingerprint back to the
+//!   `DefId` it was computed from. In other cases, too much information gets
+//!   lost during fingerprint computation.
+
+use super::{DepContext, DepKind};
+
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+
+use std::fmt;
+use std::hash::Hash;
+
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)]
+pub struct DepNode<K> {
+    pub kind: K,
+    pub hash: Fingerprint,
+}
+
+impl<K: DepKind> DepNode<K> {
+    /// Creates a new, parameterless DepNode. This method will assert
+    /// that the DepNode corresponding to the given DepKind actually
+    /// does not require any parameters.
+    pub fn new_no_params(kind: K) -> DepNode<K> {
+        debug_assert!(!kind.has_params());
+        DepNode { kind, hash: Fingerprint::ZERO }
+    }
+
+    pub fn construct<Ctxt, Key>(tcx: Ctxt, kind: K, arg: &Key) -> DepNode<K>
+    where
+        Ctxt: crate::query::QueryContext<DepKind = K>,
+        Key: DepNodeParams<Ctxt>,
+    {
+        let hash = arg.to_fingerprint(tcx);
+        let dep_node = DepNode { kind, hash };
+
+        #[cfg(debug_assertions)]
+        {
+            if !kind.can_reconstruct_query_key() && tcx.debug_dep_node() {
+                tcx.dep_graph().register_dep_node_debug_str(dep_node, || arg.to_debug_str(tcx));
+            }
+        }
+
+        dep_node
+    }
+}
+
+impl<K: DepKind> fmt::Debug for DepNode<K> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        K::debug_node(self, f)
+    }
+}
+
+pub trait DepNodeParams<Ctxt: DepContext>: fmt::Debug + Sized {
+    fn can_reconstruct_query_key() -> bool;
+
+    /// This method turns the parameters of a DepNodeConstructor into an opaque
+    /// Fingerprint to be used in DepNode.
+    /// Not all DepNodeParams support being turned into a Fingerprint (they
+    /// don't need to if the corresponding DepNode is anonymous).
+    fn to_fingerprint(&self, _: Ctxt) -> Fingerprint {
+        panic!("Not implemented. Accidentally called on anonymous node?")
+    }
+
+    fn to_debug_str(&self, _: Ctxt) -> String {
+        format!("{:?}", self)
+    }
+
+    /// This method tries to recover the query key from the given `DepNode`,
+    /// something which is needed when forcing `DepNode`s during red-green
+    /// evaluation. The query system will only call this method if
+    /// `can_reconstruct_query_key()` is `true`.
+    /// It is always valid to return `None` here, in which case incremental
+    /// compilation will treat the query as having changed instead of forcing it.
+    fn recover(tcx: Ctxt, dep_node: &DepNode<Ctxt::DepKind>) -> Option<Self>;
+}
+
+impl<Ctxt: DepContext, T> DepNodeParams<Ctxt> for T
+where
+    T: HashStable<Ctxt::StableHashingContext> + fmt::Debug,
+{
+    #[inline]
+    default fn can_reconstruct_query_key() -> bool {
+        false
+    }
+
+    default fn to_fingerprint(&self, tcx: Ctxt) -> Fingerprint {
+        let mut hcx = tcx.create_stable_hashing_context();
+        let mut hasher = StableHasher::new();
+
+        self.hash_stable(&mut hcx, &mut hasher);
+
+        hasher.finish()
+    }
+
+    default fn to_debug_str(&self, _: Ctxt) -> String {
+        format!("{:?}", *self)
+    }
+
+    default fn recover(_: Ctxt, _: &DepNode<Ctxt::DepKind>) -> Option<Self> {
+        None
+    }
+}
+
+impl<Ctxt: DepContext> DepNodeParams<Ctxt> for () {
+    fn to_fingerprint(&self, _: Ctxt) -> Fingerprint {
+        Fingerprint::ZERO
+    }
+}
+
+/// A "work product" corresponds to a `.o` (or other) file that we
+/// save in between runs. These IDs do not have a `DefId` but rather
+/// some independent path or string that persists between runs without
+/// the need to be mapped or unmapped. (This ensures we can serialize
+/// them even in the absence of a tcx.)
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Encodable, Decodable)]
+pub struct WorkProductId {
+    hash: Fingerprint,
+}
+
+impl WorkProductId {
+    pub fn from_cgu_name(cgu_name: &str) -> WorkProductId {
+        let mut hasher = StableHasher::new();
+        cgu_name.len().hash(&mut hasher);
+        cgu_name.hash(&mut hasher);
+        WorkProductId { hash: hasher.finish() }
+    }
+
+    pub fn from_fingerprint(fingerprint: Fingerprint) -> WorkProductId {
+        WorkProductId { hash: fingerprint }
+    }
+}
+
+impl<HCX> HashStable<HCX> for WorkProductId {
+    #[inline]
+    fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
+        self.hash.hash_stable(hcx, hasher)
+    }
+}

compiler/rustc_query_system/src/dep_graph/mod.rs

@@ -0,0 +1,85 @@
+pub mod debug;
+mod dep_node;
+mod graph;
+mod prev;
+mod query;
+mod serialized;
+
+pub use dep_node::{DepNode, DepNodeParams, WorkProductId};
+pub use graph::{hash_result, DepGraph, DepNodeColor, DepNodeIndex, TaskDeps, WorkProduct};
+pub use prev::PreviousDepGraph;
+pub use query::DepGraphQuery;
+pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
+
+use rustc_data_structures::profiling::SelfProfilerRef;
+use rustc_data_structures::sync::Lock;
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::Diagnostic;
+
+use std::fmt;
+use std::hash::Hash;
+
+pub trait DepContext: Copy {
+    type DepKind: self::DepKind;
+    type StableHashingContext;
+
+    /// Create a hashing context for hashing new results.
+    fn create_stable_hashing_context(&self) -> Self::StableHashingContext;
+
+    fn debug_dep_tasks(&self) -> bool;
+    fn debug_dep_node(&self) -> bool;
+
+    /// Try to force a dep node to execute and see if it's green.
+    fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
+
+    /// Return whether the current session is tainted by errors.
+    fn has_errors_or_delayed_span_bugs(&self) -> bool;
+
+    /// Return the diagnostic handler.
+    fn diagnostic(&self) -> &rustc_errors::Handler;
+
+    /// Load data from the on-disk cache.
+    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
+
+    /// Load diagnostics associated to the node in the previous session.
+    fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
+
+    /// Register diagnostics for the given node, for use in next session.
+    fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>);
+
+    /// Register diagnostics for the given node, for use in next session.
+    fn store_diagnostics_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        diagnostics: ThinVec<Diagnostic>,
+    );
+
+    /// Access the profiler.
+    fn profiler(&self) -> &SelfProfilerRef;
+}
+
+/// Describe the different families of dependency nodes.
+pub trait DepKind: Copy + fmt::Debug + Eq + Ord + Hash {
+    const NULL: Self;
+
+    /// Return whether this kind always require evaluation.
+    fn is_eval_always(&self) -> bool;
+
+    /// Return whether this kind requires additional parameters to be executed.
+    fn has_params(&self) -> bool;
+
+    /// Implementation of `std::fmt::Debug` for `DepNode`.
+    fn debug_node(node: &DepNode<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result;
+
+    /// Execute the operation with provided dependencies.
+    fn with_deps<OP, R>(deps: Option<&Lock<TaskDeps<Self>>>, op: OP) -> R
+    where
+        OP: FnOnce() -> R;
+
+    /// Access dependencies from current implicit context.
+    fn read_deps<OP>(op: OP)
+    where
+        OP: for<'a> FnOnce(Option<&'a Lock<TaskDeps<Self>>>);
+
+    fn can_reconstruct_query_key(&self) -> bool;
+}

compiler/rustc_query_system/src/dep_graph/prev.rs

@@ -0,0 +1,61 @@
+use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex};
+use super::{DepKind, DepNode};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::FxHashMap;
+
+#[derive(Debug, Encodable, Decodable)]
+pub struct PreviousDepGraph<K: DepKind> {
+    data: SerializedDepGraph<K>,
+    index: FxHashMap<DepNode<K>, SerializedDepNodeIndex>,
+}
+
+impl<K: DepKind> Default for PreviousDepGraph<K> {
+    fn default() -> Self {
+        PreviousDepGraph { data: Default::default(), index: Default::default() }
+    }
+}
+
+impl<K: DepKind> PreviousDepGraph<K> {
+    pub fn new(data: SerializedDepGraph<K>) -> PreviousDepGraph<K> {
+        let index: FxHashMap<_, _> =
+            data.nodes.iter_enumerated().map(|(idx, &dep_node)| (dep_node, idx)).collect();
+        PreviousDepGraph { data, index }
+    }
+
+    #[inline]
+    pub fn edge_targets_from(
+        &self,
+        dep_node_index: SerializedDepNodeIndex,
+    ) -> &[SerializedDepNodeIndex] {
+        self.data.edge_targets_from(dep_node_index)
+    }
+
+    #[inline]
+    pub fn index_to_node(&self, dep_node_index: SerializedDepNodeIndex) -> DepNode<K> {
+        self.data.nodes[dep_node_index]
+    }
+
+    #[inline]
+    pub fn node_to_index(&self, dep_node: &DepNode<K>) -> SerializedDepNodeIndex {
+        self.index[dep_node]
+    }
+
+    #[inline]
+    pub fn node_to_index_opt(&self, dep_node: &DepNode<K>) -> Option<SerializedDepNodeIndex> {
+        self.index.get(dep_node).cloned()
+    }
+
+    #[inline]
+    pub fn fingerprint_of(&self, dep_node: &DepNode<K>) -> Option<Fingerprint> {
+        self.index.get(dep_node).map(|&node_index| self.data.fingerprints[node_index])
+    }
+
+    #[inline]
+    pub fn fingerprint_by_index(&self, dep_node_index: SerializedDepNodeIndex) -> Fingerprint {
+        self.data.fingerprints[dep_node_index]
+    }
+
+    pub fn node_count(&self) -> usize {
+        self.index.len()
+    }
+}

compiler/rustc_query_system/src/dep_graph/query.rs

@@ -0,0 +1,74 @@
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::graph::implementation::{
+    Direction, Graph, NodeIndex, INCOMING, OUTGOING,
+};
+
+use super::{DepKind, DepNode};
+
+pub struct DepGraphQuery<K> {
+    pub graph: Graph<DepNode<K>, ()>,
+    pub indices: FxHashMap<DepNode<K>, NodeIndex>,
+}
+
+impl<K: DepKind> DepGraphQuery<K> {
+    pub fn new(nodes: &[DepNode<K>], edges: &[(DepNode<K>, DepNode<K>)]) -> DepGraphQuery<K> {
+        let mut graph = Graph::with_capacity(nodes.len(), edges.len());
+        let mut indices = FxHashMap::default();
+        for node in nodes {
+            indices.insert(*node, graph.add_node(*node));
+        }
+
+        for &(ref source, ref target) in edges {
+            let source = indices[source];
+            let target = indices[target];
+            graph.add_edge(source, target, ());
+        }
+
+        DepGraphQuery { graph, indices }
+    }
+
+    pub fn contains_node(&self, node: &DepNode<K>) -> bool {
+        self.indices.contains_key(&node)
+    }
+
+    pub fn nodes(&self) -> Vec<&DepNode<K>> {
+        self.graph.all_nodes().iter().map(|n| &n.data).collect()
+    }
+
+    pub fn edges(&self) -> Vec<(&DepNode<K>, &DepNode<K>)> {
+        self.graph
+            .all_edges()
+            .iter()
+            .map(|edge| (edge.source(), edge.target()))
+            .map(|(s, t)| (self.graph.node_data(s), self.graph.node_data(t)))
+            .collect()
+    }
+
+    fn reachable_nodes(&self, node: &DepNode<K>, direction: Direction) -> Vec<&DepNode<K>> {
+        if let Some(&index) = self.indices.get(node) {
+            self.graph.depth_traverse(index, direction).map(|s| self.graph.node_data(s)).collect()
+        } else {
+            vec![]
+        }
+    }
+
+    /// All nodes reachable from `node`. In other words, things that
+    /// will have to be recomputed if `node` changes.
+    pub fn transitive_successors(&self, node: &DepNode<K>) -> Vec<&DepNode<K>> {
+        self.reachable_nodes(node, OUTGOING)
+    }
+
+    /// All nodes that can reach `node`.
+    pub fn transitive_predecessors(&self, node: &DepNode<K>) -> Vec<&DepNode<K>> {
+        self.reachable_nodes(node, INCOMING)
+    }
+
+    /// Just the outgoing edges from `node`.
+    pub fn immediate_successors(&self, node: &DepNode<K>) -> Vec<&DepNode<K>> {
+        if let Some(&index) = self.indices.get(&node) {
+            self.graph.successor_nodes(index).map(|s| self.graph.node_data(s)).collect()
+        } else {
+            vec![]
+        }
+    }
+}

compiler/rustc_query_system/src/dep_graph/serialized.rs

@@ -0,0 +1,45 @@
+//! The data that we will serialize and deserialize.
+
+use super::{DepKind, DepNode};
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_index::vec::IndexVec;
+
+rustc_index::newtype_index! {
+    pub struct SerializedDepNodeIndex { .. }
+}
+
+/// Data for use when recompiling the **current crate**.
+#[derive(Debug, Encodable, Decodable)]
+pub struct SerializedDepGraph<K: DepKind> {
+    /// The set of all DepNodes in the graph
+    pub nodes: IndexVec<SerializedDepNodeIndex, DepNode<K>>,
+    /// The set of all Fingerprints in the graph. Each Fingerprint corresponds to
+    /// the DepNode at the same index in the nodes vector.
+    pub fingerprints: IndexVec<SerializedDepNodeIndex, Fingerprint>,
+    /// For each DepNode, stores the list of edges originating from that
+    /// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
+    /// which holds the actual DepNodeIndices of the target nodes.
+    pub edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>,
+    /// A flattened list of all edge targets in the graph. Edge sources are
+    /// implicit in edge_list_indices.
+    pub edge_list_data: Vec<SerializedDepNodeIndex>,
+}
+
+impl<K: DepKind> Default for SerializedDepGraph<K> {
+    fn default() -> Self {
+        SerializedDepGraph {
+            nodes: Default::default(),
+            fingerprints: Default::default(),
+            edge_list_indices: Default::default(),
+            edge_list_data: Default::default(),
+        }
+    }
+}
+
+impl<K: DepKind> SerializedDepGraph<K> {
+    #[inline]
+    pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] {
+        let targets = self.edge_list_indices[source];
+        &self.edge_list_data[targets.0 as usize..targets.1 as usize]
+    }
+}