Make force_from_dep_node a function pointer.

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -66,6 +66,7 @@ use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX};
 use rustc_hir::definitions::DefPathHash;
 use rustc_hir::HirId;
 use rustc_span::symbol::Symbol;
+use rustc_span::DUMMY_SP;
 use std::hash::Hash;
 
 pub use rustc_query_system::dep_graph::{DepContext, DepNodeParams};
@@ -95,6 +96,50 @@ pub struct DepKindStruct {
     // can be made a specialized associated const.
     can_reconstruct_query_key: fn() -> bool,
 
+    /// The red/green evaluation system will try to mark a specific DepNode in the
+    /// dependency graph as green by recursively trying to mark the dependencies of
+    /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
+    /// where we don't know if it is red or green and we therefore actually have
+    /// to recompute its value in order to find out. Since the only piece of
+    /// information that we have at that point is the `DepNode` we are trying to
+    /// re-evaluate, we need some way to re-run a query from just that. This is what
+    /// `force_from_dep_node()` implements.
+    ///
+    /// In the general case, a `DepNode` consists of a `DepKind` and an opaque
+    /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
+    /// is usually constructed by computing a stable hash of the query-key that the
+    /// `DepNode` corresponds to. Consequently, it is not in general possible to go
+    /// back from hash to query-key (since hash functions are not reversible). For
+    /// this reason `force_from_dep_node()` is expected to fail from time to time
+    /// because we just cannot find out, from the `DepNode` alone, what the
+    /// corresponding query-key is and therefore cannot re-run the query.
+    ///
+    /// The system deals with this case letting `try_mark_green` fail which forces
+    /// the root query to be re-evaluated.
+    ///
+    /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
+    /// Fortunately, we can use some contextual information that will allow us to
+    /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
+    /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
+    /// valid `DefPathHash`. Since we also always build a huge table that maps every
+    /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
+    /// everything we need to re-run the query.
+    ///
+    /// Take the `mir_promoted` query as an example. Like many other queries, it
+    /// just has a single parameter: the `DefId` of the item it will compute the
+    /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
+    /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
+    /// is actually a `DefPathHash`, and can therefore just look up the corresponding
+    /// `DefId` in `tcx.def_path_hash_to_def_id`.
+    ///
+    /// When you implement a new query, it will likely have a corresponding new
+    /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
+    /// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
+    /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
+    /// add it to the "We don't have enough information to reconstruct..." group in
+    /// the match below.
+    pub(super) force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
+
     /// Invoke a query to put the on-disk cached value in memory.
     pub(super) try_load_from_on_disk_cache: fn(TyCtxt<'_>, &DepNode),
 }
@@ -156,7 +201,7 @@ macro_rules! contains_eval_always_attr {
 pub mod dep_kind {
     use super::*;
     use crate::ty::query::{queries, query_keys};
-    use rustc_query_system::query::QueryDescription;
+    use rustc_query_system::query::{force_query, QueryDescription};
 
     // We use this for most things when incr. comp. is turned off.
     pub const Null: DepKindStruct = DepKindStruct {
@@ -165,6 +210,7 @@ pub mod dep_kind {
         is_eval_always: false,
 
         can_reconstruct_query_key: || true,
+        force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
         try_load_from_on_disk_cache: |_, _| {},
     };
 
@@ -175,6 +221,7 @@ pub mod dep_kind {
         is_eval_always: true,
 
         can_reconstruct_query_key: || true,
+        force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
         try_load_from_on_disk_cache: |_, _| {},
     };
 
@@ -184,6 +231,7 @@ pub mod dep_kind {
         is_eval_always: false,
 
         can_reconstruct_query_key: || false,
+        force_from_dep_node: |_, _| false,
         try_load_from_on_disk_cache: |_, _| {},
     };
 
@@ -193,6 +241,7 @@ pub mod dep_kind {
         is_eval_always: false,
 
         can_reconstruct_query_key: || false,
+        force_from_dep_node: |_, _| false,
         try_load_from_on_disk_cache: |_, _| {},
     };
 
@@ -217,6 +266,24 @@ pub mod dep_kind {
                     <query_keys::$variant<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
                 }
 
+                fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool {
+                    if !can_reconstruct_query_key() {
+                        return false;
+                    }
+
+                    if let Some(key) = recover(tcx, dep_node) {
+                        force_query::<queries::$variant<'_>, _>(
+                            tcx,
+                            key,
+                            DUMMY_SP,
+                            *dep_node
+                        );
+                        return true;
+                    }
+
+                    false
+                }
+
                 fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: &DepNode) {
                     if !can_reconstruct_query_key() {
                         return
@@ -238,6 +305,7 @@ pub mod dep_kind {
                     is_anon,
                     is_eval_always,
                     can_reconstruct_query_key,
+                    force_from_dep_node,
                     try_load_from_on_disk_cache,
                 }
             };)*

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -8,7 +8,6 @@ use rustc_hir::def_id::LocalDefId;
 
 mod dep_node;
 
-pub(crate) use rustc_query_system::dep_graph::DepNodeParams;
 pub use rustc_query_system::dep_graph::{
     debug, hash_result, DepContext, DepNodeColor, DepNodeIndex, SerializedDepNodeIndex,
     WorkProduct, WorkProductId,
@@ -155,7 +154,26 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
         }
 
         debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node);
-        ty::query::force_from_dep_node(*self, dep_node)
+
+        // We must avoid ever having to call `force_from_dep_node()` for a
+        // `DepNode::codegen_unit`:
+        // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
+        // would always end up having to evaluate the first caller of the
+        // `codegen_unit` query that *is* reconstructible. This might very well be
+        // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
+        // to re-trigger calling the `codegen_unit` query with the right key. At
+        // that point we would already have re-done all the work we are trying to
+        // avoid doing in the first place.
+        // The solution is simple: Just explicitly call the `codegen_unit` query for
+        // each CGU, right after partitioning. This way `try_mark_green` will always
+        // hit the cache instead of having to go through `force_from_dep_node`.
+        // This assertion makes sure, we actually keep applying the solution above.
+        debug_assert!(
+            dep_node.kind != DepKind::codegen_unit,
+            "calling force_from_dep_node() on DepKind::codegen_unit"
+        );
+
+        (dep_node.kind.force_from_dep_node)(*self, dep_node)
     }
 
     fn has_errors_or_delayed_span_bugs(&self) -> bool {

compiler/rustc_middle/src/ty/query/mod.rs

@@ -1,4 +1,4 @@
-use crate::dep_graph::{self, DepKind, DepNode, DepNodeParams};
+use crate::dep_graph;
 use crate::hir::exports::Export;
 use crate::hir::map;
 use crate::infer::canonical::{self, Canonical};
@@ -103,112 +103,6 @@ pub use self::profiling_support::{IntoSelfProfilingString, QueryKeyStringBuilder
 
 rustc_query_append! { [define_queries!][<'tcx>] }
 
-/// The red/green evaluation system will try to mark a specific DepNode in the
-/// dependency graph as green by recursively trying to mark the dependencies of
-/// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
-/// where we don't know if it is red or green and we therefore actually have
-/// to recompute its value in order to find out. Since the only piece of
-/// information that we have at that point is the `DepNode` we are trying to
-/// re-evaluate, we need some way to re-run a query from just that. This is what
-/// `force_from_dep_node()` implements.
-///
-/// In the general case, a `DepNode` consists of a `DepKind` and an opaque
-/// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
-/// is usually constructed by computing a stable hash of the query-key that the
-/// `DepNode` corresponds to. Consequently, it is not in general possible to go
-/// back from hash to query-key (since hash functions are not reversible). For
-/// this reason `force_from_dep_node()` is expected to fail from time to time
-/// because we just cannot find out, from the `DepNode` alone, what the
-/// corresponding query-key is and therefore cannot re-run the query.
-///
-/// The system deals with this case letting `try_mark_green` fail which forces
-/// the root query to be re-evaluated.
-///
-/// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
-/// Fortunately, we can use some contextual information that will allow us to
-/// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
-/// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
-/// valid `DefPathHash`. Since we also always build a huge table that maps every
-/// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
-/// everything we need to re-run the query.
-///
-/// Take the `mir_promoted` query as an example. Like many other queries, it
-/// just has a single parameter: the `DefId` of the item it will compute the
-/// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
-/// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
-/// is actually a `DefPathHash`, and can therefore just look up the corresponding
-/// `DefId` in `tcx.def_path_hash_to_def_id`.
-///
-/// When you implement a new query, it will likely have a corresponding new
-/// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
-/// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
-/// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
-/// add it to the "We don't have enough information to reconstruct..." group in
-/// the match below.
-pub fn force_from_dep_node<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> bool {
-    // We must avoid ever having to call `force_from_dep_node()` for a
-    // `DepNode::codegen_unit`:
-    // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
-    // would always end up having to evaluate the first caller of the
-    // `codegen_unit` query that *is* reconstructible. This might very well be
-    // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
-    // to re-trigger calling the `codegen_unit` query with the right key. At
-    // that point we would already have re-done all the work we are trying to
-    // avoid doing in the first place.
-    // The solution is simple: Just explicitly call the `codegen_unit` query for
-    // each CGU, right after partitioning. This way `try_mark_green` will always
-    // hit the cache instead of having to go through `force_from_dep_node`.
-    // This assertion makes sure, we actually keep applying the solution above.
-    debug_assert!(
-        dep_node.kind != DepKind::codegen_unit,
-        "calling force_from_dep_node() on DepKind::codegen_unit"
-    );
-
-    if !dep_node.kind.can_reconstruct_query_key() {
-        return false;
-    }
-
-    macro_rules! force_from_dep_node {
-        ($($(#[$attr:meta])* [$($modifiers:tt)*] $name:ident($K:ty),)*) => {
-            match dep_node.kind {
-                // These are inputs that are expected to be pre-allocated and that
-                // should therefore always be red or green already.
-                DepKind::CrateMetadata |
-
-                // These are anonymous nodes.
-                DepKind::TraitSelect |
-
-                // We don't have enough information to reconstruct the query key of
-                // these.
-                DepKind::CompileCodegenUnit |
-
-                // Forcing this makes no sense.
-                DepKind::Null => {
-                    bug!("force_from_dep_node: encountered {:?}", dep_node)
-                }
-
-                $(DepKind::$name => {
-                    debug_assert!(<$K as DepNodeParams<TyCtxt<'_>>>::can_reconstruct_query_key());
-
-                    if let Some(key) = <$K as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node) {
-                        force_query::<queries::$name<'_>, _>(
-                            tcx,
-                            key,
-                            DUMMY_SP,
-                            *dep_node
-                        );
-                        return true;
-                    }
-                })*
-            }
-        }
-    }
-
-    rustc_dep_node_append! { [force_from_dep_node!][] }
-
-    false
-}
-
 mod sealed {
     use super::{DefId, LocalDefId};