bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #63756 - Zoxc:sharded-dep-graph-1, r=nikomatsakis

Browse source 
Use more fine grained locks for the dep graph

Split out from https://github.com/rust-lang/rust/pull/61845.

r? @michaelwoerister cc @aturon
This commit is contained in:
bors 2019-10-16 20:49:27 +00:00
commit c8fa82c482
1 changed files with 72 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

132
src/librustc/dep_graph/graph.rs
View file

@ -3,7 +3,8 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_index::vec::{Idx, IndexVec};
use smallvec::SmallVec;
use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, Ordering};
use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering};
use std::sync::atomic::Ordering::SeqCst;
use std::env;
use std::hash::Hash;
use std::collections::hash_map::Entry;
@ -53,7 +54,7 @@ struct DepGraphData {
/// tracking. The `current` field is the dependency graph of only the
/// current compilation session: We don't merge the previous dep-graph into
/// current one anymore.
current: Lock<CurrentDepGraph>,
current: CurrentDepGraph,
/// The dep-graph from the previous compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
@ -95,7 +96,7 @@ impl DepGraph {
data: Some(Lrc::new(DepGraphData {
previous_work_products: prev_work_products,
dep_node_debug: Default::default(),
current: Lock::new(CurrentDepGraph::new(prev_graph_node_count)),
current: CurrentDepGraph::new(prev_graph_node_count),
emitting_diagnostics: Default::default(),
emitting_diagnostics_cond_var: Condvar::new(),
previous: prev_graph,
@ -117,13 +118,12 @@ impl DepGraph {
}
pub fn query(&self) -> DepGraphQuery {
let current_dep_graph = self.data.as_ref().unwrap().current.borrow();
let nodes: Vec<_> = current_dep_graph.data.iter().map(|n| n.node).collect();
let data = self.data.as_ref().unwrap().current.data.lock();
let nodes: Vec<_> = data.iter().map(|n| n.node).collect();
let mut edges = Vec::new();
for (from, edge_targets) in current_dep_graph.data.iter()
.map(|d| (d.node, &d.edges)) {
for (from, edge_targets) in data.iter().map(|d| (d.node, &d.edges)) {
for &edge_target in edge_targets.iter() {
let to = current_dep_graph.data[edge_target].node;
let to = data[edge_target].node;
edges.push((from, to));
}
}
@ -202,7 +202,7 @@ impl DepGraph {
read_set: Default::default(),
}),
|data, key, fingerprint, task| {
data.borrow_mut().complete_task(key, task.unwrap(), fingerprint)
data.complete_task(key, task.unwrap(), fingerprint)
},
hash_result)
}
@ -223,7 +223,7 @@ impl DepGraph {
self.with_task_impl(key, cx, input, true, identity_fn,
|_| None,
|data, key, fingerprint, _| {
data.borrow_mut().alloc_node(key, SmallVec::new(), fingerprint)
data.alloc_node(key, SmallVec::new(), fingerprint)
},
hash_result::<R>)
}
@ -236,7 +236,7 @@ impl DepGraph {
no_tcx: bool,
task: fn(C, A) -> R,
create_task: fn(DepNode) -> Option<TaskDeps>,
finish_task_and_alloc_depnode: fn(&Lock<CurrentDepGraph>,
finish_task_and_alloc_depnode: fn(&CurrentDepGraph,
DepNode,
Fingerprint,
Option<TaskDeps>) -> DepNodeIndex,
@ -350,7 +350,6 @@ impl DepGraph {
(r, task_deps.into_inner())
});
let dep_node_index = data.current
.borrow_mut()
.complete_anon_task(dep_kind, task_deps);
(result, dep_node_index)
} else {
@ -374,8 +373,7 @@ impl DepGraph {
self.with_task_impl(key, cx, arg, false, task,
|_| None,
|data, key, fingerprint, _| {
let mut current = data.borrow_mut();
current.alloc_node(key, smallvec![], fingerprint)
data.alloc_node(key, smallvec![], fingerprint)
},
hash_result)
}
@ -383,9 +381,9 @@ impl DepGraph {
#[inline]
pub fn read(&self, v: DepNode) {
if let Some(ref data) = self.data {
let current = data.current.borrow_mut();
if let Some(&dep_node_index) = current.node_to_node_index.get(&v) {
std::mem::drop(current);
let map = data.current.node_to_node_index.lock();
if let Some(dep_node_index) = map.get(&v).copied() {
std::mem::drop(map);
data.read_index(dep_node_index);
} else {
bug!("DepKind {:?} should be pre-allocated but isn't.", v.kind)
@ -406,8 +404,8 @@ impl DepGraph {
.as_ref()
.unwrap()
.current
.borrow_mut()
.node_to_node_index
.lock()
.get(dep_node)
.cloned()
.unwrap()
@ -416,7 +414,7 @@ impl DepGraph {
#[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool {
if let Some(ref data) = self.data {
data.current.borrow_mut().node_to_node_index.contains_key(dep_node)
data.current.node_to_node_index.lock().contains_key(dep_node)
} else {
false
}
@ -424,8 +422,8 @@ impl DepGraph {
#[inline]
pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
let current = self.data.as_ref().expect("dep graph enabled").current.borrow_mut();
current.data[dep_node_index].fingerprint
let data = self.data.as_ref().expect("dep graph enabled").current.data.lock();
data[dep_node_index].fingerprint
}
pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
@ -479,32 +477,29 @@ impl DepGraph {
pub fn edge_deduplication_data(&self) -> Option<(u64, u64)> {
if cfg!(debug_assertions) {
let current_dep_graph = self.data.as_ref().unwrap().current.borrow();
let current_dep_graph = &self.data.as_ref().unwrap().current;
Some((current_dep_graph.total_read_count,
current_dep_graph.total_duplicate_read_count))
Some((current_dep_graph.total_read_count.load(SeqCst),
current_dep_graph.total_duplicate_read_count.load(SeqCst)))
} else {
None
}
}
pub fn serialize(&self) -> SerializedDepGraph {
let current_dep_graph = self.data.as_ref().unwrap().current.borrow();
let data = self.data.as_ref().unwrap().current.data.lock();
let fingerprints: IndexVec<SerializedDepNodeIndex, _> =
current_dep_graph.data.iter().map(|d| d.fingerprint).collect();
data.iter().map(|d| d.fingerprint).collect();
let nodes: IndexVec<SerializedDepNodeIndex, _> =
current_dep_graph.data.iter().map(|d| d.node).collect();
data.iter().map(|d| d.node).collect();
let total_edge_count: usize = current_dep_graph.data.iter()
.map(|d| d.edges.len())
.sum();
let total_edge_count: usize = data.iter().map(|d| d.edges.len()).sum();
let mut edge_list_indices = IndexVec::with_capacity(nodes.len());
let mut edge_list_data = Vec::with_capacity(total_edge_count);
for (current_dep_node_index, edges) in current_dep_graph.data.iter_enumerated()
.map(|(i, d)| (i, &d.edges)) {
for (current_dep_node_index, edges) in data.iter_enumerated().map(|(i, d)| (i, &d.edges)) {
let start = edge_list_data.len() as u32;
// This should really just be a memcpy :/
edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index())));
@ -600,7 +595,7 @@ impl DepGraph {
#[cfg(not(parallel_compiler))]
{
debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node));
debug_assert!(!data.current.node_to_node_index.lock().contains_key(dep_node));
debug_assert!(data.colors.get(prev_dep_node_index).is_none());
}
@ -733,15 +728,13 @@ impl DepGraph {
// There may be multiple threads trying to mark the same dep node green concurrently
let dep_node_index = {
let mut current = data.current.borrow_mut();
// Copy the fingerprint from the previous graph,
// so we don't have to recompute it
let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index);
// We allocating an entry for the node in the current dependency graph and
// adding all the appropriate edges imported from the previous graph
current.intern_node(*dep_node, current_deps, fingerprint)
data.current.intern_node(*dep_node, current_deps, fingerprint)
};
// ... emitting any stored diagnostic ...
@ -917,9 +910,27 @@ struct DepNodeData {
fingerprint: Fingerprint,
}
/// `CurrentDepGraph` stores the dependency graph for the current session.
/// It will be populated as we run queries or tasks.
///
/// The nodes in it are identified by an index (`DepNodeIndex`).
/// The data for each node is stored in its `DepNodeData`, found in the `data` field.
///
/// We never remove nodes from the graph: they are only added.
///
/// This struct uses two locks internally. The `data` and `node_to_node_index` fields are
/// locked separately. Operations that take a `DepNodeIndex` typically just access
/// the data field.
///
/// The only operation that must manipulate both locks is adding new nodes, in which case
/// we first acquire the `node_to_node_index` lock and then, once a new node is to be inserted,
/// acquire the lock on `data.`
pub(super) struct CurrentDepGraph {
data: IndexVec<DepNodeIndex, DepNodeData>,
node_to_node_index: FxHashMap<DepNode, DepNodeIndex>,
data: Lock<IndexVec<DepNodeIndex, DepNodeData>>,
node_to_node_index: Lock<FxHashMap<DepNode, DepNodeIndex>>,
/// Used to trap when a specific edge is added to the graph.
/// This is used for debug purposes and is only active with `debug_assertions`.
#[allow(dead_code)]
forbidden_edge: Option<EdgeFilter>,
@ -936,8 +947,10 @@ pub(super) struct CurrentDepGraph {
/// the `DepGraph` is created.
anon_id_seed: Fingerprint,
total_read_count: u64,
total_duplicate_read_count: u64,
/// These are simple counters that are for profiling and
/// debugging and only active with `debug_assertions`.
total_read_count: AtomicU64,
total_duplicate_read_count: AtomicU64,
}
impl CurrentDepGraph {
@ -971,20 +984,20 @@ impl CurrentDepGraph {
let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200;
CurrentDepGraph {
data: IndexVec::with_capacity(new_node_count_estimate),
node_to_node_index: FxHashMap::with_capacity_and_hasher(
data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)),
node_to_node_index: Lock::new(FxHashMap::with_capacity_and_hasher(
new_node_count_estimate,
Default::default(),
),
)),
anon_id_seed: stable_hasher.finish(),
forbidden_edge,
total_read_count: 0,
total_duplicate_read_count: 0,
total_read_count: AtomicU64::new(0),
total_duplicate_read_count: AtomicU64::new(0),
}
}
fn complete_task(
&mut self,
&self,
node: DepNode,
task_deps: TaskDeps,
fingerprint: Fingerprint
@ -992,7 +1005,7 @@ impl CurrentDepGraph {
self.alloc_node(node, task_deps.reads, fingerprint)
}
fn complete_anon_task(&mut self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex {
fn complete_anon_task(&self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex {
debug_assert!(!kind.is_eval_always());
let mut hasher = StableHasher::new();
@ -1017,28 +1030,27 @@ impl CurrentDepGraph {
}
fn alloc_node(
&mut self,
&self,
dep_node: DepNode,
edges: SmallVec<[DepNodeIndex; 8]>,
fingerprint: Fingerprint
) -> DepNodeIndex {
debug_assert!(!self.node_to_node_index.contains_key(&dep_node));
debug_assert!(!self.node_to_node_index.lock().contains_key(&dep_node));
self.intern_node(dep_node, edges, fingerprint)
}
fn intern_node(
&mut self,
&self,
dep_node: DepNode,
edges: SmallVec<[DepNodeIndex; 8]>,
fingerprint: Fingerprint
) -> DepNodeIndex {
debug_assert_eq!(self.node_to_node_index.len(), self.data.len());
match self.node_to_node_index.entry(dep_node) {
match self.node_to_node_index.lock().entry(dep_node) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let dep_node_index = DepNodeIndex::new(self.data.len());
self.data.push(DepNodeData {
let mut data = self.data.lock();
let dep_node_index = DepNodeIndex::new(data.len());
data.push(DepNodeData {
node: dep_node,
edges,
fingerprint
@ -1057,7 +1069,7 @@ impl DepGraphData {
if let Some(task_deps) = icx.task_deps {
let mut task_deps = task_deps.lock();
if cfg!(debug_assertions) {
self.current.lock().total_read_count += 1;
self.current.total_read_count.fetch_add(1, SeqCst);
}
if task_deps.read_set.insert(source) {
task_deps.reads.push(source);
@ -1065,9 +1077,9 @@ impl DepGraphData {
#[cfg(debug_assertions)]
{
if let Some(target) = task_deps.node {
let graph = self.current.lock();
if let Some(ref forbidden_edge) = graph.forbidden_edge {
let source = graph.data[source].node;
let data = self.current.data.lock();
if let Some(ref forbidden_edge) = self.current.forbidden_edge {
let source = data[source].node;
if forbidden_edge.test(&source, &target) {
bug!("forbidden edge {:?} -> {:?} created",
source,
@ -1077,7 +1089,7 @@ impl DepGraphData {
}
}
} else if cfg!(debug_assertions) {
self.current.lock().total_duplicate_read_count += 1;
self.current.total_duplicate_read_count.fetch_add(1, SeqCst);
}
}
})