bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #63756 - Zoxc:sharded-dep-graph-1, r=nikomatsakis

Browse source 
Use more fine grained locks for the dep graph

Split out from https://github.com/rust-lang/rust/pull/61845.

r? @michaelwoerister cc @aturon
This commit is contained in:
bors 2019-10-16 20:49:27 +00:00
commit c8fa82c482
1 changed files with 72 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

132
src/librustc/dep_graph/graph.rs
View file

@ -3,7 +3,8 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_index::vec::{Idx, IndexVec}; use rustc_index::vec::{Idx, IndexVec};
use smallvec::SmallVec; use smallvec::SmallVec;
use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, Ordering}; use rustc_data_structures::sync::{Lrc, Lock, AtomicU32, AtomicU64, Ordering};
use std::sync::atomic::Ordering::SeqCst;
use std::env; use std::env;
use std::hash::Hash; use std::hash::Hash;
use std::collections::hash_map::Entry; use std::collections::hash_map::Entry;
@ -53,7 +54,7 @@ struct DepGraphData {
/// tracking. The `current` field is the dependency graph of only the /// tracking. The `current` field is the dependency graph of only the
/// current compilation session: We don't merge the previous dep-graph into /// current compilation session: We don't merge the previous dep-graph into
/// current one anymore. /// current one anymore.
current: Lock<CurrentDepGraph>, current: CurrentDepGraph,
/// The dep-graph from the previous compilation session. It contains all /// The dep-graph from the previous compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them. /// nodes and edges as well as all fingerprints of nodes that have them.
@ -95,7 +96,7 @@ impl DepGraph {
data: Some(Lrc::new(DepGraphData { data: Some(Lrc::new(DepGraphData {
previous_work_products: prev_work_products, previous_work_products: prev_work_products,
dep_node_debug: Default::default(), dep_node_debug: Default::default(),
current: Lock::new(CurrentDepGraph::new(prev_graph_node_count)), current: CurrentDepGraph::new(prev_graph_node_count),
emitting_diagnostics: Default::default(), emitting_diagnostics: Default::default(),
emitting_diagnostics_cond_var: Condvar::new(), emitting_diagnostics_cond_var: Condvar::new(),
previous: prev_graph, previous: prev_graph,
@ -117,13 +118,12 @@ impl DepGraph {
} }
pub fn query(&self) -> DepGraphQuery { pub fn query(&self) -> DepGraphQuery {
let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); let data = self.data.as_ref().unwrap().current.data.lock();
let nodes: Vec<_> = current_dep_graph.data.iter().map(|n| n.node).collect(); let nodes: Vec<_> = data.iter().map(|n| n.node).collect();
let mut edges = Vec::new(); let mut edges = Vec::new();
for (from, edge_targets) in current_dep_graph.data.iter() for (from, edge_targets) in data.iter().map(|d| (d.node, &d.edges)) {
.map(|d| (d.node, &d.edges)) {
for &edge_target in edge_targets.iter() { for &edge_target in edge_targets.iter() {
let to = current_dep_graph.data[edge_target].node; let to = data[edge_target].node;
edges.push((from, to)); edges.push((from, to));
} }
} }
@ -202,7 +202,7 @@ impl DepGraph {
read_set: Default::default(), read_set: Default::default(),
}), }),
|data, key, fingerprint, task| { |data, key, fingerprint, task| {
data.borrow_mut().complete_task(key, task.unwrap(), fingerprint) data.complete_task(key, task.unwrap(), fingerprint)
}, },
hash_result) hash_result)
} }
@ -223,7 +223,7 @@ impl DepGraph {
self.with_task_impl(key, cx, input, true, identity_fn, self.with_task_impl(key, cx, input, true, identity_fn,
|_| None, |_| None,
|data, key, fingerprint, _| { |data, key, fingerprint, _| {
data.borrow_mut().alloc_node(key, SmallVec::new(), fingerprint) data.alloc_node(key, SmallVec::new(), fingerprint)
}, },
hash_result::<R>) hash_result::<R>)
} }
@ -236,7 +236,7 @@ impl DepGraph {
no_tcx: bool, no_tcx: bool,
task: fn(C, A) -> R, task: fn(C, A) -> R,
create_task: fn(DepNode) -> Option<TaskDeps>, create_task: fn(DepNode) -> Option<TaskDeps>,
finish_task_and_alloc_depnode: fn(&Lock<CurrentDepGraph>, finish_task_and_alloc_depnode: fn(&CurrentDepGraph,
DepNode, DepNode,
Fingerprint, Fingerprint,
Option<TaskDeps>) -> DepNodeIndex, Option<TaskDeps>) -> DepNodeIndex,
@ -350,7 +350,6 @@ impl DepGraph {
(r, task_deps.into_inner()) (r, task_deps.into_inner())
}); });
let dep_node_index = data.current let dep_node_index = data.current
.borrow_mut()
.complete_anon_task(dep_kind, task_deps); .complete_anon_task(dep_kind, task_deps);
(result, dep_node_index) (result, dep_node_index)
} else { } else {
@ -374,8 +373,7 @@ impl DepGraph {
self.with_task_impl(key, cx, arg, false, task, self.with_task_impl(key, cx, arg, false, task,
|_| None, |_| None,
|data, key, fingerprint, _| { |data, key, fingerprint, _| {
let mut current = data.borrow_mut(); data.alloc_node(key, smallvec![], fingerprint)
current.alloc_node(key, smallvec![], fingerprint)
}, },
hash_result) hash_result)
} }
@ -383,9 +381,9 @@ impl DepGraph {
#[inline] #[inline]
pub fn read(&self, v: DepNode) { pub fn read(&self, v: DepNode) {
if let Some(ref data) = self.data { if let Some(ref data) = self.data {
let current = data.current.borrow_mut(); let map = data.current.node_to_node_index.lock();
if let Some(&dep_node_index) = current.node_to_node_index.get(&v) { if let Some(dep_node_index) = map.get(&v).copied() {
std::mem::drop(current); std::mem::drop(map);
data.read_index(dep_node_index); data.read_index(dep_node_index);
} else { } else {
bug!("DepKind {:?} should be pre-allocated but isn't.", v.kind) bug!("DepKind {:?} should be pre-allocated but isn't.", v.kind)
@ -406,8 +404,8 @@ impl DepGraph {
.as_ref() .as_ref()
.unwrap() .unwrap()
.current .current
.borrow_mut()
.node_to_node_index .node_to_node_index
.lock()
.get(dep_node) .get(dep_node)
.cloned() .cloned()
.unwrap() .unwrap()
@ -416,7 +414,7 @@ impl DepGraph {
#[inline] #[inline]
pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool { pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool {
if let Some(ref data) = self.data { if let Some(ref data) = self.data {
data.current.borrow_mut().node_to_node_index.contains_key(dep_node) data.current.node_to_node_index.lock().contains_key(dep_node)
} else { } else {
false false
} }
@ -424,8 +422,8 @@ impl DepGraph {
#[inline] #[inline]
pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint { pub fn fingerprint_of(&self, dep_node_index: DepNodeIndex) -> Fingerprint {
let current = self.data.as_ref().expect("dep graph enabled").current.borrow_mut(); let data = self.data.as_ref().expect("dep graph enabled").current.data.lock();
current.data[dep_node_index].fingerprint data[dep_node_index].fingerprint
} }
pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> { pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
@ -479,32 +477,29 @@ impl DepGraph {
pub fn edge_deduplication_data(&self) -> Option<(u64, u64)> { pub fn edge_deduplication_data(&self) -> Option<(u64, u64)> {
if cfg!(debug_assertions) { if cfg!(debug_assertions) {
let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); let current_dep_graph = &self.data.as_ref().unwrap().current;
Some((current_dep_graph.total_read_count, Some((current_dep_graph.total_read_count.load(SeqCst),
current_dep_graph.total_duplicate_read_count)) current_dep_graph.total_duplicate_read_count.load(SeqCst)))
} else { } else {
None None
} }
} }
pub fn serialize(&self) -> SerializedDepGraph { pub fn serialize(&self) -> SerializedDepGraph {
let current_dep_graph = self.data.as_ref().unwrap().current.borrow(); let data = self.data.as_ref().unwrap().current.data.lock();
let fingerprints: IndexVec<SerializedDepNodeIndex, _> = let fingerprints: IndexVec<SerializedDepNodeIndex, _> =
current_dep_graph.data.iter().map(|d| d.fingerprint).collect(); data.iter().map(|d| d.fingerprint).collect();
let nodes: IndexVec<SerializedDepNodeIndex, _> = let nodes: IndexVec<SerializedDepNodeIndex, _> =
current_dep_graph.data.iter().map(|d| d.node).collect(); data.iter().map(|d| d.node).collect();
let total_edge_count: usize = current_dep_graph.data.iter() let total_edge_count: usize = data.iter().map(|d| d.edges.len()).sum();
.map(|d| d.edges.len())
.sum();
let mut edge_list_indices = IndexVec::with_capacity(nodes.len()); let mut edge_list_indices = IndexVec::with_capacity(nodes.len());
let mut edge_list_data = Vec::with_capacity(total_edge_count); let mut edge_list_data = Vec::with_capacity(total_edge_count);
for (current_dep_node_index, edges) in current_dep_graph.data.iter_enumerated() for (current_dep_node_index, edges) in data.iter_enumerated().map(|(i, d)| (i, &d.edges)) {
.map(|(i, d)| (i, &d.edges)) {
let start = edge_list_data.len() as u32; let start = edge_list_data.len() as u32;
// This should really just be a memcpy :/ // This should really just be a memcpy :/
edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index()))); edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex::new(i.index())));
@ -600,7 +595,7 @@ impl DepGraph {
#[cfg(not(parallel_compiler))] #[cfg(not(parallel_compiler))]
{ {
debug_assert!(!data.current.borrow().node_to_node_index.contains_key(dep_node)); debug_assert!(!data.current.node_to_node_index.lock().contains_key(dep_node));
debug_assert!(data.colors.get(prev_dep_node_index).is_none()); debug_assert!(data.colors.get(prev_dep_node_index).is_none());
} }
@ -733,15 +728,13 @@ impl DepGraph {
// There may be multiple threads trying to mark the same dep node green concurrently // There may be multiple threads trying to mark the same dep node green concurrently
let dep_node_index = { let dep_node_index = {
let mut current = data.current.borrow_mut();
// Copy the fingerprint from the previous graph, // Copy the fingerprint from the previous graph,
// so we don't have to recompute it // so we don't have to recompute it
let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index); let fingerprint = data.previous.fingerprint_by_index(prev_dep_node_index);
// We allocating an entry for the node in the current dependency graph and // We allocating an entry for the node in the current dependency graph and
// adding all the appropriate edges imported from the previous graph // adding all the appropriate edges imported from the previous graph
current.intern_node(*dep_node, current_deps, fingerprint) data.current.intern_node(*dep_node, current_deps, fingerprint)
}; };
// ... emitting any stored diagnostic ... // ... emitting any stored diagnostic ...
@ -917,9 +910,27 @@ struct DepNodeData {
fingerprint: Fingerprint, fingerprint: Fingerprint,
} }
/// `CurrentDepGraph` stores the dependency graph for the current session.
/// It will be populated as we run queries or tasks.
///
/// The nodes in it are identified by an index (`DepNodeIndex`).
/// The data for each node is stored in its `DepNodeData`, found in the `data` field.
///
/// We never remove nodes from the graph: they are only added.
///
/// This struct uses two locks internally. The `data` and `node_to_node_index` fields are
/// locked separately. Operations that take a `DepNodeIndex` typically just access
/// the data field.
///
/// The only operation that must manipulate both locks is adding new nodes, in which case
/// we first acquire the `node_to_node_index` lock and then, once a new node is to be inserted,
/// acquire the lock on `data.`
pub(super) struct CurrentDepGraph { pub(super) struct CurrentDepGraph {
data: IndexVec<DepNodeIndex, DepNodeData>, data: Lock<IndexVec<DepNodeIndex, DepNodeData>>,
node_to_node_index: FxHashMap<DepNode, DepNodeIndex>, node_to_node_index: Lock<FxHashMap<DepNode, DepNodeIndex>>,
/// Used to trap when a specific edge is added to the graph.
/// This is used for debug purposes and is only active with `debug_assertions`.
#[allow(dead_code)] #[allow(dead_code)]
forbidden_edge: Option<EdgeFilter>, forbidden_edge: Option<EdgeFilter>,
@ -936,8 +947,10 @@ pub(super) struct CurrentDepGraph {
/// the `DepGraph` is created. /// the `DepGraph` is created.
anon_id_seed: Fingerprint, anon_id_seed: Fingerprint,
total_read_count: u64, /// These are simple counters that are for profiling and
total_duplicate_read_count: u64, /// debugging and only active with `debug_assertions`.
total_read_count: AtomicU64,
total_duplicate_read_count: AtomicU64,
} }
impl CurrentDepGraph { impl CurrentDepGraph {
@ -971,20 +984,20 @@ impl CurrentDepGraph {
let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200; let new_node_count_estimate = (prev_graph_node_count * 102) / 100 + 200;
CurrentDepGraph { CurrentDepGraph {
data: IndexVec::with_capacity(new_node_count_estimate), data: Lock::new(IndexVec::with_capacity(new_node_count_estimate)),
node_to_node_index: FxHashMap::with_capacity_and_hasher( node_to_node_index: Lock::new(FxHashMap::with_capacity_and_hasher(
new_node_count_estimate, new_node_count_estimate,
Default::default(), Default::default(),
), )),
anon_id_seed: stable_hasher.finish(), anon_id_seed: stable_hasher.finish(),
forbidden_edge, forbidden_edge,
total_read_count: 0, total_read_count: AtomicU64::new(0),
total_duplicate_read_count: 0, total_duplicate_read_count: AtomicU64::new(0),
} }
} }
fn complete_task( fn complete_task(
&mut self, &self,
node: DepNode, node: DepNode,
task_deps: TaskDeps, task_deps: TaskDeps,
fingerprint: Fingerprint fingerprint: Fingerprint
@ -992,7 +1005,7 @@ impl CurrentDepGraph {
self.alloc_node(node, task_deps.reads, fingerprint) self.alloc_node(node, task_deps.reads, fingerprint)
} }
fn complete_anon_task(&mut self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex { fn complete_anon_task(&self, kind: DepKind, task_deps: TaskDeps) -> DepNodeIndex {
debug_assert!(!kind.is_eval_always()); debug_assert!(!kind.is_eval_always());
let mut hasher = StableHasher::new(); let mut hasher = StableHasher::new();
@ -1017,28 +1030,27 @@ impl CurrentDepGraph {
} }
fn alloc_node( fn alloc_node(
&mut self, &self,
dep_node: DepNode, dep_node: DepNode,
edges: SmallVec<[DepNodeIndex; 8]>, edges: SmallVec<[DepNodeIndex; 8]>,
fingerprint: Fingerprint fingerprint: Fingerprint
) -> DepNodeIndex { ) -> DepNodeIndex {
debug_assert!(!self.node_to_node_index.contains_key(&dep_node)); debug_assert!(!self.node_to_node_index.lock().contains_key(&dep_node));
self.intern_node(dep_node, edges, fingerprint) self.intern_node(dep_node, edges, fingerprint)
} }
fn intern_node( fn intern_node(
&mut self, &self,
dep_node: DepNode, dep_node: DepNode,
edges: SmallVec<[DepNodeIndex; 8]>, edges: SmallVec<[DepNodeIndex; 8]>,
fingerprint: Fingerprint fingerprint: Fingerprint
) -> DepNodeIndex { ) -> DepNodeIndex {
debug_assert_eq!(self.node_to_node_index.len(), self.data.len()); match self.node_to_node_index.lock().entry(dep_node) {
match self.node_to_node_index.entry(dep_node) {
Entry::Occupied(entry) => *entry.get(), Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => { Entry::Vacant(entry) => {
let dep_node_index = DepNodeIndex::new(self.data.len()); let mut data = self.data.lock();
self.data.push(DepNodeData { let dep_node_index = DepNodeIndex::new(data.len());
data.push(DepNodeData {
node: dep_node, node: dep_node,
edges, edges,
fingerprint fingerprint
@ -1057,7 +1069,7 @@ impl DepGraphData {
if let Some(task_deps) = icx.task_deps { if let Some(task_deps) = icx.task_deps {
let mut task_deps = task_deps.lock(); let mut task_deps = task_deps.lock();
if cfg!(debug_assertions) { if cfg!(debug_assertions) {
self.current.lock().total_read_count += 1; self.current.total_read_count.fetch_add(1, SeqCst);
} }
if task_deps.read_set.insert(source) { if task_deps.read_set.insert(source) {
task_deps.reads.push(source); task_deps.reads.push(source);
@ -1065,9 +1077,9 @@ impl DepGraphData {
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
{ {
if let Some(target) = task_deps.node { if let Some(target) = task_deps.node {
let graph = self.current.lock(); let data = self.current.data.lock();
if let Some(ref forbidden_edge) = graph.forbidden_edge { if let Some(ref forbidden_edge) = self.current.forbidden_edge {
let source = graph.data[source].node; let source = data[source].node;
if forbidden_edge.test(&source, &target) { if forbidden_edge.test(&source, &target) {
bug!("forbidden edge {:?} -> {:?} created", bug!("forbidden edge {:?} -> {:?} created",
source, source,
@ -1077,7 +1089,7 @@ impl DepGraphData {
} }
} }
} else if cfg!(debug_assertions) { } else if cfg!(debug_assertions) {
self.current.lock().total_duplicate_read_count += 1; self.current.total_duplicate_read_count.fetch_add(1, SeqCst);
} }
} }
}) })