bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Simplify tracking the encoder state.

Browse source
This commit is contained in:
Camille GILLOT 2021-03-16 20:52:28 +01:00
parent e1c99e5fcc
commit c5c935af92
1 changed files with 72 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

120
compiler/rustc_query_system/src/dep_graph/serialized.rs
View file

@ -4,7 +4,7 @@ use super::query::DepGraphQuery;
use super::{DepKind, DepNode, DepNodeIndex}; use super::{DepKind, DepNode, DepNodeIndex};
use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::{Lock, Lrc}; use rustc_data_structures::sync::Lock;
use rustc_index::vec::{Idx, IndexVec}; use rustc_index::vec::{Idx, IndexVec};
use rustc_serialize::opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize}; use rustc_serialize::opaque::{self, FileEncodeResult, FileEncoder, IntEncodedWithFixedSize};
use rustc_serialize::{Decodable, Decoder, Encodable}; use rustc_serialize::{Decodable, Decoder, Encodable};
@ -125,52 +125,66 @@ struct Stat<K: DepKind> {
edge_counter: u64, edge_counter: u64,
} }
struct Stats<K: DepKind> { struct EncodingStatus<K: DepKind> {
stats: FxHashMap<K, Stat<K>>, encoder: FileEncoder,
total_node_count: usize, total_node_count: usize,
total_edge_count: usize, total_edge_count: usize,
result: FileEncodeResult,
stats: Option<FxHashMap<K, Stat<K>>>,
} }
#[instrument(skip(encoder, _record_graph, record_stats))] impl<K: DepKind> EncodingStatus<K> {
fn encode_node<K: DepKind>( fn new(encoder: FileEncoder, record_stats: bool) -> Self {
encoder: &mut FileEncoder, Self {
_index: DepNodeIndex, encoder,
total_edge_count: 0,
total_node_count: 0,
result: Ok(()),
stats: if record_stats { Some(FxHashMap::default()) } else { None },
}
}
#[instrument(skip(self, _record_graph))]
fn encode_node(
&mut self,
node: &NodeInfo<K>, node: &NodeInfo<K>,
_record_graph: &Option<Lrc<Lock<DepGraphQuery<K>>>>, _record_graph: &Option<Lock<DepGraphQuery<K>>>,
record_stats: &Option<Lrc<Lock<Stats<K>>>>, ) -> DepNodeIndex {
) -> FileEncodeResult { let index = DepNodeIndex::new(self.total_node_count);
self.total_node_count += 1;
let edge_count = node.edges.len();
self.total_edge_count += edge_count;
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
if let Some(record_graph) = &_record_graph { if let Some(record_graph) = &_record_graph {
// Do not ICE when a query is called from within `with_query`. // Do not ICE when a query is called from within `with_query`.
if let Some(record_graph) = &mut record_graph.try_lock() { if let Some(record_graph) = &mut record_graph.try_lock() {
record_graph.push(_index, node.node, &node.edges); record_graph.push(index, node.node, &node.edges);
} }
} }
if let Some(record_stats) = &record_stats { if let Some(stats) = &mut self.stats {
let mut stats = record_stats.lock();
let kind = node.node.kind; let kind = node.node.kind;
let edge_count = node.edges.len();
let stat = let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stats.stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1; stat.node_counter += 1;
stat.edge_counter += edge_count as u64; stat.edge_counter += edge_count as u64;
stats.total_node_count += 1;
stats.total_edge_count += edge_count;
} }
debug!(?_index, ?node); debug!(?index, ?node);
node.encode(encoder) let encoder = &mut self.encoder;
} self.result =
std::mem::replace(&mut self.result, Ok(())).and_then(|()| node.encode(encoder));
index
}
fn encode_counts( fn finish(self) -> FileEncodeResult {
mut encoder: FileEncoder, let Self { mut encoder, total_node_count, total_edge_count, result, stats: _ } = self;
node_count: usize, let () = result?;
edge_count: usize,
) -> FileEncodeResult { let node_count = total_node_count.try_into().unwrap();
let node_count = node_count.try_into().unwrap(); let edge_count = total_edge_count.try_into().unwrap();
let edge_count = edge_count.try_into().unwrap();
debug!(?node_count, ?edge_count); debug!(?node_count, ?edge_count);
debug!("position: {:?}", encoder.position()); debug!("position: {:?}", encoder.position());
@ -179,12 +193,12 @@ fn encode_counts(
debug!("position: {:?}", encoder.position()); debug!("position: {:?}", encoder.position());
// Drop the encoder so that nothing is written after the counts. // Drop the encoder so that nothing is written after the counts.
encoder.flush() encoder.flush()
}
} }
pub struct GraphEncoder<K: DepKind> { pub struct GraphEncoder<K: DepKind> {
status: Lock<(FileEncoder, DepNodeIndex, usize, FileEncodeResult)>, status: Lock<EncodingStatus<K>>,
record_graph: Option<Lrc<Lock<DepGraphQuery<K>>>>, record_graph: Option<Lock<DepGraphQuery<K>>>,
record_stats: Option<Lrc<Lock<Stats<K>>>>,
} }
impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> { impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
@ -195,21 +209,12 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
record_stats: bool, record_stats: bool,
) -> Self { ) -> Self {
let record_graph = if cfg!(debug_assertions) && record_graph { let record_graph = if cfg!(debug_assertions) && record_graph {
Some(Lrc::new(Lock::new(DepGraphQuery::new(prev_node_count)))) Some(Lock::new(DepGraphQuery::new(prev_node_count)))
} else { } else {
None None
}; };
let record_stats = if record_stats { let status = Lock::new(EncodingStatus::new(encoder, record_stats));
Some(Lrc::new(Lock::new(Stats { GraphEncoder { status, record_graph }
stats: FxHashMap::default(),
total_node_count: 0,
total_edge_count: 0,
})))
} else {
None
};
let status = Lock::new((encoder, DepNodeIndex::new(0), 0, Ok(())));
GraphEncoder { status, record_graph, record_stats }
} }
pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery<K>)) { pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery<K>)) {
@ -223,10 +228,9 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
total_read_count: u64, total_read_count: u64,
total_duplicate_read_count: u64, total_duplicate_read_count: u64,
) { ) {
if let Some(record_stats) = &self.record_stats { let status = self.status.lock();
let record_stats = record_stats.lock(); if let Some(record_stats) = &status.stats {
let mut stats: Vec<_> = record_stats.values().collect();
let mut stats: Vec<_> = record_stats.stats.values().collect();
stats.sort_by_key(|s| -(s.node_counter as i64)); stats.sort_by_key(|s| -(s.node_counter as i64));
const SEPARATOR: &str = "[incremental] --------------------------------\ const SEPARATOR: &str = "[incremental] --------------------------------\
@ -237,8 +241,8 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
eprintln!("[incremental] DepGraph Statistics"); eprintln!("[incremental] DepGraph Statistics");
eprintln!("{}", SEPARATOR); eprintln!("{}", SEPARATOR);
eprintln!("[incremental]"); eprintln!("[incremental]");
eprintln!("[incremental] Total Node Count: {}", record_stats.total_node_count); eprintln!("[incremental] Total Node Count: {}", status.total_node_count);
eprintln!("[incremental] Total Edge Count: {}", record_stats.total_edge_count); eprintln!("[incremental] Total Edge Count: {}", status.total_edge_count);
if cfg!(debug_assertions) { if cfg!(debug_assertions) {
eprintln!("[incremental] Total Edge Reads: {}", total_read_count); eprintln!("[incremental] Total Edge Reads: {}", total_read_count);
@ -257,7 +261,7 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
for stat in stats { for stat in stats {
let node_kind_ratio = let node_kind_ratio =
(100.0 * (stat.node_counter as f64)) / (record_stats.total_node_count as f64); (100.0 * (stat.node_counter as f64)) / (status.total_node_count as f64);
let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64); let node_kind_avg_edges = (stat.edge_counter as f64) / (stat.node_counter as f64);
eprintln!( eprintln!(
@ -280,23 +284,11 @@ impl<K: DepKind + Encodable<FileEncoder>> GraphEncoder<K> {
fingerprint: Fingerprint, fingerprint: Fingerprint,
edges: SmallVec<[DepNodeIndex; 8]>, edges: SmallVec<[DepNodeIndex; 8]>,
) -> DepNodeIndex { ) -> DepNodeIndex {
let &mut (ref mut encoder, ref mut next_index, ref mut edge_count, ref mut result) =
&mut *self.status.lock();
let index = next_index.clone();
next_index.increment_by(1);
*edge_count += edges.len();
*result = std::mem::replace(result, Ok(())).and_then(|()| {
let node = NodeInfo { node, fingerprint, edges }; let node = NodeInfo { node, fingerprint, edges };
encode_node(encoder, index, &node, &self.record_graph, &self.record_stats) self.status.lock().encode_node(&node, &self.record_graph)
});
index
} }
pub fn finish(self) -> FileEncodeResult { pub fn finish(self) -> FileEncodeResult {
let (encoder, node_count, edge_count, result) = self.status.into_inner(); self.status.into_inner().finish()
let () = result?;
let node_count = node_count.index();
encode_counts(encoder, node_count, edge_count)
} }
} }