bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #115301 - Zalathar:regions-vec, r=davidtwco

Browse source 
coverage: Allow each coverage statement to have multiple code regions

The original implementation of coverage instrumentation was built around the assumption that a coverage counter/expression would be associated with *up to one* code region. When it was discovered that *multiple* regions would sometimes need to share a counter, a workaround was found: for the remaining regions, the instrumentor would create a fresh expression that adds zero  to the existing counter/expression.

That got the job done, but resulted in some awkward code, and produces unnecessarily complicated coverage maps in the final binary.

---

This PR removes that tension by changing `StatementKind::Coverage`'s code region field from `Option<CodeRegion>` to `Vec<CodeRegion>`.

The changes on the codegen side are fairly straightforward. As long as each `CoverageKind::Counter` only injects one `llvm.instrprof.increment`, the rest of coverage codegen is happy to handle multiple regions mapped to the same counter/expression, with only minor option-to-vec adjustments.

On the instrumentor/mir-transform side, we can get rid of the code that creates extra (x + 0) expressions. Instead we gather all of the code regions associated with a single BCB, and inject them all into one coverage statement.

---

There are several patches here but they can be divided in to three phases:
- Preparatory work
- Actually switching over to multiple regions per coverage statement
- Cleaning up

So viewing the patches individually may be easier.
This commit is contained in:
bors 2023-10-03 18:36:21 +00:00
commit 36aab8df0a
37 changed files with 1074 additions and 1218 deletions
Download patch file Download diff file Expand all files Collapse all files

37
compiler/rustc_mir_transform/src/coverage/counters.rs
View file

@ -1,10 +1,8 @@
use super::Error;
use super::graph;
use super::spans;
use graph::{BasicCoverageBlock, BcbBranch, CoverageGraph, TraverseCoverageGraphWithLoops};
use spans::CoverageSpan;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::graph::WithNumNodes;
@ -93,14 +91,14 @@ impl CoverageCounters {
}
/// Makes [`BcbCounter`] `Counter`s and `Expressions` for the `BasicCoverageBlock`s directly or
/// indirectly associated with `CoverageSpans`, and accumulates additional `Expression`s
/// indirectly associated with coverage spans, and accumulates additional `Expression`s
/// representing intermediate values.
pub fn make_bcb_counters(
&mut self,
basic_coverage_blocks: &CoverageGraph,
coverage_spans: &[CoverageSpan],
bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool,
) -> Result<(), Error> {
MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(coverage_spans)
MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(bcb_has_coverage_spans)
}
fn make_counter(&mut self) -> BcbCounter {
@ -113,14 +111,10 @@ impl CoverageCounters {
BcbCounter::Expression { id, lhs, op, rhs }
}
pub fn make_identity_counter(&mut self, counter_operand: Operand) -> BcbCounter {
self.make_expression(counter_operand, Op::Add, Operand::Zero)
}
/// Counter IDs start from one and go up.
fn next_counter(&mut self) -> CounterId {
let next = self.next_counter_id;
self.next_counter_id = next.next_id();
self.next_counter_id = self.next_counter_id + 1;
next
}
@ -128,7 +122,7 @@ impl CoverageCounters {
/// (Counter IDs and Expression IDs are distinguished by the `Operand` enum.)
fn next_expression(&mut self) -> ExpressionId {
let next = self.next_expression_id;
self.next_expression_id = next.next_id();
self.next_expression_id = self.next_expression_id + 1;
next
}
@ -208,7 +202,7 @@ impl CoverageCounters {
}
/// Traverse the `CoverageGraph` and add either a `Counter` or `Expression` to every BCB, to be
/// injected with `CoverageSpan`s. `Expressions` have no runtime overhead, so if a viable expression
/// injected with coverage spans. `Expressions` have no runtime overhead, so if a viable expression
/// (adding or subtracting two other counters or expressions) can compute the same result as an
/// embedded counter, an `Expression` should be used.
struct MakeBcbCounters<'a> {
@ -234,17 +228,14 @@ impl<'a> MakeBcbCounters<'a> {
/// Returns any non-code-span expressions created to represent intermediate values (such as to
/// add two counters so the result can be subtracted from another counter), or an Error with
/// message for subsequent debugging.
fn make_bcb_counters(&mut self, coverage_spans: &[CoverageSpan]) -> Result<(), Error> {
fn make_bcb_counters(
&mut self,
bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool,
) -> Result<(), Error> {
debug!("make_bcb_counters(): adding a counter or expression to each BasicCoverageBlock");
let num_bcbs = self.basic_coverage_blocks.num_nodes();
let mut bcbs_with_coverage = BitSet::new_empty(num_bcbs);
for covspan in coverage_spans {
bcbs_with_coverage.insert(covspan.bcb);
}
// Walk the `CoverageGraph`. For each `BasicCoverageBlock` node with an associated
// `CoverageSpan`, add a counter. If the `BasicCoverageBlock` branches, add a counter or
// coverage span, add a counter. If the `BasicCoverageBlock` branches, add a counter or
// expression to each branch `BasicCoverageBlock` (if the branch BCB has only one incoming
// edge) or edge from the branching BCB to the branch BCB (if the branch BCB has multiple
// incoming edges).
@ -255,8 +246,8 @@ impl<'a> MakeBcbCounters<'a> {
// the current BCB is in one or more nested loops or not.
let mut traversal = TraverseCoverageGraphWithLoops::new(&self.basic_coverage_blocks);
while let Some(bcb) = traversal.next(self.basic_coverage_blocks) {
if bcbs_with_coverage.contains(bcb) {
debug!("{:?} has at least one `CoverageSpan`. Get or make its counter", bcb);
if bcb_has_coverage_spans(bcb) {
debug!("{:?} has at least one coverage span. Get or make its counter", bcb);
let branching_counter_operand = self.get_or_make_counter_operand(bcb)?;
if self.bcb_needs_branch_counters(bcb) {
@ -264,7 +255,7 @@ impl<'a> MakeBcbCounters<'a> {
}
} else {
debug!(
"{:?} does not have any `CoverageSpan`s. A counter will only be added if \
"{:?} does not have any coverage spans. A counter will only be added if \
and when a covered BCB has an expression dependency.",
bcb,
);

4
compiler/rustc_mir_transform/src/coverage/graph.rs
View file

@ -288,9 +288,9 @@ rustc_index::newtype_index! {
/// not relevant to coverage analysis. `FalseUnwind`, for example, can be treated the same as
/// a `Goto`, and merged with its successor into the same BCB.
///
/// Each BCB with at least one computed `CoverageSpan` will have no more than one `Counter`.
/// Each BCB with at least one computed coverage span will have no more than one `Counter`.
/// In some cases, a BCB's execution count can be computed by `Expression`. Additional
/// disjoint `CoverageSpan`s in a BCB can also be counted by `Expression` (by adding `ZERO`
/// disjoint coverage spans in a BCB can also be counted by `Expression` (by adding `ZERO`
/// to the BCB's primary counter or expression).
///
/// The BCB CFG is critical to simplifying the coverage analysis by ensuring graph path-based

81
compiler/rustc_mir_transform/src/coverage/mod.rs
View file

@ -9,13 +9,11 @@ mod tests;
use self::counters::{BcbCounter, CoverageCounters};
use self::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph};
use self::spans::{CoverageSpan, CoverageSpans};
use self::spans::CoverageSpans;
use crate::MirPass;
use rustc_data_structures::graph::WithNumNodes;
use rustc_data_structures::sync::Lrc;
use rustc_index::IndexVec;
use rustc_middle::hir;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::coverage::*;
@ -154,7 +152,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
let body_span = self.body_span;
////////////////////////////////////////////////////
// Compute `CoverageSpan`s from the `CoverageGraph`.
// Compute coverage spans from the `CoverageGraph`.
let coverage_spans = CoverageSpans::generate_coverage_spans(
&self.mir_body,
fn_sig_span,
@ -164,32 +162,33 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
////////////////////////////////////////////////////
// Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure
// every `CoverageSpan` has a `Counter` or `Expression` assigned to its `BasicCoverageBlock`
// every coverage span has a `Counter` or `Expression` assigned to its `BasicCoverageBlock`
// and all `Expression` dependencies (operands) are also generated, for any other
// `BasicCoverageBlock`s not already associated with a `CoverageSpan`.
// `BasicCoverageBlock`s not already associated with a coverage span.
//
// Intermediate expressions (used to compute other `Expression` values), which have no
// direct association with any `BasicCoverageBlock`, are accumulated inside `coverage_counters`.
let bcb_has_coverage_spans = |bcb| coverage_spans.bcb_has_coverage_spans(bcb);
let result = self
.coverage_counters
.make_bcb_counters(&mut self.basic_coverage_blocks, &coverage_spans);
.make_bcb_counters(&mut self.basic_coverage_blocks, bcb_has_coverage_spans);
if let Ok(()) = result {
////////////////////////////////////////////////////
// Remove the counter or edge counter from of each `CoverageSpan`s associated
// Remove the counter or edge counter from of each coverage cpan's associated
// `BasicCoverageBlock`, and inject a `Coverage` statement into the MIR.
//
// `Coverage` statements injected from `CoverageSpan`s will include the code regions
// `Coverage` statements injected from coverage spans will include the code regions
// (source code start and end positions) to be counted by the associated counter.
//
// These `CoverageSpan`-associated counters are removed from their associated
// These coverage-span-associated counters are removed from their associated
// `BasicCoverageBlock`s so that the only remaining counters in the `CoverageGraph`
// are indirect counters (to be injected next, without associated code regions).
self.inject_coverage_span_counters(coverage_spans);
self.inject_coverage_span_counters(&coverage_spans);
////////////////////////////////////////////////////
// For any remaining `BasicCoverageBlock` counters (that were not associated with
// any `CoverageSpan`), inject `Coverage` statements (_without_ code region `Span`s)
// any coverage span), inject `Coverage` statements (_without_ code region spans)
// to ensure `BasicCoverageBlock` counters that other `Expression`s may depend on
// are in fact counted, even though they don't directly contribute to counting
// their own independent code region's coverage.
@ -214,47 +213,40 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
}
}
/// Inject a counter for each `CoverageSpan`. There can be multiple `CoverageSpan`s for a given
/// BCB, but only one actual counter needs to be incremented per BCB. `bb_counters` maps each
/// `bcb` to its `Counter`, when injected. Subsequent `CoverageSpan`s for a BCB that already has
/// a `Counter` will inject an `Expression` instead, and compute its value by adding `ZERO` to
/// the BCB `Counter` value.
fn inject_coverage_span_counters(&mut self, coverage_spans: Vec<CoverageSpan>) {
/// Injects a single [`StatementKind::Coverage`] for each BCB that has one
/// or more coverage spans.
fn inject_coverage_span_counters(&mut self, coverage_spans: &CoverageSpans) {
let tcx = self.tcx;
let source_map = tcx.sess.source_map();
let body_span = self.body_span;
let file_name = Symbol::intern(&self.source_file.name.prefer_remapped().to_string_lossy());
let mut bcb_counters = IndexVec::from_elem_n(None, self.basic_coverage_blocks.num_nodes());
for covspan in coverage_spans {
let bcb = covspan.bcb;
let span = covspan.span;
let counter_kind = if let Some(&counter_operand) = bcb_counters[bcb].as_ref() {
self.coverage_counters.make_identity_counter(counter_operand)
} else if let Some(counter_kind) = self.coverage_counters.take_bcb_counter(bcb) {
bcb_counters[bcb] = Some(counter_kind.as_operand());
counter_kind
} else {
for (bcb, spans) in coverage_spans.bcbs_with_coverage_spans() {
let counter_kind = self.coverage_counters.take_bcb_counter(bcb).unwrap_or_else(|| {
bug!("Every BasicCoverageBlock should have a Counter or Expression");
};
});
let code_region = make_code_region(source_map, file_name, span, body_span);
// Convert the coverage spans into a vector of code regions to be
// associated with this BCB's coverage statement.
let code_regions = spans
.iter()
.map(|&span| make_code_region(source_map, file_name, span, body_span))
.collect::<Vec<_>>();
inject_statement(
self.mir_body,
self.make_mir_coverage_kind(&counter_kind),
self.bcb_leader_bb(bcb),
Some(code_region),
code_regions,
);
}
}
/// `inject_coverage_span_counters()` looped through the `CoverageSpan`s and injected the
/// counter from the `CoverageSpan`s `BasicCoverageBlock`, removing it from the BCB in the
/// process (via `take_counter()`).
/// At this point, any BCB with coverage counters has already had its counter injected
/// into MIR, and had its counter removed from `coverage_counters` (via `take_counter()`).
///
/// Any other counter associated with a `BasicCoverageBlock`, or its incoming edge, but not
/// associated with a `CoverageSpan`, should only exist if the counter is an `Expression`
/// associated with a coverage span, should only exist if the counter is an `Expression`
/// dependency (one of the expression operands). Collect them, and inject the additional
/// counters into the MIR, without a reportable coverage span.
fn inject_indirect_counters(&mut self) {
@ -303,7 +295,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
self.mir_body,
self.make_mir_coverage_kind(&counter_kind),
inject_to_bb,
None,
Vec::new(),
);
}
BcbCounter::Expression { .. } => inject_intermediate_expression(
@ -368,20 +360,14 @@ fn inject_statement(
mir_body: &mut mir::Body<'_>,
counter_kind: CoverageKind,
bb: BasicBlock,
some_code_region: Option<CodeRegion>,
code_regions: Vec<CodeRegion>,
) {
debug!(
" injecting statement {:?} for {:?} at code region: {:?}",
counter_kind, bb, some_code_region
);
debug!(" injecting statement {counter_kind:?} for {bb:?} at code regions: {code_regions:?}");
let data = &mut mir_body[bb];
let source_info = data.terminator().source_info;
let statement = Statement {
source_info,
kind: StatementKind::Coverage(Box::new(Coverage {
kind: counter_kind,
code_region: some_code_region,
})),
kind: StatementKind::Coverage(Box::new(Coverage { kind: counter_kind, code_regions })),
};
data.statements.insert(0, statement);
}
@ -395,7 +381,10 @@ fn inject_intermediate_expression(mir_body: &mut mir::Body<'_>, expression: Cove
let source_info = data.terminator().source_info;
let statement = Statement {
source_info,
kind: StatementKind::Coverage(Box::new(Coverage { kind: expression, code_region: None })),
kind: StatementKind::Coverage(Box::new(Coverage {
kind: expression,
code_regions: Vec::new(),
})),
};
data.statements.push(statement);
}

4
compiler/rustc_mir_transform/src/coverage/query.rs
View file

@ -93,8 +93,8 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) ->
fn covered_code_regions(tcx: TyCtxt<'_>, def_id: DefId) -> Vec<&CodeRegion> {
let body = mir_body(tcx, def_id);
all_coverage_in_mir_body(body)
// Not all coverage statements have an attached code region.
.filter_map(|coverage| coverage.code_region.as_ref())
// Coverage statements have a list of code regions (possibly empty).
.flat_map(|coverage| coverage.code_regions.as_slice())
.collect()
}

57
compiler/rustc_mir_transform/src/coverage/spans.rs
View file

@ -1,6 +1,7 @@
use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB};
use rustc_data_structures::graph::WithNumNodes;
use rustc_index::IndexVec;
use rustc_middle::mir::{
self, AggregateKind, BasicBlock, FakeReadCause, Rvalue, Statement, StatementKind, Terminator,
TerminatorKind,
@ -10,6 +11,48 @@ use rustc_span::{BytePos, ExpnKind, MacroKind, Span, Symbol};
use std::cell::OnceCell;
pub(super) struct CoverageSpans {
/// Map from BCBs to their list of coverage spans.
bcb_to_spans: IndexVec<BasicCoverageBlock, Vec<Span>>,
}
impl CoverageSpans {
pub(super) fn generate_coverage_spans(
mir_body: &mir::Body<'_>,
fn_sig_span: Span,
body_span: Span,
basic_coverage_blocks: &CoverageGraph,
) -> Self {
let coverage_spans = CoverageSpansGenerator::generate_coverage_spans(
mir_body,
fn_sig_span,
body_span,
basic_coverage_blocks,
);
// Group the coverage spans by BCB, with the BCBs in sorted order.
let mut bcb_to_spans = IndexVec::from_elem_n(Vec::new(), basic_coverage_blocks.num_nodes());
for CoverageSpan { bcb, span, .. } in coverage_spans {
bcb_to_spans[bcb].push(span);
}
Self { bcb_to_spans }
}
pub(super) fn bcb_has_coverage_spans(&self, bcb: BasicCoverageBlock) -> bool {
!self.bcb_to_spans[bcb].is_empty()
}
pub(super) fn bcbs_with_coverage_spans(
&self,
) -> impl Iterator<Item = (BasicCoverageBlock, &[Span])> {
self.bcb_to_spans.iter_enumerated().filter_map(|(bcb, spans)| {
// Only yield BCBs that have at least one coverage span.
(!spans.is_empty()).then_some((bcb, spans.as_slice()))
})
}
}
#[derive(Debug, Copy, Clone)]
pub(super) enum CoverageStatement {
Statement(BasicBlock, Span, usize),
@ -35,7 +78,7 @@ impl CoverageStatement {
/// or is subsumed by the `Span` associated with this `CoverageSpan`, and it's `BasicBlock`
/// `dominates()` the `BasicBlock`s in this `CoverageSpan`.
#[derive(Debug, Clone)]
pub(super) struct CoverageSpan {
struct CoverageSpan {
pub span: Span,
pub expn_span: Span,
pub current_macro_or_none: OnceCell<Option<Symbol>>,
@ -162,7 +205,7 @@ impl CoverageSpan {
/// * Merge spans that represent continuous (both in source code and control flow), non-branching
/// execution
/// * Carve out (leave uncovered) any span that will be counted by another MIR (notably, closures)
pub struct CoverageSpans<'a, 'tcx> {
struct CoverageSpansGenerator<'a, 'tcx> {
/// The MIR, used to look up `BasicBlockData`.
mir_body: &'a mir::Body<'tcx>,
@ -218,7 +261,7 @@ pub struct CoverageSpans<'a, 'tcx> {
refined_spans: Vec<CoverageSpan>,
}
impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
impl<'a, 'tcx> CoverageSpansGenerator<'a, 'tcx> {
/// Generate a minimal set of `CoverageSpan`s, each representing a contiguous code region to be
/// counted.
///
@ -246,7 +289,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
body_span: Span,
basic_coverage_blocks: &'a CoverageGraph,
) -> Vec<CoverageSpan> {
let mut coverage_spans = CoverageSpans {
let mut coverage_spans = Self {
mir_body,
fn_sig_span,
body_span,
@ -734,7 +777,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> {
/// If the MIR `Statement` has a span contributive to computing coverage spans,
/// return it; otherwise return `None`.
pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
match statement.kind {
// These statements have spans that are often outside the scope of the executed source code
// for their parent `BasicBlock`.
@ -781,7 +824,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span>
/// If the MIR `Terminator` has a span contributive to computing coverage spans,
/// return it; otherwise return `None`.
pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> {
fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> {
match terminator.kind {
// These terminators have spans that don't positively contribute to computing a reasonable
// span of actually executed source code. (For example, SwitchInt terminators extracted from
@ -828,7 +871,7 @@ pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Sp
/// [^1]Expansions result from Rust syntax including macros, syntactic sugar,
/// etc.).
#[inline]
pub(super) fn function_source_span(span: Span, body_span: Span) -> Span {
fn function_source_span(span: Span, body_span: Span) -> Span {
let original_span = original_sp(span, body_span).with_ctxt(body_span.ctxt());
if body_span.contains(original_span) { original_span } else { body_span }
}

34
compiler/rustc_mir_transform/src/coverage/tests.rs
View file

@ -25,8 +25,7 @@
//! to: `rustc_span::create_default_session_globals_then(|| { test_here(); })`.
use super::counters;
use super::graph;
use super::spans;
use super::graph::{self, BasicCoverageBlock};
use coverage_test_macros::let_bcb;
@ -644,39 +643,18 @@ fn test_traverse_coverage_with_loops() {
);
}
fn synthesize_body_span_from_terminators(mir_body: &Body<'_>) -> Span {
let mut some_span: Option<Span> = None;
for (_, data) in mir_body.basic_blocks.iter_enumerated() {
let term_span = data.terminator().source_info.span;
if let Some(span) = some_span.as_mut() {
*span = span.to(term_span);
} else {
some_span = Some(term_span)
}
}
some_span.expect("body must have at least one BasicBlock")
}
#[test]
fn test_make_bcb_counters() {
rustc_span::create_default_session_globals_then(|| {
let mir_body = goto_switchint();
let body_span = synthesize_body_span_from_terminators(&mir_body);
let mut basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body);
let mut coverage_spans = Vec::new();
for (bcb, data) in basic_coverage_blocks.iter_enumerated() {
if let Some(span) = spans::filtered_terminator_span(data.terminator(&mir_body)) {
coverage_spans.push(spans::CoverageSpan::for_terminator(
spans::function_source_span(span, body_span),
span,
bcb,
data.last_bb(),
));
}
}
// Historically this test would use `spans` internals to set up fake
// coverage spans for BCBs 1 and 2. Now we skip that step and just tell
// BCB counter construction that those BCBs have spans.
let bcb_has_coverage_spans = |bcb: BasicCoverageBlock| (1..=2).contains(&bcb.as_usize());
let mut coverage_counters = counters::CoverageCounters::new(&basic_coverage_blocks);
coverage_counters
.make_bcb_counters(&mut basic_coverage_blocks, &coverage_spans)
.make_bcb_counters(&mut basic_coverage_blocks, bcb_has_coverage_spans)
.expect("should be Ok");
assert_eq!(coverage_counters.intermediate_expressions.len(), 0);

10
compiler/rustc_mir_transform/src/simplify.rs
View file

@ -441,21 +441,23 @@ fn save_unreachable_coverage(
let dead_block = &basic_blocks[dead_block];
for statement in &dead_block.statements {
let StatementKind::Coverage(coverage) = &statement.kind else { continue };
let Some(code_region) = &coverage.code_region else { continue };
if coverage.code_regions.is_empty() {
continue;
};
let instance = statement.source_info.scope.inlined_instance(source_scopes);
if live.contains(&instance) {
retained_coverage.push((statement.source_info, code_region.clone()));
retained_coverage.push((statement.source_info, coverage.code_regions.clone()));
}
}
}
let start_block = &mut basic_blocks[START_BLOCK];
start_block.statements.extend(retained_coverage.into_iter().map(
|(source_info, code_region)| Statement {
|(source_info, code_regions)| Statement {
source_info,
kind: StatementKind::Coverage(Box::new(Coverage {
kind: CoverageKind::Unreachable,
code_region: Some(code_region),
code_regions,
})),
},
));