bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

coverage: Defer part of counter-creation until codegen

Browse source
This commit is contained in:
Zalathar 2025-01-22 13:55:08 +11:00
parent ee7dc06cf1
commit 20d051ec87
19 changed files with 205 additions and 305 deletions
Download patch file Download diff file Expand all files Collapse all files

13
compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs
View file

@ -54,7 +54,7 @@ pub(crate) fn prepare_covfun_record<'tcx>(
let fn_cov_info = tcx.instance_mir(instance.def).function_coverage_info.as_deref()?;
let ids_info = tcx.coverage_ids_info(instance.def)?;
let expressions = prepare_expressions(fn_cov_info, ids_info, is_used);
let expressions = prepare_expressions(ids_info, is_used);
let mut covfun = CovfunRecord {
mangled_function_name: tcx.symbol_name(instance).name,
@ -76,11 +76,7 @@ pub(crate) fn prepare_covfun_record<'tcx>(
}
/// Convert the function's coverage-counter expressions into a form suitable for FFI.
fn prepare_expressions(
fn_cov_info: &FunctionCoverageInfo,
ids_info: &CoverageIdsInfo,
is_used: bool,
) -> Vec<ffi::CounterExpression> {
fn prepare_expressions(ids_info: &CoverageIdsInfo, is_used: bool) -> Vec<ffi::CounterExpression> {
// If any counters or expressions were removed by MIR opts, replace their
// terms with zero.
let counter_for_term = |term| {
@ -95,7 +91,7 @@ fn prepare_expressions(
// producing the final coverage map, so there's no need to do the same
// thing on the Rust side unless we're confident we can do much better.
// (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.)
fn_cov_info
ids_info
.expressions
.iter()
.map(move |&Expression { lhs, op, rhs }| ffi::CounterExpression {
@ -142,8 +138,7 @@ fn fill_region_tables<'tcx>(
// If the mapping refers to counters/expressions that were removed by
// MIR opts, replace those occurrences with zero.
let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter {
let term =
fn_cov_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term");
let term = ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term");
let term = if is_zero_term(term) { CovTerm::Zero } else { term };
ffi::Counter::from_term(term)
};

19
compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
View file

@ -160,17 +160,10 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!(
"marker statement {kind:?} should have been removed by CleanupPostBorrowck"
),
CoverageKind::CounterIncrement { id } => {
// The number of counters passed to `llvm.instrprof.increment` might
// be smaller than the number originally inserted by the instrumentor,
// if some high-numbered counters were removed by MIR optimizations.
// If so, LLVM's profiler runtime will use fewer physical counters.
CoverageKind::VirtualCounter { bcb }
if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) =>
{
let num_counters = ids_info.num_counters_after_mir_opts();
assert!(
num_counters as usize <= function_coverage_info.num_counters,
"num_counters disagreement: query says {num_counters} but function info only has {}",
function_coverage_info.num_counters
);
let fn_name = bx.get_pgo_func_name_var(instance);
let hash = bx.const_u64(function_coverage_info.function_source_hash);
@ -182,10 +175,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
);
bx.instrprof_increment(fn_name, hash, num_counters, index);
}
CoverageKind::ExpressionUsed { id: _ } => {
// Expression-used statements are markers that are handled by
// `coverage_ids_info`, so there's nothing to codegen here.
}
// If a BCB doesn't have an associated physical counter, there's nothing to codegen.
CoverageKind::VirtualCounter { .. } => {}
CoverageKind::CondBitmapUpdate { index, decision_depth } => {
let cond_bitmap = coverage_cx
.try_get_mcdc_condition_bitmap(&instance, decision_depth)

1
compiler/rustc_codegen_llvm/src/lib.rs
View file

@ -13,6 +13,7 @@
#![feature(extern_types)]
#![feature(file_buffered)]
#![feature(hash_raw_entry)]
#![feature(if_let_guard)]
#![feature(impl_trait_in_assoc_type)]
#![feature(iter_intersperse)]
#![feature(let_chains)]

70
compiler/rustc_middle/src/mir/coverage.rs
View file

@ -2,8 +2,9 @@
use std::fmt::{self, Debug, Formatter};
use rustc_index::IndexVec;
use rustc_data_structures::fx::FxIndexMap;
use rustc_index::bit_set::DenseBitSet;
use rustc_index::{Idx, IndexVec};
use rustc_macros::{HashStable, TyDecodable, TyEncodable};
use rustc_span::Span;
@ -103,23 +104,12 @@ pub enum CoverageKind {
/// Should be erased before codegen (at some point after `InstrumentCoverage`).
BlockMarker { id: BlockMarkerId },
/// Marks the point in MIR control flow represented by a coverage counter.
/// Marks its enclosing basic block with the ID of the coverage graph node
/// that it was part of during the `InstrumentCoverage` MIR pass.
///
/// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR.
///
/// If this statement does not survive MIR optimizations, any mappings that
/// refer to this counter can have those references simplified to zero.
CounterIncrement { id: CounterId },
/// Marks the point in MIR control-flow represented by a coverage expression.
///
/// If this statement does not survive MIR optimizations, any mappings that
/// refer to this expression can have those references simplified to zero.
///
/// (This is only inserted for expression IDs that are directly used by
/// mappings. Intermediate expressions with no direct mappings are
/// retained/zeroed based on whether they are transitively used.)
ExpressionUsed { id: ExpressionId },
/// During codegen, this might be lowered to `llvm.instrprof.increment` or
/// to a no-op, depending on the outcome of counter-creation.
VirtualCounter { bcb: BasicCoverageBlock },
/// Marks the point in MIR control flow represented by a evaluated condition.
///
@ -138,8 +128,7 @@ impl Debug for CoverageKind {
match self {
SpanMarker => write!(fmt, "SpanMarker"),
BlockMarker { id } => write!(fmt, "BlockMarker({:?})", id.index()),
CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
VirtualCounter { bcb } => write!(fmt, "VirtualCounter({bcb:?})"),
CondBitmapUpdate { index, decision_depth } => {
write!(fmt, "CondBitmapUpdate(index={:?}, depth={:?})", index, decision_depth)
}
@ -208,11 +197,12 @@ pub struct FunctionCoverageInfo {
pub function_source_hash: u64,
pub body_span: Span,
pub num_counters: usize,
pub expressions: IndexVec<ExpressionId, Expression>,
/// Used in conjunction with `priority_list` to create physical counters
/// and counter expressions, after MIR optimizations.
pub node_flow_data: NodeFlowData<BasicCoverageBlock>,
pub priority_list: Vec<BasicCoverageBlock>,
pub mappings: Vec<Mapping>,
pub term_for_bcb: IndexVec<BasicCoverageBlock, Option<CovTerm>>,
pub mcdc_bitmap_bits: usize,
/// The depth of the deepest decision is used to know how many
@ -281,15 +271,18 @@ pub struct MCDCDecisionSpan {
pub num_conditions: usize,
}
/// Summarizes coverage IDs inserted by the `InstrumentCoverage` MIR pass
/// (for compiler option `-Cinstrument-coverage`), after MIR optimizations
/// have had a chance to potentially remove some of them.
/// Contains information needed during codegen, obtained by inspecting the
/// function's MIR after MIR optimizations.
///
/// Used by the `coverage_ids_info` query.
/// Returned by the `coverage_ids_info` query.
#[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable)]
pub struct CoverageIdsInfo {
pub counters_seen: DenseBitSet<CounterId>,
pub zero_expressions: DenseBitSet<ExpressionId>,
pub phys_counter_for_node: FxIndexMap<BasicCoverageBlock, CounterId>,
pub term_for_bcb: IndexVec<BasicCoverageBlock, Option<CovTerm>>,
pub expressions: IndexVec<ExpressionId, Expression>,
}
impl CoverageIdsInfo {
@ -334,3 +327,28 @@ rustc_index::newtype_index! {
const START_BCB = 0;
}
}
/// Data representing a view of some underlying graph, in which each node's
/// successors have been merged into a single "supernode".
///
/// The resulting supernodes have no obvious meaning on their own.
/// However, merging successor nodes means that a node's out-edges can all
/// be combined into a single out-edge, whose flow is the same as the flow
/// (execution count) of its corresponding node in the original graph.
///
/// With all node flows now in the original graph now represented as edge flows
/// in the merged graph, it becomes possible to analyze the original node flows
/// using techniques for analyzing edge flows.
#[derive(Clone, Debug)]
#[derive(TyEncodable, TyDecodable, Hash, HashStable)]
pub struct NodeFlowData<Node: Idx> {
/// Maps each node to the supernode that contains it, indicated by some
/// arbitrary "root" node that is part of that supernode.
pub supernodes: IndexVec<Node, Node>,
/// For each node, stores the single supernode that all of its successors
/// have been merged into.
///
/// (Note that each node in a supernode can potentially have a _different_
/// successor supernode from its peers.)
pub succ_supernodes: IndexVec<Node, Node>,
}

6
compiler/rustc_middle/src/mir/pretty.rs
View file

@ -619,13 +619,9 @@ fn write_function_coverage_info(
function_coverage_info: &coverage::FunctionCoverageInfo,
w: &mut dyn io::Write,
) -> io::Result<()> {
let coverage::FunctionCoverageInfo { body_span, expressions, mappings, .. } =
function_coverage_info;
let coverage::FunctionCoverageInfo { body_span, mappings, .. } = function_coverage_info;
writeln!(w, "{INDENT}coverage body span: {body_span:?}")?;
for (id, expression) in expressions.iter_enumerated() {
writeln!(w, "{INDENT}coverage {id:?} => {expression:?};")?;
}
for coverage::Mapping { kind, span } in mappings {
writeln!(w, "{INDENT}coverage {kind:?} => {span:?};")?;
}

13
compiler/rustc_middle/src/query/mod.rs
View file

@ -615,9 +615,16 @@ rustc_queries! {
feedable
}
/// Summarizes coverage IDs inserted by the `InstrumentCoverage` MIR pass
/// (for compiler option `-Cinstrument-coverage`), after MIR optimizations
/// have had a chance to potentially remove some of them.
/// Scans through a function's MIR after MIR optimizations, to prepare the
/// information needed by codegen when `-Cinstrument-coverage` is active.
///
/// This includes the details of where to insert `llvm.instrprof.increment`
/// intrinsics, and the expression tables to be embedded in the function's
/// coverage metadata.
///
/// FIXME(Zalathar): This query's purpose has drifted a bit and should
/// probably be renamed, but that can wait until after the potential
/// follow-ups to #136053 have settled down.
///
/// Returns `None` for functions that were not instrumented.
query coverage_ids_info(key: ty::InstanceKind<'tcx>) -> Option<&'tcx mir::coverage::CoverageIdsInfo> {

69
compiler/rustc_mir_transform/src/coverage/counters.rs
View file

@ -2,7 +2,6 @@ use std::cmp::Ordering;
use either::Either;
use itertools::Itertools;
use rustc_data_structures::captures::Captures;
use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
use rustc_data_structures::graph::DirectedGraph;
use rustc_index::IndexVec;
@ -11,31 +10,35 @@ use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId,
use crate::coverage::counters::balanced_flow::BalancedFlowGraph;
use crate::coverage::counters::node_flow::{
CounterTerm, NodeCounters, make_node_counters, node_flow_data_for_balanced_graph,
CounterTerm, NodeCounters, NodeFlowData, node_flow_data_for_balanced_graph,
};
use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph};
mod balanced_flow;
mod node_flow;
pub(crate) mod node_flow;
mod union_find;
/// Ensures that each BCB node needing a counter has one, by creating physical
/// counters or counter expressions for nodes as required.
pub(super) fn make_bcb_counters(
graph: &CoverageGraph,
bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>,
) -> CoverageCounters {
/// Struct containing the results of [`prepare_bcb_counters_data`].
pub(crate) struct BcbCountersData {
pub(crate) node_flow_data: NodeFlowData<BasicCoverageBlock>,
pub(crate) priority_list: Vec<BasicCoverageBlock>,
}
/// Analyzes the coverage graph to create intermediate data structures that
/// will later be used (during codegen) to create physical counters or counter
/// expressions for each BCB node that needs one.
pub(crate) fn prepare_bcb_counters_data(graph: &CoverageGraph) -> BcbCountersData {
// Create the derived graphs that are necessary for subsequent steps.
let balanced_graph = BalancedFlowGraph::for_graph(graph, |n| !graph[n].is_out_summable);
let node_flow_data = node_flow_data_for_balanced_graph(&balanced_graph);
// Use those graphs to determine which nodes get physical counters, and how
// to compute the execution counts of other nodes from those counters.
// Also create a "priority list" of coverage graph nodes, to help determine
// which ones get physical counters or counter expressions. This needs to
// be done now, because the later parts of the counter-creation process
// won't have access to the original coverage graph.
let priority_list = make_node_flow_priority_list(graph, balanced_graph);
let node_counters = make_node_counters(&node_flow_data, &priority_list);
// Convert the counters into a form suitable for embedding into MIR.
transcribe_counters(&node_counters, bcb_needs_counter)
BcbCountersData { node_flow_data, priority_list }
}
/// Arranges the nodes in `balanced_graph` into a list, such that earlier nodes
@ -74,7 +77,7 @@ fn make_node_flow_priority_list(
}
// Converts node counters into a form suitable for embedding into MIR.
fn transcribe_counters(
pub(crate) fn transcribe_counters(
old: &NodeCounters<BasicCoverageBlock>,
bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>,
) -> CoverageCounters {
@ -129,7 +132,7 @@ fn transcribe_counters(
pub(super) struct CoverageCounters {
/// List of places where a counter-increment statement should be injected
/// into MIR, each with its corresponding counter ID.
phys_counter_for_node: FxIndexMap<BasicCoverageBlock, CounterId>,
pub(crate) phys_counter_for_node: FxIndexMap<BasicCoverageBlock, CounterId>,
next_counter_id: CounterId,
/// Coverage counters/expressions that are associated with individual BCBs.
@ -137,7 +140,7 @@ pub(super) struct CoverageCounters {
/// Table of expression data, associating each expression ID with its
/// corresponding operator (+ or -) and its LHS/RHS operands.
expressions: IndexVec<ExpressionId, Expression>,
pub(crate) expressions: IndexVec<ExpressionId, Expression>,
/// Remember expressions that have already been created (or simplified),
/// so that we don't create unnecessary duplicates.
expressions_memo: FxHashMap<Expression, CovTerm>,
@ -188,12 +191,6 @@ impl CoverageCounters {
self.make_expression(lhs, Op::Subtract, rhs_sum)
}
pub(super) fn num_counters(&self) -> usize {
let num_counters = self.phys_counter_for_node.len();
assert_eq!(num_counters, self.next_counter_id.as_usize());
num_counters
}
fn set_node_counter(&mut self, bcb: BasicCoverageBlock, counter: CovTerm) -> CovTerm {
let existing = self.node_counters[bcb].replace(counter);
assert!(
@ -202,30 +199,4 @@ impl CoverageCounters {
);
counter
}
/// Returns an iterator over all the nodes in the coverage graph that
/// should have a counter-increment statement injected into MIR, along with
/// each site's corresponding counter ID.
pub(super) fn counter_increment_sites(
&self,
) -> impl Iterator<Item = (CounterId, BasicCoverageBlock)> + Captures<'_> {
self.phys_counter_for_node.iter().map(|(&site, &id)| (id, site))
}
/// Returns an iterator over the subset of BCB nodes that have been associated
/// with a counter *expression*, along with the ID of that expression.
pub(super) fn bcb_nodes_with_coverage_expressions(
&self,
) -> impl Iterator<Item = (BasicCoverageBlock, ExpressionId)> + Captures<'_> {
self.node_counters.iter_enumerated().filter_map(|(bcb, &counter)| match counter {
// Yield the BCB along with its associated expression ID.
Some(CovTerm::Expression(id)) => Some((bcb, id)),
// This BCB is associated with a counter or nothing, so skip it.
Some(CovTerm::Counter { .. } | CovTerm::Zero) | None => None,
})
}
pub(super) fn into_expressions(self) -> IndexVec<ExpressionId, Expression> {
self.expressions
}
}

25
compiler/rustc_mir_transform/src/coverage/counters/node_flow.rs
View file

@ -9,6 +9,7 @@
use rustc_data_structures::graph;
use rustc_index::bit_set::DenseBitSet;
use rustc_index::{Idx, IndexSlice, IndexVec};
pub(crate) use rustc_middle::mir::coverage::NodeFlowData;
use rustc_middle::mir::coverage::Op;
use crate::coverage::counters::union_find::UnionFind;
@ -16,30 +17,6 @@ use crate::coverage::counters::union_find::UnionFind;
#[cfg(test)]
mod tests;
/// Data representing a view of some underlying graph, in which each node's
/// successors have been merged into a single "supernode".
///
/// The resulting supernodes have no obvious meaning on their own.
/// However, merging successor nodes means that a node's out-edges can all
/// be combined into a single out-edge, whose flow is the same as the flow
/// (execution count) of its corresponding node in the original graph.
///
/// With all node flows now in the original graph now represented as edge flows
/// in the merged graph, it becomes possible to analyze the original node flows
/// using techniques for analyzing edge flows.
#[derive(Debug)]
pub(crate) struct NodeFlowData<Node: Idx> {
/// Maps each node to the supernode that contains it, indicated by some
/// arbitrary "root" node that is part of that supernode.
supernodes: IndexVec<Node, Node>,
/// For each node, stores the single supernode that all of its successors
/// have been merged into.
///
/// (Note that each node in a supernode can potentially have a _different_
/// successor supernode from its peers.)
succ_supernodes: IndexVec<Node, Node>,
}
/// Creates a "merged" view of an underlying graph.
///
/// The given graph is assumed to have [“balanced flow”](balanced-flow),

61
compiler/rustc_mir_transform/src/coverage/mappings.rs
View file

@ -1,9 +1,7 @@
use std::collections::BTreeSet;
use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::graph::DirectedGraph;
use rustc_index::IndexVec;
use rustc_index::bit_set::DenseBitSet;
use rustc_middle::mir::coverage::{
BlockMarkerId, BranchSpan, ConditionId, ConditionInfo, CoverageInfoHi, CoverageKind,
};
@ -63,10 +61,6 @@ const MCDC_MAX_BITMAP_SIZE: usize = i32::MAX as usize;
#[derive(Default)]
pub(super) struct ExtractedMappings {
/// Store our own copy of [`CoverageGraph::num_nodes`], so that we don't
/// need access to the whole graph when allocating per-BCB data. This is
/// only public so that other code can still use exhaustive destructuring.
pub(super) num_bcbs: usize,
pub(super) code_mappings: Vec<CodeMapping>,
pub(super) branch_pairs: Vec<BranchPair>,
pub(super) mcdc_bitmap_bits: usize,
@ -118,7 +112,6 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>(
);
ExtractedMappings {
num_bcbs: graph.num_nodes(),
code_mappings,
branch_pairs,
mcdc_bitmap_bits,
@ -127,60 +120,6 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>(
}
}
impl ExtractedMappings {
pub(super) fn all_bcbs_with_counter_mappings(&self) -> DenseBitSet<BasicCoverageBlock> {
// Fully destructure self to make sure we don't miss any fields that have mappings.
let Self {
num_bcbs,
code_mappings,
branch_pairs,
mcdc_bitmap_bits: _,
mcdc_degraded_branches,
mcdc_mappings,
} = self;
// Identify which BCBs have one or more mappings.
let mut bcbs_with_counter_mappings = DenseBitSet::new_empty(*num_bcbs);
let mut insert = |bcb| {
bcbs_with_counter_mappings.insert(bcb);
};
for &CodeMapping { span: _, bcb } in code_mappings {
insert(bcb);
}
for &BranchPair { true_bcb, false_bcb, .. } in branch_pairs {
insert(true_bcb);
insert(false_bcb);
}
for &MCDCBranch { true_bcb, false_bcb, .. } in mcdc_degraded_branches
.iter()
.chain(mcdc_mappings.iter().map(|(_, branches)| branches.into_iter()).flatten())
{
insert(true_bcb);
insert(false_bcb);
}
// MC/DC decisions refer to BCBs, but don't require those BCBs to have counters.
if bcbs_with_counter_mappings.is_empty() {
debug_assert!(
mcdc_mappings.is_empty(),
"A function with no counter mappings shouldn't have any decisions: {mcdc_mappings:?}",
);
}
bcbs_with_counter_mappings
}
/// Returns the set of BCBs that have one or more `Code` mappings.
pub(super) fn bcbs_with_ordinary_code_mappings(&self) -> DenseBitSet<BasicCoverageBlock> {
let mut bcbs = DenseBitSet::new_empty(self.num_bcbs);
for &CodeMapping { span: _, bcb } in &self.code_mappings {
bcbs.insert(bcb);
}
bcbs
}
}
fn resolve_block_markers(
coverage_info_hi: &CoverageInfoHi,
mir_body: &mir::Body<'_>,

70
compiler/rustc_mir_transform/src/coverage/mod.rs
View file

@ -21,7 +21,7 @@ use rustc_span::Span;
use rustc_span::def_id::LocalDefId;
use tracing::{debug, debug_span, trace};
use crate::coverage::counters::CoverageCounters;
use crate::coverage::counters::BcbCountersData;
use crate::coverage::graph::CoverageGraph;
use crate::coverage::mappings::ExtractedMappings;
@ -82,29 +82,21 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir:
let extracted_mappings =
mappings::extract_all_mapping_info_from_mir(tcx, mir_body, &hir_info, &graph);
////////////////////////////////////////////////////
// Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure
// every coverage span has a `Counter` or `Expression` assigned to its `BasicCoverageBlock`
// and all `Expression` dependencies (operands) are also generated, for any other
// `BasicCoverageBlock`s not already associated with a coverage span.
let bcbs_with_counter_mappings = extracted_mappings.all_bcbs_with_counter_mappings();
if bcbs_with_counter_mappings.is_empty() {
// No relevant spans were found in MIR, so skip instrumenting this function.
return;
}
let coverage_counters = counters::make_bcb_counters(&graph, &bcbs_with_counter_mappings);
let mappings = create_mappings(&extracted_mappings);
if mappings.is_empty() {
// No spans could be converted into valid mappings, so skip this function.
debug!("no spans could be converted into valid mappings; skipping");
return;
}
let term_for_bcb = coverage_counters.node_counters.clone();
inject_coverage_statements(mir_body, &graph, &extracted_mappings, &coverage_counters);
// Use the coverage graph to prepare intermediate data that will eventually
// be used to assign physical counters and counter expressions to points in
// the control-flow graph
let BcbCountersData { node_flow_data, priority_list } =
counters::prepare_bcb_counters_data(&graph);
// Inject coverage statements into MIR.
inject_coverage_statements(mir_body, &graph);
inject_mcdc_statements(mir_body, &graph, &extracted_mappings);
let mcdc_num_condition_bitmaps = extracted_mappings
@ -118,11 +110,10 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir:
function_source_hash: hir_info.function_source_hash,
body_span: hir_info.body_span,
num_counters: coverage_counters.num_counters(),
expressions: coverage_counters.into_expressions(),
node_flow_data,
priority_list,
mappings,
term_for_bcb,
mcdc_bitmap_bits: extracted_mappings.mcdc_bitmap_bits,
mcdc_num_condition_bitmaps,
@ -137,7 +128,6 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir:
fn create_mappings(extracted_mappings: &ExtractedMappings) -> Vec<Mapping> {
// Fully destructure the mappings struct to make sure we don't miss any kinds.
let ExtractedMappings {
num_bcbs: _,
code_mappings,
branch_pairs,
mcdc_bitmap_bits: _,
@ -215,41 +205,11 @@ fn create_mappings(extracted_mappings: &ExtractedMappings) -> Vec<Mapping> {
mappings
}
/// For each BCB node or BCB edge that has an associated coverage counter,
/// inject any necessary coverage statements into MIR.
fn inject_coverage_statements<'tcx>(
mir_body: &mut mir::Body<'tcx>,
graph: &CoverageGraph,
extracted_mappings: &ExtractedMappings,
coverage_counters: &CoverageCounters,
) {
// Inject counter-increment statements into MIR.
for (id, bcb) in coverage_counters.counter_increment_sites() {
let target_bb = graph[bcb].leader_bb();
inject_statement(mir_body, CoverageKind::CounterIncrement { id }, target_bb);
}
// For each counter expression that is directly associated with at least one
// span, we inject an "expression-used" statement, so that coverage codegen
// can check whether the injected statement survived MIR optimization.
// (BCB edges can't have spans, so we only need to process BCB nodes here.)
//
// We only do this for ordinary `Code` mappings, because branch and MC/DC
// mappings might have expressions that don't correspond to any single
// point in the control-flow graph.
//
// See the code in `rustc_codegen_llvm::coverageinfo::map_data` that deals
// with "expressions seen" and "zero terms".
let eligible_bcbs = extracted_mappings.bcbs_with_ordinary_code_mappings();
for (bcb, expression_id) in coverage_counters
.bcb_nodes_with_coverage_expressions()
.filter(|&(bcb, _)| eligible_bcbs.contains(bcb))
{
inject_statement(
mir_body,
CoverageKind::ExpressionUsed { id: expression_id },
graph[bcb].leader_bb(),
);
/// Inject any necessary coverage statements into MIR, so that they influence codegen.
fn inject_coverage_statements<'tcx>(mir_body: &mut mir::Body<'tcx>, graph: &CoverageGraph) {
for (bcb, data) in graph.iter_enumerated() {
let target_bb = data.leader_bb();
inject_statement(mir_body, CoverageKind::VirtualCounter { bcb }, target_bb);
}
}

112
compiler/rustc_mir_transform/src/coverage/query.rs
View file

@ -1,9 +1,10 @@
use rustc_data_structures::captures::Captures;
use rustc_index::IndexSlice;
use rustc_index::bit_set::DenseBitSet;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::coverage::{
CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, ExpressionId,
FunctionCoverageInfo, MappingKind, Op,
BasicCoverageBlock, CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression,
ExpressionId, MappingKind, Op,
};
use rustc_middle::mir::{Body, Statement, StatementKind};
use rustc_middle::ty::{self, TyCtxt};
@ -12,6 +13,9 @@ use rustc_span::def_id::LocalDefId;
use rustc_span::sym;
use tracing::trace;
use crate::coverage::counters::node_flow::make_node_counters;
use crate::coverage::counters::{CoverageCounters, transcribe_counters};
/// Registers query/hook implementations related to coverage.
pub(crate) fn provide(providers: &mut Providers) {
providers.hooks.is_eligible_for_coverage = is_eligible_for_coverage;
@ -89,41 +93,85 @@ fn coverage_ids_info<'tcx>(
let mir_body = tcx.instance_mir(instance_def);
let fn_cov_info = mir_body.function_coverage_info.as_deref()?;
let mut counters_seen = DenseBitSet::new_empty(fn_cov_info.num_counters);
let mut expressions_seen = DenseBitSet::new_filled(fn_cov_info.expressions.len());
// For each expression ID that is directly used by one or more mappings,
// mark it as not-yet-seen. This indicates that we expect to see a
// corresponding `ExpressionUsed` statement during MIR traversal.
for mapping in fn_cov_info.mappings.iter() {
// Currently we only worry about ordinary code mappings.
// For branch and MC/DC mappings, expressions might not correspond
// to any particular point in the control-flow graph.
// (Keep this in sync with the injection of `ExpressionUsed`
// statements in the `InstrumentCoverage` MIR pass.)
if let MappingKind::Code { bcb } = mapping.kind
&& let Some(CovTerm::Expression(id)) = fn_cov_info.term_for_bcb[bcb]
{
expressions_seen.remove(id);
}
}
// Scan through the final MIR to see which BCBs survived MIR opts.
// Any BCB not in this set was optimized away.
let mut bcbs_seen = DenseBitSet::new_empty(fn_cov_info.priority_list.len());
for kind in all_coverage_in_mir_body(mir_body) {
match *kind {
CoverageKind::CounterIncrement { id } => {
counters_seen.insert(id);
}
CoverageKind::ExpressionUsed { id } => {
expressions_seen.insert(id);
CoverageKind::VirtualCounter { bcb } => {
bcbs_seen.insert(bcb);
}
_ => {}
}
}
let zero_expressions =
identify_zero_expressions(fn_cov_info, &counters_seen, &expressions_seen);
// Determine the set of BCBs that are referred to by mappings, and therefore
// need a counter. Any node not in this set will only get a counter if it
// is part of the counter expression for a node that is in the set.
let mut bcb_needs_counter =
DenseBitSet::<BasicCoverageBlock>::new_empty(fn_cov_info.priority_list.len());
for mapping in &fn_cov_info.mappings {
match mapping.kind {
MappingKind::Code { bcb } => {
bcb_needs_counter.insert(bcb);
}
MappingKind::Branch { true_bcb, false_bcb } => {
bcb_needs_counter.insert(true_bcb);
bcb_needs_counter.insert(false_bcb);
}
MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params: _ } => {
bcb_needs_counter.insert(true_bcb);
bcb_needs_counter.insert(false_bcb);
}
MappingKind::MCDCDecision(_) => {}
}
}
Some(CoverageIdsInfo { counters_seen, zero_expressions })
let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &fn_cov_info.priority_list);
let coverage_counters = transcribe_counters(&node_counters, &bcb_needs_counter);
let mut counters_seen = DenseBitSet::new_empty(coverage_counters.node_counters.len());
let mut expressions_seen = DenseBitSet::new_filled(coverage_counters.expressions.len());
// For each expression ID that is directly used by one or more mappings,
// mark it as not-yet-seen. This indicates that we expect to see a
// corresponding `VirtualCounter` statement during MIR traversal.
for mapping in fn_cov_info.mappings.iter() {
// Currently we only worry about ordinary code mappings.
// For branch and MC/DC mappings, expressions might not correspond
// to any particular point in the control-flow graph.
if let MappingKind::Code { bcb } = mapping.kind
&& let Some(CovTerm::Expression(id)) = coverage_counters.node_counters[bcb]
{
expressions_seen.remove(id);
}
}
for bcb in bcbs_seen.iter() {
if let Some(&id) = coverage_counters.phys_counter_for_node.get(&bcb) {
counters_seen.insert(id);
}
if let Some(CovTerm::Expression(id)) = coverage_counters.node_counters[bcb] {
expressions_seen.insert(id);
}
}
let zero_expressions = identify_zero_expressions(
&coverage_counters.expressions,
&counters_seen,
&expressions_seen,
);
let CoverageCounters { phys_counter_for_node, node_counters, expressions, .. } =
coverage_counters;
Some(CoverageIdsInfo {
counters_seen,
zero_expressions,
phys_counter_for_node,
term_for_bcb: node_counters,
expressions,
})
}
fn all_coverage_in_mir_body<'a, 'tcx>(
@ -150,7 +198,7 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool {
/// already being performed by the Rust-side expression renumbering, so that
/// the resulting coverage mappings don't get worse.
fn identify_zero_expressions(
fn_cov_info: &FunctionCoverageInfo,
expressions: &IndexSlice<ExpressionId, Expression>,
counters_seen: &DenseBitSet<CounterId>,
expressions_seen: &DenseBitSet<ExpressionId>,
) -> DenseBitSet<ExpressionId> {
@ -158,13 +206,13 @@ fn identify_zero_expressions(
// have zero as both of their operands, and will therefore always have
// a value of zero. Other expressions that refer to these as operands
// can have those operands replaced with `CovTerm::Zero`.
let mut zero_expressions = DenseBitSet::new_empty(fn_cov_info.expressions.len());
let mut zero_expressions = DenseBitSet::new_empty(expressions.len());
// Simplify a copy of each expression based on lower-numbered expressions,
// and then update the set of always-zero expressions if necessary.
// (By construction, expressions can only refer to other expressions
// that have lower IDs, so one pass is sufficient.)
for (id, expression) in fn_cov_info.expressions.iter_enumerated() {
for (id, expression) in expressions.iter_enumerated() {
if !expressions_seen.contains(id) {
// If an expression was not seen, it must have been optimized away,
// so any operand that refers to it can be replaced with zero.

3
compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
View file

@ -137,8 +137,7 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
// These coverage statements should not exist prior to coverage instrumentation.
StatementKind::Coverage(
CoverageKind::CounterIncrement { .. }
| CoverageKind::ExpressionUsed { .. }
CoverageKind::VirtualCounter { .. }
| CoverageKind::CondBitmapUpdate { .. }
| CoverageKind::TestVectorBitmapUpdate { .. },
) => bug!(

14
tests/mir-opt/coverage/branch_match_arms.main.InstrumentCoverage.diff
View file

@ -27,9 +27,6 @@
}
+ coverage body span: $DIR/branch_match_arms.rs:14:11: 21:2 (#0)
+ coverage ExpressionId(0) => Expression { lhs: Counter(1), op: Add, rhs: Counter(2) };
+ coverage ExpressionId(1) => Expression { lhs: Expression(0), op: Add, rhs: Counter(3) };
+ coverage ExpressionId(2) => Expression { lhs: Counter(0), op: Subtract, rhs: Expression(1) };
+ coverage Code { bcb: bcb0 } => $DIR/branch_match_arms.rs:14:1: 15:21 (#0);
+ coverage Code { bcb: bcb1 } => $DIR/branch_match_arms.rs:16:17: 16:33 (#0);
+ coverage Code { bcb: bcb3 } => $DIR/branch_match_arms.rs:17:17: 17:33 (#0);
@ -38,7 +35,7 @@
+ coverage Code { bcb: bcb2 } => $DIR/branch_match_arms.rs:21:2: 21:2 (#0);
+
bb0: {
+ Coverage::CounterIncrement(0);
+ Coverage::VirtualCounter(bcb0);
StorageLive(_1);
_1 = Enum::A(const 0_u32);
PlaceMention(_1);
@ -52,22 +49,22 @@
}
bb2: {
+ Coverage::CounterIncrement(1);
+ Coverage::VirtualCounter(bcb1);
falseEdge -> [real: bb8, imaginary: bb3];
}
bb3: {
+ Coverage::CounterIncrement(2);
+ Coverage::VirtualCounter(bcb3);
falseEdge -> [real: bb7, imaginary: bb4];
}
bb4: {
+ Coverage::CounterIncrement(3);
+ Coverage::VirtualCounter(bcb4);
falseEdge -> [real: bb6, imaginary: bb5];
}
bb5: {
+ Coverage::ExpressionUsed(2);
+ Coverage::VirtualCounter(bcb5);
StorageLive(_9);
_9 = copy ((_1 as A).0: u32);
StorageLive(_10);
@ -124,6 +121,7 @@
}
bb13: {
+ Coverage::VirtualCounter(bcb2);
StorageDead(_1);
return;
}

2
tests/mir-opt/coverage/instrument_coverage.bar.InstrumentCoverage.diff
View file

@ -8,7 +8,7 @@
+ coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:29:1: 31:2 (#0);
+
bb0: {
+ Coverage::CounterIncrement(0);
+ Coverage::VirtualCounter(bcb0);
_0 = const true;
return;
}

8
tests/mir-opt/coverage/instrument_coverage.main.InstrumentCoverage.diff
View file

@ -8,7 +8,6 @@
let mut _3: !;
+ coverage body span: $DIR/instrument_coverage.rs:14:11: 20:2 (#0)
+ coverage ExpressionId(0) => Expression { lhs: Counter(1), op: Subtract, rhs: Counter(0) };
+ coverage Code { bcb: bcb0 } => $DIR/instrument_coverage.rs:14:1: 14:11 (#0);
+ coverage Code { bcb: bcb1 } => $DIR/instrument_coverage.rs:16:12: 16:17 (#0);
+ coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:17:13: 17:18 (#0);
@ -16,12 +15,12 @@
+ coverage Code { bcb: bcb2 } => $DIR/instrument_coverage.rs:20:2: 20:2 (#0);
+
bb0: {
+ Coverage::CounterIncrement(0);
+ Coverage::VirtualCounter(bcb0);
goto -> bb1;
}
bb1: {
+ Coverage::CounterIncrement(1);
+ Coverage::VirtualCounter(bcb1);
falseUnwind -> [real: bb2, unwind: bb6];
}
@ -35,13 +34,14 @@
}
bb4: {
+ Coverage::VirtualCounter(bcb2);
_0 = const ();
StorageDead(_2);
return;
}
bb5: {
+ Coverage::ExpressionUsed(0);
+ Coverage::VirtualCounter(bcb3);
_1 = const ();
StorageDead(_2);
goto -> bb1;

4
tests/mir-opt/coverage/instrument_coverage.rs
View file

@ -10,7 +10,7 @@
// CHECK: coverage body span:
// CHECK: coverage Code { bcb: bcb{{[0-9]+}} } =>
// CHECK: bb0:
// CHECK: Coverage::CounterIncrement
// CHECK: Coverage::VirtualCounter
fn main() {
loop {
if bar() {
@ -24,7 +24,7 @@ fn main() {
// CHECK: coverage body span:
// CHECK: coverage Code { bcb: bcb{{[0-9]+}} } =>
// CHECK: bb0:
// CHECK: Coverage::CounterIncrement
// CHECK: Coverage::VirtualCounter
#[inline(never)]
fn bar() -> bool {
true

8
tests/mir-opt/coverage/instrument_coverage_cleanup.main.CleanupPostBorrowck.diff
View file

@ -8,7 +8,6 @@
coverage branch { true: BlockMarkerId(0), false: BlockMarkerId(1) } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0)
coverage body span: $DIR/instrument_coverage_cleanup.rs:13:11: 15:2 (#0)
coverage ExpressionId(0) => Expression { lhs: Counter(0), op: Subtract, rhs: Counter(1) };
coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0);
coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0);
coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0);
@ -16,7 +15,7 @@
coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0);
bb0: {
Coverage::CounterIncrement(0);
Coverage::VirtualCounter(bcb0);
- Coverage::SpanMarker;
+ nop;
StorageLive(_1);
@ -28,7 +27,7 @@
}
bb2: {
Coverage::CounterIncrement(1);
Coverage::VirtualCounter(bcb1);
- Coverage::BlockMarker(1);
+ nop;
_0 = const ();
@ -36,7 +35,7 @@
}
bb3: {
Coverage::ExpressionUsed(0);
Coverage::VirtualCounter(bcb3);
- Coverage::BlockMarker(0);
+ nop;
_0 = const ();
@ -44,6 +43,7 @@
}
bb4: {
Coverage::VirtualCounter(bcb2);
StorageDead(_1);
return;
}

8
tests/mir-opt/coverage/instrument_coverage_cleanup.main.InstrumentCoverage.diff
View file

@ -8,7 +8,6 @@
coverage branch { true: BlockMarkerId(0), false: BlockMarkerId(1) } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0)
+ coverage body span: $DIR/instrument_coverage_cleanup.rs:13:11: 15:2 (#0)
+ coverage ExpressionId(0) => Expression { lhs: Counter(0), op: Subtract, rhs: Counter(1) };
+ coverage Code { bcb: bcb0 } => $DIR/instrument_coverage_cleanup.rs:13:1: 14:36 (#0);
+ coverage Code { bcb: bcb3 } => $DIR/instrument_coverage_cleanup.rs:14:37: 14:39 (#0);
+ coverage Code { bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:39: 14:39 (#0);
@ -16,7 +15,7 @@
+ coverage Branch { true_bcb: bcb3, false_bcb: bcb1 } => $DIR/instrument_coverage_cleanup.rs:14:8: 14:36 (#0);
+
bb0: {
+ Coverage::CounterIncrement(0);
+ Coverage::VirtualCounter(bcb0);
Coverage::SpanMarker;
StorageLive(_1);
_1 = std::hint::black_box::<bool>(const true) -> [return: bb1, unwind: bb5];
@ -27,20 +26,21 @@
}
bb2: {
+ Coverage::CounterIncrement(1);
+ Coverage::VirtualCounter(bcb1);
Coverage::BlockMarker(1);
_0 = const ();
goto -> bb4;
}
bb3: {
+ Coverage::ExpressionUsed(0);
+ Coverage::VirtualCounter(bcb3);
Coverage::BlockMarker(0);
_0 = const ();
goto -> bb4;
}
bb4: {
+ Coverage::VirtualCounter(bcb2);
StorageDead(_1);
return;
}

4
tests/mir-opt/coverage/instrument_coverage_cleanup.rs
View file

@ -3,7 +3,7 @@
// but leaves the statements that were added by InstrumentCoverage.
//
// Removed statement kinds: BlockMarker, SpanMarker
// Retained statement kinds: CounterIncrement, ExpressionUsed
// Retained statement kinds: VirtualCounter
//@ test-mir-pass: InstrumentCoverage
//@ compile-flags: -Cinstrument-coverage -Zcoverage-options=branch -Zno-profiler-runtime
@ -16,6 +16,6 @@ fn main() {
// CHECK-NOT: Coverage::BlockMarker
// CHECK-NOT: Coverage::SpanMarker
// CHECK: Coverage::CounterIncrement
// CHECK: Coverage::VirtualCounter
// CHECK-NOT: Coverage::BlockMarker
// CHECK-NOT: Coverage::SpanMarker