Rollup merge of #136053 - Zalathar:defer-counters, r=saethlin
coverage: Defer part of counter-creation until codegen Follow-up to #135481 and #135873. One of the pleasant properties of the new counter-assignment algorithm is that we can stop partway through the process, store the intermediate state in MIR, and then resume the rest of the algorithm during codegen. This lets it take into account which parts of the control-flow graph were eliminated by MIR opts, resulting in fewer physical counters and simpler counter expressions. Those improvements end up completely obsoleting much larger chunks of code that were previously responsible for cleaning up the coverage metadata after MIR opts, while also doing a more thorough cleanup job. (That change also unlocks some further simplifications that I've kept out of this PR to limit its scope.)
This commit is contained in:
Jubilee
GitHub
commit
7f8108afc8
28 changed files with 356 additions and 665 deletions
|
|
@ -11,7 +11,8 @@ use rustc_codegen_ssa::traits::{
|
|||
BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods,
|
||||
};
|
||||
use rustc_middle::mir::coverage::{
|
||||
CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, MappingKind, Op,
|
||||
BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping,
|
||||
MappingKind, Op,
|
||||
};
|
||||
use rustc_middle::ty::{Instance, TyCtxt};
|
||||
use rustc_span::Span;
|
||||
|
|
@ -53,7 +54,7 @@ pub(crate) fn prepare_covfun_record<'tcx>(
|
|||
let fn_cov_info = tcx.instance_mir(instance.def).function_coverage_info.as_deref()?;
|
||||
let ids_info = tcx.coverage_ids_info(instance.def)?;
|
||||
|
||||
let expressions = prepare_expressions(fn_cov_info, ids_info, is_used);
|
||||
let expressions = prepare_expressions(ids_info);
|
||||
|
||||
let mut covfun = CovfunRecord {
|
||||
mangled_function_name: tcx.symbol_name(instance).name,
|
||||
|
|
@ -75,26 +76,14 @@ pub(crate) fn prepare_covfun_record<'tcx>(
|
|||
}
|
||||
|
||||
/// Convert the function's coverage-counter expressions into a form suitable for FFI.
|
||||
fn prepare_expressions(
|
||||
fn_cov_info: &FunctionCoverageInfo,
|
||||
ids_info: &CoverageIdsInfo,
|
||||
is_used: bool,
|
||||
) -> Vec<ffi::CounterExpression> {
|
||||
// If any counters or expressions were removed by MIR opts, replace their
|
||||
// terms with zero.
|
||||
let counter_for_term = |term| {
|
||||
if !is_used || ids_info.is_zero_term(term) {
|
||||
ffi::Counter::ZERO
|
||||
} else {
|
||||
ffi::Counter::from_term(term)
|
||||
}
|
||||
};
|
||||
fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec<ffi::CounterExpression> {
|
||||
let counter_for_term = ffi::Counter::from_term;
|
||||
|
||||
// We know that LLVM will optimize out any unused expressions before
|
||||
// producing the final coverage map, so there's no need to do the same
|
||||
// thing on the Rust side unless we're confident we can do much better.
|
||||
// (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.)
|
||||
fn_cov_info
|
||||
ids_info
|
||||
.expressions
|
||||
.iter()
|
||||
.map(move |&Expression { lhs, op, rhs }| ffi::CounterExpression {
|
||||
|
|
@ -136,11 +125,16 @@ fn fill_region_tables<'tcx>(
|
|||
|
||||
// For each counter/region pair in this function+file, convert it to a
|
||||
// form suitable for FFI.
|
||||
let is_zero_term = |term| !covfun.is_used || ids_info.is_zero_term(term);
|
||||
for &Mapping { ref kind, span } in &fn_cov_info.mappings {
|
||||
// If the mapping refers to counters/expressions that were removed by
|
||||
// MIR opts, replace those occurrences with zero.
|
||||
let kind = kind.map_terms(|term| if is_zero_term(term) { CovTerm::Zero } else { term });
|
||||
// If this function is unused, replace all counters with zero.
|
||||
let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter {
|
||||
let term = if covfun.is_used {
|
||||
ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term")
|
||||
} else {
|
||||
CovTerm::Zero
|
||||
};
|
||||
ffi::Counter::from_term(term)
|
||||
};
|
||||
|
||||
// Convert the `Span` into coordinates that we can pass to LLVM, or
|
||||
// discard the span if conversion fails. In rare, cases _all_ of a
|
||||
|
|
@ -154,23 +148,22 @@ fn fill_region_tables<'tcx>(
|
|||
continue;
|
||||
}
|
||||
|
||||
match kind {
|
||||
MappingKind::Code(term) => {
|
||||
code_regions
|
||||
.push(ffi::CodeRegion { cov_span, counter: ffi::Counter::from_term(term) });
|
||||
match *kind {
|
||||
MappingKind::Code { bcb } => {
|
||||
code_regions.push(ffi::CodeRegion { cov_span, counter: counter_for_bcb(bcb) });
|
||||
}
|
||||
MappingKind::Branch { true_term, false_term } => {
|
||||
MappingKind::Branch { true_bcb, false_bcb } => {
|
||||
branch_regions.push(ffi::BranchRegion {
|
||||
cov_span,
|
||||
true_counter: ffi::Counter::from_term(true_term),
|
||||
false_counter: ffi::Counter::from_term(false_term),
|
||||
true_counter: counter_for_bcb(true_bcb),
|
||||
false_counter: counter_for_bcb(false_bcb),
|
||||
});
|
||||
}
|
||||
MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => {
|
||||
MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params } => {
|
||||
mcdc_branch_regions.push(ffi::MCDCBranchRegion {
|
||||
cov_span,
|
||||
true_counter: ffi::Counter::from_term(true_term),
|
||||
false_counter: ffi::Counter::from_term(false_term),
|
||||
true_counter: counter_for_bcb(true_bcb),
|
||||
false_counter: counter_for_bcb(false_bcb),
|
||||
mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -160,21 +160,12 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
|
|||
CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!(
|
||||
"marker statement {kind:?} should have been removed by CleanupPostBorrowck"
|
||||
),
|
||||
CoverageKind::CounterIncrement { id } => {
|
||||
// The number of counters passed to `llvm.instrprof.increment` might
|
||||
// be smaller than the number originally inserted by the instrumentor,
|
||||
// if some high-numbered counters were removed by MIR optimizations.
|
||||
// If so, LLVM's profiler runtime will use fewer physical counters.
|
||||
let num_counters = ids_info.num_counters_after_mir_opts();
|
||||
assert!(
|
||||
num_counters as usize <= function_coverage_info.num_counters,
|
||||
"num_counters disagreement: query says {num_counters} but function info only has {}",
|
||||
function_coverage_info.num_counters
|
||||
);
|
||||
|
||||
CoverageKind::VirtualCounter { bcb }
|
||||
if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) =>
|
||||
{
|
||||
let fn_name = bx.get_pgo_func_name_var(instance);
|
||||
let hash = bx.const_u64(function_coverage_info.function_source_hash);
|
||||
let num_counters = bx.const_u32(num_counters);
|
||||
let num_counters = bx.const_u32(ids_info.num_counters);
|
||||
let index = bx.const_u32(id.as_u32());
|
||||
debug!(
|
||||
"codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
|
||||
|
|
@ -182,10 +173,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
|
|||
);
|
||||
bx.instrprof_increment(fn_name, hash, num_counters, index);
|
||||
}
|
||||
CoverageKind::ExpressionUsed { id: _ } => {
|
||||
// Expression-used statements are markers that are handled by
|
||||
// `coverage_ids_info`, so there's nothing to codegen here.
|
||||
}
|
||||
// If a BCB doesn't have an associated physical counter, there's nothing to codegen.
|
||||
CoverageKind::VirtualCounter { .. } => {}
|
||||
CoverageKind::CondBitmapUpdate { index, decision_depth } => {
|
||||
let cond_bitmap = coverage_cx
|
||||
.try_get_mcdc_condition_bitmap(&instance, decision_depth)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
#![feature(extern_types)]
|
||||
#![feature(file_buffered)]
|
||||
#![feature(hash_raw_entry)]
|
||||
#![feature(if_let_guard)]
|
||||
#![feature(impl_trait_in_assoc_type)]
|
||||
#![feature(iter_intersperse)]
|
||||
#![feature(let_chains)]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue