coverage: Collect a function's coverage mappings into a single list

This is an intermediate step towards being able to store all of a function's
mappings in function coverage info.

compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs

@@ -1,9 +1,10 @@
 use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind};
 
 use rustc_data_structures::fx::FxIndexSet;
+use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_middle::mir::coverage::{
-    CodeRegion, CounterId, CovTerm, ExpressionId, FunctionCoverageInfo, Op,
+    CodeRegion, CounterId, CovTerm, ExpressionId, FunctionCoverageInfo, Mapping, Op,
 };
 use rustc_middle::ty::Instance;
 
@@ -12,28 +13,21 @@ pub struct Expression {
     lhs: CovTerm,
     op: Op,
     rhs: CovTerm,
-    code_regions: Vec<CodeRegion>,
 }
 
-/// Collects all of the coverage regions associated with (a) injected counters, (b) counter
+/// Holds all of the coverage mapping data associated with a function instance,
-/// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero),
+/// collected during traversal of `Coverage` statements in the function's MIR.
-/// for a given Function. This struct also stores the `function_source_hash`,
-/// computed during instrumentation, and forwarded with counters.
-///
-/// Note, it may be important to understand LLVM's definitions of `unreachable` regions versus "gap
-/// regions" (or "gap areas"). A gap region is a code region within a counted region (either counter
-/// or expression), but the line or lines in the gap region are not executable (such as lines with
-/// only whitespace or comments). According to LLVM Code Coverage Mapping documentation, "A count
-/// for a gap area is only used as the line execution count if there are no other regions on a
-/// line."
 #[derive(Debug)]
 pub struct FunctionCoverage<'tcx> {
     /// Coverage info that was attached to this function by the instrumentor.
     function_coverage_info: &'tcx FunctionCoverageInfo,
     is_used: bool,
-    counters: IndexVec<CounterId, Option<Vec<CodeRegion>>>,
+
+    /// Tracks which counters have been seen, to avoid duplicate mappings
+    /// that might be introduced by MIR inlining.
+    counters_seen: BitSet<CounterId>,
     expressions: IndexVec<ExpressionId, Option<Expression>>,
-    unreachable_regions: Vec<CodeRegion>,
+    mappings: Vec<Mapping>,
 }
 
 impl<'tcx> FunctionCoverage<'tcx> {
@@ -67,9 +61,9 @@ impl<'tcx> FunctionCoverage<'tcx> {
         Self {
             function_coverage_info,
             is_used,
-            counters: IndexVec::from_elem_n(None, num_counters),
+            counters_seen: BitSet::new_empty(num_counters),
             expressions: IndexVec::from_elem_n(None, num_expressions),
-            unreachable_regions: Vec::new(),
+            mappings: Vec::new(),
         }
     }
 
@@ -81,19 +75,8 @@ impl<'tcx> FunctionCoverage<'tcx> {
     /// Adds code regions to be counted by an injected counter intrinsic.
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn add_counter(&mut self, id: CounterId, code_regions: &[CodeRegion]) {
-        if code_regions.is_empty() {
+        if self.counters_seen.insert(id) {
-            return;
+            self.add_mappings(CovTerm::Counter(id), code_regions);
-        }
-
-        let slot = &mut self.counters[id];
-        match slot {
-            None => *slot = Some(code_regions.to_owned()),
-            // If this counter ID slot has already been filled, it should
-            // contain identical information.
-            Some(ref previous_regions) => assert_eq!(
-                previous_regions, code_regions,
-                "add_counter: code regions for id changed"
-            ),
         }
     }
 
@@ -121,10 +104,13 @@ impl<'tcx> FunctionCoverage<'tcx> {
             self,
         );
 
-        let expression = Expression { lhs, op, rhs, code_regions: code_regions.to_owned() };
+        let expression = Expression { lhs, op, rhs };
         let slot = &mut self.expressions[expression_id];
         match slot {
-            None => *slot = Some(expression),
+            None => {
+                *slot = Some(expression);
+                self.add_mappings(CovTerm::Expression(expression_id), code_regions);
+            }
             // If this expression ID slot has already been filled, it should
             // contain identical information.
             Some(ref previous_expression) => assert_eq!(
@@ -138,7 +124,25 @@ impl<'tcx> FunctionCoverage<'tcx> {
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn add_unreachable_regions(&mut self, code_regions: &[CodeRegion]) {
         assert!(!code_regions.is_empty(), "unreachable regions always have code regions");
-        self.unreachable_regions.extend_from_slice(code_regions);
+        self.add_mappings(CovTerm::Zero, code_regions);
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn add_mappings(&mut self, term: CovTerm, code_regions: &[CodeRegion]) {
+        self.mappings
+            .extend(code_regions.iter().cloned().map(|code_region| Mapping { term, code_region }));
+    }
+
+    pub(crate) fn finalize(&mut self) {
+        self.simplify_expressions();
+
+        // Reorder the collected mappings so that counter mappings are first and
+        // zero mappings are last, matching the historical order.
+        self.mappings.sort_by_key(|mapping| match mapping.term {
+            CovTerm::Counter(_) => 0,
+            CovTerm::Expression(_) => 1,
+            CovTerm::Zero => u8::MAX,
+        });
     }
 
     /// Perform some simplifications to make the final coverage mappings
@@ -147,7 +151,7 @@ impl<'tcx> FunctionCoverage<'tcx> {
     /// This method mainly exists to preserve the simplifications that were
     /// already being performed by the Rust-side expression renumbering, so that
     /// the resulting coverage mappings don't get worse.
-    pub(crate) fn simplify_expressions(&mut self) {
+    fn simplify_expressions(&mut self) {
         // The set of expressions that either were optimized out entirely, or
         // have zero as both of their operands, and will therefore always have
         // a value of zero. Other expressions that refer to these as operands
@@ -212,27 +216,10 @@ impl<'tcx> FunctionCoverage<'tcx> {
         assert_eq!(self.expressions.len(), counter_expressions.len());
 
         let counter_regions = self.counter_regions();
-        let expression_regions = self.expression_regions();
-        let unreachable_regions = self.unreachable_regions();
 
-        let counter_regions =
-            counter_regions.chain(expression_regions.into_iter().chain(unreachable_regions));
         (counter_expressions, counter_regions)
     }
 
-    fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> {
-        self.counters
-            .iter_enumerated()
-            // Filter out counter IDs that we never saw during MIR traversal.
-            // This can happen if a counter was optimized out by MIR transforms
-            // (and replaced with `CoverageKind::Unreachable` instead).
-            .filter_map(|(id, maybe_code_regions)| Some((id, maybe_code_regions.as_ref()?)))
-            .flat_map(|(id, code_regions)| {
-                let counter = Counter::counter_value_reference(id);
-                code_regions.iter().map(move |region| (counter, region))
-            })
-    }
-
     /// Convert this function's coverage expression data into a form that can be
     /// passed through FFI to LLVM.
     fn counter_expressions(&self) -> Vec<CounterExpression> {
@@ -266,24 +253,12 @@ impl<'tcx> FunctionCoverage<'tcx> {
             .collect::<Vec<_>>()
     }
 
-    fn expression_regions(&self) -> Vec<(Counter, &CodeRegion)> {
+    /// Converts this function's coverage mappings into an intermediate form
-        // Find all of the expression IDs that weren't optimized out AND have
+    /// that will be used by `mapgen` when preparing for FFI.
-        // one or more attached code regions, and return the corresponding
+    fn counter_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> {
-        // mappings as counter/region pairs.
+        self.mappings.iter().map(|&Mapping { term, ref code_region }| {
-        self.expressions
+            let counter = Counter::from_term(term);
-            .iter_enumerated()
+            (counter, code_region)
-            .filter_map(|(id, maybe_expression)| {
-                let code_regions = &maybe_expression.as_ref()?.code_regions;
-                Some((id, code_regions))
         })
-            .flat_map(|(id, code_regions)| {
-                let counter = Counter::expression(id);
-                code_regions.iter().map(move |code_region| (counter, code_region))
-            })
-            .collect::<Vec<_>>()
-    }
-
-    fn unreachable_regions(&self) -> impl Iterator<Item = (Counter, &CodeRegion)> {
-        self.unreachable_regions.iter().map(|region| (Counter::ZERO, region))
     }
 }

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -61,7 +61,7 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) {
     let mut function_data = Vec::new();
     for (instance, mut function_coverage) in function_coverage_map {
         debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance);
-        function_coverage.simplify_expressions();
+        function_coverage.finalize();
         let function_coverage = function_coverage;
 
         let mangled_function_name = tcx.symbol_name(instance).name;
@@ -169,10 +169,11 @@ fn encode_mappings_for_function(
     let mut virtual_file_mapping = IndexVec::<u32, u32>::new();
     let mut mapping_regions = Vec::with_capacity(counter_regions.len());
 
-    // Sort the list of (counter, region) mapping pairs by region, so that they
+    // Sort and group the list of (counter, region) mapping pairs by filename.
-    // can be grouped by filename. Prepare file IDs for each filename, and
+    // (Preserve any further ordering imposed by `FunctionCoverage`.)
-    // prepare the mapping data so that we can pass it through FFI to LLVM.
+    // Prepare file IDs for each filename, and prepare the mapping data so that
-    counter_regions.sort_by_key(|(_counter, region)| *region);
+    // we can pass it through FFI to LLVM.
+    counter_regions.sort_by_key(|(_counter, region)| region.file_name);
     for counter_regions_for_file in
         counter_regions.group_by(|(_, a), (_, b)| a.file_name == b.file_name)
     {

compiler/rustc_middle/src/mir/coverage.rs

@@ -135,6 +135,20 @@ impl Op {
     }
 }
 
+#[derive(Clone, Debug)]
+#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub struct Mapping {
+    pub code_region: CodeRegion,
+
+    /// Indicates whether this mapping uses a counter value, expression value,
+    /// or zero value.
+    ///
+    /// FIXME: When we add support for mapping kinds other than `Code`
+    /// (e.g. branch regions, expansion regions), replace this with a dedicated
+    /// mapping-kind enum.
+    pub term: CovTerm,
+}
+
 /// Stores per-function coverage information attached to a `mir::Body`,
 /// to be used in conjunction with the individual coverage statements injected
 /// into the function's basic blocks.