bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Workaround for inconsistent order of llvm-cov results on Windows

Browse source
This commit is contained in:
Rich Kadel 2020-12-02 09:43:46 -08:00
parent d96f351fa3
commit b0c140a55b
4 changed files with 50 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/test/run-make-fulldeps/coverage-reports/Makefile
View file

@ -22,6 +22,43 @@ ifeq ($(LLVM_COV_DEBUG), 1)
DEBUG_FLAG=--debug
endif
# FIXME(richkadel): I'm adding `--ignore-filename-regex=` line(s) for specific test(s) that produce
# `llvm-cov` results for multiple files (for example `uses_crate.rs` and `used_crate/mod.rs`) as a
# workaround for two problems causing tests to fail on Windows:
#
# 1. When multiple files appear in the `llvm-cov show` results, each file's coverage results can
# appear in different a different order. Whether this is random or, somehow, platform-specific,
# the Windows output flips the order of the files, compared to Linux. In the `uses_crate.rs`
# test, the only test-unique (interesting) results we care about are the results for only one
# of the two files, `mod/uses_crate.rs`, so the workaround is to ignore all but this one file.
# In the future, we may want a more sophisticated solution that splits apart `llvm-cov show`
# results into separate results files for each result (taking care not to create new file
# paths that might be too long for Windows MAX_PATH limits when creating these new sub-results,
# as well).
# 2. When multiple files appear in the `llvm-cov show` results, the results for each file are
# prefixed with their filename, including platform-specific path separators (`\` for Windows,
# and `/` everywhere else). This could be filtered or normalized of course, but by ignoring
# coverage results for all but one of the file, the filenames are no longer included anyway.
# If this changes (if/when we decide to support `llvm-cov show` results for multiple files),
# the file path separator differences may need to be addressed.
#
# Since this is only a workaround, I decided to implement the override by adding an option for
# each file to be ignored, using a `--ignore-filename-regex=` entry for each one, rather than
# implement some more sophisticated solution with a new custom test directive in the test file
# itself (similar to `expect-exit-status`) because that would add a lot of complexity and still
# be a workaround, with the same result, with no benefit.
#
# Yes these `--ignore-filename-regex=` options are included in all invocations of `llvm-cov show`
# for now, but it is effectively ignored for all tests that don't include this file anyway.
#
# Note that it's also possible the `_counters.<test>.txt` and `<test>.json` files may order
# results from multiple files inconsistently, which might also have to be accomodated if and when
# we allow `llvm-cov` to produce results for multiple files. (The path separators appear to be
# normalized to `/` in those files, thankfully.) But since we are ignoring results for all but one
# file, this workaround addresses those potential issues as well.
LLVM_COV_IGNORE_FILES=\
--ignore-filename-regex=uses_crate.rs
# When generating `expected_*` results (using `x.py test --bless`), the `--debug` flag is forced.
# If assertions are disabled, the command will fail with an error, rather than attempt to generate
# only partial results.
@ -76,6 +113,7 @@ endif
# Generate a coverage report using `llvm-cov show`.
"$(LLVM_BIN_DIR)"/llvm-cov show \
$(DEBUG_FLAG) \
$(LLVM_COV_IGNORE_FILES) \
--Xdemangler="$(RUST_DEMANGLER)" \
--show-line-counts-or-regions \
--instr-profile="$(TMPDIR)"/$@.profdata \
@ -133,6 +171,7 @@ endif
# Generate a coverage report in JSON, using `llvm-cov export`, and fail if
# there are differences from the expected output.
"$(LLVM_BIN_DIR)"/llvm-cov export \
$(LLVM_COV_IGNORE_FILES) \
--summary-only \
--instr-profile="$(TMPDIR)"/$@.profdata \
$(call BIN,"$(TMPDIR)"/$@) \

48
src/test/run-make-fulldeps/coverage-reports/expected_export_coverage.uses_crate.json
View file

@ -27,55 +27,29 @@
"percent": 37.5
}
}
},
{
"filename": "../coverage/uses_crate.rs",
"summary": {
"functions": {
"count": 1,
"covered": 1,
"percent": 100
},
"instantiations": {
"count": 1,
"covered": 1,
"percent": 100
},
"lines": {
"count": 6,
"covered": 6,
"percent": 100
},
"regions": {
"count": 1,
"covered": 1,
"notcovered": 0,
"percent": 100
}
}
}
],
"totals": {
"functions": {
"count": 3,
"covered": 3,
"percent": 100
},
"instantiations": {
"count": 4,
"covered": 4,
"percent": 100
},
"instantiations": {
"count": 5,
"covered": 5,
"percent": 100
},
"lines": {
"count": 37,
"covered": 20,
"percent": 54.054054054054056
"count": 31,
"covered": 14,
"percent": 45.16129032258064
},
"regions": {
"count": 17,
"covered": 7,
"count": 16,
"covered": 6,
"notcovered": 10,
"percent": 41.17647058823529
"percent": 37.5
}
}
}

13
src/test/run-make-fulldeps/coverage-reports/expected_show_coverage.uses_crate.txt
View file

@ -1,4 +1,3 @@
../coverage/used_crate/mod.rs:
1| |#![allow(unused_assignments, unused_variables)]
2| |
3| |use std::fmt::Debug;
@ -55,15 +54,3 @@
42| 0| }
43| 0|}
../coverage/uses_crate.rs:
1| |#![allow(unused_assignments, unused_variables)]
2| |
3| |mod used_crate;
4| |
5| 1|fn main() {
6| 1| used_crate::used_function();
7| 1| let some_vec = vec![1, 2, 3, 4];
8| 1| used_crate::used_generic_function(&some_vec);
9| 1| used_crate::used_twice_generic_function(some_vec);
10| 1|}

5
src/test/run-make-fulldeps/coverage-reports/expected_show_coverage_counters.uses_crate.txt
View file

@ -73,8 +73,3 @@ Combined regions:
21:1 -> 23:2 (count=1)
Segment at 21:1 (count = 1), RegionEntry
Segment at 23:2 (count = 0), Skipped
Emitting segments for file: ../coverage/uses_crate.rs
Combined regions:
5:1 -> 10:2 (count=1)
Segment at 5:1 (count = 1), RegionEntry
Segment at 10:2 (count = 0), Skipped