From bc4c89c10a02abda1c2c83b7fcde756c84913e5f Mon Sep 17 00:00:00 2001 From: Graydon Hoare Date: Mon, 15 Jul 2013 18:51:20 -0700 Subject: [PATCH] compiletest: Add support for metrics and ratchet modes. --- src/compiletest/common.rs | 9 ++++++++ src/compiletest/compiletest.rs | 40 ++++++++++++++++++++++++++++------ src/compiletest/runtest.rs | 28 ++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/compiletest/common.rs b/src/compiletest/common.rs index df00286c87f..4add16fd7a9 100644 --- a/src/compiletest/common.rs +++ b/src/compiletest/common.rs @@ -58,6 +58,15 @@ pub struct config { // Write out a parseable log of tests that were run logfile: Option, + // Write out a json file containing any metrics of the run + save_metrics: Option, + + // Write and ratchet a metrics file + ratchet_metrics: Option, + + // Percent change in metrics to consider noise + ratchet_noise_percent: Option, + // A command line to prefix program execution with, // for running under valgrind runtool: Option<~str>, diff --git a/src/compiletest/compiletest.rs b/src/compiletest/compiletest.rs index a411e714247..39dc55b44f4 100644 --- a/src/compiletest/compiletest.rs +++ b/src/compiletest/compiletest.rs @@ -17,6 +17,7 @@ extern mod extra; use std::os; +use std::f64; use extra::getopts; use extra::getopts::groups::{optopt, optflag, reqopt}; @@ -66,6 +67,10 @@ pub fn parse_config(args: ~[~str]) -> config { optopt("", "rustcflags", "flags to pass to rustc", "FLAGS"), optflag("", "verbose", "run tests verbosely, showing all output"), optopt("", "logfile", "file to log test execution to", "FILE"), + optopt("", "save-metrics", "file to save metrics to", "FILE"), + optopt("", "ratchet-metrics", "file to ratchet metrics against", "FILE"), + optopt("", "ratchet-noise-percent", + "percent change in metrics to consider noise", "N"), optflag("", "jit", "run tests under the JIT"), optflag("", "newrt", "run tests on the new runtime / scheduler"), optopt("", "target", "the target to build for", "TARGET"), @@ -116,6 +121,13 @@ pub fn parse_config(args: ~[~str]) -> config { Some(copy matches.free[0]) } else { None }, logfile: getopts::opt_maybe_str(matches, "logfile").map(|s| Path(*s)), + save_metrics: getopts::opt_maybe_str(matches, "save-metrics").map(|s| Path(*s)), + ratchet_metrics: + getopts::opt_maybe_str(matches, "ratchet-metrics").map(|s| Path(*s)), + ratchet_noise_percent: + getopts::opt_maybe_str(matches, + "ratchet-noise-percent").map(|s| + f64::from_str(*s).get()), runtool: getopts::opt_maybe_str(matches, "runtool"), rustcflags: getopts::opt_maybe_str(matches, "rustcflags"), jit: getopts::opt_present(matches, "jit"), @@ -215,10 +227,10 @@ pub fn test_opts(config: &config) -> test::TestOpts { run_ignored: config.run_ignored, logfile: copy config.logfile, run_tests: true, - run_benchmarks: false, - ratchet_metrics: None, - ratchet_noise_percent: None, - save_metrics: None, + run_benchmarks: true, + ratchet_metrics: copy config.ratchet_metrics, + ratchet_noise_percent: copy config.ratchet_noise_percent, + save_metrics: copy config.save_metrics, } } @@ -231,7 +243,13 @@ pub fn make_tests(config: &config) -> ~[test::TestDescAndFn] { let file = copy *file; debug!("inspecting file %s", file.to_str()); if is_test(config, file) { - tests.push(make_test(config, file)) + let t = do make_test(config, file) { + match config.mode { + mode_codegen => make_metrics_test_closure(config, file), + _ => make_test_closure(config, file) + } + }; + tests.push(t) } } tests @@ -260,14 +278,15 @@ pub fn is_test(config: &config, testfile: &Path) -> bool { return valid; } -pub fn make_test(config: &config, testfile: &Path) -> test::TestDescAndFn { +pub fn make_test(config: &config, testfile: &Path, + f: &fn()->test::TestFn) -> test::TestDescAndFn { test::TestDescAndFn { desc: test::TestDesc { name: make_test_name(config, testfile), ignore: header::is_test_ignored(config, testfile), should_fail: false }, - testfn: make_test_closure(config, testfile), + testfn: f(), } } @@ -291,3 +310,10 @@ pub fn make_test_closure(config: &config, testfile: &Path) -> test::TestFn { let testfile = Cell::new(testfile.to_str()); test::DynTestFn(|| { runtest::run(config.take(), testfile.take()) }) } + +pub fn make_metrics_test_closure(config: &config, testfile: &Path) -> test::TestFn { + use std::cell::Cell; + let config = Cell::new(copy *config); + let testfile = Cell::new(testfile.to_str()); + test::DynMetricFn(|mm| { runtest::run_metrics(config.take(), testfile.take(), mm) }) +} diff --git a/src/compiletest/runtest.rs b/src/compiletest/runtest.rs index dee07c6de49..a51ab820856 100644 --- a/src/compiletest/runtest.rs +++ b/src/compiletest/runtest.rs @@ -25,7 +25,14 @@ use std::os; use std::uint; use std::vec; +use extra::test::MetricMap; + pub fn run(config: config, testfile: ~str) { + let mut _mm = MetricMap::new(); + run_metrics(config, testfile, &mut _mm); +} + +pub fn run_metrics(config: config, testfile: ~str, mm: &mut MetricMap) { if config.verbose { // We're going to be dumping a lot of info. Start on a new line. io::stdout().write_str("\n\n"); @@ -40,7 +47,7 @@ pub fn run(config: config, testfile: ~str) { mode_run_pass => run_rpass_test(&config, &props, &testfile), mode_pretty => run_pretty_test(&config, &props, &testfile), mode_debug_info => run_debuginfo_test(&config, &props, &testfile), - mode_codegen => run_codegen_test(&config, &props, &testfile) + mode_codegen => run_codegen_test(&config, &props, &testfile, mm) } } @@ -906,7 +913,14 @@ fn disassemble_extract(config: &config, _props: &TestProps, } -fn run_codegen_test(config: &config, props: &TestProps, testfile: &Path) { +fn count_extracted_lines(p: &Path) -> uint { + let x = io::read_whole_file_str(&p.with_filetype("ll")).get(); + x.line_iter().len_() +} + + +fn run_codegen_test(config: &config, props: &TestProps, + testfile: &Path, mm: &mut MetricMap) { if config.llvm_bin_path.is_none() { fatal(~"missing --llvm-bin-path"); @@ -947,7 +961,17 @@ fn run_codegen_test(config: &config, props: &TestProps, testfile: &Path) { fatal_ProcRes(~"disassembling extract failed", &ProcRes); } + let base = output_base_name(config, testfile); + let base_extract = append_suffix_to_stem(&base, "extract"); + let base_clang = append_suffix_to_stem(&base, "clang"); + let base_clang_extract = append_suffix_to_stem(&base_clang, "extract"); + let base_lines = count_extracted_lines(&base_extract); + let clang_lines = count_extracted_lines(&base_clang_extract); + + mm.insert_metric("clang-codegen-ratio", + (base_lines as f64) / (clang_lines as f64), + 0.001); }