Auto merge of #64158 - tmandry:libtest-panic-abort, r=alexcrichton

panic=abort support in libtest

Add experimental support for tests compiled with panic=abort. Enabled with `-Z panic_abort_tests`.

r? @alexcrichton
cc @cramertj

src/librustc/session/config.rs

@@ -1279,6 +1279,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "show extended diagnostic help"),
     terminal_width: Option<usize> = (None, parse_opt_uint, [UNTRACKED],
         "set the current terminal width"),
+    panic_abort_tests: bool = (false, parse_bool, [TRACKED],
+        "support compiling tests with panic=abort"),
     continue_parse_after_error: bool = (false, parse_bool, [TRACKED],
         "attempt to recover from parse errors (experimental)"),
     dep_tasks: bool = (false, parse_bool, [UNTRACKED],

src/librustc_interface/passes.rs

@@ -440,6 +440,9 @@ fn configure_and_expand_inner<'a>(
             &mut krate,
             sess.diagnostic(),
             &sess.features_untracked(),
+            sess.panic_strategy(),
+            sess.target.target.options.panic_strategy,
+            sess.opts.debugging_opts.panic_abort_tests,
         )
     });
 

src/libsyntax_ext/test_harness.rs

@@ -2,6 +2,7 @@
 
 use log::debug;
 use smallvec::{smallvec, SmallVec};
+use rustc_target::spec::PanicStrategy;
 use syntax::ast::{self, Ident};
 use syntax::attr;
 use syntax::entry::{self, EntryPointType};
@@ -25,6 +26,7 @@ struct Test {
 
 struct TestCtxt<'a> {
     ext_cx: ExtCtxt<'a>,
+    panic_strategy: PanicStrategy,
     def_site: Span,
     test_cases: Vec<Test>,
     reexport_test_harness_main: Option<Symbol>,
@@ -40,6 +42,9 @@ pub fn inject(
     krate: &mut ast::Crate,
     span_diagnostic: &errors::Handler,
     features: &Features,
+    panic_strategy: PanicStrategy,
+    platform_panic_strategy: PanicStrategy,
+    enable_panic_abort_tests: bool,
 ) {
     // Check for #![reexport_test_harness_main = "some_name"] which gives the
     // main test function the name `some_name` without hygiene. This needs to be
@@ -53,8 +58,22 @@ pub fn inject(
     let test_runner = get_test_runner(span_diagnostic, &krate);
 
     if should_test {
+        let panic_strategy = match (panic_strategy, enable_panic_abort_tests) {
+            (PanicStrategy::Abort, true) =>
+                PanicStrategy::Abort,
+            (PanicStrategy::Abort, false) if panic_strategy == platform_panic_strategy => {
+                // Silently allow compiling with panic=abort on these platforms,
+                // but with old behavior (abort if a test fails).
+                PanicStrategy::Unwind
+            }
+            (PanicStrategy::Abort, false) => {
+                span_diagnostic.err("building tests with panic=abort is not yet supported");
+                PanicStrategy::Unwind
+            }
+            (PanicStrategy::Unwind, _) => PanicStrategy::Unwind,
+        };
         generate_test_harness(sess, resolver, reexport_test_harness_main,
-                              krate, features, test_runner)
+                              krate, features, panic_strategy, test_runner)
     }
 }
 
@@ -183,6 +202,7 @@ fn generate_test_harness(sess: &ParseSess,
                          reexport_test_harness_main: Option<Symbol>,
                          krate: &mut ast::Crate,
                          features: &Features,
+                         panic_strategy: PanicStrategy,
                          test_runner: Option<ast::Path>) {
     let mut econfig = ExpansionConfig::default("test".to_string());
     econfig.features = Some(features);
@@ -203,6 +223,7 @@ fn generate_test_harness(sess: &ParseSess,
 
     let cx = TestCtxt {
         ext_cx,
+        panic_strategy,
         def_site,
         test_cases: Vec::new(),
         reexport_test_harness_main,
@@ -248,9 +269,14 @@ fn mk_main(cx: &mut TestCtxt<'_>) -> P<ast::Item> {
     let ecx = &cx.ext_cx;
     let test_id = Ident::new(sym::test, sp);
 
+    let runner_name = match cx.panic_strategy {
+        PanicStrategy::Unwind => "test_main_static",
+        PanicStrategy::Abort => "test_main_static_abort",
+    };
+
     // test::test_main_static(...)
     let mut test_runner = cx.test_runner.clone().unwrap_or(
-        ecx.path(sp, vec![test_id, ecx.ident_of("test_main_static", sp)]));
+        ecx.path(sp, vec![test_id, ecx.ident_of(runner_name, sp)]));
 
     test_runner.span = sp;
 

src/libtest/formatters/mod.rs

@@ -22,3 +22,12 @@ pub(crate) trait OutputFormatter {
     ) -> io::Result<()>;
     fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool>;
 }
+
+pub(crate) fn write_stderr_delimiter(test_output: &mut Vec<u8>, test_name: &TestName) {
+    match test_output.last() {
+        Some(b'\n') => (),
+        Some(_) => test_output.push(b'\n'),
+        None => (),
+    }
+    write!(test_output, "---- {} stderr ----\n", test_name).unwrap();
+}

src/libtest/lib.rs

@@ -21,7 +21,8 @@
 #![unstable(feature = "test", issue = "50297")]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/", test(attr(deny(warnings))))]
 #![feature(asm)]
-#![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc, rustc_private))]
+#![cfg_attr(any(unix, target_os = "cloudabi"), feature(libc))]
+#![feature(rustc_private)]
 #![feature(nll)]
 #![feature(set_stdio)]
 #![feature(panic_unwind)]
@@ -34,16 +35,6 @@ use getopts;
 extern crate libc;
 use term;
 
-// FIXME(#54291): rustc and/or LLVM don't yet support building with panic-unwind
-//                on aarch64-pc-windows-msvc, or thumbv7a-pc-windows-msvc
-//                so we don't link libtest against libunwind (for the time being)
-//                even though it means that libtest won't be fully functional on
-//                these platforms.
-//
-// See also: https://github.com/rust-lang/rust/issues/54190#issuecomment-422904437
-#[cfg(not(all(windows, any(target_arch = "aarch64", target_arch = "arm"))))]
-extern crate panic_unwind;
-
 pub use self::ColorConfig::*;
 use self::NamePadding::*;
 use self::OutputLocation::*;
@@ -61,10 +52,10 @@ use std::fmt;
 use std::fs::File;
 use std::io;
 use std::io::prelude::*;
-use std::panic::{catch_unwind, AssertUnwindSafe};
+use std::panic::{self, catch_unwind, AssertUnwindSafe, PanicInfo};
 use std::path::PathBuf;
 use std::process;
-use std::process::Termination;
+use std::process::{ExitStatus, Command, Termination};
 use std::sync::mpsc::{channel, Sender};
 use std::sync::{Arc, Mutex};
 use std::thread;
@@ -76,13 +67,21 @@ mod tests;
 const TEST_WARN_TIMEOUT_S: u64 = 60;
 const QUIET_MODE_MAX_COLUMN: usize = 100; // insert a '\n' after 100 tests in quiet mode
 
+const SECONDARY_TEST_INVOKER_VAR: &'static str = "__RUST_TEST_INVOKE";
+
+// Return codes for secondary process.
+// Start somewhere other than 0 so we know the return code means what we think
+// it means.
+const TR_OK: i32 = 50;
+const TR_FAILED: i32 = 51;
+
 // to be used by rustc to compile tests in libtest
 pub mod test {
     pub use crate::{
         assert_test_result, filter_tests, parse_opts, run_test, test_main, test_main_static,
-        Bencher, DynTestFn, DynTestName, Metric, MetricMap, Options, RunIgnored, ShouldPanic,
+        Bencher, DynTestFn, DynTestName, Metric, MetricMap, Options, RunIgnored, RunStrategy,
-        StaticBenchFn, StaticTestFn, StaticTestName, TestDesc, TestDescAndFn, TestName, TestOpts,
+        ShouldPanic, StaticBenchFn, StaticTestFn, StaticTestName, TestDesc, TestDescAndFn, TestName,
-        TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk,
+        TestOpts, TestResult, TrFailed, TrFailedMsg, TrIgnored, TrOk,
     };
 }
 
@@ -257,12 +256,14 @@ impl Metric {
 #[derive(Copy, Clone, Debug)]
 pub struct Options {
     display_output: bool,
+    panic_abort: bool,
 }
 
 impl Options {
     pub fn new() -> Options {
         Options {
             display_output: false,
+            panic_abort: false,
         }
     }
 
@@ -270,6 +271,11 @@ impl Options {
         self.display_output = display_output;
         self
     }
+
+    pub fn panic_abort(mut self, panic_abort: bool) -> Options {
+        self.panic_abort = panic_abort;
+        self
+    }
 }
 
 // The default console test runner. It accepts the command line
@@ -303,32 +309,66 @@ pub fn test_main(args: &[String], tests: Vec<TestDescAndFn>, options: Option<Opt
     }
 }
 
-// A variant optimized for invocation with a static test vector.
+/// A variant optimized for invocation with a static test vector.
-// This will panic (intentionally) when fed any dynamic tests, because
+/// This will panic (intentionally) when fed any dynamic tests.
-// it is copying the static values out into a dynamic vector and cannot
+///
-// copy dynamic values. It is doing this because from this point on
+/// This is the entry point for the main function generated by `rustc --test`
-// a Vec<TestDescAndFn> is used in order to effect ownership-transfer
+/// when panic=unwind.
-// semantics into parallel test runners, which in turn requires a Vec<>
-// rather than a &[].
 pub fn test_main_static(tests: &[&TestDescAndFn]) {
     let args = env::args().collect::<Vec<_>>();
-    let owned_tests = tests
+    let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
-        .iter()
-        .map(|t| match t.testfn {
-            StaticTestFn(f) => TestDescAndFn {
-                testfn: StaticTestFn(f),
-                desc: t.desc.clone(),
-            },
-            StaticBenchFn(f) => TestDescAndFn {
-                testfn: StaticBenchFn(f),
-                desc: t.desc.clone(),
-            },
-            _ => panic!("non-static tests passed to test::test_main_static"),
-        })
-        .collect();
     test_main(&args, owned_tests, None)
 }
 
+/// A variant optimized for invocation with a static test vector.
+/// This will panic (intentionally) when fed any dynamic tests.
+///
+/// Runs tests in panic=abort mode, which involves spawning subprocesses for
+/// tests.
+///
+/// This is the entry point for the main function generated by `rustc --test`
+/// when panic=abort.
+pub fn test_main_static_abort(tests: &[&TestDescAndFn]) {
+    // If we're being run in SpawnedSecondary mode, run the test here. run_test
+    // will then exit the process.
+    if let Ok(name) = env::var(SECONDARY_TEST_INVOKER_VAR) {
+        let test = tests
+            .iter()
+            .filter(|test| test.desc.name.as_slice() == name)
+            .map(make_owned_test)
+            .next()
+            .expect("couldn't find a test with the provided name");
+        let TestDescAndFn { desc, testfn } = test;
+        let testfn = match testfn {
+            StaticTestFn(f) => f,
+            _ => panic!("only static tests are supported"),
+        };
+        run_test_in_spawned_subprocess(desc, Box::new(testfn));
+    }
+
+    let args = env::args().collect::<Vec<_>>();
+    let owned_tests: Vec<_> = tests.iter().map(make_owned_test).collect();
+    test_main(&args, owned_tests, Some(Options::new().panic_abort(true)))
+}
+
+/// Clones static values for putting into a dynamic vector, which test_main()
+/// needs to hand out ownership of tests to parallel test runners.
+///
+/// This will panic when fed any dynamic tests, because they cannot be cloned.
+fn make_owned_test(test: &&TestDescAndFn) -> TestDescAndFn {
+    match test.testfn {
+        StaticTestFn(f) => TestDescAndFn {
+            testfn: StaticTestFn(f),
+            desc: test.desc.clone(),
+        },
+        StaticBenchFn(f) => TestDescAndFn {
+            testfn: StaticBenchFn(f),
+            desc: test.desc.clone(),
+        },
+        _ => panic!("non-static tests passed to test::test_main_static"),
+    }
+}
+
 /// Invoked when unit tests terminate. Should panic if the unit
 /// Tests is considered a failure. By default, invokes `report()`
 /// and checks for a `0` result.
@@ -1062,6 +1102,18 @@ impl Write for Sink {
     }
 }
 
+#[derive(Clone, Copy)]
+pub enum RunStrategy {
+    /// Runs the test in the current process, and sends the result back over the
+    /// supplied channel.
+    InProcess,
+
+    /// Spawns a subprocess to run the test, and sends the result back over the
+    /// supplied channel. Requires argv[0] to exist and point to the binary
+    /// that's currently running.
+    SpawnPrimary,
+}
+
 pub fn run_tests<F>(opts: &TestOpts, tests: Vec<TestDescAndFn>, mut callback: F) -> io::Result<()>
 where
     F: FnMut(TestEvent) -> io::Result<()>,
@@ -1109,6 +1161,11 @@ where
     let mut pending = 0;
 
     let (tx, rx) = channel::<MonitorMsg>();
+    let run_strategy = if opts.options.panic_abort {
+        RunStrategy::SpawnPrimary
+    } else {
+        RunStrategy::InProcess
+    };
 
     let mut running_tests: TestMap = HashMap::default();
 
@@ -1145,7 +1202,7 @@ where
         while !remaining.is_empty() {
             let test = remaining.pop().unwrap();
             callback(TeWait(test.desc.clone()))?;
-            run_test(opts, !opts.run_tests, test, tx.clone(), Concurrent::No);
+            run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::No);
             let (test, result, exec_time, stdout) = rx.recv().unwrap();
             callback(TeResult(test, result, exec_time, stdout))?;
         }
@@ -1156,7 +1213,7 @@ where
                 let timeout = Instant::now() + Duration::from_secs(TEST_WARN_TIMEOUT_S);
                 running_tests.insert(test.desc.clone(), timeout);
                 callback(TeWait(test.desc.clone()))?; //here no pad
-                run_test(opts, !opts.run_tests, test, tx.clone(), Concurrent::Yes);
+                run_test(opts, !opts.run_tests, test, run_strategy, tx.clone(), Concurrent::Yes);
                 pending += 1;
             }
 
@@ -1188,7 +1245,7 @@ where
         // All benchmarks run at the end, in serial.
         for b in filtered_benchs {
             callback(TeWait(b.desc.clone()))?;
-            run_test(opts, false, b, tx.clone(), Concurrent::No);
+            run_test(opts, false, b, run_strategy, tx.clone(), Concurrent::No);
             let (test, result, exec_time, stdout) = rx.recv().unwrap();
             callback(TeResult(test, result, exec_time, stdout))?;
         }
@@ -1415,64 +1472,38 @@ pub fn run_test(
     opts: &TestOpts,
     force_ignore: bool,
     test: TestDescAndFn,
+    strategy: RunStrategy,
     monitor_ch: Sender<MonitorMsg>,
     concurrency: Concurrent,
 ) {
     let TestDescAndFn { desc, testfn } = test;
 
-    let ignore_because_panic_abort = cfg!(target_arch = "wasm32")
+    let ignore_because_no_process_support = cfg!(target_arch = "wasm32")
         && !cfg!(target_os = "emscripten")
         && desc.should_panic != ShouldPanic::No;
 
-    if force_ignore || desc.ignore || ignore_because_panic_abort {
+    if force_ignore || desc.ignore || ignore_because_no_process_support {
         monitor_ch.send((desc, TrIgnored, None, Vec::new())).unwrap();
         return;
     }
 
     fn run_test_inner(
         desc: TestDesc,
-        monitor_ch: Sender<MonitorMsg>,
         nocapture: bool,
         report_time: bool,
+        strategy: RunStrategy,
+        monitor_ch: Sender<MonitorMsg>,
         testfn: Box<dyn FnOnce() + Send>,
         concurrency: Concurrent,
     ) {
-        // Buffer for capturing standard I/O
-        let data = Arc::new(Mutex::new(Vec::new()));
-        let data2 = data.clone();
-
         let name = desc.name.clone();
+
         let runtest = move || {
-            let oldio = if !nocapture {
+            match strategy {
-                Some((
+                RunStrategy::InProcess =>
-                    io::set_print(Some(Box::new(Sink(data2.clone())))),
+                    run_test_in_process(desc, nocapture, report_time, testfn, monitor_ch),
-                    io::set_panic(Some(Box::new(Sink(data2)))),
+                RunStrategy::SpawnPrimary => spawn_test_subprocess(desc, report_time, monitor_ch),
-                ))
+            }
-            } else {
-                None
-            };
-
-            let start = if report_time {
-                Some(Instant::now())
-            } else {
-                None
-            };
-            let result = catch_unwind(AssertUnwindSafe(testfn));
-            let exec_time = start.map(|start| {
-                let duration = start.elapsed();
-                TestExecTime(duration)
-            });
-
-            if let Some((printio, panicio)) = oldio {
-                io::set_print(printio);
-                io::set_panic(panicio);
-            };
-
-            let test_result = calc_result(&desc, result);
-            let stdout = data.lock().unwrap().to_vec();
-            monitor_ch
-                .send((desc.clone(), test_result, exec_time, stdout))
-                .unwrap();
         };
 
         // If the platform is single-threaded we're just going to run
@@ -1489,31 +1520,38 @@ pub fn run_test(
 
     match testfn {
         DynBenchFn(bencher) => {
+            // Benchmarks aren't expected to panic, so we run them all in-process.
             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
                 bencher.run(harness)
             });
         }
         StaticBenchFn(benchfn) => {
+            // Benchmarks aren't expected to panic, so we run them all in-process.
             crate::bench::benchmark(desc, monitor_ch, opts.nocapture, |harness| {
                 (benchfn.clone())(harness)
             });
         }
         DynTestFn(f) => {
-            let cb = move || __rust_begin_short_backtrace(f);
+            match strategy {
+                RunStrategy::InProcess => (),
+                _ => panic!("Cannot run dynamic test fn out-of-process"),
+            };
             run_test_inner(
                 desc,
-                monitor_ch,
                 opts.nocapture,
                 opts.report_time,
-                Box::new(cb),
+                strategy,
-                concurrency,
+                monitor_ch,
-            )
+                Box::new(move || __rust_begin_short_backtrace(f)),
+                concurrency
+            );
         }
         StaticTestFn(f) => run_test_inner(
             desc,
-            monitor_ch,
             opts.nocapture,
             opts.report_time,
+            strategy,
+            monitor_ch,
             Box::new(move || __rust_begin_short_backtrace(f)),
             concurrency,
         ),
@@ -1526,7 +1564,9 @@ fn __rust_begin_short_backtrace<F: FnOnce()>(f: F) {
     f()
 }
 
-fn calc_result(desc: &TestDesc, task_result: Result<(), Box<dyn Any + Send>>) -> TestResult {
+fn calc_result<'a>(desc: &TestDesc,
+                   task_result: Result<(), &'a (dyn Any + 'static + Send)>)
+-> TestResult {
     match (&desc.should_panic, task_result) {
         (&ShouldPanic::No, Ok(())) | (&ShouldPanic::Yes, Err(_)) => TrOk,
         (&ShouldPanic::YesWithMessage(msg), Err(ref err)) => {
@@ -1552,6 +1592,150 @@ fn calc_result(desc: &TestDesc, task_result: Result<(), Box<dyn Any + Send>>) ->
     }
 }
 
+fn get_result_from_exit_code(desc: &TestDesc, code: i32) -> TestResult {
+    match (desc.allow_fail, code) {
+        (_, TR_OK) => TrOk,
+        (true, TR_FAILED) => TrAllowedFail,
+        (false, TR_FAILED) => TrFailed,
+        (_, _) => TrFailedMsg(format!("got unexpected return code {}", code)),
+    }
+}
+
+fn run_test_in_process(desc: TestDesc,
+                       nocapture: bool,
+                       report_time: bool,
+                       testfn: Box<dyn FnOnce() + Send>,
+                       monitor_ch: Sender<MonitorMsg>) {
+    // Buffer for capturing standard I/O
+    let data = Arc::new(Mutex::new(Vec::new()));
+
+    let oldio = if !nocapture {
+        Some((
+            io::set_print(Some(Box::new(Sink(data.clone())))),
+            io::set_panic(Some(Box::new(Sink(data.clone())))),
+        ))
+    } else {
+        None
+    };
+
+    let start = if report_time {
+        Some(Instant::now())
+    } else {
+        None
+    };
+    let result = catch_unwind(AssertUnwindSafe(testfn));
+    let exec_time = start.map(|start| {
+        let duration = start.elapsed();
+        TestExecTime(duration)
+    });
+
+    if let Some((printio, panicio)) = oldio {
+        io::set_print(printio);
+        io::set_panic(panicio);
+    }
+
+    let test_result = match result {
+        Ok(()) => calc_result(&desc, Ok(())),
+        Err(e) => calc_result(&desc, Err(e.as_ref())),
+    };
+    let stdout = data.lock().unwrap().to_vec();
+    monitor_ch.send((desc.clone(), test_result, exec_time, stdout)).unwrap();
+}
+
+fn spawn_test_subprocess(desc: TestDesc, report_time: bool, monitor_ch: Sender<MonitorMsg>) {
+    let (result, test_output, exec_time) = (|| {
+        let args = env::args().collect::<Vec<_>>();
+        let current_exe = &args[0];
+
+        let start = if report_time {
+            Some(Instant::now())
+        } else {
+            None
+        };
+        let output = match Command::new(current_exe)
+            .env(SECONDARY_TEST_INVOKER_VAR, desc.name.as_slice())
+            .output() {
+                Ok(out) => out,
+                Err(e) => {
+                    let err = format!("Failed to spawn {} as child for test: {:?}", args[0], e);
+                    return (TrFailed, err.into_bytes(), None);
+                }
+            };
+        let exec_time = start.map(|start| {
+            let duration = start.elapsed();
+            TestExecTime(duration)
+        });
+
+        let std::process::Output { stdout, stderr, status } = output;
+        let mut test_output = stdout;
+        formatters::write_stderr_delimiter(&mut test_output, &desc.name);
+        test_output.extend_from_slice(&stderr);
+
+        let result = match (|| -> Result<TestResult, String> {
+            let exit_code = get_exit_code(status)?;
+            Ok(get_result_from_exit_code(&desc, exit_code))
+        })() {
+            Ok(r) => r,
+            Err(e) => {
+                write!(&mut test_output, "Unexpected error: {}", e).unwrap();
+                TrFailed
+            }
+        };
+
+        (result, test_output, exec_time)
+    })();
+
+    monitor_ch.send((desc.clone(), result, exec_time, test_output)).unwrap();
+}
+
+fn run_test_in_spawned_subprocess(desc: TestDesc, testfn: Box<dyn FnOnce() + Send>) -> ! {
+    let builtin_panic_hook = panic::take_hook();
+    let record_result = Arc::new(move |panic_info: Option<&'_ PanicInfo<'_>>| {
+        let test_result = match panic_info {
+            Some(info) => calc_result(&desc, Err(info.payload())),
+            None => calc_result(&desc, Ok(())),
+        };
+
+        // We don't support serializing TrFailedMsg, so just
+        // print the message out to stderr.
+        if let TrFailedMsg(msg) = &test_result {
+            eprintln!("{}", msg);
+        }
+
+        if let Some(info) = panic_info {
+            builtin_panic_hook(info);
+        }
+
+        if let TrOk = test_result {
+            process::exit(TR_OK);
+        } else {
+            process::exit(TR_FAILED);
+        }
+    });
+    let record_result2 = record_result.clone();
+    panic::set_hook(Box::new(move |info| record_result2(Some(&info))));
+    testfn();
+    record_result(None);
+    unreachable!("panic=abort callback should have exited the process")
+}
+
+#[cfg(not(unix))]
+fn get_exit_code(status: ExitStatus) -> Result<i32, String> {
+    status.code().ok_or("received no exit code from child process".into())
+}
+
+#[cfg(unix)]
+fn get_exit_code(status: ExitStatus) -> Result<i32, String> {
+    use std::os::unix::process::ExitStatusExt;
+    match status.code() {
+        Some(code) => Ok(code),
+        None => match status.signal() {
+            Some(signal) => Err(format!("child process exited with signal {}", signal)),
+            None => Err("child process exited with unknown signal".into()),
+        }
+    }
+}
+
 #[derive(Clone, PartialEq)]
 pub struct MetricMap(BTreeMap<String, Metric>);
 
@@ -1700,7 +1884,9 @@ where
 }
 
 pub mod bench {
-    use super::{BenchMode, BenchSamples, Bencher, MonitorMsg, Sender, Sink, TestDesc, TestResult};
+    use super::{
+        BenchMode, BenchSamples, Bencher, MonitorMsg, Sender, Sink, TestDesc, TestResult
+    };
     use crate::stats;
     use std::cmp;
     use std::io;
@@ -1718,12 +1904,10 @@ pub mod bench {
         };
 
         let data = Arc::new(Mutex::new(Vec::new()));
-        let data2 = data.clone();
-
         let oldio = if !nocapture {
             Some((
-                io::set_print(Some(Box::new(Sink(data2.clone())))),
+                io::set_print(Some(Box::new(Sink(data.clone())))),
-                io::set_panic(Some(Box::new(Sink(data2)))),
+                io::set_panic(Some(Box::new(Sink(data.clone())))),
             ))
         } else {
             None
@@ -1734,7 +1918,7 @@ pub mod bench {
         if let Some((printio, panicio)) = oldio {
             io::set_print(printio);
             io::set_panic(panicio);
-        };
+        }
 
         let test_result = match result {
             //bs.bench(f) {

src/libtest/tests.rs

@@ -1,7 +1,7 @@
 use super::*;
 
 use crate::test::{
-    filter_tests, parse_opts, run_test, DynTestFn, DynTestName, MetricMap, RunIgnored,
+    filter_tests, parse_opts, run_test, DynTestFn, DynTestName, MetricMap, RunIgnored, RunStrategy,
     ShouldPanic, StaticTestName, TestDesc, TestDescAndFn, TestOpts, TrFailedMsg,
     TrIgnored, TrOk,
 };
@@ -67,7 +67,7 @@ pub fn do_not_run_ignored_tests() {
         testfn: DynTestFn(Box::new(f)),
     };
     let (tx, rx) = channel();
-    run_test(&TestOpts::new(), false, desc, tx, Concurrent::No);
+    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, res, _, _) = rx.recv().unwrap();
     assert!(res != TrOk);
 }
@@ -85,7 +85,7 @@ pub fn ignored_tests_result_in_ignored() {
         testfn: DynTestFn(Box::new(f)),
     };
     let (tx, rx) = channel();
-    run_test(&TestOpts::new(), false, desc, tx, Concurrent::No);
+    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, res, _, _) = rx.recv().unwrap();
     assert!(res == TrIgnored);
 }
@@ -105,7 +105,7 @@ fn test_should_panic() {
         testfn: DynTestFn(Box::new(f)),
     };
     let (tx, rx) = channel();
-    run_test(&TestOpts::new(), false, desc, tx, Concurrent::No);
+    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, res, _, _) = rx.recv().unwrap();
     assert!(res == TrOk);
 }
@@ -125,7 +125,7 @@ fn test_should_panic_good_message() {
         testfn: DynTestFn(Box::new(f)),
     };
     let (tx, rx) = channel();
-    run_test(&TestOpts::new(), false, desc, tx, Concurrent::No);
+    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, res, _, _) = rx.recv().unwrap();
     assert!(res == TrOk);
 }
@@ -147,7 +147,7 @@ fn test_should_panic_bad_message() {
         testfn: DynTestFn(Box::new(f)),
     };
     let (tx, rx) = channel();
-    run_test(&TestOpts::new(), false, desc, tx, Concurrent::No);
+    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, res, _, _) = rx.recv().unwrap();
     assert!(res == TrFailedMsg(format!("{} '{}'", failed_msg, expected)));
 }
@@ -165,7 +165,7 @@ fn test_should_panic_but_succeeds() {
         testfn: DynTestFn(Box::new(f)),
     };
     let (tx, rx) = channel();
-    run_test(&TestOpts::new(), false, desc, tx, Concurrent::No);
+    run_test(&TestOpts::new(), false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, res, _, _) = rx.recv().unwrap();
     assert!(res == TrFailedMsg("test did not panic as expected".to_string()));
 }
@@ -186,7 +186,7 @@ fn report_time_test_template(report_time: bool) -> Option<TestExecTime> {
         ..TestOpts::new()
     };
     let (tx, rx) = channel();
-    run_test(&test_opts, false, desc, tx, Concurrent::No);
+    run_test(&test_opts, false, desc, RunStrategy::InProcess, tx, Concurrent::No);
     let (_, _, exec_time, _) = rx.recv().unwrap();
     exec_time
 }

src/test/ui/panic-runtime/libtest-unwinds.rs

@@ -1,10 +0,0 @@
-// error-pattern:is not compiled with this crate's panic strategy `abort`
-// compile-flags:-C panic=abort
-// ignore-wasm32-bare compiled with panic=abort by default
-
-#![feature(test)]
-
-extern crate test;
-
-fn main() {
-}

src/test/ui/panic-runtime/libtest-unwinds.stderr

@@ -1,4 +0,0 @@
-error: the linked panic runtime `panic_unwind` is not compiled with this crate's panic strategy `abort`
-
-error: aborting due to previous error
-

src/test/ui/test-panic-abort-disabled.rs

@@ -0,0 +1,20 @@
+// error-pattern:building tests with panic=abort is not yet supported
+// no-prefer-dynamic
+// compile-flags: --test -Cpanic=abort
+// run-flags: --test-threads=1
+
+// ignore-wasm no panic or subprocess support
+// ignore-emscripten no panic or subprocess support
+
+#![cfg(test)]
+
+#[test]
+fn it_works() {
+    assert_eq!(1 + 1, 2);
+}
+
+#[test]
+#[should_panic]
+fn it_panics() {
+    assert_eq!(1 + 1, 4);
+}

src/test/ui/test-panic-abort-disabled.stderr

@@ -0,0 +1,4 @@
+error: building tests with panic=abort is not yet supported
+
+error: aborting due to previous error
+

src/test/ui/test-panic-abort.rs

@@ -0,0 +1,36 @@
+// no-prefer-dynamic
+// compile-flags: --test -Cpanic=abort -Zpanic_abort_tests
+// run-flags: --test-threads=1
+// run-fail
+// check-run-results
+
+// ignore-wasm no panic or subprocess support
+// ignore-emscripten no panic or subprocess support
+
+#![cfg(test)]
+
+use std::io::Write;
+
+#[test]
+fn it_works() {
+    assert_eq!(1 + 1, 2);
+}
+
+#[test]
+#[should_panic]
+fn it_panics() {
+    assert_eq!(1 + 1, 4);
+}
+
+#[test]
+fn it_fails() {
+    println!("hello, world");
+    writeln!(std::io::stdout(), "testing123").unwrap();
+    writeln!(std::io::stderr(), "testing321").unwrap();
+    assert_eq!(1 + 1, 5);
+}
+
+#[test]
+fn it_exits() {
+    std::process::exit(123);
+}

src/test/ui/test-panic-abort.run.stdout

@@ -0,0 +1,29 @@
+
+running 4 tests
+test it_exits ... FAILED
+test it_fails ... FAILED
+test it_panics ... ok
+test it_works ... ok
+
+failures:
+
+---- it_exits stdout ----
+---- it_exits stderr ----
+note: got unexpected return code 123
+---- it_fails stdout ----
+hello, world
+testing123
+---- it_fails stderr ----
+testing321
+thread 'main' panicked at 'assertion failed: `(left == right)`
+  left: `2`,
+ right: `5`', $DIR/test-panic-abort.rs:30:5
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace.
+
+
+failures:
+    it_exits
+    it_fails
+
+test result: FAILED. 2 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out
+

src/tools/compiletest/src/common.rs

@@ -100,6 +100,7 @@ pub enum PassMode {
     Check,
     Build,
     Run,
+    RunFail,
 }
 
 impl FromStr for PassMode {
@@ -120,6 +121,7 @@ impl fmt::Display for PassMode {
             PassMode::Check => "check",
             PassMode::Build => "build",
             PassMode::Run => "run",
+            PassMode::RunFail => "run-fail",
         };
         fmt::Display::fmt(s, f)
     }

src/tools/compiletest/src/header.rs

@@ -610,6 +610,11 @@ impl TestProps {
                 panic!("`run-pass` header is only supported in UI tests")
             }
             Some(PassMode::Run)
+        } else if config.parse_name_directive(ln, "run-fail") {
+            if config.mode != Mode::Ui {
+                panic!("`run-fail` header is only supported in UI tests")
+            }
+            Some(PassMode::RunFail)
         } else {
             None
         };

src/tools/compiletest/src/runtest.rs

@@ -326,6 +326,14 @@ impl<'test> TestCx<'test> {
         self.props.pass_mode(self.config)
     }
 
+    fn should_run(&self) -> bool {
+        let pass_mode = self.pass_mode();
+        match self.config.mode {
+            Ui => pass_mode == Some(PassMode::Run) || pass_mode == Some(PassMode::RunFail),
+            mode => panic!("unimplemented for mode {:?}", mode),
+        }
+    }
+
     fn should_run_successfully(&self) -> bool {
         let pass_mode = self.pass_mode();
         match self.config.mode {
@@ -1534,7 +1542,7 @@ impl<'test> TestCx<'test> {
     fn compile_test(&self) -> ProcRes {
         // Only use `make_exe_name` when the test ends up being executed.
         let will_execute = match self.config.mode {
-            Ui => self.should_run_successfully(),
+            Ui => self.should_run(),
             Incremental => self.revision.unwrap().starts_with("r"),
             RunFail | RunPassValgrind | MirOpt |
             DebugInfoCdb | DebugInfoGdbLldb | DebugInfoGdb | DebugInfoLldb => true,
@@ -3107,7 +3115,7 @@ impl<'test> TestCx<'test> {
 
         let expected_errors = errors::load_errors(&self.testpaths.file, self.revision);
 
-        if self.should_run_successfully() {
+        if self.should_run() {
             let proc_res = self.exec_compiled_test();
             let run_output_errors = if self.props.check_run_results {
                 self.load_compare_outputs(&proc_res, TestOutput::Run, explicit)
@@ -3120,8 +3128,14 @@ impl<'test> TestCx<'test> {
                     &proc_res,
                 );
             }
-            if !proc_res.status.success() {
+            if self.should_run_successfully() {
-                self.fatal_proc_rec("test run failed!", &proc_res);
+                if !proc_res.status.success() {
+                    self.fatal_proc_rec("test run failed!", &proc_res);
+                }
+            } else {
+                if proc_res.status.success() {
+                    self.fatal_proc_rec("test run succeeded!", &proc_res);
+                }
             }
         }