diff --git a/src/test/run-make/test-harness/Makefile b/src/test/run-make/test-harness/Makefile
new file mode 100644
index 00000000000..4517af8e24b
--- /dev/null
+++ b/src/test/run-make/test-harness/Makefile
@@ -0,0 +1,7 @@
+-include ../tools.mk
+
+all:
+	# check that #[ignore(cfg(...))] does the right thing.
+	$(RUSTC) --test test-ignore-cfg.rs --cfg ignorecfg
+	$(call RUN,test-ignore-cfg) | grep 'shouldnotignore ... ok'
+	$(call RUN,test-ignore-cfg) | grep 'shouldignore ... ignored'
diff --git a/src/test/run-make/test-harness/test-ignore-cfg.rs b/src/test/run-make/test-harness/test-ignore-cfg.rs
new file mode 100644
index 00000000000..a8f88cc8544
--- /dev/null
+++ b/src/test/run-make/test-harness/test-ignore-cfg.rs
@@ -0,0 +1,19 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[test]
+#[ignore(cfg(ignorecfg))]
+fn shouldignore() {
+}
+
+#[test]
+#[ignore(cfg(noignorecfg))]
+fn shouldnotignore() {
+}
diff --git a/src/test/run-pass/test-ignore-cfg.rs b/src/test/run-pass/test-ignore-cfg.rs
deleted file mode 100644
index b36fbca2da0..00000000000
--- a/src/test/run-pass/test-ignore-cfg.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-
-// compile-flags: --test --cfg ignorecfg
-// ignore-pretty: does not work well with `--test`
-
-#[test]
-#[ignore(cfg(ignorecfg))]
-fn shouldignore() {
-}
-
-#[test]
-#[ignore(cfg(noignorecfg))]
-fn shouldnotignore() {
-}
-
-#[test]
-fn checktests() {
-    // Pull the tests out of the secreturn test module
-    let tests = __test::TESTS;
-
-    assert!(
-        tests.iter().any(|t| t.desc.name.to_string().as_slice() == "shouldignore" &&
-                         t.desc.ignore));
-
-    assert!(
-        tests.iter().any(|t| t.desc.name.to_string().as_slice() == "shouldnotignore" &&
-                         !t.desc.ignore));
-}