diff --git a/tests/ui/feature-gates/feature-gate-link-arg-attribute.stderr b/tests/ui/feature-gates/feature-gate-link-arg-attribute.in_attr.stderr
similarity index 91%
rename from tests/ui/feature-gates/feature-gate-link-arg-attribute.stderr
rename to tests/ui/feature-gates/feature-gate-link-arg-attribute.in_attr.stderr
index 8cbad78478e..3ea8c89bf3a 100644
--- a/tests/ui/feature-gates/feature-gate-link-arg-attribute.stderr
+++ b/tests/ui/feature-gates/feature-gate-link-arg-attribute.in_attr.stderr
@@ -1,5 +1,5 @@
 error[E0658]: link kind `link-arg` is unstable
-  --> $DIR/feature-gate-link-arg-attribute.rs:1:15
+  --> $DIR/feature-gate-link-arg-attribute.rs:5:15
    |
 LL | #[link(kind = "link-arg", name = "foo")]
    |               ^^^^^^^^^^
diff --git a/tests/ui/feature-gates/feature-gate-link-arg-attribute.in_flag.stderr b/tests/ui/feature-gates/feature-gate-link-arg-attribute.in_flag.stderr
new file mode 100644
index 00000000000..4d65db3c66d
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-link-arg-attribute.in_flag.stderr
@@ -0,0 +1,2 @@
+error: unknown linking modifier `link-arg`, expected one of: bundle, verbatim, whole-archive, as-needed
+
diff --git a/tests/ui/feature-gates/feature-gate-link-arg-attribute.rs b/tests/ui/feature-gates/feature-gate-link-arg-attribute.rs
index 9036095fbc4..c12ff5b04dc 100644
--- a/tests/ui/feature-gates/feature-gate-link-arg-attribute.rs
+++ b/tests/ui/feature-gates/feature-gate-link-arg-attribute.rs
@@ -1,5 +1,9 @@
+//@ revisions: in_attr in_flag
+//@[in_flag] compile-flags: -l dylib:+link-arg=foo
+
+#[cfg(in_attr)]
 #[link(kind = "link-arg", name = "foo")]
-//~^ ERROR link kind `link-arg` is unstable
+//[in_attr]~^ ERROR link kind `link-arg` is unstable
 extern "C" {}
 
 fn main() {}
diff --git a/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.stderr b/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.in_attr.stderr
similarity index 90%
rename from tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.stderr
rename to tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.in_attr.stderr
index 216477828e7..11b7abc883e 100644
--- a/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.stderr
+++ b/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.in_attr.stderr
@@ -1,5 +1,5 @@
 error[E0658]: linking modifier `as-needed` is unstable
-  --> $DIR/feature-gate-native_link_modifiers_as_needed.rs:1:50
+  --> $DIR/feature-gate-native_link_modifiers_as_needed.rs:5:50
    |
 LL | #[link(name = "foo", kind = "dylib", modifiers = "+as-needed")]
    |                                                  ^^^^^^^^^^^^
diff --git a/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.in_flag.stderr b/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.in_flag.stderr
new file mode 100644
index 00000000000..1f0b9d04ef6
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.in_flag.stderr
@@ -0,0 +1,2 @@
+error: linking modifier `as-needed` is unstable and only accepted on the nightly compiler
+
diff --git a/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.rs b/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.rs
index 132bc6ab04a..c2965e42f27 100644
--- a/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.rs
+++ b/tests/ui/feature-gates/feature-gate-native_link_modifiers_as_needed.rs
@@ -1,5 +1,9 @@
+//@ revisions: in_attr in_flag
+//@[in_flag] compile-flags: -l dylib:+as-needed=foo
+
+#[cfg(in_attr)]
 #[link(name = "foo", kind = "dylib", modifiers = "+as-needed")]
-//~^ ERROR: linking modifier `as-needed` is unstable
+//[in_attr]~^ ERROR: linking modifier `as-needed` is unstable
 extern "C" {}
 
 fn main() {}