Auto merge of #116035 - lqd:mcp-510-target-specs, r=petrochenkov

Allow target specs to use an LLD flavor, and self-contained linking components

This PR allows:
- target specs to use an LLD linker-flavor: this is needed to switch `x86_64-unknown-linux-gnu` to using LLD, and is currently not possible because the current flavor json serialization fails to roundtrip on the modern linker-flavors. This can e.g. be seen in https://github.com/rust-lang/rust/pull/115622#discussion_r1321312880 which explains where an `Lld::Yes` is ultimately deserialized into an `Lld::No`.
- target specs to declare self-contained linking components: this is needed to switch `x86_64-unknown-linux-gnu` to using `rust-lld`
- adds an end-to-end test of a custom target json simulating `x86_64-unknown-linux-gnu` being switched to using `rust-lld`
- disables codegen backends from participating because they don't support `-Zgcc-ld=lld` which is the basis of mcp510.

r? `@petrochenkov:` if the approach discussed https://github.com/rust-lang/rust/pull/115622#discussion_r1329403467 and on zulip would work for you: basically, see if we can emit only modern linker flavors in the json specs, but accept both old and new flavors while reading them, to fix the roundtrip issue.

The backwards compatible `LinkSelfContainedDefault` variants are still serialized and deserialized in `crt-objects-fallback`, while the spec equivalent of e.g. `-Clink-self-contained=+linker` is serialized into a different json object (with future-proofing to incorporate `crt-objects-fallback`  in the future).

---

I've been test-driving this in https://github.com/rust-lang/rust/pull/113382 to test actually switching `x86_64-unknown-linux-gnu`  to `rust-lld` (and fix what needs to be fixed in CI, bootstrap, etc), and it seems to work fine.

compiler/rustc_target/src/spec/crt_objects.rs

@@ -40,11 +40,9 @@
 //! but not gcc's. As a result rustc cannot link with C++ static libraries (#36710)
 //! when linking in self-contained mode.
 
-use crate::json::{Json, ToJson};
 use crate::spec::LinkOutputKind;
 use std::borrow::Cow;
 use std::collections::BTreeMap;
-use std::str::FromStr;
 
 pub type CrtObjects = BTreeMap<LinkOutputKind, Vec<Cow<'static, str>>>;
 
@@ -123,39 +121,3 @@ pub(super) fn pre_wasi_self_contained() -> CrtObjects {
 pub(super) fn post_wasi_self_contained() -> CrtObjects {
     new(&[])
 }
-
-/// Which logic to use to determine whether to use self-contained linking mode
-/// if `-Clink-self-contained` is not specified explicitly.
-#[derive(Clone, Copy, PartialEq, Hash, Debug)]
-pub enum LinkSelfContainedDefault {
-    False,
-    True,
-    Musl,
-    Mingw,
-}
-
-impl FromStr for LinkSelfContainedDefault {
-    type Err = ();
-
-    fn from_str(s: &str) -> Result<LinkSelfContainedDefault, ()> {
-        Ok(match s {
-            "false" => LinkSelfContainedDefault::False,
-            "true" | "wasm" => LinkSelfContainedDefault::True,
-            "musl" => LinkSelfContainedDefault::Musl,
-            "mingw" => LinkSelfContainedDefault::Mingw,
-            _ => return Err(()),
-        })
-    }
-}
-
-impl ToJson for LinkSelfContainedDefault {
-    fn to_json(&self) -> Json {
-        match *self {
-            LinkSelfContainedDefault::False => "false",
-            LinkSelfContainedDefault::True => "true",
-            LinkSelfContainedDefault::Musl => "musl",
-            LinkSelfContainedDefault::Mingw => "mingw",
-        }
-        .to_json()
-    }
-}

compiler/rustc_target/src/spec/linux_musl_base.rs

@@ -1,5 +1,5 @@
-use crate::spec::crt_objects::{self, LinkSelfContainedDefault};
-use crate::spec::TargetOptions;
+use crate::spec::crt_objects;
+use crate::spec::{LinkSelfContainedDefault, TargetOptions};
 
 pub fn opts() -> TargetOptions {
     let mut base = super::linux_base::opts();
@@ -7,7 +7,7 @@ pub fn opts() -> TargetOptions {
     base.env = "musl".into();
     base.pre_link_objects_self_contained = crt_objects::pre_musl_self_contained();
     base.post_link_objects_self_contained = crt_objects::post_musl_self_contained();
-    base.link_self_contained = LinkSelfContainedDefault::Musl;
+    base.link_self_contained = LinkSelfContainedDefault::InferredForMusl;
 
     // These targets statically link libc by default
     base.crt_static_default = true;

compiler/rustc_target/src/spec/mod.rs

@@ -38,7 +38,7 @@ use crate::abi::call::Conv;
 use crate::abi::{Endian, Integer, Size, TargetDataLayout, TargetDataLayoutErrors};
 use crate::json::{Json, ToJson};
 use crate::spec::abi::{lookup as lookup_abi, Abi};
-use crate::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
+use crate::spec::crt_objects::CrtObjects;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_fs_util::try_canonicalize;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
@@ -278,6 +278,7 @@ impl LinkerFlavor {
         }
     }
 
+    /// Returns the corresponding backwards-compatible CLI flavor.
     fn to_cli(self) -> LinkerFlavorCli {
         match self {
             LinkerFlavor::Gnu(Cc::Yes, _)
@@ -298,6 +299,20 @@ impl LinkerFlavor {
         }
     }
 
+    /// Returns the modern CLI flavor that is the counterpart of this flavor.
+    fn to_cli_counterpart(self) -> LinkerFlavorCli {
+        match self {
+            LinkerFlavor::Gnu(cc, lld) => LinkerFlavorCli::Gnu(cc, lld),
+            LinkerFlavor::Darwin(cc, lld) => LinkerFlavorCli::Darwin(cc, lld),
+            LinkerFlavor::WasmLld(cc) => LinkerFlavorCli::WasmLld(cc),
+            LinkerFlavor::Unix(cc) => LinkerFlavorCli::Unix(cc),
+            LinkerFlavor::Msvc(lld) => LinkerFlavorCli::Msvc(lld),
+            LinkerFlavor::EmCc => LinkerFlavorCli::EmCc,
+            LinkerFlavor::Bpf => LinkerFlavorCli::Bpf,
+            LinkerFlavor::Ptx => LinkerFlavorCli::Ptx,
+        }
+    }
+
     fn infer_cli_hints(cli: LinkerFlavorCli) -> (Option<Cc>, Option<Lld>) {
         match cli {
             LinkerFlavorCli::Gnu(cc, lld) | LinkerFlavorCli::Darwin(cc, lld) => {
@@ -520,6 +535,98 @@ impl ToJson for LinkerFlavorCli {
     }
 }
 
+/// The different `-Clink-self-contained` options that can be specified in a target spec:
+/// - enabling or disabling in bulk
+/// - some target-specific pieces of inference to determine whether to use self-contained linking
+///   if `-Clink-self-contained` is not specified explicitly (e.g. on musl/mingw)
+/// - explicitly enabling some of the self-contained linking components, e.g. the linker component
+///   to use `rust-lld`
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub enum LinkSelfContainedDefault {
+    /// The target spec explicitly enables self-contained linking.
+    True,
+
+    /// The target spec explicitly disables self-contained linking.
+    False,
+
+    /// The target spec requests that the self-contained mode is inferred, in the context of musl.
+    InferredForMusl,
+
+    /// The target spec requests that the self-contained mode is inferred, in the context of mingw.
+    InferredForMingw,
+
+    /// The target spec explicitly enables a list of self-contained linking components: e.g. for
+    /// targets opting into a subset of components like the CLI's `-C link-self-contained=+linker`.
+    WithComponents(LinkSelfContainedComponents),
+}
+
+/// Parses a backwards-compatible `-Clink-self-contained` option string, without components.
+impl FromStr for LinkSelfContainedDefault {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<LinkSelfContainedDefault, ()> {
+        Ok(match s {
+            "false" => LinkSelfContainedDefault::False,
+            "true" | "wasm" => LinkSelfContainedDefault::True,
+            "musl" => LinkSelfContainedDefault::InferredForMusl,
+            "mingw" => LinkSelfContainedDefault::InferredForMingw,
+            _ => return Err(()),
+        })
+    }
+}
+
+impl ToJson for LinkSelfContainedDefault {
+    fn to_json(&self) -> Json {
+        match *self {
+            LinkSelfContainedDefault::WithComponents(components) => {
+                // Serialize the components in a json object's `components` field, to prepare for a
+                // future where `crt-objects-fallback` is removed from the json specs and
+                // incorporated as a field here.
+                let mut map = BTreeMap::new();
+                map.insert("components", components);
+                map.to_json()
+            }
+
+            // Stable backwards-compatible values
+            LinkSelfContainedDefault::True => "true".to_json(),
+            LinkSelfContainedDefault::False => "false".to_json(),
+            LinkSelfContainedDefault::InferredForMusl => "musl".to_json(),
+            LinkSelfContainedDefault::InferredForMingw => "mingw".to_json(),
+        }
+    }
+}
+
+impl LinkSelfContainedDefault {
+    /// Returns whether the target spec has self-contained linking explicitly disabled. Used to emit
+    /// errors if the user then enables it on the CLI.
+    pub fn is_disabled(self) -> bool {
+        self == LinkSelfContainedDefault::False
+    }
+
+    /// Returns whether the target spec explictly requests self-contained linking, i.e. not via
+    /// inference.
+    pub fn is_linker_enabled(self) -> bool {
+        match self {
+            LinkSelfContainedDefault::True => true,
+            LinkSelfContainedDefault::False => false,
+            LinkSelfContainedDefault::WithComponents(c) => {
+                c.contains(LinkSelfContainedComponents::LINKER)
+            }
+            _ => false,
+        }
+    }
+
+    /// Returns the key to use when serializing the setting to json:
+    /// - individual components in a `link-self-contained` object value
+    /// - the other variants as a backwards-compatible `crt-objects-fallback` string
+    fn json_key(self) -> &'static str {
+        match self {
+            LinkSelfContainedDefault::WithComponents(_) => "link-self-contained",
+            _ => "crt-objects-fallback",
+        }
+    }
+}
+
 bitflags::bitflags! {
     #[derive(Default)]
     /// The `-C link-self-contained` components that can individually be enabled or disabled.
@@ -579,6 +686,21 @@ impl LinkSelfContainedComponents {
             LinkSelfContainedComponents::MINGW,
         ]
     }
+
+    /// Returns whether at least a component is enabled.
+    pub fn are_any_components_enabled(self) -> bool {
+        !self.is_empty()
+    }
+
+    /// Returns whether `LinkSelfContainedComponents::LINKER` is enabled.
+    pub fn is_linker_enabled(self) -> bool {
+        self.contains(LinkSelfContainedComponents::LINKER)
+    }
+
+    /// Returns whether `LinkSelfContainedComponents::CRT_OBJECTS` is enabled.
+    pub fn is_crt_objects_enabled(self) -> bool {
+        self.contains(LinkSelfContainedComponents::CRT_OBJECTS)
+    }
 }
 
 impl IntoIterator for LinkSelfContainedComponents {
@@ -594,6 +716,22 @@ impl IntoIterator for LinkSelfContainedComponents {
     }
 }
 
+impl ToJson for LinkSelfContainedComponents {
+    fn to_json(&self) -> Json {
+        let components: Vec<_> = Self::all_components()
+            .into_iter()
+            .filter(|c| self.contains(*c))
+            .map(|c| {
+                // We can unwrap because we're iterating over all the known singular components,
+                // not an actual set of flags where `as_str` can fail.
+                c.as_str().unwrap().to_owned()
+            })
+            .collect();
+
+        components.to_json()
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Hash, Encodable, Decodable, HashStable_Generic)]
 pub enum PanicStrategy {
     Unwind,
@@ -1770,6 +1908,8 @@ pub struct TargetOptions {
     /// Same as `(pre|post)_link_objects`, but when self-contained linking mode is enabled.
     pub pre_link_objects_self_contained: CrtObjects,
     pub post_link_objects_self_contained: CrtObjects,
+    /// Behavior for the self-contained linking mode: inferred for some targets, or explicitly
+    /// enabled (in bulk, or with individual components).
     pub link_self_contained: LinkSelfContainedDefault,
 
     /// Linker arguments that are passed *before* any user-defined libraries.
@@ -2170,7 +2310,7 @@ impl TargetOptions {
     }
 
     fn update_to_cli(&mut self) {
-        self.linker_flavor_json = self.linker_flavor.to_cli();
+        self.linker_flavor_json = self.linker_flavor.to_cli_counterpart();
         self.lld_flavor_json = self.linker_flavor.lld_flavor();
         self.linker_is_gnu_json = self.linker_flavor.is_gnu();
         for (args, args_json) in [
@@ -2180,8 +2320,10 @@ impl TargetOptions {
             (&self.late_link_args_static, &mut self.late_link_args_static_json),
             (&self.post_link_args, &mut self.post_link_args_json),
         ] {
-            *args_json =
-                args.iter().map(|(flavor, args)| (flavor.to_cli(), args.clone())).collect();
+            *args_json = args
+                .iter()
+                .map(|(flavor, args)| (flavor.to_cli_counterpart(), args.clone()))
+                .collect();
         }
     }
 }
@@ -2724,8 +2866,43 @@ impl Target {
                 }
                 Ok::<(), String>(())
             } );
-
-            ($key_name:ident = $json_name:expr, link_self_contained) => ( {
+            ($key_name:ident, link_self_contained_components) => ( {
+                // Skeleton of what needs to be parsed:
+                //
+                // ```
+                // $name: {
+                //     "components": [
+                //         <array of strings>
+                //     ]
+                // }
+                // ```
+                let name = (stringify!($key_name)).replace("_", "-");
+                if let Some(o) = obj.remove(&name) {
+                    if let Some(o) = o.as_object() {
+                        let component_array = o.get("components")
+                            .ok_or_else(|| format!("{name}: expected a \
+                                JSON object with a `components` field."))?;
+                        let component_array = component_array.as_array()
+                            .ok_or_else(|| format!("{name}.components: expected a JSON array"))?;
+                        let mut components = LinkSelfContainedComponents::empty();
+                        for s in component_array {
+                            components |= match s.as_str() {
+                                Some(s) => {
+                                    LinkSelfContainedComponents::from_str(s)
+                                        .ok_or_else(|| format!("unknown \
+                                        `-Clink-self-contained` component: {s}"))?
+                                },
+                                _ => return Err(format!("not a string: {:?}", s)),
+                            };
+                        }
+                        base.$key_name = LinkSelfContainedDefault::WithComponents(components);
+                    } else {
+                        incorrect_type.push(name)
+                    }
+                }
+                Ok::<(), String>(())
+            } );
+            ($key_name:ident = $json_name:expr, link_self_contained_backwards_compatible) => ( {
                 let name = $json_name;
                 obj.remove(name).and_then(|o| o.as_str().and_then(|s| {
                     match s.parse::<LinkSelfContainedDefault>() {
@@ -2878,7 +3055,13 @@ impl Target {
         key!(post_link_objects = "post-link-objects", link_objects);
         key!(pre_link_objects_self_contained = "pre-link-objects-fallback", link_objects);
         key!(post_link_objects_self_contained = "post-link-objects-fallback", link_objects);
-        key!(link_self_contained = "crt-objects-fallback", link_self_contained)?;
+        // Deserializes the backwards-compatible variants of `-Clink-self-contained`
+        key!(
+            link_self_contained = "crt-objects-fallback",
+            link_self_contained_backwards_compatible
+        )?;
+        // Deserializes the components variant of `-Clink-self-contained`
+        key!(link_self_contained, link_self_contained_components)?;
         key!(pre_link_args_json = "pre-link-args", link_args);
         key!(late_link_args_json = "late-link-args", link_args);
         key!(late_link_args_dynamic_json = "late-link-args-dynamic", link_args);
@@ -3134,7 +3317,6 @@ impl ToJson for Target {
         target_option_val!(post_link_objects);
         target_option_val!(pre_link_objects_self_contained, "pre-link-objects-fallback");
         target_option_val!(post_link_objects_self_contained, "post-link-objects-fallback");
-        target_option_val!(link_self_contained, "crt-objects-fallback");
         target_option_val!(link_args - pre_link_args_json, "pre-link-args");
         target_option_val!(link_args - late_link_args_json, "late-link-args");
         target_option_val!(link_args - late_link_args_dynamic_json, "late-link-args-dynamic");
@@ -3231,6 +3413,10 @@ impl ToJson for Target {
             d.insert("default-adjusted-cabi".into(), Abi::name(abi).to_json());
         }
 
+        // Serializing `-Clink-self-contained` needs a dynamic key to support the
+        // backwards-compatible variants.
+        d.insert(self.link_self_contained.json_key().into(), self.link_self_contained.to_json());
+
         Json::Object(d)
     }
 }

compiler/rustc_target/src/spec/tests/tests_impl.rs

@@ -97,7 +97,7 @@ impl Target {
             );
         }
 
-        if self.link_self_contained == LinkSelfContainedDefault::False {
+        if self.link_self_contained.is_disabled() {
             assert!(
                 self.pre_link_objects_self_contained.is_empty()
                     && self.post_link_objects_self_contained.is_empty()

compiler/rustc_target/src/spec/wasm32_wasi.rs

@@ -72,7 +72,8 @@
 //! best we can with this target. Don't start relying on too much here unless
 //! you know what you're getting in to!
 
-use super::crt_objects::{self, LinkSelfContainedDefault};
+use super::crt_objects;
+use super::LinkSelfContainedDefault;
 use super::{wasm_base, Cc, LinkerFlavor, Target};
 
 pub fn target() -> Target {

compiler/rustc_target/src/spec/wasm32_wasi_preview1_threads.rs

@@ -72,8 +72,8 @@
 //! best we can with this target. Don't start relying on too much here unless
 //! you know what you're getting in to!
 
-use super::crt_objects::{self, LinkSelfContainedDefault};
-use super::{wasm_base, Cc, LinkerFlavor, Target};
+use super::{crt_objects, wasm_base};
+use super::{Cc, LinkSelfContainedDefault, LinkerFlavor, Target};
 
 pub fn target() -> Target {
     let mut options = wasm_base::options();

compiler/rustc_target/src/spec/wasm_base.rs

@@ -1,4 +1,4 @@
-use super::crt_objects::LinkSelfContainedDefault;
+use super::LinkSelfContainedDefault;
 use super::{cvs, Cc, LinkerFlavor, PanicStrategy, RelocModel, TargetOptions, TlsModel};
 
 pub fn options() -> TargetOptions {

compiler/rustc_target/src/spec/windows_gnu_base.rs

@@ -1,4 +1,5 @@
-use crate::spec::crt_objects::{self, LinkSelfContainedDefault};
+use crate::spec::crt_objects;
+use crate::spec::LinkSelfContainedDefault;
 use crate::spec::{cvs, Cc, DebuginfoKind, LinkerFlavor, Lld, SplitDebuginfo, TargetOptions};
 use std::borrow::Cow;
 
@@ -90,7 +91,7 @@ pub fn opts() -> TargetOptions {
         post_link_objects: crt_objects::post_mingw(),
         pre_link_objects_self_contained: crt_objects::pre_mingw_self_contained(),
         post_link_objects_self_contained: crt_objects::post_mingw_self_contained(),
-        link_self_contained: LinkSelfContainedDefault::Mingw,
+        link_self_contained: LinkSelfContainedDefault::InferredForMingw,
         late_link_args,
         late_link_args_dynamic,
         late_link_args_static,