Auto merge of #77762 - pietroalbini:dist-build-manifest, r=Mark-Simulacrum

Dist build manifest

This PR makes two changes that should remove a significant chunk of the time spent in our release process: cloning the `rust-lang/rust` monorepo, all its submodules, and building `bootstrap` to then invoke `build-manifest`:

* `build-manifest` doesn't rely on a clone of the monorepo being present anymore. The only remaining bit of information it fetched from it (the Rust version) is instead bundled in the binary.
* A new "component" is added, `build-manifest`. That component includes a prebuilt version of the tool, and it's *not* included in the Rustup manifest. This will allow `promote-release` to directly invoke the tool without interacting with our build system.
* The Linux x86_64 CI is changed to also build the component mentioned above. It's the only CI builder tasked to do so, and to cleanly support this a new `--include-default-paths` flag was added to `./x.py`.
* The `BUILD_MANIFEST_NUM_THREADS` environment variable is added to configure the number of threads at runtime.

This PR is best reviewed commit-by-commit.
r? `@Mark-Simulacrum`

Cargo.lock

@@ -243,6 +243,7 @@ dependencies = [
  "anyhow",
  "flate2",
  "hex 0.4.2",
+ "num_cpus",
  "rayon",
  "serde",
  "serde_json",

src/bootstrap/builder.rs

@@ -193,37 +193,37 @@ impl StepDescription {
             );
         }
 
-        if paths.is_empty() {
+        if paths.is_empty() || builder.config.include_default_paths {
-            for (desc, should_run) in v.iter().zip(should_runs) {
+            for (desc, should_run) in v.iter().zip(&should_runs) {
                 if desc.default && should_run.is_really_default {
                     for pathset in &should_run.paths {
                         desc.maybe_run(builder, pathset);
                     }
                 }
             }
-        } else {
+        }
-            for path in paths {
-                // strip CurDir prefix if present
-                let path = match path.strip_prefix(".") {
-                    Ok(p) => p,
-                    Err(_) => path,
-                };
 
-                let mut attempted_run = false;
+        for path in paths {
-                for (desc, should_run) in v.iter().zip(&should_runs) {
+            // strip CurDir prefix if present
-                    if let Some(suite) = should_run.is_suite_path(path) {
+            let path = match path.strip_prefix(".") {
-                        attempted_run = true;
+                Ok(p) => p,
-                        desc.maybe_run(builder, suite);
+                Err(_) => path,
-                    } else if let Some(pathset) = should_run.pathset_for_path(path) {
+            };
-                        attempted_run = true;
-                        desc.maybe_run(builder, pathset);
-                    }
-                }
 
-                if !attempted_run {
+            let mut attempted_run = false;
-                    panic!("error: no rules matched {}", path.display());
+            for (desc, should_run) in v.iter().zip(&should_runs) {
+                if let Some(suite) = should_run.is_suite_path(path) {
+                    attempted_run = true;
+                    desc.maybe_run(builder, suite);
+                } else if let Some(pathset) = should_run.pathset_for_path(path) {
+                    attempted_run = true;
+                    desc.maybe_run(builder, pathset);
                 }
             }
+
+            if !attempted_run {
+                panic!("error: no rules matched {}", path.display());
+            }
         }
     }
 }
@@ -462,6 +462,7 @@ impl<'a> Builder<'a> {
                 dist::LlvmTools,
                 dist::RustDev,
                 dist::Extended,
+                dist::BuildManifest,
                 dist::HashSign
             ),
             Kind::Install => describe!(

src/bootstrap/config.rs

@@ -61,6 +61,7 @@ pub struct Config {
     pub profiler: bool,
     pub ignore_git: bool,
     pub exclude: Vec<PathBuf>,
+    pub include_default_paths: bool,
     pub rustc_error_format: Option<String>,
     pub json_output: bool,
     pub test_compare_mode: bool,
@@ -532,6 +533,7 @@ impl Config {
 
         let mut config = Config::default_opts();
         config.exclude = flags.exclude;
+        config.include_default_paths = flags.include_default_paths;
         config.rustc_error_format = flags.rustc_error_format;
         config.json_output = flags.json_output;
         config.on_fail = flags.on_fail;

src/bootstrap/dist.rs

@@ -2353,7 +2353,6 @@ impl Step for HashSign {
         cmd.arg(today.trim());
         cmd.arg(addr);
         cmd.arg(&builder.config.channel);
-        cmd.arg(&builder.src);
         cmd.env("BUILD_MANIFEST_LEGACY", "1");
 
         builder.create_dir(&distdir(builder));
@@ -2584,3 +2583,70 @@ impl Step for RustDev {
         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)))
     }
 }
+
+/// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
+/// release process to avoid cloning the monorepo and building stuff.
+///
+/// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct BuildManifest {
+    pub target: TargetSelection,
+}
+
+impl Step for BuildManifest {
+    type Output = PathBuf;
+    const DEFAULT: bool = false;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/build-manifest")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(BuildManifest { target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> PathBuf {
+        let build_manifest = builder.tool_exe(Tool::BuildManifest);
+
+        let name = pkgname(builder, "build-manifest");
+        let tmp = tmpdir(builder);
+
+        // Prepare the image.
+        let image = tmp.join("build-manifest-image");
+        let image_bin = image.join("bin");
+        let _ = fs::remove_dir_all(&image);
+        t!(fs::create_dir_all(&image_bin));
+        builder.install(&build_manifest, &image_bin.join("build-manifest"), 0o755);
+
+        // Prepare the overlay.
+        let overlay = tmp.join("build-manifest-overlay");
+        let _ = fs::remove_dir_all(&overlay);
+        builder.create_dir(&overlay);
+        builder.create(&overlay.join("version"), &builder.rust_version());
+        for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] {
+            builder.install(&builder.src.join(file), &overlay, 0o644);
+        }
+
+        // Create the final tarball.
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=build-manifest installed.")
+            .arg("--image-dir")
+            .arg(&image)
+            .arg("--work-dir")
+            .arg(&tmpdir(builder))
+            .arg("--output-dir")
+            .arg(&distdir(builder))
+            .arg("--non-installed-overlay")
+            .arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, self.target.triple))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=build-manifest");
+
+        builder.run(&mut cmd);
+        distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple))
+    }
+}

src/bootstrap/flags.rs

@@ -30,6 +30,7 @@ pub struct Flags {
     pub cmd: Subcommand,
     pub incremental: bool,
     pub exclude: Vec<PathBuf>,
+    pub include_default_paths: bool,
     pub rustc_error_format: Option<String>,
     pub json_output: bool,
     pub dry_run: bool,
@@ -137,6 +138,11 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
         opts.optmulti("", "host", "host targets to build", "HOST");
         opts.optmulti("", "target", "target targets to build", "TARGET");
         opts.optmulti("", "exclude", "build paths to exclude", "PATH");
+        opts.optflag(
+            "",
+            "include-default-paths",
+            "include default paths in addition to the provided ones",
+        );
         opts.optopt("", "on-fail", "command to run on failure", "CMD");
         opts.optflag("", "dry-run", "dry run; don't build anything");
         opts.optopt(
@@ -618,6 +624,7 @@ Arguments:
                 .into_iter()
                 .map(|p| p.into())
                 .collect::<Vec<_>>(),
+            include_default_paths: matches.opt_present("include-default-paths"),
             deny_warnings: parse_deny_warnings(&matches),
             llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map(
                 |s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"),

src/bootstrap/run.rs

@@ -77,7 +77,6 @@ impl Step for BuildManifest {
         cmd.arg(today.trim());
         cmd.arg(addr);
         cmd.arg(&builder.config.channel);
-        cmd.arg(&builder.src);
 
         builder.create_dir(&distdir(builder));
         builder.run(&mut cmd);

src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile

@@ -98,7 +98,9 @@ ENV RUST_CONFIGURE_ARGS \
       --set llvm.thin-lto=true \
       --set llvm.ninja=false \
       --set rust.jemalloc
-ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS \
+        --include-default-paths \
+        src/tools/build-manifest
 ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
 
 # This is the only builder which will create source tarballs

src/tools/build-manifest/Cargo.toml

@@ -14,3 +14,4 @@ tar = "0.4.29"
 sha2 = "0.9.1"
 rayon = "1.3.1"
 hex = "0.4.2"
+num_cpus = "1.13.0"

src/tools/build-manifest/README.md

@@ -21,8 +21,8 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
 
 ```
 $ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
-    CHANNEL path/to/rust/repo
+    CHANNEL VERSION
 ```
 
 Remember to replace `CHANNEL` with the channel you produced dist artifacts of
-and `path/to/rust/repo` with the path to your checkout of the Rust repository.
+and `VERSION` with the current Rust version.

src/tools/build-manifest/src/main.rs

@@ -205,15 +205,20 @@ fn main() {
     //
     // Once the old release process is fully decommissioned, the environment variable, all the
     // related code in this tool and ./x.py dist hash-and-sign can be removed.
-    let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok();
+    let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some();
 
-    // Avoid overloading the old server in legacy mode.
+    let num_threads = if legacy {
-    if legacy {
+        // Avoid overloading the old server in legacy mode.
-        rayon::ThreadPoolBuilder::new()
+        1
-            .num_threads(1)
+    } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
-            .build_global()
+        num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
-            .expect("failed to initialize Rayon");
+    } else {
-    }
+        num_cpus::get()
+    };
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(num_threads)
+        .build_global()
+        .expect("failed to initialize Rayon");
 
     let mut args = env::args().skip(1);
     let input = PathBuf::from(args.next().unwrap());
@@ -221,7 +226,6 @@ fn main() {
     let date = args.next().unwrap();
     let s3_address = args.next().unwrap();
     let channel = args.next().unwrap();
-    let monorepo_path = args.next().unwrap();
 
     // Do not ask for a passphrase while manually testing
     let mut passphrase = String::new();
@@ -231,7 +235,7 @@ fn main() {
     }
 
     Builder {
-        versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),
+        versions: Versions::new(&channel, &input).unwrap(),
 
         input,
         output,

src/tools/build-manifest/src/versions.rs

@@ -1,4 +1,4 @@
-use anyhow::{Context, Error};
+use anyhow::Error;
 use flate2::read::GzDecoder;
 use std::collections::HashMap;
 use std::fs::File;
@@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
 use tar::Archive;
 
 const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
+const RUSTC_VERSION: &str = include_str!("../../../version");
 
 #[derive(Debug, Hash, Eq, PartialEq, Clone)]
 pub(crate) enum PkgType {
@@ -87,26 +88,13 @@ pub(crate) struct VersionInfo {
 
 pub(crate) struct Versions {
     channel: String,
-    rustc_version: String,
     dist_path: PathBuf,
     versions: HashMap<PkgType, VersionInfo>,
 }
 
 impl Versions {
-    pub(crate) fn new(
+    pub(crate) fn new(channel: &str, dist_path: &Path) -> Result<Self, Error> {
-        channel: &str,
+        Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() })
-        dist_path: &Path,
-        monorepo_root: &Path,
-    ) -> Result<Self, Error> {
-        Ok(Self {
-            channel: channel.into(),
-            rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
-                .context("failed to read the rustc version from src/version")?
-                .trim()
-                .to_string(),
-            dist_path: dist_path.into(),
-            versions: HashMap::new(),
-        })
     }
 
     pub(crate) fn channel(&self) -> &str {
@@ -184,10 +172,10 @@ impl Versions {
     ) -> Result<String, Error> {
         let component_name = package.tarball_component_name();
         let version = match self.channel.as_str() {
-            "stable" => self.rustc_version.clone(),
+            "stable" => RUSTC_VERSION.into(),
             "beta" => "beta".into(),
             "nightly" => "nightly".into(),
-            _ => format!("{}-dev", self.rustc_version),
+            _ => format!("{}-dev", RUSTC_VERSION),
         };
 
         if package.target_independent() {
@@ -198,6 +186,6 @@ impl Versions {
     }
 
     pub(crate) fn rustc_version(&self) -> &str {
-        &self.rustc_version
+        RUSTC_VERSION
     }
 }