Rollup merge of #83478 - jyn514:fine-grained-files, r=Mark-Simulacrum

rustdoc: Add unstable option to only emit shared/crate-specific files

The intended use case is for docs.rs, which can now copy exactly the
files it cares about, rather than having to guess based on whether they
have a resource suffix or not. In particular, some files have a resource
suffix but cannot be shared between crates: https://github.com/rust-lang/docs.rs/pull/1312#issuecomment-798783688

The end goal is to fix rust-lang/docs.rs#1327 by reverting rust-lang/docs.rs#1324.

This obsoletes `--print=unversioned-files`, which I plan to remove as
soon as docs.rs stops using it.

I recommend reviewing this one commit at a time.

r? ``@GuillaumeGomez`` cc ``@Nemo157`` ``@pietroalbini``

src/librustdoc/config.rs

@@ -3,6 +3,7 @@ use std::convert::TryFrom;
 use std::ffi::OsStr;
 use std::fmt;
 use std::path::PathBuf;
+use std::str::FromStr;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_session::config::{self, parse_crate_types_from_list, parse_externs, CrateType};
@@ -266,6 +267,34 @@ crate struct RenderOptions {
     /// If `true`, generate a JSON file in the crate folder instead of HTML redirection files.
     crate generate_redirect_map: bool,
     crate unstable_features: rustc_feature::UnstableFeatures,
+    crate emit: Vec<EmitType>,
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+crate enum EmitType {
+    Unversioned,
+    Toolchain,
+    InvocationSpecific,
+}
+
+impl FromStr for EmitType {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        use EmitType::*;
+        match s {
+            "unversioned-shared-resources" => Ok(Unversioned),
+            "toolchain-shared-resources" => Ok(Toolchain),
+            "invocation-specific" => Ok(InvocationSpecific),
+            _ => Err(()),
+        }
+    }
+}
+
+impl RenderOptions {
+    crate fn should_emit_crate(&self) -> bool {
+        self.emit.is_empty() || self.emit.contains(&EmitType::InvocationSpecific)
+    }
 }
 
 impl Options {
@@ -334,6 +363,19 @@ impl Options {
         // check for deprecated options
         check_deprecated_options(&matches, &diag);
 
+        let mut emit = Vec::new();
+        for list in matches.opt_strs("emit") {
+            for kind in list.split(',') {
+                match kind.parse() {
+                    Ok(kind) => emit.push(kind),
+                    Err(()) => {
+                        diag.err(&format!("unrecognized emission type: {}", kind));
+                        return Err(1);
+                    }
+                }
+            }
+        }
+
         let to_check = matches.opt_strs("check-theme");
         if !to_check.is_empty() {
             let paths = theme::load_css_paths(static_files::themes::LIGHT.as_bytes());
@@ -641,6 +683,7 @@ impl Options {
                 unstable_features: rustc_feature::UnstableFeatures::from_environment(
                     crate_name.as_deref(),
                 ),
+                emit,
             },
             crate_name,
             output_format,

src/librustdoc/formats/renderer.rs

@@ -63,10 +63,15 @@ crate fn run_format<'tcx, T: FormatRenderer<'tcx>>(
 ) -> Result<(), Error> {
     let prof = &tcx.sess.prof;
 
+    let emit_crate = options.should_emit_crate();
     let (mut format_renderer, krate) = prof
         .extra_verbose_generic_activity("create_renderer", T::descr())
         .run(|| T::init(krate, options, edition, cache, tcx))?;
 
+    if !emit_crate {
+        return Ok(());
+    }
+
     // Render the crate documentation
     let crate_name = krate.name;
     let mut work = vec![(format_renderer.make_child_renderer(), krate.module)];

src/librustdoc/html/render/context.rs

@@ -79,17 +79,6 @@ crate struct Context<'tcx> {
 rustc_data_structures::static_assert_size!(Context<'_>, 152);
 
 impl<'tcx> Context<'tcx> {
-    pub(super) fn path(&self, filename: &str) -> PathBuf {
-        // We use splitn vs Path::extension here because we might get a filename
-        // like `style.min.css` and we want to process that into
-        // `style-suffix.min.css`.  Path::extension would just return `css`
-        // which would result in `style.min-suffix.css` which isn't what we
-        // want.
-        let (base, ext) = filename.split_once('.').unwrap();
-        let filename = format!("{}{}.{}", base, self.shared.resource_suffix, ext);
-        self.dst.join(&filename)
-    }
-
     pub(super) fn tcx(&self) -> TyCtxt<'tcx> {
         self.shared.tcx
     }
@@ -301,6 +290,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
     ) -> Result<(Self, clean::Crate), Error> {
         // need to save a copy of the options for rendering the index page
         let md_opts = options.clone();
+        let emit_crate = options.should_emit_crate();
         let RenderOptions {
             output,
             external_html,
@@ -406,7 +396,9 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
 
         let dst = output;
         scx.ensure_dir(&dst)?;
-        krate = sources::render(&dst, &mut scx, krate)?;
+        if emit_crate {
+            krate = sources::render(&dst, &mut scx, krate)?;
+        }
 
         // Build our search index
         let index = build_index(&krate, &mut cache, tcx);
@@ -489,7 +481,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
             |buf: &mut Buffer| all.print(buf),
             &self.shared.style_files,
         );
-        self.shared.fs.write(&final_file, v.as_bytes())?;
+        self.shared.fs.write(final_file, v.as_bytes())?;
 
         // Generating settings page.
         page.title = "Rustdoc settings";

src/librustdoc/html/render/write_shared.rs

@@ -13,8 +13,8 @@ use serde::Serialize;
 
 use super::{collect_paths_for_type, ensure_trailing_slash, Context, BASIC_KEYWORDS};
 use crate::clean::Crate;
-use crate::config::RenderOptions;
+use crate::config::{EmitType, RenderOptions};
-use crate::docfs::{DocFS, PathError};
+use crate::docfs::PathError;
 use crate::error::Error;
 use crate::formats::FormatRenderer;
 use crate::html::{layout, static_files};
@@ -40,6 +40,102 @@ crate static FILES_UNVERSIONED: Lazy<FxHashMap<&str, &[u8]>> = Lazy::new(|| {
     }
 });
 
+enum SharedResource<'a> {
+    /// This file will never change, no matter what toolchain is used to build it.
+    ///
+    /// It does not have a resource suffix.
+    Unversioned { name: &'static str },
+    /// This file may change depending on the toolchain.
+    ///
+    /// It has a resource suffix.
+    ToolchainSpecific { basename: &'static str },
+    /// This file may change for any crate within a build, or based on the CLI arguments.
+    ///
+    /// This differs from normal invocation-specific files because it has a resource suffix.
+    InvocationSpecific { basename: &'a str },
+}
+
+impl SharedResource<'_> {
+    fn extension(&self) -> Option<&OsStr> {
+        use SharedResource::*;
+        match self {
+            Unversioned { name }
+            | ToolchainSpecific { basename: name }
+            | InvocationSpecific { basename: name } => Path::new(name).extension(),
+        }
+    }
+
+    fn path(&self, cx: &Context<'_>) -> PathBuf {
+        match self {
+            SharedResource::Unversioned { name } => cx.dst.join(name),
+            SharedResource::ToolchainSpecific { basename } => cx.suffix_path(basename),
+            SharedResource::InvocationSpecific { basename } => cx.suffix_path(basename),
+        }
+    }
+
+    fn should_emit(&self, emit: &[EmitType]) -> bool {
+        if emit.is_empty() {
+            return true;
+        }
+        let kind = match self {
+            SharedResource::Unversioned { .. } => EmitType::Unversioned,
+            SharedResource::ToolchainSpecific { .. } => EmitType::Toolchain,
+            SharedResource::InvocationSpecific { .. } => EmitType::InvocationSpecific,
+        };
+        emit.contains(&kind)
+    }
+}
+
+impl Context<'_> {
+    fn suffix_path(&self, filename: &str) -> PathBuf {
+        // We use splitn vs Path::extension here because we might get a filename
+        // like `style.min.css` and we want to process that into
+        // `style-suffix.min.css`.  Path::extension would just return `css`
+        // which would result in `style.min-suffix.css` which isn't what we
+        // want.
+        let (base, ext) = filename.split_once('.').unwrap();
+        let filename = format!("{}{}.{}", base, self.shared.resource_suffix, ext);
+        self.dst.join(&filename)
+    }
+
+    fn write_shared<C: AsRef<[u8]>>(
+        &self,
+        resource: SharedResource<'_>,
+        contents: C,
+        emit: &[EmitType],
+    ) -> Result<(), Error> {
+        if resource.should_emit(emit) {
+            self.shared.fs.write(resource.path(self), contents)
+        } else {
+            Ok(())
+        }
+    }
+
+    fn write_minify(
+        &self,
+        resource: SharedResource<'_>,
+        contents: &str,
+        minify: bool,
+        emit: &[EmitType],
+    ) -> Result<(), Error> {
+        let tmp;
+        let contents = if minify {
+            tmp = if resource.extension() == Some(&OsStr::new("css")) {
+                minifier::css::minify(contents).map_err(|e| {
+                    Error::new(format!("failed to minify CSS file: {}", e), resource.path(self))
+                })?
+            } else {
+                minifier::js::minify(contents)
+            };
+            tmp.as_bytes()
+        } else {
+            contents.as_bytes()
+        };
+
+        self.write_shared(resource, contents, emit)
+    }
+}
+
 pub(super) fn write_shared(
     cx: &Context<'_>,
     krate: &Crate,
@@ -52,27 +148,31 @@ pub(super) fn write_shared(
     let lock_file = cx.dst.join(".lock");
     let _lock = try_err!(flock::Lock::new(&lock_file, true, true, true), &lock_file);
 
+    // The weird `: &_` is to work around a borrowck bug: https://github.com/rust-lang/rust/issues/41078#issuecomment-293646723
+    let write_minify = |p, c: &_| {
+        cx.write_minify(
+            SharedResource::ToolchainSpecific { basename: p },
+            c,
+            options.enable_minification,
+            &options.emit,
+        )
+    };
+    // Toolchain resources should never be dynamic.
+    let write_toolchain = |p: &'static _, c: &'static _| {
+        cx.write_shared(SharedResource::ToolchainSpecific { basename: p }, c, &options.emit)
+    };
+
+    // Crate resources should always be dynamic.
+    let write_crate = |p: &_, make_content: &dyn Fn() -> Result<Vec<u8>, Error>| {
+        let content = make_content()?;
+        cx.write_shared(SharedResource::InvocationSpecific { basename: p }, content, &options.emit)
+    };
+
     // Add all the static files. These may already exist, but we just
     // overwrite them anyway to make sure that they're fresh and up-to-date.
-
+    write_minify("rustdoc.css", static_files::RUSTDOC_CSS)?;
-    write_minify(
+    write_minify("settings.css", static_files::SETTINGS_CSS)?;
-        &cx.shared.fs,
+    write_minify("noscript.css", static_files::NOSCRIPT_CSS)?;
-        cx.path("rustdoc.css"),
-        static_files::RUSTDOC_CSS,
-        options.enable_minification,
-    )?;
-    write_minify(
-        &cx.shared.fs,
-        cx.path("settings.css"),
-        static_files::SETTINGS_CSS,
-        options.enable_minification,
-    )?;
-    write_minify(
-        &cx.shared.fs,
-        cx.path("noscript.css"),
-        static_files::NOSCRIPT_CSS,
-        options.enable_minification,
-    )?;
 
     // To avoid "light.css" to be overwritten, we'll first run over the received themes and only
     // then we'll run over the "official" styles.
@@ -85,106 +185,73 @@ pub(super) fn write_shared(
 
         // Handle the official themes
         match theme {
-            "light" => write_minify(
+            "light" => write_minify("light.css", static_files::themes::LIGHT)?,
-                &cx.shared.fs,
+            "dark" => write_minify("dark.css", static_files::themes::DARK)?,
-                cx.path("light.css"),
+            "ayu" => write_minify("ayu.css", static_files::themes::AYU)?,
-                static_files::themes::LIGHT,
-                options.enable_minification,
-            )?,
-            "dark" => write_minify(
-                &cx.shared.fs,
-                cx.path("dark.css"),
-                static_files::themes::DARK,
-                options.enable_minification,
-            )?,
-            "ayu" => write_minify(
-                &cx.shared.fs,
-                cx.path("ayu.css"),
-                static_files::themes::AYU,
-                options.enable_minification,
-            )?,
             _ => {
                 // Handle added third-party themes
-                let content = try_err!(fs::read(&entry.path), &entry.path);
+                let filename = format!("{}.{}", theme, extension);
-                cx.shared
+                write_crate(&filename, &|| Ok(try_err!(fs::read(&entry.path), &entry.path)))?;
-                    .fs
-                    .write(cx.path(&format!("{}.{}", theme, extension)), content.as_slice())?;
             }
         };
 
         themes.insert(theme.to_owned());
     }
 
-    let write = |p, c| cx.shared.fs.write(p, c);
     if (*cx.shared).layout.logo.is_empty() {
-        write(cx.path("rust-logo.png"), static_files::RUST_LOGO)?;
+        write_toolchain("rust-logo.png", static_files::RUST_LOGO)?;
     }
     if (*cx.shared).layout.favicon.is_empty() {
-        write(cx.path("favicon.svg"), static_files::RUST_FAVICON_SVG)?;
+        write_toolchain("favicon.svg", static_files::RUST_FAVICON_SVG)?;
-        write(cx.path("favicon-16x16.png"), static_files::RUST_FAVICON_PNG_16)?;
+        write_toolchain("favicon-16x16.png", static_files::RUST_FAVICON_PNG_16)?;
-        write(cx.path("favicon-32x32.png"), static_files::RUST_FAVICON_PNG_32)?;
+        write_toolchain("favicon-32x32.png", static_files::RUST_FAVICON_PNG_32)?;
     }
-    write(cx.path("brush.svg"), static_files::BRUSH_SVG)?;
+    write_toolchain("brush.svg", static_files::BRUSH_SVG)?;
-    write(cx.path("wheel.svg"), static_files::WHEEL_SVG)?;
+    write_toolchain("wheel.svg", static_files::WHEEL_SVG)?;
-    write(cx.path("down-arrow.svg"), static_files::DOWN_ARROW_SVG)?;
+    write_toolchain("down-arrow.svg", static_files::DOWN_ARROW_SVG)?;
 
     let mut themes: Vec<&String> = themes.iter().collect();
     themes.sort();
 
+    // FIXME: this should probably not be a toolchain file since it depends on `--theme`.
+    // But it seems a shame to copy it over and over when it's almost always the same.
+    // Maybe we can change the representation to move this out of main.js?
     write_minify(
-        &cx.shared.fs,
+        "main.js",
-        cx.path("main.js"),
         &static_files::MAIN_JS.replace(
             "/* INSERT THEMES HERE */",
             &format!(" = {}", serde_json::to_string(&themes).unwrap()),
         ),
-        options.enable_minification,
-    )?;
-    write_minify(
-        &cx.shared.fs,
-        cx.path("settings.js"),
-        static_files::SETTINGS_JS,
-        options.enable_minification,
     )?;
+    write_minify("settings.js", static_files::SETTINGS_JS)?;
     if cx.shared.include_sources {
-        write_minify(
+        write_minify("source-script.js", static_files::sidebar::SOURCE_SCRIPT)?;
-            &cx.shared.fs,
-            cx.path("source-script.js"),
-            static_files::sidebar::SOURCE_SCRIPT,
-            options.enable_minification,
-        )?;
     }
 
     {
         write_minify(
-            &cx.shared.fs,
+            "storage.js",
-            cx.path("storage.js"),
             &format!(
                 "var resourcesSuffix = \"{}\";{}",
                 cx.shared.resource_suffix,
                 static_files::STORAGE_JS
             ),
-            options.enable_minification,
         )?;
     }
 
     if let Some(ref css) = cx.shared.layout.css_file_extension {
-        let out = cx.path("theme.css");
         let buffer = try_err!(fs::read_to_string(css), css);
-        if !options.enable_minification {
+        // This varies based on the invocation, so it can't go through the write_minify wrapper.
-            cx.shared.fs.write(&out, &buffer)?;
+        cx.write_minify(
-        } else {
+            SharedResource::InvocationSpecific { basename: "theme.css" },
-            write_minify(&cx.shared.fs, out, &buffer, options.enable_minification)?;
+            &buffer,
-        }
+            options.enable_minification,
+            &options.emit,
+        )?;
     }
-    write_minify(
+    write_minify("normalize.css", static_files::NORMALIZE_CSS)?;
-        &cx.shared.fs,
+    for (name, contents) in &*FILES_UNVERSIONED {
-        cx.path("normalize.css"),
+        cx.write_shared(SharedResource::Unversioned { name }, contents, &options.emit)?;
-        static_files::NORMALIZE_CSS,
-        options.enable_minification,
-    )?;
-    for (file, contents) in &*FILES_UNVERSIONED {
-        write(cx.dst.join(file), contents)?;
     }
 
     fn collect(path: &Path, krate: &str, key: &str) -> io::Result<(Vec<String>, Vec<String>)> {
@@ -312,19 +379,22 @@ pub(super) fn write_shared(
         }
 
         let dst = cx.dst.join(&format!("source-files{}.js", cx.shared.resource_suffix));
-        let (mut all_sources, _krates) =
+        let make_sources = || {
-            try_err!(collect(&dst, &krate.name.as_str(), "sourcesIndex"), &dst);
+            let (mut all_sources, _krates) =
-        all_sources.push(format!(
+                try_err!(collect(&dst, &krate.name.as_str(), "sourcesIndex"), &dst);
-            "sourcesIndex[\"{}\"] = {};",
+            all_sources.push(format!(
-            &krate.name,
+                "sourcesIndex[\"{}\"] = {};",
-            hierarchy.to_json_string()
+                &krate.name,
-        ));
+                hierarchy.to_json_string()
-        all_sources.sort();
+            ));
-        let v = format!(
+            all_sources.sort();
-            "var N = null;var sourcesIndex = {{}};\n{}\ncreateSourceSidebar();\n",
+            Ok(format!(
-            all_sources.join("\n")
+                "var N = null;var sourcesIndex = {{}};\n{}\ncreateSourceSidebar();\n",
-        );
+                all_sources.join("\n")
-        cx.shared.fs.write(&dst, v.as_bytes())?;
+            )
+            .into_bytes())
+        };
+        write_crate("source-files.js", &make_sources)?;
     }
 
     // Update the search index and crate list.
@@ -337,17 +407,17 @@ pub(super) fn write_shared(
     // Sort the indexes by crate so the file will be generated identically even
     // with rustdoc running in parallel.
     all_indexes.sort();
-    {
+    write_crate("search-index.js", &|| {
         let mut v = String::from("var searchIndex = JSON.parse('{\\\n");
         v.push_str(&all_indexes.join(",\\\n"));
         v.push_str("\\\n}');\ninitSearch(searchIndex);");
-        cx.shared.fs.write(&dst, &v)?;
+        Ok(v.into_bytes())
-    }
+    })?;
 
-    let crate_list_dst = cx.dst.join(&format!("crates{}.js", cx.shared.resource_suffix));
+    write_crate("crates.js", &|| {
-    let crate_list =
+        let krates = krates.iter().map(|k| format!("\"{}\"", k)).join(",");
-        format!("window.ALL_CRATES = [{}];", krates.iter().map(|k| format!("\"{}\"", k)).join(","));
+        Ok(format!("window.ALL_CRATES = [{}];", krates).into_bytes())
-    cx.shared.fs.write(&crate_list_dst, &crate_list)?;
+    })?;
 
     if options.enable_index_page {
         if let Some(index_page) = options.index_page.clone() {
@@ -481,21 +551,3 @@ pub(super) fn write_shared(
     }
     Ok(())
 }
-
-fn write_minify(
-    fs: &DocFS,
-    dst: PathBuf,
-    contents: &str,
-    enable_minification: bool,
-) -> Result<(), Error> {
-    if enable_minification {
-        if dst.extension() == Some(&OsStr::new("css")) {
-            let res = try_none!(minifier::css::minify(contents).ok(), &dst);
-            fs.write(dst, res.as_bytes())
-        } else {
-            fs.write(dst, minifier::js::minify(contents).as_bytes())
-        }
-    } else {
-        fs.write(dst, contents.as_bytes())
-    }
-}

src/librustdoc/lib.rs

@@ -527,6 +527,14 @@ fn opts() -> Vec<RustcOptGroup> {
         unstable("print", |o| {
             o.optmulti("", "print", "Rustdoc information to print on stdout", "[unversioned-files]")
         }),
+        unstable("emit", |o| {
+            o.optmulti(
+                "",
+                "emit",
+                "Comma separated list of types of output for rustdoc to emit",
+                "[unversioned-shared-resources,toolchain-shared-resources,invocation-specific]",
+            )
+        }),
     ]
 }
 

src/test/run-make/emit-shared-files/Makefile

@@ -0,0 +1,46 @@
+-include ../../run-make-fulldeps/tools.mk
+
+INVOCATION_ONLY = $(TMPDIR)/invocation-only
+TOOLCHAIN_ONLY = $(TMPDIR)/toolchain-only
+ALL_SHARED = $(TMPDIR)/all-shared
+
+all: invocation-only toolchain-only all-shared
+
+invocation-only:
+	$(RUSTDOC) -Z unstable-options --emit=invocation-specific --output $(INVOCATION_ONLY) --resource-suffix=-xxx --theme y.css --extend-css z.css x.rs
+	[ -e $(INVOCATION_ONLY)/search-index-xxx.js ]
+	[ -e $(INVOCATION_ONLY)/settings.html ]
+	[ -e $(INVOCATION_ONLY)/x/all.html ]
+	[ -e $(INVOCATION_ONLY)/x/index.html ]
+	[ -e $(INVOCATION_ONLY)/theme-xxx.css ] # generated from z.css
+	! [ -e $(INVOCATION_ONLY)/storage-xxx.js ]
+	! [ -e $(INVOCATION_ONLY)/SourceSerifPro-It.ttf.woff ]
+
+	# FIXME: this probably shouldn't have a suffix
+	[ -e $(INVOCATION_ONLY)/y-xxx.css ]
+	# FIXME: this is technically incorrect (see `write_shared`)
+	! [ -e $(INVOCATION_ONLY)/main-xxx.js ]
+
+toolchain-only:
+	$(RUSTDOC) -Z unstable-options --emit=toolchain-shared-resources --output $(TOOLCHAIN_ONLY) --resource-suffix=-xxx --extend-css z.css x.rs
+	[ -e $(TOOLCHAIN_ONLY)/storage-xxx.js ]
+	! [ -e $(TOOLCHAIN_ONLY)/SourceSerifPro-It.ttf.woff ]
+	! [ -e $(TOOLCHAIN_ONLY)/search-index-xxx.js ]
+	! [ -e $(TOOLCHAIN_ONLY)/x/index.html ]
+	! [ -e $(TOOLCHAIN_ONLY)/theme.css ]
+
+	[ -e $(TOOLCHAIN_ONLY)/main-xxx.js ]
+	! [ -e $(TOOLCHAIN_ONLY)/y-xxx.css ]
+
+all-shared:
+	$(RUSTDOC) -Z unstable-options --emit=toolchain-shared-resources,unversioned-shared-resources --output $(ALL_SHARED) --resource-suffix=-xxx --extend-css z.css x.rs
+	[ -e $(ALL_SHARED)/storage-xxx.js ]
+	[ -e $(ALL_SHARED)/SourceSerifPro-It.ttf.woff ]
+	! [ -e $(ALL_SHARED)/search-index-xxx.js ]
+	! [ -e $(ALL_SHARED)/settings.html ]
+	! [ -e $(ALL_SHARED)/x ]
+	! [ -e $(ALL_SHARED)/src ]
+	! [ -e $(ALL_SHARED)/theme.css ]
+
+	[ -e $(ALL_SHARED)/main-xxx.js ]
+	! [ -e $(ALL_SHARED)/y-xxx.css ]

src/test/run-make/emit-shared-files/x.rs

@@ -0,0 +1 @@
+// nothing to see here