bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

rustdoc: add hash to filename of toolchain files

Browse source 
All static files used by rustdoc are now stored in static.files/ and
include a hash of their contents. They no longer include the contents of
the --resource-suffix flag. This clarifies caching semantics. Anything
in static.files can use Cache-Control: immutable because any updates
will show up as a new URL.

Invocation-specific files like crates-NN.js, search-index-NN.js,
and sidebar-items-NN.js still get the resource suffix.

The --disable-minification flag is removed because it would vary the
output of static files based on invocation flags. Instead, for
rustdoc development purposes it's preferable to symlink static files
to a non-minified copy for quick iteration.
This commit is contained in:
Jacob Hoffman-Andrews 2022-10-24 01:28:55 -07:00
parent 68c836a904
commit f9e1f6ffdf
18 changed files with 267 additions and 509 deletions
Download patch file Download diff file Expand all files Collapse all files

288
src/librustdoc/html/render/write_shared.rs
View file

@ -1,10 +1,8 @@
use std::ffi::OsStr;
use std::fs::{self, File};
use std::io::prelude::*;
use std::io::{self, BufReader};
use std::path::{Component, Path, PathBuf};
use std::path::{Component, Path};
use std::rc::Rc;
use std::sync::LazyLock as Lazy;
use itertools::Itertools;
use rustc_data_structures::flock;
@ -20,123 +18,19 @@ use crate::error::Error;
use crate::html::{layout, static_files};
use crate::{try_err, try_none};
static FILES_UNVERSIONED: Lazy<FxHashMap<&str, &[u8]>> = Lazy::new(|| {
map! {
"FiraSans-Regular.woff2" => static_files::fira_sans::REGULAR,
"FiraSans-Medium.woff2" => static_files::fira_sans::MEDIUM,
"FiraSans-LICENSE.txt" => static_files::fira_sans::LICENSE,
"SourceSerif4-Regular.ttf.woff2" => static_files::source_serif_4::REGULAR,
"SourceSerif4-Bold.ttf.woff2" => static_files::source_serif_4::BOLD,
"SourceSerif4-It.ttf.woff2" => static_files::source_serif_4::ITALIC,
"SourceSerif4-LICENSE.md" => static_files::source_serif_4::LICENSE,
"SourceCodePro-Regular.ttf.woff2" => static_files::source_code_pro::REGULAR,
"SourceCodePro-Semibold.ttf.woff2" => static_files::source_code_pro::SEMIBOLD,
"SourceCodePro-It.ttf.woff2" => static_files::source_code_pro::ITALIC,
"SourceCodePro-LICENSE.txt" => static_files::source_code_pro::LICENSE,
"NanumBarunGothic.ttf.woff2" => static_files::nanum_barun_gothic::REGULAR,
"NanumBarunGothic-LICENSE.txt" => static_files::nanum_barun_gothic::LICENSE,
"LICENSE-MIT.txt" => static_files::LICENSE_MIT,
"LICENSE-APACHE.txt" => static_files::LICENSE_APACHE,
"COPYRIGHT.txt" => static_files::COPYRIGHT,
}
});
enum SharedResource<'a> {
/// This file will never change, no matter what toolchain is used to build it.
///
/// It does not have a resource suffix.
Unversioned { name: &'static str },
/// This file may change depending on the toolchain.
///
/// It has a resource suffix.
ToolchainSpecific { basename: &'static str },
/// This file may change for any crate within a build, or based on the CLI arguments.
///
/// This differs from normal invocation-specific files because it has a resource suffix.
InvocationSpecific { basename: &'a str },
}
impl SharedResource<'_> {
fn extension(&self) -> Option<&OsStr> {
use SharedResource::*;
match self {
Unversioned { name }
| ToolchainSpecific { basename: name }
| InvocationSpecific { basename: name } => Path::new(name).extension(),
}
}
fn path(&self, cx: &Context<'_>) -> PathBuf {
match self {
SharedResource::Unversioned { name } => cx.dst.join(name),
SharedResource::ToolchainSpecific { basename } => cx.suffix_path(basename),
SharedResource::InvocationSpecific { basename } => cx.suffix_path(basename),
}
}
fn should_emit(&self, emit: &[EmitType]) -> bool {
if emit.is_empty() {
return true;
}
let kind = match self {
SharedResource::Unversioned { .. } => EmitType::Unversioned,
SharedResource::ToolchainSpecific { .. } => EmitType::Toolchain,
SharedResource::InvocationSpecific { .. } => EmitType::InvocationSpecific,
};
emit.contains(&kind)
}
}
impl Context<'_> {
fn suffix_path(&self, filename: &str) -> PathBuf {
// We use splitn vs Path::extension here because we might get a filename
// like `style.min.css` and we want to process that into
// `style-suffix.min.css`. Path::extension would just return `css`
// which would result in `style.min-suffix.css` which isn't what we
// want.
let (base, ext) = filename.split_once('.').unwrap();
let filename = format!("{}{}.{}", base, self.shared.resource_suffix, ext);
self.dst.join(&filename)
}
fn write_shared(
&self,
resource: SharedResource<'_>,
contents: impl 'static + Send + AsRef<[u8]>,
emit: &[EmitType],
) -> Result<(), Error> {
if resource.should_emit(emit) {
self.shared.fs.write(resource.path(self), contents)
} else {
Ok(())
}
}
fn write_minify(
&self,
resource: SharedResource<'_>,
contents: impl 'static + Send + AsRef<str> + AsRef<[u8]>,
minify: bool,
emit: &[EmitType],
) -> Result<(), Error> {
if minify {
let contents = contents.as_ref();
let contents = if resource.extension() == Some(OsStr::new("css")) {
minifier::css::minify(contents)
.map_err(|e| {
Error::new(format!("failed to minify CSS file: {}", e), resource.path(self))
})?
.to_string()
} else {
minifier::js::minify(contents).to_string()
};
self.write_shared(resource, contents, emit)
} else {
self.write_shared(resource, contents, emit)
}
}
}
/// Rustdoc writes out two kinds of shared files:
/// - Static files, which are embedded in the rustdoc binary and are written with a
/// filename that includes a hash of their contents. These will always have a new
/// URL if the contents change, so they are safe to cache with the
/// `Cache-Control: immutable` directive. They are written under the static.files/
/// directory and are written when --emit-type is empty (default) or contains
/// "toolchain-specific".
/// - Invocation specific files. These are generated based on the crate(s) being
/// documented. Their filenames need to be predictable without knowing their
/// contents, so they do not include a hash in their filename and are not safe to
/// cache with `Cache-Control: immutable`. They include the contents of the
/// --resource-suffix flag and are emitted when --emit-type is empty (default)
/// or contains "invocation-specific".
pub(super) fn write_shared(
cx: &mut Context<'_>,
krate: &Crate,
@ -149,139 +43,51 @@ pub(super) fn write_shared(
let lock_file = cx.dst.join(".lock");
let _lock = try_err!(flock::Lock::new(&lock_file, true, true, true), &lock_file);
// Minified resources are usually toolchain resources. If they're not, they should use `cx.write_minify` directly.
fn write_minify(
basename: &'static str,
contents: impl 'static + Send + AsRef<str> + AsRef<[u8]>,
cx: &Context<'_>,
options: &RenderOptions,
) -> Result<(), Error> {
cx.write_minify(
SharedResource::ToolchainSpecific { basename },
contents,
options.enable_minification,
&options.emit,
)
}
// Toolchain resources should never be dynamic.
let write_toolchain = |p: &'static _, c: &'static _| {
cx.write_shared(SharedResource::ToolchainSpecific { basename: p }, c, &options.emit)
};
// Crate resources should always be dynamic.
let write_crate = |p: &_, make_content: &dyn Fn() -> Result<Vec<u8>, Error>| {
// InvocationSpecific resources should always be dynamic.
let write_invocation_specific = |p: &str, make_content: &dyn Fn() -> Result<Vec<u8>, Error>| {
let content = make_content()?;
cx.write_shared(SharedResource::InvocationSpecific { basename: p }, content, &options.emit)
if options.emit.is_empty() || options.emit.contains(&EmitType::InvocationSpecific) {
let output_filename = static_files::suffix_path(p, &cx.shared.resource_suffix);
cx.shared.fs.write(cx.dst.join(output_filename), content)
} else {
Ok(())
}
};
// Given "foo.svg", return e.g. "url(\"foo1.58.0.svg\")"
fn ver_url(cx: &Context<'_>, basename: &'static str) -> String {
format!(
"url(\"{}\")",
SharedResource::ToolchainSpecific { basename }
.path(cx)
.file_name()
.unwrap()
.to_str()
.unwrap()
)
}
// We use the AUTOREPLACE mechanism to inject into our static JS and CSS certain
// values that are only known at doc build time. Since this mechanism is somewhat
// surprising when reading the code, please limit it to rustdoc.css.
write_minify(
"rustdoc.css",
static_files::RUSTDOC_CSS
.replace(
"/* AUTOREPLACE: */url(\"toggle-minus.svg\")",
&ver_url(cx, "toggle-minus.svg"),
)
.replace("/* AUTOREPLACE: */url(\"toggle-plus.svg\")", &ver_url(cx, "toggle-plus.svg"))
.replace("/* AUTOREPLACE: */url(\"down-arrow.svg\")", &ver_url(cx, "down-arrow.svg")),
cx,
options,
)?;
// Add all the static files. These may already exist, but we just
// overwrite them anyway to make sure that they're fresh and up-to-date.
write_minify("settings.css", static_files::SETTINGS_CSS, cx, options)?;
write_minify("noscript.css", static_files::NOSCRIPT_CSS, cx, options)?;
// To avoid "light.css" to be overwritten, we'll first run over the received themes and only
// then we'll run over the "official" styles.
let mut themes: FxHashSet<String> = FxHashSet::default();
cx.shared
.fs
.create_dir_all(cx.dst.join("static.files"))
.map_err(|e| PathError::new(e, "static.files"))?;
// Handle added third-party themes
for entry in &cx.shared.style_files {
let theme = entry.basename()?;
let extension =
try_none!(try_none!(entry.path.extension(), &entry.path).to_str(), &entry.path);
// Handle the official themes
match theme.as_str() {
"light" => write_minify("light.css", static_files::themes::LIGHT, cx, options)?,
"dark" => write_minify("dark.css", static_files::themes::DARK, cx, options)?,
"ayu" => write_minify("ayu.css", static_files::themes::AYU, cx, options)?,
_ => {
// Handle added third-party themes
let filename = format!("{}.{}", theme, extension);
write_crate(&filename, &|| Ok(try_err!(fs::read(&entry.path), &entry.path)))?;
}
};
// Skip the official themes. They are written below as part of STATIC_FILES_LIST.
if matches!(theme.as_str(), "light" | "dark" | "ayu") {
continue;
}
themes.insert(theme.to_owned());
}
if (*cx.shared).layout.logo.is_empty() {
write_toolchain("rust-logo.svg", static_files::RUST_LOGO_SVG)?;
}
if (*cx.shared).layout.favicon.is_empty() {
write_toolchain("favicon.svg", static_files::RUST_FAVICON_SVG)?;
write_toolchain("favicon-16x16.png", static_files::RUST_FAVICON_PNG_16)?;
write_toolchain("favicon-32x32.png", static_files::RUST_FAVICON_PNG_32)?;
}
write_toolchain("wheel.svg", static_files::WHEEL_SVG)?;
write_toolchain("clipboard.svg", static_files::CLIPBOARD_SVG)?;
write_toolchain("down-arrow.svg", static_files::DOWN_ARROW_SVG)?;
write_toolchain("toggle-minus.svg", static_files::TOGGLE_MINUS_PNG)?;
write_toolchain("toggle-plus.svg", static_files::TOGGLE_PLUS_PNG)?;
let mut themes: Vec<&String> = themes.iter().collect();
themes.sort();
write_minify("main.js", static_files::MAIN_JS, cx, options)?;
write_minify("search.js", static_files::SEARCH_JS, cx, options)?;
write_minify("settings.js", static_files::SETTINGS_JS, cx, options)?;
if cx.include_sources {
write_minify("source-script.js", static_files::sidebar::SOURCE_SCRIPT, cx, options)?;
}
write_minify("storage.js", static_files::STORAGE_JS, cx, options)?;
if cx.shared.layout.scrape_examples_extension {
cx.write_minify(
SharedResource::InvocationSpecific { basename: "scrape-examples.js" },
static_files::SCRAPE_EXAMPLES_JS,
options.enable_minification,
&options.emit,
)?;
let bytes = try_err!(fs::read(&entry.path), &entry.path);
let filename = format!("{}{}.{}", theme, cx.shared.resource_suffix, extension);
cx.shared.fs.write(cx.dst.join(filename), bytes)?;
}
// When the user adds their own CSS files with --extend-css, we write that as an
// invocation-specific file (that is, with a resource suffix).
if let Some(ref css) = cx.shared.layout.css_file_extension {
let buffer = try_err!(fs::read_to_string(css), css);
// This varies based on the invocation, so it can't go through the write_minify wrapper.
cx.write_minify(
SharedResource::InvocationSpecific { basename: "theme.css" },
buffer,
options.enable_minification,
&options.emit,
)?;
let path = static_files::suffix_path("theme.css", &cx.shared.resource_suffix);
cx.shared.fs.write(cx.dst.join(path), buffer)?;
}
write_minify("normalize.css", static_files::NORMALIZE_CSS, cx, options)?;
for (name, contents) in &*FILES_UNVERSIONED {
cx.write_shared(SharedResource::Unversioned { name }, contents, &options.emit)?;
if options.emit.is_empty() || options.emit.contains(&EmitType::Toolchain) {
for f in static_files::STATIC_FILES_LIST {
let filename = static_files::static_filename(f.filename, f.bytes);
cx.shared.fs.write(cx.dst.join(filename), f.minified())?;
}
}
/// Read a file and return all lines that match the `"{crate}":{data},` format,
@ -463,7 +269,7 @@ pub(super) fn write_shared(
v.push_str("\\\n}');\ncreateSourceSidebar();\n");
Ok(v.into_bytes())
};
write_crate("source-files.js", &make_sources)?;
write_invocation_specific("source-files.js", &make_sources)?;
}
// Update the search index and crate list.
@ -477,7 +283,7 @@ pub(super) fn write_shared(
// Sort the indexes by crate so the file will be generated identically even
// with rustdoc running in parallel.
all_indexes.sort();
write_crate("search-index.js", &|| {
write_invocation_specific("search-index.js", &|| {
let mut v = String::from("var searchIndex = JSON.parse('{\\\n");
v.push_str(&all_indexes.join(",\\\n"));
v.push_str(
@ -490,7 +296,7 @@ if (typeof exports !== 'undefined') {exports.searchIndex = searchIndex};
Ok(v.into_bytes())
})?;
write_crate("crates.js", &|| {
write_invocation_specific("crates.js", &|| {
let krates = krates.iter().map(|k| format!("\"{}\"", k)).join(",");
Ok(format!("window.ALL_CRATES = [{}];", krates).into_bytes())
})?;