bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

rustc_metadata: Preprocess search paths for better performance

Browse source 
Over in Zed we've noticed that loading crates for a large-ish workspace can take almost 200ms. We've pinned it down to how rustc searches for paths, as it performs a linear search over the list of candidate paths. In our case the candidate list had about 20k entries which we had to iterate over for each dependency being loaded.

This commit introduces a simple FilesIndex that's just a sorted Vec under the hood. Since crates are looked up by both prefix and suffix, we perform a range search on said Vec (which constraints the search space based on prefix) and follow up with a linear scan of entries with matching suffixes.
FilesIndex is also pre-filtered before any queries are performed using available target information; query prefixes/sufixes are based on the target we are compiling for, so we can remove entries that can never match up front.

Overall, this commit brings down build time for us in dev scenarios by about 6%.
100ms might not seem like much, but this is a constant cost that each of our workspace crates has to pay, even when said crate is miniscule.
This commit is contained in:
Piotr Osiewicz 2024-11-11 18:34:46 +01:00
parent 251dc8ad84
commit 42e71bb8ea
8 changed files with 137 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

3
compiler/rustc_metadata/src/creader.rs
View file

@ -507,7 +507,8 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
locator.is_proc_macro = true;
locator.target = &self.sess.host;
locator.tuple = TargetTuple::from_tuple(config::host_tuple());
locator.filesearch = self.sess.host_filesearch(path_kind);
locator.filesearch = self.sess.host_filesearch();
locator.path_kind = path_kind;
let Some(host_result) = self.load(locator)? else {
return Ok(None);

84
compiler/rustc_metadata/src/locator.rs
View file

@ -253,9 +253,10 @@ pub(crate) struct CrateLocator<'a> {
extra_filename: Option<&'a str>,
pub target: &'a Target,
pub tuple: TargetTuple,
pub filesearch: FileSearch<'a>,
pub filesearch: &'a FileSearch,
pub is_proc_macro: bool,
pub path_kind: PathKind,
// Mutable in-progress state or output.
crate_rejections: CrateRejections,
}
@ -339,7 +340,8 @@ impl<'a> CrateLocator<'a> {
extra_filename,
target: &sess.target,
tuple: sess.opts.target_triple.clone(),
filesearch: sess.target_filesearch(path_kind),
filesearch: sess.target_filesearch(),
path_kind,
is_proc_macro: false,
crate_rejections: CrateRejections::default(),
}
@ -407,47 +409,49 @@ impl<'a> CrateLocator<'a> {
// given that `extra_filename` comes from the `-C extra-filename`
// option and thus can be anything, and the incorrect match will be
// handled safely in `extract_one`.
for search_path in self.filesearch.search_paths() {
for search_path in self.filesearch.search_paths(self.path_kind) {
debug!("searching {}", search_path.dir.display());
for spf in search_path.files.iter() {
debug!("testing {}", spf.path.display());
let spf = &search_path.files;
let f = &spf.file_name_str;
let (hash, kind) = if let Some(f) = f.strip_prefix(rlib_prefix)
&& let Some(f) = f.strip_suffix(rlib_suffix)
{
(f, CrateFlavor::Rlib)
} else if let Some(f) = f.strip_prefix(rmeta_prefix)
&& let Some(f) = f.strip_suffix(rmeta_suffix)
{
(f, CrateFlavor::Rmeta)
} else if let Some(f) = f.strip_prefix(dylib_prefix)
&& let Some(f) = f.strip_suffix(dylib_suffix.as_ref())
{
(f, CrateFlavor::Dylib)
} else {
if f.starts_with(staticlib_prefix) && f.ends_with(staticlib_suffix.as_ref()) {
self.crate_rejections.via_kind.push(CrateMismatch {
path: spf.path.clone(),
got: "static".to_string(),
});
let mut should_check_staticlibs = true;
for (prefix, suffix, kind) in [
(rlib_prefix.as_str(), rlib_suffix, CrateFlavor::Rlib),
(rmeta_prefix.as_str(), rmeta_suffix, CrateFlavor::Rmeta),
(dylib_prefix, dylib_suffix, CrateFlavor::Dylib),
] {
if prefix == staticlib_prefix && suffix == staticlib_suffix {
should_check_staticlibs = false;
}
if let Some(matches) = spf.query(prefix, suffix) {
for (hash, spf) in matches {
info!("lib candidate: {}", spf.path.display());
let (rlibs, rmetas, dylibs) =
candidates.entry(hash.to_string()).or_default();
let path =
try_canonicalize(&spf.path).unwrap_or_else(|_| spf.path.to_path_buf());
if seen_paths.contains(&path) {
continue;
};
seen_paths.insert(path.clone());
match kind {
CrateFlavor::Rlib => rlibs.insert(path, search_path.kind),
CrateFlavor::Rmeta => rmetas.insert(path, search_path.kind),
CrateFlavor::Dylib => dylibs.insert(path, search_path.kind),
};
}
continue;
};
info!("lib candidate: {}", spf.path.display());
let (rlibs, rmetas, dylibs) = candidates.entry(hash.to_string()).or_default();
let path = try_canonicalize(&spf.path).unwrap_or_else(|_| spf.path.clone());
if seen_paths.contains(&path) {
continue;
};
seen_paths.insert(path.clone());
match kind {
CrateFlavor::Rlib => rlibs.insert(path, search_path.kind),
CrateFlavor::Rmeta => rmetas.insert(path, search_path.kind),
CrateFlavor::Dylib => dylibs.insert(path, search_path.kind),
};
}
}
if let Some(static_matches) = should_check_staticlibs
.then(|| spf.query(staticlib_prefix, staticlib_suffix))
.flatten()
{
for (_, spf) in static_matches {
self.crate_rejections.via_kind.push(CrateMismatch {
path: spf.path.to_path_buf(),
got: "static".to_string(),
});
}
}
}

4
compiler/rustc_metadata/src/native_libs.rs
View file

@ -28,10 +28,10 @@ pub fn walk_native_lib_search_dirs<R>(
mut f: impl FnMut(&Path, bool /*is_framework*/) -> ControlFlow<R>,
) -> ControlFlow<R> {
// Library search paths explicitly supplied by user (`-L` on the command line).
for search_path in sess.target_filesearch(PathKind::Native).cli_search_paths() {
for search_path in sess.target_filesearch().cli_search_paths(PathKind::Native) {
f(&search_path.dir, false)?;
}
for search_path in sess.target_filesearch(PathKind::Framework).cli_search_paths() {
for search_path in sess.target_filesearch().cli_search_paths(PathKind::Framework) {
// Frameworks are looked up strictly in framework-specific paths.
if search_path.kind != PathKind::All {
f(&search_path.dir, true)?;