Auto merge of #96544 - m-ysk:feature/issue-96358, r=cjgillot
Stop keeping metadata in memory before writing it to disk Fixes #96358 I created this PR according with the instruction given in the issue except for the following points: - While the issue says "Write metadata into the temporary file in `encode_and_write_metadata` even if `!need_metadata_file`", I could not do that. That is because though I tried to do that and run `x.py test`, I got a lot of test failures as follows. <details> <summary>List of failed tests</summary> <pre> <code> failures: [ui] src/test/ui/json-multiple.rs [ui] src/test/ui/json-options.rs [ui] src/test/ui/rmeta/rmeta-rpass.rs [ui] src/test/ui/save-analysis/emit-notifications.rs [ui] src/test/ui/svh/changing-crates.rs [ui] src/test/ui/svh/svh-change-lit.rs [ui] src/test/ui/svh/svh-change-significant-cfg.rs [ui] src/test/ui/svh/svh-change-trait-bound.rs [ui] src/test/ui/svh/svh-change-type-arg.rs [ui] src/test/ui/svh/svh-change-type-ret.rs [ui] src/test/ui/svh/svh-change-type-static.rs [ui] src/test/ui/svh/svh-use-trait.rs test result: FAILED. 12915 passed; 12 failed; 100 ignored; 0 measured; 0 filtered out; finished in 71.41s Some tests failed in compiletest suite=ui mode=ui host=x86_64-unknown-linux-gnu target=x86_64-unknown-linux-gnu Build completed unsuccessfully in 0:01:58 </code> </pre> </details> - I could not resolve the extra tasks about `create_rmeta_file` and `create_compressed_metadata_file` for my lack of ability.
This commit is contained in:
bors
commit
1ba1fec234
16 changed files with 300 additions and 147 deletions
|
|
@ -5,18 +5,15 @@ use crate::util;
|
|||
use ast::CRATE_NODE_ID;
|
||||
use rustc_ast::{self as ast, visit};
|
||||
use rustc_borrowck as mir_borrowck;
|
||||
use rustc_codegen_ssa::back::link::emit_metadata;
|
||||
use rustc_codegen_ssa::traits::CodegenBackend;
|
||||
use rustc_data_structures::parallel;
|
||||
use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
|
||||
use rustc_data_structures::temp_dir::MaybeTempDir;
|
||||
use rustc_errors::{Applicability, ErrorGuaranteed, MultiSpan, PResult};
|
||||
use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand};
|
||||
use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE};
|
||||
use rustc_hir::def_id::StableCrateId;
|
||||
use rustc_hir::definitions::Definitions;
|
||||
use rustc_lint::{BufferedEarlyLint, EarlyCheckNode, LintStore};
|
||||
use rustc_metadata::creader::CStore;
|
||||
use rustc_metadata::{encode_metadata, EncodedMetadata};
|
||||
use rustc_middle::arena::Arena;
|
||||
use rustc_middle::dep_graph::DepGraph;
|
||||
use rustc_middle::ty::query::{ExternProviders, Providers};
|
||||
|
|
@ -29,14 +26,13 @@ use rustc_query_impl::{OnDiskCache, Queries as TcxQueries};
|
|||
use rustc_resolve::{Resolver, ResolverArenas};
|
||||
use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType};
|
||||
use rustc_session::cstore::{CrateStoreDyn, MetadataLoader, MetadataLoaderDyn};
|
||||
use rustc_session::output::{filename_for_input, filename_for_metadata};
|
||||
use rustc_session::output::filename_for_input;
|
||||
use rustc_session::search_paths::PathKind;
|
||||
use rustc_session::{Limit, Session};
|
||||
use rustc_span::symbol::{sym, Symbol};
|
||||
use rustc_span::FileName;
|
||||
use rustc_trait_selection::traits;
|
||||
use rustc_typeck as typeck;
|
||||
use tempfile::Builder as TempFileBuilder;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use std::any::Any;
|
||||
|
|
@ -993,69 +989,6 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn encode_and_write_metadata(
|
||||
tcx: TyCtxt<'_>,
|
||||
outputs: &OutputFilenames,
|
||||
) -> (EncodedMetadata, bool) {
|
||||
#[derive(PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum MetadataKind {
|
||||
None,
|
||||
Uncompressed,
|
||||
Compressed,
|
||||
}
|
||||
|
||||
let metadata_kind = tcx
|
||||
.sess
|
||||
.crate_types()
|
||||
.iter()
|
||||
.map(|ty| match *ty {
|
||||
CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None,
|
||||
|
||||
CrateType::Rlib => MetadataKind::Uncompressed,
|
||||
|
||||
CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed,
|
||||
})
|
||||
.max()
|
||||
.unwrap_or(MetadataKind::None);
|
||||
|
||||
let metadata = match metadata_kind {
|
||||
MetadataKind::None => EncodedMetadata::new(),
|
||||
MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx),
|
||||
};
|
||||
|
||||
let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata");
|
||||
|
||||
let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata);
|
||||
if need_metadata_file {
|
||||
let crate_name = tcx.crate_name(LOCAL_CRATE);
|
||||
let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs);
|
||||
// To avoid races with another rustc process scanning the output directory,
|
||||
// we need to write the file somewhere else and atomically move it to its
|
||||
// final destination, with an `fs::rename` call. In order for the rename to
|
||||
// always succeed, the temporary file needs to be on the same filesystem,
|
||||
// which is why we create it inside the output directory specifically.
|
||||
let metadata_tmpdir = TempFileBuilder::new()
|
||||
.prefix("rmeta")
|
||||
.tempdir_in(out_filename.parent().unwrap())
|
||||
.unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
|
||||
let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
|
||||
let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir);
|
||||
if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) {
|
||||
tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
|
||||
}
|
||||
if tcx.sess.opts.json_artifact_notifications {
|
||||
tcx.sess
|
||||
.parse_sess
|
||||
.span_diagnostic
|
||||
.emit_artifact_notification(&out_filename, "metadata");
|
||||
}
|
||||
}
|
||||
|
||||
let need_metadata_module = metadata_kind == MetadataKind::Compressed;
|
||||
|
||||
(metadata, need_metadata_module)
|
||||
}
|
||||
|
||||
/// Runs the codegen backend, after which the AST and analysis can
|
||||
/// be discarded.
|
||||
pub fn start_codegen<'tcx>(
|
||||
|
|
@ -1065,7 +998,8 @@ pub fn start_codegen<'tcx>(
|
|||
) -> Box<dyn Any> {
|
||||
info!("Pre-codegen\n{:?}", tcx.debug_stats());
|
||||
|
||||
let (metadata, need_metadata_module) = encode_and_write_metadata(tcx, outputs);
|
||||
let (metadata, need_metadata_module) =
|
||||
rustc_metadata::fs::encode_and_write_metadata(tcx, outputs);
|
||||
|
||||
let codegen = tcx.sess.time("codegen_crate", move || {
|
||||
codegen_backend.codegen_crate(tcx, metadata, need_metadata_module)
|
||||
|
|
|
|||
|
|
@ -650,24 +650,6 @@ pub fn build_output_filenames(
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
|
||||
std::fs::rename(src, dst)
|
||||
}
|
||||
|
||||
/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems
|
||||
/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully"
|
||||
/// write back the source file before committing the rename in case a developer forgot some of
|
||||
/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates.
|
||||
///
|
||||
/// To avoid triggering this heuristic we delete the destination first, if it exists.
|
||||
/// The cost of an extra syscall is much lower than getting descheduled for the sync IO.
|
||||
#[cfg(target_os = "linux")]
|
||||
pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> {
|
||||
let _ = std::fs::remove_file(dst);
|
||||
std::fs::rename(src, dst)
|
||||
}
|
||||
|
||||
/// Returns a version string such as "1.46.0 (04488afe3 2020-08-24)"
|
||||
pub fn version_str() -> Option<&'static str> {
|
||||
option_env!("CFG_VERSION")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue