bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Decode SourceFile out of order.

Browse source
This commit is contained in:
Camille GILLOT 2022-08-06 23:00:49 +02:00
parent f20ceb1c6f
commit a09e9c99a4
4 changed files with 148 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

179
compiler/rustc_metadata/src/rmeta/decoder.rs
View file

@ -37,6 +37,7 @@ use rustc_span::symbol::{sym, Ident, Symbol};
use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP};
use proc_macro::bridge::client::ProcMacro;
use std::cell::RefCell;
use std::io;
use std::iter::TrustedLen;
use std::mem;
@ -99,7 +100,7 @@ pub(crate) struct CrateMetadata {
/// Proc macro descriptions for this crate, if it's a proc macro crate.
raw_proc_macros: Option<&'static [ProcMacro]>,
/// Source maps for code from the crate.
source_map_import_info: OnceCell<Vec<ImportedSourceFile>>,
source_map_import_info: RefCell<Vec<Option<ImportedSourceFile>>>,
/// For every definition in this crate, maps its `DefPathHash` to its `DefIndex`.
def_path_hash_map: DefPathHashMapRef<'static>,
/// Likewise for ExpnHash.
@ -143,7 +144,8 @@ pub(crate) struct CrateMetadata {
}
/// Holds information about a rustc_span::SourceFile imported from another crate.
/// See `imported_source_files()` for more information.
/// See `imported_source_file()` for more information.
#[derive(Clone)]
struct ImportedSourceFile {
/// This SourceFile's byte-offset within the source_map of its original crate
original_start_pos: rustc_span::BytePos,
@ -528,7 +530,7 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
};
// Index of the file in the corresponding crate's list of encoded files.
let metadata_index = usize::decode(decoder);
let metadata_index = u32::decode(decoder);
// There are two possibilities here:
// 1. This is a 'local span', which is located inside a `SourceFile`
@ -556,10 +558,10 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
// to be based on the *foreign* crate (e.g. crate C), not the crate
// we are writing metadata for (e.g. crate B). This allows us to
// treat the 'local' and 'foreign' cases almost identically during deserialization:
// we can call `imported_source_files` for the proper crate, and binary search
// we can call `imported_source_file` for the proper crate, and binary search
// through the returned slice using our span.
let imported_source_files = if tag == TAG_VALID_SPAN_LOCAL {
decoder.cdata().imported_source_files(sess)
let source_file = if tag == TAG_VALID_SPAN_LOCAL {
decoder.cdata().imported_source_file(metadata_index, sess)
} else {
// When we encode a proc-macro crate, all `Span`s should be encoded
// with `TAG_VALID_SPAN_LOCAL`
@ -587,13 +589,9 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
decoder.last_source_file_index = 0;
let foreign_data = decoder.cdata().cstore.get_crate_data(cnum);
foreign_data.imported_source_files(sess)
foreign_data.imported_source_file(metadata_index, sess)
};
// Optimize for the case that most spans within a translated item
// originate from the same source_file.
let source_file = &imported_source_files[metadata_index];
// Make sure our binary search above is correct.
debug_assert!(
lo >= source_file.original_start_pos && lo <= source_file.original_end_pos,
@ -1438,7 +1436,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
///
/// Proc macro crates don't currently export spans, so this function does not have
/// to work for them.
fn imported_source_files(self, sess: &Session) -> &'a [ImportedSourceFile] {
fn imported_source_file(self, source_file_index: u32, sess: &Session) -> ImportedSourceFile {
fn filter<'a>(sess: &Session, path: Option<&'a Path>) -> Option<&'a Path> {
path.filter(|_| {
// Only spend time on further checks if we have what to translate *to*.
@ -1526,94 +1524,97 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
}
};
self.cdata.source_map_import_info.get_or_init(|| {
let external_source_map = self.root.source_map.decode(self);
let mut import_info = self.cdata.source_map_import_info.borrow_mut();
for _ in import_info.len()..=(source_file_index as usize) {
import_info.push(None);
}
import_info[source_file_index as usize]
.get_or_insert_with(|| {
let source_file_to_import = self
.root
.source_map
.get(self, source_file_index)
.expect("missing source file")
.decode(self);
external_source_map
.enumerate()
.map(|(source_file_index, source_file_to_import)| {
// We can't reuse an existing SourceFile, so allocate a new one
// containing the information we need.
let rustc_span::SourceFile {
mut name,
src_hash,
start_pos,
end_pos,
lines,
multibyte_chars,
non_narrow_chars,
normalized_pos,
name_hash,
..
} = source_file_to_import;
// We can't reuse an existing SourceFile, so allocate a new one
// containing the information we need.
let rustc_span::SourceFile {
mut name,
src_hash,
start_pos,
end_pos,
lines,
multibyte_chars,
non_narrow_chars,
normalized_pos,
name_hash,
..
} = source_file_to_import;
// If this file is under $sysroot/lib/rustlib/src/ but has not been remapped
// during rust bootstrapping by `remap-debuginfo = true`, and the user
// wish to simulate that behaviour by -Z simulate-remapped-rust-src-base,
// then we change `name` to a similar state as if the rust was bootstrapped
// with `remap-debuginfo = true`.
// This is useful for testing so that tests about the effects of
// `try_to_translate_virtual_to_real` don't have to worry about how the
// compiler is bootstrapped.
if let Some(virtual_dir) =
&sess.opts.unstable_opts.simulate_remapped_rust_src_base
{
if let Some(real_dir) = &sess.opts.real_rust_source_base_dir {
if let rustc_span::FileName::Real(ref mut old_name) = name {
if let rustc_span::RealFileName::LocalPath(local) = old_name {
if let Ok(rest) = local.strip_prefix(real_dir) {
*old_name = rustc_span::RealFileName::Remapped {
local_path: None,
virtual_name: virtual_dir.join(rest),
};
}
// If this file is under $sysroot/lib/rustlib/src/ but has not been remapped
// during rust bootstrapping by `remap-debuginfo = true`, and the user
// wish to simulate that behaviour by -Z simulate-remapped-rust-src-base,
// then we change `name` to a similar state as if the rust was bootstrapped
// with `remap-debuginfo = true`.
// This is useful for testing so that tests about the effects of
// `try_to_translate_virtual_to_real` don't have to worry about how the
// compiler is bootstrapped.
if let Some(virtual_dir) = &sess.opts.unstable_opts.simulate_remapped_rust_src_base
{
if let Some(real_dir) = &sess.opts.real_rust_source_base_dir {
if let rustc_span::FileName::Real(ref mut old_name) = name {
if let rustc_span::RealFileName::LocalPath(local) = old_name {
if let Ok(rest) = local.strip_prefix(real_dir) {
*old_name = rustc_span::RealFileName::Remapped {
local_path: None,
virtual_name: virtual_dir.join(rest),
};
}
}
}
}
}
// If this file's path has been remapped to `/rustc/$hash`,
// we might be able to reverse that (also see comments above,
// on `try_to_translate_virtual_to_real`).
try_to_translate_virtual_to_real(&mut name);
// If this file's path has been remapped to `/rustc/$hash`,
// we might be able to reverse that (also see comments above,
// on `try_to_translate_virtual_to_real`).
try_to_translate_virtual_to_real(&mut name);
let source_length = (end_pos - start_pos).to_usize();
let source_length = (end_pos - start_pos).to_usize();
let local_version = sess.source_map().new_imported_source_file(
name,
src_hash,
name_hash,
source_length,
self.cnum,
lines,
multibyte_chars,
non_narrow_chars,
normalized_pos,
start_pos,
end_pos,
source_file_index
.try_into()
.expect("cannot import more than U32_MAX files"),
);
debug!(
"CrateMetaData::imported_source_files alloc \
let local_version = sess.source_map().new_imported_source_file(
name,
src_hash,
name_hash,
source_length,
self.cnum,
lines,
multibyte_chars,
non_narrow_chars,
normalized_pos,
start_pos,
end_pos,
source_file_index,
);
debug!(
"CrateMetaData::imported_source_files alloc \
source_file {:?} original (start_pos {:?} end_pos {:?}) \
translated (start_pos {:?} end_pos {:?})",
local_version.name,
start_pos,
end_pos,
local_version.start_pos,
local_version.end_pos
);
local_version.name,
start_pos,
end_pos,
local_version.start_pos,
local_version.end_pos
);
ImportedSourceFile {
original_start_pos: start_pos,
original_end_pos: end_pos,
translated_source_file: local_version,
}
})
.collect()
})
ImportedSourceFile {
original_start_pos: start_pos,
original_end_pos: end_pos,
translated_source_file: local_version,
}
})
.clone()
}
fn get_generator_diagnostic_data(
@ -1676,7 +1677,7 @@ impl CrateMetadata {
trait_impls,
incoherent_impls: Default::default(),
raw_proc_macros,
source_map_import_info: OnceCell::new(),
source_map_import_info: RefCell::new(Vec::new()),
def_path_hash_map,
expn_hash_map: Default::default(),
alloc_decoding_state,

5
compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
View file

@ -675,6 +675,9 @@ impl CrateStore for CStore {
}
fn import_source_files(&self, sess: &Session, cnum: CrateNum) {
self.get_crate_data(cnum).imported_source_files(sess);
let cdata = self.get_crate_data(cnum);
for file_index in 0..cdata.root.source_map.size() {
cdata.imported_source_file(file_index as u32, sess);
}
}
}

105
compiler/rustc_metadata/src/rmeta/encoder.rs
View file

@ -272,7 +272,7 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for Span {
// Introduce a new scope so that we drop the 'lock()' temporary
match &*s.source_file_cache.0.external_src.lock() {
ExternalSource::Foreign { original_start_pos, metadata_index, .. } => {
(*original_start_pos, *metadata_index as usize)
(*original_start_pos, *metadata_index)
}
src => panic!("Unexpected external source {:?}", src),
}
@ -286,6 +286,8 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for Span {
let source_files =
s.required_source_files.as_mut().expect("Already encoded SourceMap!");
let (source_file_index, _) = source_files.insert_full(s.source_file_cache.1);
let source_file_index: u32 =
source_file_index.try_into().expect("cannot export more than U32_MAX files");
(TAG_VALID_SPAN_LOCAL, span.lo, span.hi, source_file_index)
};
@ -452,7 +454,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
self.lazy(DefPathHashMapRef::BorrowedFromTcx(self.tcx.def_path_hash_to_def_index_map()))
}
fn encode_source_map(&mut self) -> LazyArray<rustc_span::SourceFile> {
fn encode_source_map(&mut self) -> LazyTable<u32, LazyValue<rustc_span::SourceFile>> {
let source_map = self.tcx.sess.source_map();
let all_source_files = source_map.files();
@ -463,65 +465,64 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
let working_directory = &self.tcx.sess.opts.working_dir;
let mut adapted = TableBuilder::default();
// Only serialize `SourceFile`s that were used during the encoding of a `Span`.
//
// The order in which we encode source files is important here: the on-disk format for
// `Span` contains the index of the corresponding `SourceFile`.
let adapted = required_source_files
.iter()
.map(|&source_file_index| &all_source_files[source_file_index])
.map(|source_file| {
// Don't serialize imported `SourceFile`s, unless we're in a proc-macro crate.
assert!(!source_file.is_imported() || self.is_proc_macro);
for (on_disk_index, &source_file_index) in required_source_files.iter().enumerate() {
let source_file = &all_source_files[source_file_index];
// Don't serialize imported `SourceFile`s, unless we're in a proc-macro crate.
assert!(!source_file.is_imported() || self.is_proc_macro);
// At export time we expand all source file paths to absolute paths because
// downstream compilation sessions can have a different compiler working
// directory, so relative paths from this or any other upstream crate
// won't be valid anymore.
//
// At this point we also erase the actual on-disk path and only keep
// the remapped version -- as is necessary for reproducible builds.
match source_file.name {
FileName::Real(ref original_file_name) => {
let adapted_file_name =
source_map.path_mapping().to_embeddable_absolute_path(
original_file_name.clone(),
working_directory,
);
// At export time we expand all source file paths to absolute paths because
// downstream compilation sessions can have a different compiler working
// directory, so relative paths from this or any other upstream crate
// won't be valid anymore.
//
// At this point we also erase the actual on-disk path and only keep
// the remapped version -- as is necessary for reproducible builds.
let mut source_file = match source_file.name {
FileName::Real(ref original_file_name) => {
let adapted_file_name = source_map
.path_mapping()
.to_embeddable_absolute_path(original_file_name.clone(), working_directory);
if adapted_file_name != *original_file_name {
let mut adapted: SourceFile = (**source_file).clone();
adapted.name = FileName::Real(adapted_file_name);
adapted.name_hash = {
let mut hasher: StableHasher = StableHasher::new();
adapted.name.hash(&mut hasher);
hasher.finish::<u128>()
};
Lrc::new(adapted)
} else {
// Nothing to adapt
source_file.clone()
}
if adapted_file_name != *original_file_name {
let mut adapted: SourceFile = (**source_file).clone();
adapted.name = FileName::Real(adapted_file_name);
adapted.name_hash = {
let mut hasher: StableHasher = StableHasher::new();
adapted.name.hash(&mut hasher);
hasher.finish::<u128>()
};
Lrc::new(adapted)
} else {
// Nothing to adapt
source_file.clone()
}
// expanded code, not from a file
_ => source_file.clone(),
}
})
.map(|mut source_file| {
// We're serializing this `SourceFile` into our crate metadata,
// so mark it as coming from this crate.
// This also ensures that we don't try to deserialize the
// `CrateNum` for a proc-macro dependency - since proc macro
// dependencies aren't loaded when we deserialize a proc-macro,
// trying to remap the `CrateNum` would fail.
if self.is_proc_macro {
Lrc::make_mut(&mut source_file).cnum = LOCAL_CRATE;
}
source_file
})
.collect::<Vec<_>>();
// expanded code, not from a file
_ => source_file.clone(),
};
self.lazy_array(adapted.iter().map(|rc| &**rc))
// We're serializing this `SourceFile` into our crate metadata,
// so mark it as coming from this crate.
// This also ensures that we don't try to deserialize the
// `CrateNum` for a proc-macro dependency - since proc macro
// dependencies aren't loaded when we deserialize a proc-macro,
// trying to remap the `CrateNum` would fail.
if self.is_proc_macro {
Lrc::make_mut(&mut source_file).cnum = LOCAL_CRATE;
}
let on_disk_index: u32 =
on_disk_index.try_into().expect("cannot export more than U32_MAX files");
adapted.set(on_disk_index, self.lazy(source_file));
}
adapted.encode(&mut self.opaque)
}
fn encode_crate_root(&mut self) -> LazyValue<CrateRoot> {

2
compiler/rustc_metadata/src/rmeta/mod.rs
View file

@ -249,7 +249,7 @@ pub(crate) struct CrateRoot {
def_path_hash_map: LazyValue<DefPathHashMapRef<'static>>,
source_map: LazyArray<rustc_span::SourceFile>,
source_map: LazyTable<u32, LazyValue<rustc_span::SourceFile>>,
compiler_builtins: bool,
needs_allocator: bool,