bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Decode SourceFile out of order.

Browse source
This commit is contained in:
Camille GILLOT 2022-08-06 23:00:49 +02:00
parent f20ceb1c6f
commit a09e9c99a4
4 changed files with 148 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

179
compiler/rustc_metadata/src/rmeta/decoder.rs
View file

@ -37,6 +37,7 @@ use rustc_span::symbol::{sym, Ident, Symbol};
use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP}; use rustc_span::{self, BytePos, ExpnId, Pos, Span, SyntaxContext, DUMMY_SP};
use proc_macro::bridge::client::ProcMacro; use proc_macro::bridge::client::ProcMacro;
use std::cell::RefCell;
use std::io; use std::io;
use std::iter::TrustedLen; use std::iter::TrustedLen;
use std::mem; use std::mem;
@ -99,7 +100,7 @@ pub(crate) struct CrateMetadata {
/// Proc macro descriptions for this crate, if it's a proc macro crate. /// Proc macro descriptions for this crate, if it's a proc macro crate.
raw_proc_macros: Option<&'static [ProcMacro]>, raw_proc_macros: Option<&'static [ProcMacro]>,
/// Source maps for code from the crate. /// Source maps for code from the crate.
source_map_import_info: OnceCell<Vec<ImportedSourceFile>>, source_map_import_info: RefCell<Vec<Option<ImportedSourceFile>>>,
/// For every definition in this crate, maps its `DefPathHash` to its `DefIndex`. /// For every definition in this crate, maps its `DefPathHash` to its `DefIndex`.
def_path_hash_map: DefPathHashMapRef<'static>, def_path_hash_map: DefPathHashMapRef<'static>,
/// Likewise for ExpnHash. /// Likewise for ExpnHash.
@ -143,7 +144,8 @@ pub(crate) struct CrateMetadata {
} }
/// Holds information about a rustc_span::SourceFile imported from another crate. /// Holds information about a rustc_span::SourceFile imported from another crate.
/// See `imported_source_files()` for more information. /// See `imported_source_file()` for more information.
#[derive(Clone)]
struct ImportedSourceFile { struct ImportedSourceFile {
/// This SourceFile's byte-offset within the source_map of its original crate /// This SourceFile's byte-offset within the source_map of its original crate
original_start_pos: rustc_span::BytePos, original_start_pos: rustc_span::BytePos,
@ -528,7 +530,7 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
}; };
// Index of the file in the corresponding crate's list of encoded files. // Index of the file in the corresponding crate's list of encoded files.
let metadata_index = usize::decode(decoder); let metadata_index = u32::decode(decoder);
// There are two possibilities here: // There are two possibilities here:
// 1. This is a 'local span', which is located inside a `SourceFile` // 1. This is a 'local span', which is located inside a `SourceFile`
@ -556,10 +558,10 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
// to be based on the *foreign* crate (e.g. crate C), not the crate // to be based on the *foreign* crate (e.g. crate C), not the crate
// we are writing metadata for (e.g. crate B). This allows us to // we are writing metadata for (e.g. crate B). This allows us to
// treat the 'local' and 'foreign' cases almost identically during deserialization: // treat the 'local' and 'foreign' cases almost identically during deserialization:
// we can call `imported_source_files` for the proper crate, and binary search // we can call `imported_source_file` for the proper crate, and binary search
// through the returned slice using our span. // through the returned slice using our span.
let imported_source_files = if tag == TAG_VALID_SPAN_LOCAL { let source_file = if tag == TAG_VALID_SPAN_LOCAL {
decoder.cdata().imported_source_files(sess) decoder.cdata().imported_source_file(metadata_index, sess)
} else { } else {
// When we encode a proc-macro crate, all `Span`s should be encoded // When we encode a proc-macro crate, all `Span`s should be encoded
// with `TAG_VALID_SPAN_LOCAL` // with `TAG_VALID_SPAN_LOCAL`
@ -587,13 +589,9 @@ impl<'a, 'tcx> Decodable<DecodeContext<'a, 'tcx>> for Span {
decoder.last_source_file_index = 0; decoder.last_source_file_index = 0;
let foreign_data = decoder.cdata().cstore.get_crate_data(cnum); let foreign_data = decoder.cdata().cstore.get_crate_data(cnum);
foreign_data.imported_source_files(sess) foreign_data.imported_source_file(metadata_index, sess)
}; };
// Optimize for the case that most spans within a translated item
// originate from the same source_file.
let source_file = &imported_source_files[metadata_index];
// Make sure our binary search above is correct. // Make sure our binary search above is correct.
debug_assert!( debug_assert!(
lo >= source_file.original_start_pos && lo <= source_file.original_end_pos, lo >= source_file.original_start_pos && lo <= source_file.original_end_pos,
@ -1438,7 +1436,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
/// ///
/// Proc macro crates don't currently export spans, so this function does not have /// Proc macro crates don't currently export spans, so this function does not have
/// to work for them. /// to work for them.
fn imported_source_files(self, sess: &Session) -> &'a [ImportedSourceFile] { fn imported_source_file(self, source_file_index: u32, sess: &Session) -> ImportedSourceFile {
fn filter<'a>(sess: &Session, path: Option<&'a Path>) -> Option<&'a Path> { fn filter<'a>(sess: &Session, path: Option<&'a Path>) -> Option<&'a Path> {
path.filter(|_| { path.filter(|_| {
// Only spend time on further checks if we have what to translate *to*. // Only spend time on further checks if we have what to translate *to*.
@ -1526,94 +1524,97 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
} }
}; };
self.cdata.source_map_import_info.get_or_init(|| { let mut import_info = self.cdata.source_map_import_info.borrow_mut();
let external_source_map = self.root.source_map.decode(self); for _ in import_info.len()..=(source_file_index as usize) {
import_info.push(None);
}
import_info[source_file_index as usize]
.get_or_insert_with(|| {
let source_file_to_import = self
.root
.source_map
.get(self, source_file_index)
.expect("missing source file")
.decode(self);
external_source_map // We can't reuse an existing SourceFile, so allocate a new one
.enumerate() // containing the information we need.
.map(|(source_file_index, source_file_to_import)| { let rustc_span::SourceFile {
// We can't reuse an existing SourceFile, so allocate a new one mut name,
// containing the information we need. src_hash,
let rustc_span::SourceFile { start_pos,
mut name, end_pos,
src_hash, lines,
start_pos, multibyte_chars,
end_pos, non_narrow_chars,
lines, normalized_pos,
multibyte_chars, name_hash,
non_narrow_chars, ..
normalized_pos, } = source_file_to_import;
name_hash,
..
} = source_file_to_import;
// If this file is under $sysroot/lib/rustlib/src/ but has not been remapped // If this file is under $sysroot/lib/rustlib/src/ but has not been remapped
// during rust bootstrapping by `remap-debuginfo = true`, and the user // during rust bootstrapping by `remap-debuginfo = true`, and the user
// wish to simulate that behaviour by -Z simulate-remapped-rust-src-base, // wish to simulate that behaviour by -Z simulate-remapped-rust-src-base,
// then we change `name` to a similar state as if the rust was bootstrapped // then we change `name` to a similar state as if the rust was bootstrapped
// with `remap-debuginfo = true`. // with `remap-debuginfo = true`.
// This is useful for testing so that tests about the effects of // This is useful for testing so that tests about the effects of
// `try_to_translate_virtual_to_real` don't have to worry about how the // `try_to_translate_virtual_to_real` don't have to worry about how the
// compiler is bootstrapped. // compiler is bootstrapped.
if let Some(virtual_dir) = if let Some(virtual_dir) = &sess.opts.unstable_opts.simulate_remapped_rust_src_base
&sess.opts.unstable_opts.simulate_remapped_rust_src_base {
{ if let Some(real_dir) = &sess.opts.real_rust_source_base_dir {
if let Some(real_dir) = &sess.opts.real_rust_source_base_dir { if let rustc_span::FileName::Real(ref mut old_name) = name {
if let rustc_span::FileName::Real(ref mut old_name) = name { if let rustc_span::RealFileName::LocalPath(local) = old_name {
if let rustc_span::RealFileName::LocalPath(local) = old_name { if let Ok(rest) = local.strip_prefix(real_dir) {
if let Ok(rest) = local.strip_prefix(real_dir) { *old_name = rustc_span::RealFileName::Remapped {
*old_name = rustc_span::RealFileName::Remapped { local_path: None,
local_path: None, virtual_name: virtual_dir.join(rest),
virtual_name: virtual_dir.join(rest), };
};
}
} }
} }
} }
} }
}
// If this file's path has been remapped to `/rustc/$hash`, // If this file's path has been remapped to `/rustc/$hash`,
// we might be able to reverse that (also see comments above, // we might be able to reverse that (also see comments above,
// on `try_to_translate_virtual_to_real`). // on `try_to_translate_virtual_to_real`).
try_to_translate_virtual_to_real(&mut name); try_to_translate_virtual_to_real(&mut name);
let source_length = (end_pos - start_pos).to_usize(); let source_length = (end_pos - start_pos).to_usize();
let local_version = sess.source_map().new_imported_source_file( let local_version = sess.source_map().new_imported_source_file(
name, name,
src_hash, src_hash,
name_hash, name_hash,
source_length, source_length,
self.cnum, self.cnum,
lines, lines,
multibyte_chars, multibyte_chars,
non_narrow_chars, non_narrow_chars,
normalized_pos, normalized_pos,
start_pos, start_pos,
end_pos, end_pos,
source_file_index source_file_index,
.try_into() );
.expect("cannot import more than U32_MAX files"), debug!(
); "CrateMetaData::imported_source_files alloc \
debug!(
"CrateMetaData::imported_source_files alloc \
source_file {:?} original (start_pos {:?} end_pos {:?}) \ source_file {:?} original (start_pos {:?} end_pos {:?}) \
translated (start_pos {:?} end_pos {:?})", translated (start_pos {:?} end_pos {:?})",
local_version.name, local_version.name,
start_pos, start_pos,
end_pos, end_pos,
local_version.start_pos, local_version.start_pos,
local_version.end_pos local_version.end_pos
); );
ImportedSourceFile { ImportedSourceFile {
original_start_pos: start_pos, original_start_pos: start_pos,
original_end_pos: end_pos, original_end_pos: end_pos,
translated_source_file: local_version, translated_source_file: local_version,
} }
}) })
.collect() .clone()
})
} }
fn get_generator_diagnostic_data( fn get_generator_diagnostic_data(
@ -1676,7 +1677,7 @@ impl CrateMetadata {
trait_impls, trait_impls,
incoherent_impls: Default::default(), incoherent_impls: Default::default(),
raw_proc_macros, raw_proc_macros,
source_map_import_info: OnceCell::new(), source_map_import_info: RefCell::new(Vec::new()),
def_path_hash_map, def_path_hash_map,
expn_hash_map: Default::default(), expn_hash_map: Default::default(),
alloc_decoding_state, alloc_decoding_state,

5
compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
View file

@ -675,6 +675,9 @@ impl CrateStore for CStore {
} }
fn import_source_files(&self, sess: &Session, cnum: CrateNum) { fn import_source_files(&self, sess: &Session, cnum: CrateNum) {
self.get_crate_data(cnum).imported_source_files(sess); let cdata = self.get_crate_data(cnum);
for file_index in 0..cdata.root.source_map.size() {
cdata.imported_source_file(file_index as u32, sess);
}
} }
} }

105
compiler/rustc_metadata/src/rmeta/encoder.rs
View file

@ -272,7 +272,7 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for Span {
// Introduce a new scope so that we drop the 'lock()' temporary // Introduce a new scope so that we drop the 'lock()' temporary
match &*s.source_file_cache.0.external_src.lock() { match &*s.source_file_cache.0.external_src.lock() {
ExternalSource::Foreign { original_start_pos, metadata_index, .. } => { ExternalSource::Foreign { original_start_pos, metadata_index, .. } => {
(*original_start_pos, *metadata_index as usize) (*original_start_pos, *metadata_index)
} }
src => panic!("Unexpected external source {:?}", src), src => panic!("Unexpected external source {:?}", src),
} }
@ -286,6 +286,8 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for Span {
let source_files = let source_files =
s.required_source_files.as_mut().expect("Already encoded SourceMap!"); s.required_source_files.as_mut().expect("Already encoded SourceMap!");
let (source_file_index, _) = source_files.insert_full(s.source_file_cache.1); let (source_file_index, _) = source_files.insert_full(s.source_file_cache.1);
let source_file_index: u32 =
source_file_index.try_into().expect("cannot export more than U32_MAX files");
(TAG_VALID_SPAN_LOCAL, span.lo, span.hi, source_file_index) (TAG_VALID_SPAN_LOCAL, span.lo, span.hi, source_file_index)
}; };
@ -452,7 +454,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
self.lazy(DefPathHashMapRef::BorrowedFromTcx(self.tcx.def_path_hash_to_def_index_map())) self.lazy(DefPathHashMapRef::BorrowedFromTcx(self.tcx.def_path_hash_to_def_index_map()))
} }
fn encode_source_map(&mut self) -> LazyArray<rustc_span::SourceFile> { fn encode_source_map(&mut self) -> LazyTable<u32, LazyValue<rustc_span::SourceFile>> {
let source_map = self.tcx.sess.source_map(); let source_map = self.tcx.sess.source_map();
let all_source_files = source_map.files(); let all_source_files = source_map.files();
@ -463,65 +465,64 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
let working_directory = &self.tcx.sess.opts.working_dir; let working_directory = &self.tcx.sess.opts.working_dir;
let mut adapted = TableBuilder::default();
// Only serialize `SourceFile`s that were used during the encoding of a `Span`. // Only serialize `SourceFile`s that were used during the encoding of a `Span`.
// //
// The order in which we encode source files is important here: the on-disk format for // The order in which we encode source files is important here: the on-disk format for
// `Span` contains the index of the corresponding `SourceFile`. // `Span` contains the index of the corresponding `SourceFile`.
let adapted = required_source_files for (on_disk_index, &source_file_index) in required_source_files.iter().enumerate() {
.iter() let source_file = &all_source_files[source_file_index];
.map(|&source_file_index| &all_source_files[source_file_index]) // Don't serialize imported `SourceFile`s, unless we're in a proc-macro crate.
.map(|source_file| { assert!(!source_file.is_imported() || self.is_proc_macro);
// Don't serialize imported `SourceFile`s, unless we're in a proc-macro crate.
assert!(!source_file.is_imported() || self.is_proc_macro);
// At export time we expand all source file paths to absolute paths because // At export time we expand all source file paths to absolute paths because
// downstream compilation sessions can have a different compiler working // downstream compilation sessions can have a different compiler working
// directory, so relative paths from this or any other upstream crate // directory, so relative paths from this or any other upstream crate
// won't be valid anymore. // won't be valid anymore.
// //
// At this point we also erase the actual on-disk path and only keep // At this point we also erase the actual on-disk path and only keep
// the remapped version -- as is necessary for reproducible builds. // the remapped version -- as is necessary for reproducible builds.
match source_file.name { let mut source_file = match source_file.name {
FileName::Real(ref original_file_name) => { FileName::Real(ref original_file_name) => {
let adapted_file_name = let adapted_file_name = source_map
source_map.path_mapping().to_embeddable_absolute_path( .path_mapping()
original_file_name.clone(), .to_embeddable_absolute_path(original_file_name.clone(), working_directory);
working_directory,
);
if adapted_file_name != *original_file_name { if adapted_file_name != *original_file_name {
let mut adapted: SourceFile = (**source_file).clone(); let mut adapted: SourceFile = (**source_file).clone();
adapted.name = FileName::Real(adapted_file_name); adapted.name = FileName::Real(adapted_file_name);
adapted.name_hash = { adapted.name_hash = {
let mut hasher: StableHasher = StableHasher::new(); let mut hasher: StableHasher = StableHasher::new();
adapted.name.hash(&mut hasher); adapted.name.hash(&mut hasher);
hasher.finish::<u128>() hasher.finish::<u128>()
}; };
Lrc::new(adapted) Lrc::new(adapted)
} else { } else {
// Nothing to adapt // Nothing to adapt
source_file.clone() source_file.clone()
}
} }
// expanded code, not from a file
_ => source_file.clone(),
} }
}) // expanded code, not from a file
.map(|mut source_file| { _ => source_file.clone(),
// We're serializing this `SourceFile` into our crate metadata, };
// so mark it as coming from this crate.
// This also ensures that we don't try to deserialize the
// `CrateNum` for a proc-macro dependency - since proc macro
// dependencies aren't loaded when we deserialize a proc-macro,
// trying to remap the `CrateNum` would fail.
if self.is_proc_macro {
Lrc::make_mut(&mut source_file).cnum = LOCAL_CRATE;
}
source_file
})
.collect::<Vec<_>>();
self.lazy_array(adapted.iter().map(|rc| &**rc)) // We're serializing this `SourceFile` into our crate metadata,
// so mark it as coming from this crate.
// This also ensures that we don't try to deserialize the
// `CrateNum` for a proc-macro dependency - since proc macro
// dependencies aren't loaded when we deserialize a proc-macro,
// trying to remap the `CrateNum` would fail.
if self.is_proc_macro {
Lrc::make_mut(&mut source_file).cnum = LOCAL_CRATE;
}
let on_disk_index: u32 =
on_disk_index.try_into().expect("cannot export more than U32_MAX files");
adapted.set(on_disk_index, self.lazy(source_file));
}
adapted.encode(&mut self.opaque)
} }
fn encode_crate_root(&mut self) -> LazyValue<CrateRoot> { fn encode_crate_root(&mut self) -> LazyValue<CrateRoot> {

2
compiler/rustc_metadata/src/rmeta/mod.rs
View file

@ -249,7 +249,7 @@ pub(crate) struct CrateRoot {
def_path_hash_map: LazyValue<DefPathHashMapRef<'static>>, def_path_hash_map: LazyValue<DefPathHashMapRef<'static>>,
source_map: LazyArray<rustc_span::SourceFile>, source_map: LazyTable<u32, LazyValue<rustc_span::SourceFile>>,
compiler_builtins: bool, compiler_builtins: bool,
needs_allocator: bool, needs_allocator: bool,