bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Auto merge of #100101 - BelovDV:issue-99429, r=petrochenkov

Browse source 
change rlib format to distinguish native dependencies

Another one method to solve problem mentioned in #99429.

Changed .rlib format, it contains all bundled native libraries as archieves.
At link time rlib is unpacked and native dependencies linked separately.
New behavior hidden under separate_native_rlib_dependencies flag.
This commit is contained in:
bors 2022-09-13 04:00:24 +00:00
commit 9da4644d56
23 changed files with 328 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

72
compiler/rustc_codegen_ssa/src/back/archive.rs
View file

@ -1,44 +1,16 @@
use rustc_data_structures::fx::FxHashSet;
use rustc_data_structures::memmap::Mmap;
use rustc_session::cstore::DllImport;
use rustc_session::Session;
use rustc_span::symbol::Symbol;
use object::read::archive::ArchiveFile;
use std::fmt::Display;
use std::fs::File;
use std::io;
use std::path::{Path, PathBuf};
pub(super) fn find_library(
name: &str,
verbatim: bool,
search_paths: &[PathBuf],
sess: &Session,
) -> PathBuf {
// On Windows, static libraries sometimes show up as libfoo.a and other
// times show up as foo.lib
let oslibname = if verbatim {
name.to_string()
} else {
format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix)
};
let unixlibname = format!("lib{}.a", name);
for path in search_paths {
debug!("looking for {} inside {:?}", name, path);
let test = path.join(&oslibname);
if test.exists() {
return test;
}
if oslibname != unixlibname {
let test = path.join(&unixlibname);
if test.exists() {
return test;
}
}
}
sess.fatal(&format!(
"could not find native static library `{}`, \
perhaps an -L flag is missing?",
name
));
}
pub trait ArchiveBuilderBuilder {
fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a>;
@ -54,6 +26,36 @@ pub trait ArchiveBuilderBuilder {
dll_imports: &[DllImport],
tmpdir: &Path,
) -> PathBuf;
fn extract_bundled_libs(
&self,
rlib: &Path,
outdir: &Path,
bundled_lib_file_names: &FxHashSet<Symbol>,
) -> Result<(), String> {
let message = |msg: &str, e: &dyn Display| format!("{} '{}': {}", msg, &rlib.display(), e);
let archive_map = unsafe {
Mmap::map(File::open(rlib).map_err(|e| message("failed to open file", &e))?)
.map_err(|e| message("failed to mmap file", &e))?
};
let archive = ArchiveFile::parse(&*archive_map)
.map_err(|e| message("failed to parse archive", &e))?;
for entry in archive.members() {
let entry = entry.map_err(|e| message("failed to read entry", &e))?;
let data = entry
.data(&*archive_map)
.map_err(|e| message("failed to get data from archive member", &e))?;
let name = std::str::from_utf8(entry.name())
.map_err(|e| message("failed to convert name", &e))?;
if !bundled_lib_file_names.contains(&Symbol::intern(name)) {
continue; // We need to extract only native libraries.
}
std::fs::write(&outdir.join(&name), data)
.map_err(|e| message("failed to write file", &e))?;
}
Ok(())
}
}
pub trait ArchiveBuilder<'a> {

98
compiler/rustc_codegen_ssa/src/back/link.rs
View file

@ -1,11 +1,13 @@
use rustc_arena::TypedArena;
use rustc_ast::CRATE_NODE_ID;
use rustc_data_structures::fx::FxHashSet;
use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::memmap::Mmap;
use rustc_data_structures::temp_dir::MaybeTempDir;
use rustc_errors::{ErrorGuaranteed, Handler};
use rustc_fs_util::fix_windows_verbatim_for_gcc;
use rustc_hir::def_id::CrateNum;
use rustc_metadata::find_native_static_library;
use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
use rustc_middle::middle::dependency_format::Linkage;
use rustc_middle::middle::exported_symbols::SymbolExportKind;
@ -24,7 +26,7 @@ use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo};
use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target};
use super::archive::{find_library, ArchiveBuilder, ArchiveBuilderBuilder};
use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
use super::command::Command;
use super::linker::{self, Linker};
use super::metadata::{create_rmeta_file, MetadataPosition};
@ -307,6 +309,9 @@ fn link_rlib<'a>(
}
}
// Used if packed_bundled_libs flag enabled.
let mut packed_bundled_libs = Vec::new();
// Note that in this loop we are ignoring the value of `lib.cfg`. That is,
// we may not be configured to actually include a static library if we're
// adding it here. That's because later when we consume this rlib we'll
@ -325,6 +330,8 @@ fn link_rlib<'a>(
// metadata of the rlib we're generating somehow.
for lib in codegen_results.crate_info.used_libraries.iter() {
match lib.kind {
NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
if flavor == RlibFlavor::Normal =>
{
@ -348,7 +355,16 @@ fn link_rlib<'a>(
}
if let Some(name) = lib.name {
let location =
find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess);
find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
packed_bundled_libs.push(find_native_static_library(
lib.filename.unwrap().as_str(),
Some(true),
&lib_search_paths,
sess,
));
continue;
}
ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|e| {
sess.fatal(&format!(
"failed to add native library {}: {}",
@ -403,6 +419,12 @@ fn link_rlib<'a>(
ab.add_file(&trailing_metadata);
}
// Add all bundled static native library dependencies.
// Archives added to the end of .rlib archive, see comment above for the reason.
for lib in packed_bundled_libs {
ab.add_file(&lib)
}
return Ok(ab);
}
@ -2341,7 +2363,15 @@ fn add_upstream_rust_crates<'a>(
let src = &codegen_results.crate_info.used_crate_source[&cnum];
match data[cnum.as_usize() - 1] {
_ if codegen_results.crate_info.profiler_runtime == Some(cnum) => {
add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
add_static_crate(
cmd,
sess,
archive_builder_builder,
codegen_results,
tmpdir,
cnum,
&Default::default(),
);
}
// compiler-builtins are always placed last to ensure that they're
// linked correctly.
@ -2351,7 +2381,23 @@ fn add_upstream_rust_crates<'a>(
}
Linkage::NotLinked | Linkage::IncludedFromDylib => {}
Linkage::Static => {
add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
let bundled_libs = if sess.opts.unstable_opts.packed_bundled_libs {
codegen_results.crate_info.native_libraries[&cnum]
.iter()
.filter_map(|lib| lib.filename)
.collect::<FxHashSet<_>>()
} else {
Default::default()
};
add_static_crate(
cmd,
sess,
archive_builder_builder,
codegen_results,
tmpdir,
cnum,
&bundled_libs,
);
// Link static native libs with "-bundle" modifier only if the crate they originate from
// is being linked statically to the current crate. If it's linked dynamically
@ -2362,6 +2408,14 @@ fn add_upstream_rust_crates<'a>(
// external build system already has the native dependencies defined, and it
// will provide them to the linker itself.
if sess.opts.unstable_opts.link_native_libraries {
if sess.opts.unstable_opts.packed_bundled_libs {
// If rlib contains native libs as archives, unpack them to tmpdir.
let rlib = &src.rlib.as_ref().unwrap().0;
archive_builder_builder
.extract_bundled_libs(rlib, tmpdir, &bundled_libs)
.unwrap_or_else(|e| sess.fatal(e));
}
let mut last = (None, NativeLibKind::Unspecified, None);
for lib in &codegen_results.crate_info.native_libraries[&cnum] {
let Some(name) = lib.name else {
@ -2411,10 +2465,17 @@ fn add_upstream_rust_crates<'a>(
| NativeLibKind::Framework { .. }
| NativeLibKind::Unspecified
| NativeLibKind::RawDylib => {}
NativeLibKind::Static {
bundle: Some(true) | None,
whole_archive: _,
} => {}
NativeLibKind::Static { bundle: Some(true) | None, whole_archive } => {
if sess.opts.unstable_opts.packed_bundled_libs {
// If rlib contains native libs as archives, they are unpacked to tmpdir.
let path = tmpdir.join(lib.filename.unwrap().as_str());
if whole_archive == Some(true) {
cmd.link_whole_rlib(&path);
} else {
cmd.link_rlib(&path);
}
}
}
}
}
}
@ -2429,7 +2490,15 @@ fn add_upstream_rust_crates<'a>(
// was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic`
// is used)
if let Some(cnum) = compiler_builtins {
add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
add_static_crate(
cmd,
sess,
archive_builder_builder,
codegen_results,
tmpdir,
cnum,
&Default::default(),
);
}
// Converts a library file-stem into a cc -l argument
@ -2462,6 +2531,7 @@ fn add_upstream_rust_crates<'a>(
codegen_results: &CodegenResults,
tmpdir: &Path,
cnum: CrateNum,
bundled_lib_file_names: &FxHashSet<Symbol>,
) {
let src = &codegen_results.crate_info.used_crate_source[&cnum];
let cratepath = &src.rlib.as_ref().unwrap().0;
@ -2490,6 +2560,7 @@ fn add_upstream_rust_crates<'a>(
let dst = tmpdir.join(cratepath.file_name().unwrap());
let name = cratepath.file_name().unwrap().to_str().unwrap();
let name = &name[3..name.len() - 5]; // chop off lib/.rlib
let bundled_lib_file_names = bundled_lib_file_names.clone();
sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
let canonical_name = name.replace('-', "_");
@ -2523,6 +2594,15 @@ fn add_upstream_rust_crates<'a>(
let skip_because_lto =
upstream_rust_objects_already_included && is_rust_object && is_builtins;
// We skip native libraries because:
// 1. This native libraries won't be used from the generated rlib,
// so we can throw them away to avoid the copying work.
// 2. We can't allow it to be a single remaining entry in archive
// as some linkers may complain on that.
if bundled_lib_file_names.contains(&Symbol::intern(f)) {
return true;
}
if skip_because_cfg_say_so || skip_because_lto {
return true;
}

4
compiler/rustc_codegen_ssa/src/back/linker.rs
View file

@ -1,4 +1,3 @@
use super::archive;
use super::command::Command;
use super::symbol_export;
use rustc_span::symbol::sym;
@ -11,6 +10,7 @@ use std::path::{Path, PathBuf};
use std::{env, mem, str};
use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
use rustc_metadata::find_native_static_library;
use rustc_middle::middle::dependency_format::Linkage;
use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind};
use rustc_middle::ty::TyCtxt;
@ -514,7 +514,7 @@ impl<'a> Linker for GccLinker<'a> {
// -force_load is the macOS equivalent of --whole-archive, but it
// involves passing the full path to the library to link.
self.linker_arg("-force_load");
let lib = archive::find_library(lib, verbatim, search_path, &self.sess);
let lib = find_native_static_library(lib, Some(verbatim), search_path, &self.sess);
self.linker_arg(&lib);
}
}

2
compiler/rustc_codegen_ssa/src/lib.rs
View file

@ -112,6 +112,7 @@ bitflags::bitflags! {
pub struct NativeLib {
pub kind: NativeLibKind,
pub name: Option<Symbol>,
pub filename: Option<Symbol>,
pub cfg: Option<ast::MetaItem>,
pub verbatim: Option<bool>,
pub dll_imports: Vec<cstore::DllImport>,
@ -121,6 +122,7 @@ impl From<&cstore::NativeLib> for NativeLib {
fn from(lib: &cstore::NativeLib) -> Self {
NativeLib {
kind: lib.kind,
filename: lib.filename,
name: lib.name,
cfg: lib.cfg.clone(),
verbatim: lib.verbatim,