bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Auto merge of #121644 - oli-obk:unique_static_innards2, r=RalfJung,nnethercote

Browse source 
Ensure nested allocations in statics neither get deduplicated nor duplicated

This PR generates new `DefId`s for nested allocations in static items and feeds all the right queries to make the compiler believe these are regular `static` items. I chose this design, because all other designs are fragile and make the compiler horribly complex for such a niche use case.

At present this wrecks incremental compilation performance *in case nested allocations exist* (because any query creating a `DefId` will be recomputed and never loaded from the cache). This will be resolved later in https://github.com/rust-lang/rust/pull/115613 . All other statics are unaffected by this change and will not have performance regressions (heh, famous last words)

This PR contains various smaller refactorings that can be pulled out into separate PRs. It is best reviewed commit-by-commit. The last commit is where the actual magic happens.

r? `@RalfJung` on the const interner and engine changes

fixes https://github.com/rust-lang/rust/issues/79738
This commit is contained in:
bors 2024-03-12 10:29:15 +00:00
commit 3b85d2c7fc
66 changed files with 461 additions and 170 deletions
Download patch file Download diff file Expand all files Collapse all files

110
compiler/rustc_codegen_llvm/src/consts.rs
View file

@ -9,6 +9,7 @@ use crate::type_::Type;
use crate::type_of::LayoutLlvmExt;
use crate::value::Value;
use rustc_codegen_ssa::traits::*;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::DefId;
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
use rustc_middle::mir::interpret::{
@ -17,7 +18,7 @@ use rustc_middle::mir::interpret::{
};
use rustc_middle::mir::mono::MonoItem;
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::{self, Instance, Ty};
use rustc_middle::ty::{self, Instance};
use rustc_middle::{bug, span_bug};
use rustc_session::config::Lto;
use rustc_target::abi::{
@ -114,7 +115,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
cx.const_struct(&llvals, true)
}
pub fn codegen_static_initializer<'ll, 'tcx>(
fn codegen_static_initializer<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
def_id: DefId,
) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
@ -147,11 +148,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align:
fn check_and_apply_linkage<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
attrs: &CodegenFnAttrs,
ty: Ty<'tcx>,
llty: &'ll Type,
sym: &str,
def_id: DefId,
) -> &'ll Value {
let llty = cx.layout_of(ty).llvm_type(cx);
if let Some(linkage) = attrs.import_linkage {
debug!("get_static: sym={} linkage={:?}", sym, linkage);
@ -226,9 +226,28 @@ impl<'ll> CodegenCx<'ll, '_> {
}
}
#[instrument(level = "debug", skip(self))]
pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value {
let instance = Instance::mono(self.tcx, def_id);
if let Some(&g) = self.instances.borrow().get(&instance) {
trace!(?instance);
let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
// Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
// the llvm type from the actual evaluated initializer.
let llty = if nested {
self.type_i8()
} else {
let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
trace!(?ty);
self.layout_of(ty).llvm_type(self)
};
self.get_static_inner(def_id, llty)
}
#[instrument(level = "debug", skip(self, llty))]
pub(crate) fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value {
if let Some(&g) = self.instances.borrow().get(&Instance::mono(self.tcx, def_id)) {
trace!("used cached value");
return g;
}
@ -240,14 +259,12 @@ impl<'ll> CodegenCx<'ll, '_> {
statics defined in the same CGU, but did not for `{def_id:?}`"
);
let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
let sym = self.tcx.symbol_name(instance).name;
let sym = self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name;
let fn_attrs = self.tcx.codegen_fn_attrs(def_id);
debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs);
debug!(?sym, ?fn_attrs);
let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) {
let llty = self.layout_of(ty).llvm_type(self);
if let Some(g) = self.get_declared_value(sym) {
if self.val_ty(g) != self.type_ptr() {
span_bug!(self.tcx.def_span(def_id), "Conflicting types for static");
@ -264,7 +281,7 @@ impl<'ll> CodegenCx<'ll, '_> {
g
} else {
check_and_apply_linkage(self, fn_attrs, ty, sym, def_id)
check_and_apply_linkage(self, fn_attrs, llty, sym, def_id)
};
// Thread-local statics in some other crate need to *always* be linked
@ -332,34 +349,18 @@ impl<'ll> CodegenCx<'ll, '_> {
}
}
self.instances.borrow_mut().insert(instance, g);
self.instances.borrow_mut().insert(Instance::mono(self.tcx, def_id), g);
g
}
}
impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
if let Some(&gv) = self.const_globals.borrow().get(&cv) {
unsafe {
// Upgrade the alignment in cases where the same constant is used with different
// alignment requirements
let llalign = align.bytes() as u32;
if llalign > llvm::LLVMGetAlignment(gv) {
llvm::LLVMSetAlignment(gv, llalign);
}
}
return gv;
}
let gv = self.static_addr_of_mut(cv, align, kind);
unsafe {
llvm::LLVMSetGlobalConstant(gv, True);
}
self.const_globals.borrow_mut().insert(cv, gv);
gv
}
fn codegen_static(&self, def_id: DefId, is_mutable: bool) {
fn codegen_static_item(&self, def_id: DefId) {
unsafe {
assert!(
llvm::LLVMGetInitializer(
self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap()
)
.is_none()
);
let attrs = self.tcx.codegen_fn_attrs(def_id);
let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else {
@ -368,13 +369,11 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
};
let alloc = alloc.inner();
let g = self.get_static(def_id);
let val_llty = self.val_ty(v);
let instance = Instance::mono(self.tcx, def_id);
let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
let llty = self.layout_of(ty).llvm_type(self);
let g = self.get_static_inner(def_id, val_llty);
let llty = self.val_ty(g);
let g = if val_llty == llty {
g
} else {
@ -409,16 +408,15 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
self.statics_to_rauw.borrow_mut().push((g, new_g));
new_g
};
set_global_alignment(self, g, self.align_of(ty));
set_global_alignment(self, g, alloc.align);
llvm::LLVMSetInitializer(g, v);
if self.should_assume_dso_local(g, true) {
llvm::LLVMRustSetDSOLocal(g, true);
}
// As an optimization, all shared statics which do not have interior
// mutability are placed into read-only memory.
if !is_mutable && self.type_is_freeze(ty) {
// Forward the allocation's mutability (picked by the const interner) to LLVM.
if alloc.mutability.is_not() {
llvm::LLVMSetGlobalConstant(g, llvm::True);
}
@ -541,6 +539,32 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
}
}
}
}
impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
if let Some(&gv) = self.const_globals.borrow().get(&cv) {
unsafe {
// Upgrade the alignment in cases where the same constant is used with different
// alignment requirements
let llalign = align.bytes() as u32;
if llalign > llvm::LLVMGetAlignment(gv) {
llvm::LLVMSetAlignment(gv, llalign);
}
}
return gv;
}
let gv = self.static_addr_of_mut(cv, align, kind);
unsafe {
llvm::LLVMSetGlobalConstant(gv, True);
}
self.const_globals.borrow_mut().insert(cv, gv);
gv
}
fn codegen_static(&self, def_id: DefId) {
self.codegen_static_item(def_id)
}
/// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
fn add_used_global(&self, global: &'ll Value) {

6
compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
View file

@ -26,6 +26,7 @@ use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind;
use rustc_codegen_ssa::traits::*;
use rustc_fs_util::path_to_c_string;
use rustc_hir::def::CtorKind;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
use rustc_middle::bug;
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
@ -1309,6 +1310,11 @@ pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, glo
};
let is_local_to_unit = is_node_local_to_unit(cx, def_id);
let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else { bug!() };
if nested {
return;
}
let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all());
let type_di_node = type_di_node(cx, variable_type);
let var_name = tcx.item_name(def_id);

11
compiler/rustc_codegen_llvm/src/mono_item.rs
View file

@ -5,7 +5,9 @@ use crate::errors::SymbolAlreadyDefined;
use crate::llvm;
use crate::type_of::LayoutLlvmExt;
use rustc_codegen_ssa::traits::*;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
use rustc_middle::bug;
use rustc_middle::mir::mono::{Linkage, Visibility};
use rustc_middle::ty::layout::{FnAbiOf, LayoutOf};
use rustc_middle::ty::{self, Instance, TypeVisitableExt};
@ -21,7 +23,14 @@ impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> {
symbol_name: &str,
) {
let instance = Instance::mono(self.tcx, def_id);
let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all());
let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
// Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out
// the llvm type from the actual evaluated initializer.
let ty = if nested {
self.tcx.types.unit
} else {
instance.ty(self.tcx, ty::ParamEnv::reveal_all())
};
let llty = self.layout_of(ty).llvm_type(self);
let g = self.define_global(symbol_name, llty).unwrap_or_else(|| {