Extract fn layout_of_enum
This commit is contained in:
Jubilee Young
parent
f116bc6e27
commit
b525f76bb5
1 changed files with 496 additions and 476 deletions
|
|
@ -291,7 +291,135 @@ pub trait LayoutCalculator {
|
||||||
// structs. (We have also handled univariant enums
|
// structs. (We have also handled univariant enums
|
||||||
// that allow representation optimization.)
|
// that allow representation optimization.)
|
||||||
assert!(is_enum);
|
assert!(is_enum);
|
||||||
|
layout_of_enum(
|
||||||
|
self,
|
||||||
|
repr,
|
||||||
|
variants,
|
||||||
|
discr_range_of_repr,
|
||||||
|
discriminants,
|
||||||
|
dont_niche_optimize_enum,
|
||||||
|
dl,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn layout_of_union<
|
||||||
|
'a,
|
||||||
|
FieldIdx: Idx,
|
||||||
|
VariantIdx: Idx,
|
||||||
|
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug,
|
||||||
|
>(
|
||||||
|
&self,
|
||||||
|
repr: &ReprOptions,
|
||||||
|
variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
|
||||||
|
) -> Option<LayoutS<FieldIdx, VariantIdx>> {
|
||||||
|
let dl = self.current_data_layout();
|
||||||
|
let dl = dl.borrow();
|
||||||
|
let mut align = if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };
|
||||||
|
let mut max_repr_align = repr.align;
|
||||||
|
|
||||||
|
// If all the non-ZST fields have the same ABI and union ABI optimizations aren't
|
||||||
|
// disabled, we can use that common ABI for the union as a whole.
|
||||||
|
struct AbiMismatch;
|
||||||
|
let mut common_non_zst_abi_and_align = if repr.inhibit_union_abi_opt() {
|
||||||
|
// Can't optimize
|
||||||
|
Err(AbiMismatch)
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut size = Size::ZERO;
|
||||||
|
let only_variant = &variants[VariantIdx::new(0)];
|
||||||
|
for field in only_variant {
|
||||||
|
if field.is_unsized() {
|
||||||
|
self.delayed_bug("unsized field in union".to_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
align = align.max(field.align);
|
||||||
|
max_repr_align = max_repr_align.max(field.max_repr_align);
|
||||||
|
size = cmp::max(size, field.size);
|
||||||
|
|
||||||
|
if field.is_zst() {
|
||||||
|
// Nothing more to do for ZST fields
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Ok(common) = common_non_zst_abi_and_align {
|
||||||
|
// Discard valid range information and allow undef
|
||||||
|
let field_abi = field.abi.to_union();
|
||||||
|
|
||||||
|
if let Some((common_abi, common_align)) = common {
|
||||||
|
if common_abi != field_abi {
|
||||||
|
// Different fields have different ABI: disable opt
|
||||||
|
common_non_zst_abi_and_align = Err(AbiMismatch);
|
||||||
|
} else {
|
||||||
|
// Fields with the same non-Aggregate ABI should also
|
||||||
|
// have the same alignment
|
||||||
|
if !matches!(common_abi, Abi::Aggregate { .. }) {
|
||||||
|
assert_eq!(
|
||||||
|
common_align, field.align.abi,
|
||||||
|
"non-Aggregate field with matching ABI but differing alignment"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// First non-ZST field: record its ABI and alignment
|
||||||
|
common_non_zst_abi_and_align = Ok(Some((field_abi, field.align.abi)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(pack) = repr.pack {
|
||||||
|
align = align.min(AbiAndPrefAlign::new(pack));
|
||||||
|
}
|
||||||
|
// The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
|
||||||
|
// See documentation on `LayoutS::unadjusted_abi_align`.
|
||||||
|
let unadjusted_abi_align = align.abi;
|
||||||
|
if let Some(repr_align) = repr.align {
|
||||||
|
align = align.max(AbiAndPrefAlign::new(repr_align));
|
||||||
|
}
|
||||||
|
// `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.
|
||||||
|
let align = align;
|
||||||
|
|
||||||
|
// If all non-ZST fields have the same ABI, we may forward that ABI
|
||||||
|
// for the union as a whole, unless otherwise inhibited.
|
||||||
|
let abi = match common_non_zst_abi_and_align {
|
||||||
|
Err(AbiMismatch) | Ok(None) => Abi::Aggregate { sized: true },
|
||||||
|
Ok(Some((abi, _))) => {
|
||||||
|
if abi.inherent_align(dl).map(|a| a.abi) != Some(align.abi) {
|
||||||
|
// Mismatched alignment (e.g. union is #[repr(packed)]): disable opt
|
||||||
|
Abi::Aggregate { sized: true }
|
||||||
|
} else {
|
||||||
|
abi
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Some(LayoutS {
|
||||||
|
variants: Variants::Single { index: VariantIdx::new(0) },
|
||||||
|
fields: FieldsShape::Union(NonZeroUsize::new(only_variant.len())?),
|
||||||
|
abi,
|
||||||
|
largest_niche: None,
|
||||||
|
align,
|
||||||
|
size: size.align_to(align.abi),
|
||||||
|
max_repr_align,
|
||||||
|
unadjusted_abi_align,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn layout_of_enum<'a, LC, FieldIdx: Idx, VariantIdx: Idx, F>(
|
||||||
|
layout_calc: &LC,
|
||||||
|
repr: &ReprOptions,
|
||||||
|
variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
|
||||||
|
discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
|
||||||
|
discriminants: impl Iterator<Item = (VariantIdx, i128)>,
|
||||||
|
dont_niche_optimize_enum: bool,
|
||||||
|
dl: &TargetDataLayout,
|
||||||
|
) -> Option<LayoutS<FieldIdx, VariantIdx>>
|
||||||
|
where
|
||||||
|
LC: LayoutCalculator + ?Sized,
|
||||||
|
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug,
|
||||||
|
{
|
||||||
// Until we've decided whether to use the tagged or
|
// Until we've decided whether to use the tagged or
|
||||||
// niche filling LayoutS, we don't want to intern the
|
// niche filling LayoutS, we don't want to intern the
|
||||||
// variant layouts, so we can't store them in the
|
// variant layouts, so we can't store them in the
|
||||||
|
|
@ -318,7 +446,7 @@ pub trait LayoutCalculator {
|
||||||
let mut variant_layouts = variants
|
let mut variant_layouts = variants
|
||||||
.iter_enumerated()
|
.iter_enumerated()
|
||||||
.map(|(j, v)| {
|
.map(|(j, v)| {
|
||||||
let mut st = self.univariant(dl, v, repr, StructKind::AlwaysSized)?;
|
let mut st = layout_calc.univariant(dl, v, repr, StructKind::AlwaysSized)?;
|
||||||
st.variants = Variants::Single { index: j };
|
st.variants = Variants::Single { index: j };
|
||||||
|
|
||||||
align = align.max(st.align);
|
align = align.max(st.align);
|
||||||
|
|
@ -512,7 +640,7 @@ pub trait LayoutCalculator {
|
||||||
let mut layout_variants = variants
|
let mut layout_variants = variants
|
||||||
.iter_enumerated()
|
.iter_enumerated()
|
||||||
.map(|(i, field_layouts)| {
|
.map(|(i, field_layouts)| {
|
||||||
let mut st = self.univariant(
|
let mut st = layout_calc.univariant(
|
||||||
dl,
|
dl,
|
||||||
field_layouts,
|
field_layouts,
|
||||||
repr,
|
repr,
|
||||||
|
|
@ -679,7 +807,7 @@ pub trait LayoutCalculator {
|
||||||
// Common prim might be uninit.
|
// Common prim might be uninit.
|
||||||
Scalar::Union { value: prim }
|
Scalar::Union { value: prim }
|
||||||
};
|
};
|
||||||
let pair = self.scalar_pair::<FieldIdx, VariantIdx>(tag, prim_scalar);
|
let pair = layout_calc.scalar_pair::<FieldIdx, VariantIdx>(tag, prim_scalar);
|
||||||
let pair_offsets = match pair.fields {
|
let pair_offsets = match pair.fields {
|
||||||
FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
|
FieldsShape::Arbitrary { ref offsets, ref memory_index } => {
|
||||||
assert_eq!(memory_index.raw, [0, 1]);
|
assert_eq!(memory_index.raw, [0, 1]);
|
||||||
|
|
@ -725,10 +853,7 @@ pub trait LayoutCalculator {
|
||||||
tag_field: 0,
|
tag_field: 0,
|
||||||
variants: IndexVec::new(),
|
variants: IndexVec::new(),
|
||||||
},
|
},
|
||||||
fields: FieldsShape::Arbitrary {
|
fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() },
|
||||||
offsets: [Size::ZERO].into(),
|
|
||||||
memory_index: [0].into(),
|
|
||||||
},
|
|
||||||
largest_niche,
|
largest_niche,
|
||||||
abi,
|
abi,
|
||||||
align,
|
align,
|
||||||
|
|
@ -769,111 +894,6 @@ pub trait LayoutCalculator {
|
||||||
Some(best_layout.layout)
|
Some(best_layout.layout)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn layout_of_union<
|
|
||||||
'a,
|
|
||||||
FieldIdx: Idx,
|
|
||||||
VariantIdx: Idx,
|
|
||||||
F: Deref<Target = &'a LayoutS<FieldIdx, VariantIdx>> + fmt::Debug,
|
|
||||||
>(
|
|
||||||
&self,
|
|
||||||
repr: &ReprOptions,
|
|
||||||
variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
|
|
||||||
) -> Option<LayoutS<FieldIdx, VariantIdx>> {
|
|
||||||
let dl = self.current_data_layout();
|
|
||||||
let dl = dl.borrow();
|
|
||||||
let mut align = if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };
|
|
||||||
let mut max_repr_align = repr.align;
|
|
||||||
|
|
||||||
// If all the non-ZST fields have the same ABI and union ABI optimizations aren't
|
|
||||||
// disabled, we can use that common ABI for the union as a whole.
|
|
||||||
struct AbiMismatch;
|
|
||||||
let mut common_non_zst_abi_and_align = if repr.inhibit_union_abi_opt() {
|
|
||||||
// Can't optimize
|
|
||||||
Err(AbiMismatch)
|
|
||||||
} else {
|
|
||||||
Ok(None)
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut size = Size::ZERO;
|
|
||||||
let only_variant = &variants[VariantIdx::new(0)];
|
|
||||||
for field in only_variant {
|
|
||||||
if field.is_unsized() {
|
|
||||||
self.delayed_bug("unsized field in union".to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
align = align.max(field.align);
|
|
||||||
max_repr_align = max_repr_align.max(field.max_repr_align);
|
|
||||||
size = cmp::max(size, field.size);
|
|
||||||
|
|
||||||
if field.is_zst() {
|
|
||||||
// Nothing more to do for ZST fields
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Ok(common) = common_non_zst_abi_and_align {
|
|
||||||
// Discard valid range information and allow undef
|
|
||||||
let field_abi = field.abi.to_union();
|
|
||||||
|
|
||||||
if let Some((common_abi, common_align)) = common {
|
|
||||||
if common_abi != field_abi {
|
|
||||||
// Different fields have different ABI: disable opt
|
|
||||||
common_non_zst_abi_and_align = Err(AbiMismatch);
|
|
||||||
} else {
|
|
||||||
// Fields with the same non-Aggregate ABI should also
|
|
||||||
// have the same alignment
|
|
||||||
if !matches!(common_abi, Abi::Aggregate { .. }) {
|
|
||||||
assert_eq!(
|
|
||||||
common_align, field.align.abi,
|
|
||||||
"non-Aggregate field with matching ABI but differing alignment"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// First non-ZST field: record its ABI and alignment
|
|
||||||
common_non_zst_abi_and_align = Ok(Some((field_abi, field.align.abi)));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(pack) = repr.pack {
|
|
||||||
align = align.min(AbiAndPrefAlign::new(pack));
|
|
||||||
}
|
|
||||||
// The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
|
|
||||||
// See documentation on `LayoutS::unadjusted_abi_align`.
|
|
||||||
let unadjusted_abi_align = align.abi;
|
|
||||||
if let Some(repr_align) = repr.align {
|
|
||||||
align = align.max(AbiAndPrefAlign::new(repr_align));
|
|
||||||
}
|
|
||||||
// `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.
|
|
||||||
let align = align;
|
|
||||||
|
|
||||||
// If all non-ZST fields have the same ABI, we may forward that ABI
|
|
||||||
// for the union as a whole, unless otherwise inhibited.
|
|
||||||
let abi = match common_non_zst_abi_and_align {
|
|
||||||
Err(AbiMismatch) | Ok(None) => Abi::Aggregate { sized: true },
|
|
||||||
Ok(Some((abi, _))) => {
|
|
||||||
if abi.inherent_align(dl).map(|a| a.abi) != Some(align.abi) {
|
|
||||||
// Mismatched alignment (e.g. union is #[repr(packed)]): disable opt
|
|
||||||
Abi::Aggregate { sized: true }
|
|
||||||
} else {
|
|
||||||
abi
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
Some(LayoutS {
|
|
||||||
variants: Variants::Single { index: VariantIdx::new(0) },
|
|
||||||
fields: FieldsShape::Union(NonZeroUsize::new(only_variant.len())?),
|
|
||||||
abi,
|
|
||||||
largest_niche: None,
|
|
||||||
align,
|
|
||||||
size: size.align_to(align.abi),
|
|
||||||
max_repr_align,
|
|
||||||
unadjusted_abi_align,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Determines towards which end of a struct layout optimizations will try to place the best niches.
|
/// Determines towards which end of a struct layout optimizations will try to place the best niches.
|
||||||
enum NicheBias {
|
enum NicheBias {
|
||||||
Start,
|
Start,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue