Increase Span from 4 bytes to 8 bytes.
This increases the size of some important types, such as `ast::Expr` and `mir::Statement`. However, it drastically reduces how much the interner is used, and the fields are more natural sizes that don't require bit operations to extract. As a result, instruction counts drop across a range of workloads, by as much as 12% for incremental "check" builds of `script-servo`. Peak memory usage goes up a little for some cases, but down by more for some other cases -- as much as 18% for non-incremental builds of `packed-simd`. The commit also: - removes the `repr(packed)`, because it has negligible effect, but can cause undefined behaviour; - replaces explicit impls of common traits (`Copy`, `PartialEq`, etc.) with derived ones.
This commit is contained in:
Nicholas Nethercote
parent
314a79cd80
commit
fd7f605365
4 changed files with 90 additions and 103 deletions
|
|
@ -9,122 +9,106 @@ use crate::{BytePos, SpanData};
|
|||
use crate::hygiene::SyntaxContext;
|
||||
|
||||
use rustc_data_structures::fx::FxHashMap;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
/// A compressed span.
|
||||
/// Contains either fields of `SpanData` inline if they are small, or index into span interner.
|
||||
/// The primary goal of `Span` is to be as small as possible and fit into other structures
|
||||
/// (that's why it uses `packed` as well). Decoding speed is the second priority.
|
||||
/// See `SpanData` for the info on span fields in decoded representation.
|
||||
#[repr(packed)]
|
||||
pub struct Span(u32);
|
||||
///
|
||||
/// `SpanData` is 12 bytes, which is a bit too big to stick everywhere. `Span`
|
||||
/// is a form that only takes up 8 bytes, with less space for the length and
|
||||
/// context. The vast majority (99.9%+) of `SpanData` instances will fit within
|
||||
/// those 8 bytes; any `SpanData` whose fields don't fit into a `Span` are
|
||||
/// stored in a separate interner table, and the `Span` will index into that
|
||||
/// table. Interning is rare enough that the cost is low, but common enough
|
||||
/// that the code is exercised regularly.
|
||||
///
|
||||
/// An earlier version of this code used only 4 bytes for `Span`, but that was
|
||||
/// slower because only 80--90% of spans could be stored inline (even less in
|
||||
/// very large crates) and so the interner was used a lot more.
|
||||
///
|
||||
/// Inline (compressed) format:
|
||||
/// - `span.base_or_index == span_data.lo`
|
||||
/// - `span.len_or_tag == len == span_data.hi - span_data.lo` (must be `<= MAX_LEN`)
|
||||
/// - `span.ctxt == span_data.ctxt` (must be `<= MAX_CTXT`)
|
||||
///
|
||||
/// Interned format:
|
||||
/// - `span.base_or_index == index` (indexes into the interner table)
|
||||
/// - `span.len_or_tag == LEN_TAG` (high bit set, all other bits are zero)
|
||||
/// - `span.ctxt == 0`
|
||||
///
|
||||
/// The inline form uses 0 for the tag value (rather than 1) so that we don't
|
||||
/// need to mask out the tag bit when getting the length, and so that the
|
||||
/// dummy span can be all zeroes.
|
||||
///
|
||||
/// Notes about the choice of field sizes:
|
||||
/// - `base` is 32 bits in both `Span` and `SpanData`, which means that `base`
|
||||
/// values never cause interning. The number of bits needed for `base`
|
||||
/// depends on the crate size. 32 bits allows up to 4 GiB of code in a crate.
|
||||
/// `script-servo` is the largest crate in `rustc-perf`, requiring 26 bits
|
||||
/// for some spans.
|
||||
/// - `len` is 15 bits in `Span` (a u16, minus 1 bit for the tag) and 32 bits
|
||||
/// in `SpanData`, which means that large `len` values will cause interning.
|
||||
/// The number of bits needed for `len` does not depend on the crate size.
|
||||
/// The most common number of bits for `len` are 0--7, with a peak usually at
|
||||
/// 3 or 4, and then it drops off quickly from 8 onwards. 15 bits is enough
|
||||
/// for 99.99%+ of cases, but larger values (sometimes 20+ bits) might occur
|
||||
/// dozens of times in a typical crate.
|
||||
/// - `ctxt` is 16 bits in `Span` and 32 bits in `SpanData`, which means that
|
||||
/// large `ctxt` values will cause interning. The number of bits needed for
|
||||
/// `ctxt` values depend partly on the crate size and partly on the form of
|
||||
/// the code. No crates in `rustc-perf` need more than 15 bits for `ctxt`,
|
||||
/// but larger crates might need more than 16 bits.
|
||||
///
|
||||
#[derive(Clone, Copy, Eq, PartialEq, Hash)]
|
||||
pub struct Span {
|
||||
base_or_index: u32,
|
||||
len_or_tag: u16,
|
||||
ctxt_or_zero: u16
|
||||
}
|
||||
|
||||
impl Copy for Span {}
|
||||
impl Clone for Span {
|
||||
#[inline]
|
||||
fn clone(&self) -> Span {
|
||||
*self
|
||||
}
|
||||
}
|
||||
impl PartialEq for Span {
|
||||
#[inline]
|
||||
fn eq(&self, other: &Span) -> bool {
|
||||
let a = self.0;
|
||||
let b = other.0;
|
||||
a == b
|
||||
}
|
||||
}
|
||||
impl Eq for Span {}
|
||||
impl Hash for Span {
|
||||
#[inline]
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
let a = self.0;
|
||||
a.hash(state)
|
||||
}
|
||||
}
|
||||
const LEN_TAG: u16 = 0b1000_0000_0000_0000;
|
||||
const MAX_LEN: u32 = 0b0111_1111_1111_1111;
|
||||
const MAX_CTXT: u32 = 0b1111_1111_1111_1111;
|
||||
|
||||
/// Dummy span, both position and length are zero, syntax context is zero as well.
|
||||
/// This span is kept inline and encoded with format 0.
|
||||
pub const DUMMY_SP: Span = Span(0);
|
||||
pub const DUMMY_SP: Span = Span { base_or_index: 0, len_or_tag: 0, ctxt_or_zero: 0 };
|
||||
|
||||
impl Span {
|
||||
#[inline]
|
||||
pub fn new(lo: BytePos, hi: BytePos, ctxt: SyntaxContext) -> Self {
|
||||
encode(&match lo <= hi {
|
||||
true => SpanData { lo, hi, ctxt },
|
||||
false => SpanData { lo: hi, hi: lo, ctxt },
|
||||
})
|
||||
pub fn new(mut lo: BytePos, mut hi: BytePos, ctxt: SyntaxContext) -> Self {
|
||||
if lo > hi {
|
||||
std::mem::swap(&mut lo, &mut hi);
|
||||
}
|
||||
|
||||
let (base, len, ctxt2) = (lo.0, hi.0 - lo.0, ctxt.as_u32());
|
||||
|
||||
if len <= MAX_LEN && ctxt2 <= MAX_CTXT {
|
||||
// Inline format.
|
||||
Span { base_or_index: base, len_or_tag: len as u16, ctxt_or_zero: ctxt2 as u16 }
|
||||
} else {
|
||||
// Interned format.
|
||||
let index = with_span_interner(|interner| interner.intern(&SpanData { lo, hi, ctxt }));
|
||||
Span { base_or_index: index, len_or_tag: LEN_TAG, ctxt_or_zero: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn data(self) -> SpanData {
|
||||
decode(self)
|
||||
if self.len_or_tag != LEN_TAG {
|
||||
// Inline format.
|
||||
debug_assert!(self.len_or_tag as u32 <= MAX_LEN);
|
||||
SpanData {
|
||||
lo: BytePos(self.base_or_index),
|
||||
hi: BytePos(self.base_or_index + self.len_or_tag as u32),
|
||||
ctxt: SyntaxContext::from_u32(self.ctxt_or_zero as u32),
|
||||
}
|
||||
} else {
|
||||
// Interned format.
|
||||
debug_assert!(self.ctxt_or_zero == 0);
|
||||
let index = self.base_or_index;
|
||||
with_span_interner(|interner| *interner.get(index))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tags
|
||||
const TAG_INLINE: u32 = 0;
|
||||
const TAG_INTERNED: u32 = 1;
|
||||
const TAG_MASK: u32 = 1;
|
||||
|
||||
// Fields indexes
|
||||
const BASE_INDEX: usize = 0;
|
||||
const LEN_INDEX: usize = 1;
|
||||
const CTXT_INDEX: usize = 2;
|
||||
|
||||
// Tag = 0, inline format.
|
||||
// -------------------------------------------------------------
|
||||
// | base 31:7 | len 6:1 | ctxt (currently 0 bits) | tag 0:0 |
|
||||
// -------------------------------------------------------------
|
||||
// Since there are zero bits for ctxt, only SpanData with a 0 SyntaxContext
|
||||
// can be inline.
|
||||
const INLINE_SIZES: [u32; 3] = [25, 6, 0];
|
||||
const INLINE_OFFSETS: [u32; 3] = [7, 1, 1];
|
||||
|
||||
// Tag = 1, interned format.
|
||||
// ------------------------
|
||||
// | index 31:1 | tag 0:0 |
|
||||
// ------------------------
|
||||
const INTERNED_INDEX_SIZE: u32 = 31;
|
||||
const INTERNED_INDEX_OFFSET: u32 = 1;
|
||||
|
||||
#[inline]
|
||||
fn encode(sd: &SpanData) -> Span {
|
||||
let (base, len, ctxt) = (sd.lo.0, sd.hi.0 - sd.lo.0, sd.ctxt.as_u32());
|
||||
|
||||
let val = if (base >> INLINE_SIZES[BASE_INDEX]) == 0 &&
|
||||
(len >> INLINE_SIZES[LEN_INDEX]) == 0 &&
|
||||
(ctxt >> INLINE_SIZES[CTXT_INDEX]) == 0 {
|
||||
(base << INLINE_OFFSETS[BASE_INDEX]) | (len << INLINE_OFFSETS[LEN_INDEX]) |
|
||||
(ctxt << INLINE_OFFSETS[CTXT_INDEX]) | TAG_INLINE
|
||||
} else {
|
||||
let index = with_span_interner(|interner| interner.intern(sd));
|
||||
(index << INTERNED_INDEX_OFFSET) | TAG_INTERNED
|
||||
};
|
||||
Span(val)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn decode(span: Span) -> SpanData {
|
||||
let val = span.0;
|
||||
|
||||
// Extract a field at position `pos` having size `size`.
|
||||
let extract = |pos: u32, size: u32| {
|
||||
let mask = ((!0u32) as u64 >> (32 - size)) as u32; // Can't shift u32 by 32
|
||||
(val >> pos) & mask
|
||||
};
|
||||
|
||||
let (base, len, ctxt) = if val & TAG_MASK == TAG_INLINE {(
|
||||
extract(INLINE_OFFSETS[BASE_INDEX], INLINE_SIZES[BASE_INDEX]),
|
||||
extract(INLINE_OFFSETS[LEN_INDEX], INLINE_SIZES[LEN_INDEX]),
|
||||
extract(INLINE_OFFSETS[CTXT_INDEX], INLINE_SIZES[CTXT_INDEX]),
|
||||
)} else {
|
||||
let index = extract(INTERNED_INDEX_OFFSET, INTERNED_INDEX_SIZE);
|
||||
return with_span_interner(|interner| *interner.get(index));
|
||||
};
|
||||
SpanData { lo: BytePos(base), hi: BytePos(base + len), ctxt: SyntaxContext::from_u32(ctxt) }
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct SpanInterner {
|
||||
spans: FxHashMap<SpanData, u32>,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue