Refactor parser lookahead buffer and increase its size
This commit is contained in:
parent
7bccb829d0
commit
65ff4ca294
2 changed files with 41 additions and 38 deletions
|
@ -74,6 +74,12 @@ pub struct TokenAndSpan {
|
||||||
pub sp: Span,
|
pub sp: Span,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Default for TokenAndSpan {
|
||||||
|
fn default() -> Self {
|
||||||
|
TokenAndSpan { tok: token::Underscore, sp: syntax_pos::DUMMY_SP }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct StringReader<'a> {
|
pub struct StringReader<'a> {
|
||||||
pub span_diagnostic: &'a Handler,
|
pub span_diagnostic: &'a Handler,
|
||||||
/// The absolute offset within the codemap of the next character to read
|
/// The absolute offset within the codemap of the next character to read
|
||||||
|
|
|
@ -245,6 +245,22 @@ enum PrevTokenKind {
|
||||||
Other,
|
Other,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Simple circular buffer used for keeping few next tokens.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct LookaheadBuffer {
|
||||||
|
buffer: [TokenAndSpan; LOOKAHEAD_BUFFER_CAPACITY],
|
||||||
|
start: usize,
|
||||||
|
end: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
const LOOKAHEAD_BUFFER_CAPACITY: usize = 8;
|
||||||
|
|
||||||
|
impl LookaheadBuffer {
|
||||||
|
fn len(&self) -> usize {
|
||||||
|
(LOOKAHEAD_BUFFER_CAPACITY + self.end - self.start) % LOOKAHEAD_BUFFER_CAPACITY
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* ident is handled by common.rs */
|
/* ident is handled by common.rs */
|
||||||
|
|
||||||
pub struct Parser<'a> {
|
pub struct Parser<'a> {
|
||||||
|
@ -258,9 +274,7 @@ pub struct Parser<'a> {
|
||||||
pub cfg: CrateConfig,
|
pub cfg: CrateConfig,
|
||||||
/// the previous token kind
|
/// the previous token kind
|
||||||
prev_token_kind: PrevTokenKind,
|
prev_token_kind: PrevTokenKind,
|
||||||
pub buffer: [TokenAndSpan; 4],
|
lookahead_buffer: LookaheadBuffer,
|
||||||
pub buffer_start: isize,
|
|
||||||
pub buffer_end: isize,
|
|
||||||
pub tokens_consumed: usize,
|
pub tokens_consumed: usize,
|
||||||
pub restrictions: Restrictions,
|
pub restrictions: Restrictions,
|
||||||
pub quote_depth: usize, // not (yet) related to the quasiquoter
|
pub quote_depth: usize, // not (yet) related to the quasiquoter
|
||||||
|
@ -356,10 +370,6 @@ impl<'a> Parser<'a> {
|
||||||
_ => PathBuf::from(sess.codemap().span_to_filename(span)),
|
_ => PathBuf::from(sess.codemap().span_to_filename(span)),
|
||||||
};
|
};
|
||||||
directory.pop();
|
directory.pop();
|
||||||
let placeholder = TokenAndSpan {
|
|
||||||
tok: token::Underscore,
|
|
||||||
sp: span,
|
|
||||||
};
|
|
||||||
|
|
||||||
Parser {
|
Parser {
|
||||||
reader: rdr,
|
reader: rdr,
|
||||||
|
@ -369,14 +379,7 @@ impl<'a> Parser<'a> {
|
||||||
span: span,
|
span: span,
|
||||||
prev_span: span,
|
prev_span: span,
|
||||||
prev_token_kind: PrevTokenKind::Other,
|
prev_token_kind: PrevTokenKind::Other,
|
||||||
buffer: [
|
lookahead_buffer: Default::default(),
|
||||||
placeholder.clone(),
|
|
||||||
placeholder.clone(),
|
|
||||||
placeholder.clone(),
|
|
||||||
placeholder.clone(),
|
|
||||||
],
|
|
||||||
buffer_start: 0,
|
|
||||||
buffer_end: 0,
|
|
||||||
tokens_consumed: 0,
|
tokens_consumed: 0,
|
||||||
restrictions: Restrictions::empty(),
|
restrictions: Restrictions::empty(),
|
||||||
quote_depth: 0,
|
quote_depth: 0,
|
||||||
|
@ -937,19 +940,13 @@ impl<'a> Parser<'a> {
|
||||||
_ => PrevTokenKind::Other,
|
_ => PrevTokenKind::Other,
|
||||||
};
|
};
|
||||||
|
|
||||||
let next = if self.buffer_start == self.buffer_end {
|
let next = if self.lookahead_buffer.start == self.lookahead_buffer.end {
|
||||||
self.reader.real_token()
|
self.reader.real_token()
|
||||||
} else {
|
} else {
|
||||||
// Avoid token copies with `replace`.
|
// Avoid token copies with `replace`.
|
||||||
let buffer_start = self.buffer_start as usize;
|
let old_start = self.lookahead_buffer.start;
|
||||||
let next_index = (buffer_start + 1) & 3;
|
self.lookahead_buffer.start = (old_start + 1) % LOOKAHEAD_BUFFER_CAPACITY;
|
||||||
self.buffer_start = next_index as isize;
|
mem::replace(&mut self.lookahead_buffer.buffer[old_start], Default::default())
|
||||||
|
|
||||||
let placeholder = TokenAndSpan {
|
|
||||||
tok: token::Underscore,
|
|
||||||
sp: self.span,
|
|
||||||
};
|
|
||||||
mem::replace(&mut self.buffer[buffer_start], placeholder)
|
|
||||||
};
|
};
|
||||||
self.span = next.sp;
|
self.span = next.sp;
|
||||||
self.token = next.tok;
|
self.token = next.tok;
|
||||||
|
@ -982,21 +979,22 @@ impl<'a> Parser<'a> {
|
||||||
self.expected_tokens.clear();
|
self.expected_tokens.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn buffer_length(&mut self) -> isize {
|
pub fn look_ahead<R, F>(&mut self, dist: usize, f: F) -> R where
|
||||||
if self.buffer_start <= self.buffer_end {
|
|
||||||
return self.buffer_end - self.buffer_start;
|
|
||||||
}
|
|
||||||
return (4 - self.buffer_start) + self.buffer_end;
|
|
||||||
}
|
|
||||||
pub fn look_ahead<R, F>(&mut self, distance: usize, f: F) -> R where
|
|
||||||
F: FnOnce(&token::Token) -> R,
|
F: FnOnce(&token::Token) -> R,
|
||||||
{
|
{
|
||||||
let dist = distance as isize;
|
if dist == 0 {
|
||||||
while self.buffer_length() < dist {
|
f(&self.token)
|
||||||
self.buffer[self.buffer_end as usize] = self.reader.real_token();
|
} else if dist < LOOKAHEAD_BUFFER_CAPACITY {
|
||||||
self.buffer_end = (self.buffer_end + 1) & 3;
|
while self.lookahead_buffer.len() < dist {
|
||||||
|
self.lookahead_buffer.buffer[self.lookahead_buffer.end] = self.reader.real_token();
|
||||||
|
self.lookahead_buffer.end =
|
||||||
|
(self.lookahead_buffer.end + 1) % LOOKAHEAD_BUFFER_CAPACITY;
|
||||||
|
}
|
||||||
|
let index = (self.lookahead_buffer.start + dist - 1) % LOOKAHEAD_BUFFER_CAPACITY;
|
||||||
|
f(&self.lookahead_buffer.buffer[index].tok)
|
||||||
|
} else {
|
||||||
|
self.bug("lookahead distance is too large");
|
||||||
}
|
}
|
||||||
f(&self.buffer[((self.buffer_start + dist - 1) & 3) as usize].tok)
|
|
||||||
}
|
}
|
||||||
pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> {
|
pub fn fatal(&self, m: &str) -> DiagnosticBuilder<'a> {
|
||||||
self.sess.span_diagnostic.struct_span_fatal(self.span, m)
|
self.sess.span_diagnostic.struct_span_fatal(self.span, m)
|
||||||
|
@ -1118,7 +1116,6 @@ impl<'a> Parser<'a> {
|
||||||
Ok(ast::TyKind::ImplTrait(bounds))
|
Ok(ast::TyKind::ImplTrait(bounds))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pub fn parse_ty_path(&mut self) -> PResult<'a, TyKind> {
|
pub fn parse_ty_path(&mut self) -> PResult<'a, TyKind> {
|
||||||
Ok(TyKind::Path(None, self.parse_path(PathStyle::Type)?))
|
Ok(TyKind::Path(None, self.parse_path(PathStyle::Type)?))
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue