bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Remove the loop from Parser::bump().

Browse source 
The loop is there to handle a `NoDelim` open/close token. This commit
changes `TokenCursor::inlined_next` so it never returns such a token.
This is a performance win because the conditional test in `bump()` is
removed.

If the parser needs changing in the future to handle `NoDelim` tokens,
then `inlined_next()` can easily be changed to return them.
This commit is contained in:
Nicholas Nethercote 2022-04-20 12:22:42 +10:00
parent 3cd5e34617
commit b09522a634
1 changed files with 34 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

63
compiler/rustc_parse/src/parser/mod.rs
View file

@ -242,12 +242,17 @@ struct TokenCursorFrame {
delim: token::DelimToken, delim: token::DelimToken,
span: DelimSpan, span: DelimSpan,
tree_cursor: tokenstream::Cursor, tree_cursor: tokenstream::Cursor,
close_delim: bool, need_to_produce_close_delim: bool,
} }
impl TokenCursorFrame { impl TokenCursorFrame {
fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream, close_delim: bool) -> Self { fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self {
TokenCursorFrame { delim, span, tree_cursor: tts.into_trees(), close_delim } TokenCursorFrame {
delim,
span,
tree_cursor: tts.into_trees(),
need_to_produce_close_delim: delim != DelimToken::NoDelim,
}
} }
} }
@ -261,28 +266,32 @@ impl TokenCursor {
fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) { fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
loop { loop {
if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing() { if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing() {
return match tree { match tree {
TokenTree::Token(token) => match (desugar_doc_comments, &token) { TokenTree::Token(token) => match (desugar_doc_comments, &token) {
(true, &Token { kind: token::DocComment(_, attr_style, data), span }) => { (true, &Token { kind: token::DocComment(_, attr_style, data), span }) => {
self.desugar(attr_style, data, span) return self.desugar(attr_style, data, span);
} }
_ => (token, spacing), _ => return (token, spacing),
}, },
TokenTree::Delimited(sp, delim, tts) => { TokenTree::Delimited(sp, delim, tts) => {
// Set `open_delim` to true here because we deal with it immediately. // Set `open_delim` to true here because we deal with it immediately.
let frame = TokenCursorFrame::new(sp, delim, tts, false); let frame = TokenCursorFrame::new(sp, delim, tts);
self.stack.push(mem::replace(&mut self.frame, frame)); self.stack.push(mem::replace(&mut self.frame, frame));
(Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone) if delim != DelimToken::NoDelim {
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
}
// No open delimeter to return; continue on to the next iteration.
} }
}; };
} else if !self.frame.close_delim { } else if self.frame.need_to_produce_close_delim {
self.frame.close_delim = true; self.frame.need_to_produce_close_delim = false;
return ( return (
Token::new(token::CloseDelim(self.frame.delim), self.frame.span.close), Token::new(token::CloseDelim(self.frame.delim), self.frame.span.close),
Spacing::Alone, Spacing::Alone,
); );
} else if let Some(frame) = self.stack.pop() { } else if let Some(frame) = self.stack.pop() {
self.frame = frame; self.frame = frame;
// Back to the parent frame; continue on to the next iteration.
} else { } else {
return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone); return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
} }
@ -333,7 +342,6 @@ impl TokenCursor {
.cloned() .cloned()
.collect::<TokenStream>() .collect::<TokenStream>()
}, },
true,
), ),
)); ));
@ -422,7 +430,7 @@ impl<'a> Parser<'a> {
desugar_doc_comments: bool, desugar_doc_comments: bool,
subparser_name: Option<&'static str>, subparser_name: Option<&'static str>,
) -> Self { ) -> Self {
let start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens, true); let start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens);
let mut parser = Parser { let mut parser = Parser {
sess, sess,
@ -993,24 +1001,21 @@ impl<'a> Parser<'a> {
/// Advance the parser by one token. /// Advance the parser by one token.
pub fn bump(&mut self) { pub fn bump(&mut self) {
let fallback_span = self.token.span; let fallback_span = self.token.span;
loop { let (mut next, spacing) = self.token_cursor.inlined_next(self.desugar_doc_comments);
let (mut next, spacing) = self.token_cursor.inlined_next(self.desugar_doc_comments); self.token_cursor.num_next_calls += 1;
self.token_cursor.num_next_calls += 1; // We've retrieved an token from the underlying
// We've retrieved an token from the underlying // cursor, so we no longer need to worry about
// cursor, so we no longer need to worry about // an unglued token. See `break_and_eat` for more details
// an unglued token. See `break_and_eat` for more details self.token_cursor.break_last_token = false;
self.token_cursor.break_last_token = false; if next.span.is_dummy() {
if next.span.is_dummy() { // Tweak the location for better diagnostics, but keep syntactic context intact.
// Tweak the location for better diagnostics, but keep syntactic context intact. next.span = fallback_span.with_ctxt(next.span.ctxt());
next.span = fallback_span.with_ctxt(next.span.ctxt());
}
if !matches!(
next.kind,
token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim)
) {
return self.inlined_bump_with((next, spacing));
}
} }
debug_assert!(!matches!(
next.kind,
token::OpenDelim(token::NoDelim) | token::CloseDelim(token::NoDelim)
));
self.inlined_bump_with((next, spacing))
} }
/// Look-ahead `dist` tokens of `self.token` and get access to that token there. /// Look-ahead `dist` tokens of `self.token` and get access to that token there.