bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Remove TokenStreamBuilder.

Browse source 
It's now only used in one function. Also, the "should we glue the
tokens?" check is only necessary when pushing a `TokenTree::Token`, not
when pushing a `TokenTree::Delimited`.

As part of this, we now do the "should we glue the tokens?" check
immediately, which avoids having look back at the previous token. It
also puts all the logic dealing with token gluing in a single place.
This commit is contained in:
Nicholas Nethercote 2022-09-30 16:51:35 +10:00
parent 8d0754d602
commit a822d08bd1
1 changed files with 20 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

57
compiler/rustc_parse/src/lexer/tokentrees.rs
View file

@ -47,13 +47,13 @@ impl<'a> TokenTreesReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s. // Parse a stream of tokens into a list of `TokenTree`s.
fn parse_token_trees(&mut self, is_top_level: bool) -> PResult<'a, TokenStream> { fn parse_token_trees(&mut self, is_top_level: bool) -> PResult<'a, TokenStream> {
self.token = self.string_reader.next_token().0; self.token = self.string_reader.next_token().0;
let mut buf = TokenStreamBuilder::default(); let mut buf = Vec::new();
loop { loop {
match self.token.kind { match self.token.kind {
token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)), token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
token::CloseDelim(delim) => { token::CloseDelim(delim) => {
return if !is_top_level { return if !is_top_level {
Ok(buf.into_token_stream()) Ok(TokenStream::new(buf))
} else { } else {
Err(self.close_delim_err(delim)) Err(self.close_delim_err(delim))
}; };
@ -62,21 +62,28 @@ impl<'a> TokenTreesReader<'a> {
if !is_top_level { if !is_top_level {
self.eof_err().emit(); self.eof_err().emit();
} }
return Ok(buf.into_token_stream()); return Ok(TokenStream::new(buf));
} }
_ => { _ => {
// `this_spacing` for the returned token refers to whether the token is // Get the next normal token. This might require getting multiple adjacent
// immediately followed by another op token. It is determined by the // single-char tokens and joining them together.
// next token: its kind and its `preceded_by_whitespace` status. let (this_spacing, next_tok) = loop {
let (next_tok, is_next_tok_preceded_by_whitespace) = let (next_tok, is_next_tok_preceded_by_whitespace) =
self.string_reader.next_token(); self.string_reader.next_token();
let this_spacing = if is_next_tok_preceded_by_whitespace || !next_tok.is_op() { if !is_next_tok_preceded_by_whitespace {
Spacing::Alone if let Some(glued) = self.token.glue(&next_tok) {
} else { self.token = glued;
Spacing::Joint } else {
let this_spacing =
if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
break (this_spacing, next_tok);
}
} else {
break (Spacing::Alone, next_tok);
}
}; };
let this_tok = std::mem::replace(&mut self.token, next_tok); let this_tok = std::mem::replace(&mut self.token, next_tok);
buf.push(TokenTree::Token(this_tok, this_spacing)) buf.push(TokenTree::Token(this_tok, this_spacing));
} }
} }
} }
@ -249,27 +256,3 @@ impl<'a> TokenTreesReader<'a> {
err err
} }
} }
#[derive(Default)]
struct TokenStreamBuilder {
buf: Vec<TokenTree>,
}
impl TokenStreamBuilder {
#[inline(always)]
fn push(&mut self, tree: TokenTree) {
if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last()
&& let TokenTree::Token(token, joint) = &tree
&& let Some(glued) = prev_token.glue(token)
{
self.buf.pop();
self.buf.push(TokenTree::Token(glued, *joint));
} else {
self.buf.push(tree)
}
}
fn into_token_stream(self) -> TokenStream {
TokenStream::new(self.buf)
}
}