bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Add rustc_lexer::TokenKind::Eof.

Browse source 
For alignment with `rust_ast::TokenKind::Eof`. Plus it's a bit faster,
due to less `Option` manipulation in `StringReader::next_token`.
This commit is contained in:
Nicholas Nethercote 2022-09-26 13:06:15 +10:00
parent cc0022a363
commit da84f0f4c3
3 changed files with 20 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

10
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -80,14 +80,7 @@ impl<'a> StringReader<'a> {
// Skip trivial (whitespace & comments) tokens
loop {
let token = match self.cursor.advance_token() {
Some(token) => token,
None => {
let span = self.mk_sp(self.pos, self.pos);
return (Token::new(token::Eof, span), preceded_by_whitespace);
}
};
let token = self.cursor.advance_token();
let start = self.pos;
self.pos = self.pos + BytePos(token.len);
@ -327,6 +320,7 @@ impl<'a> StringReader<'a> {
err.emit();
token?
}
rustc_lexer::TokenKind::Eof => token::Eof,
})
}