bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Remove TokenKind::InvalidPrefix.

Browse source 
It was added in #123752 to handle some cases involving emoji, but it
isn't necessary because it's always treated the same as
`TokenKind::InvalidIdent`. This commit removes it, which makes things a
little simpler.
This commit is contained in:
Nicholas Nethercote 2024-11-19 15:55:34 +11:00
parent 2c7c3697db
commit e9a0c3c98c
4 changed files with 14 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

5
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -213,7 +213,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
let ident = Symbol::intern(lifetime_name);
token::Lifetime(ident, IdentIsRaw::No)
}
rustc_lexer::TokenKind::InvalidIdent | rustc_lexer::TokenKind::InvalidPrefix
rustc_lexer::TokenKind::InvalidIdent
// Do not recover an identifier with emoji if the codepoint is a confusable
// with a recoverable substitution token, like `➖`.
if !UNICODE_ARRAY.iter().any(|&(c, _, _)| {
@ -359,8 +359,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
rustc_lexer::TokenKind::Percent => token::BinOp(token::Percent),
rustc_lexer::TokenKind::Unknown
| rustc_lexer::TokenKind::InvalidIdent
| rustc_lexer::TokenKind::InvalidPrefix => {
| rustc_lexer::TokenKind::InvalidIdent => {
// Don't emit diagnostics for sequences of the same invalid token
if swallow_next_invalid > 0 {
swallow_next_invalid -= 1;