bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

extract common code

Browse source
This commit is contained in:
Deadbeef 2023-07-25 09:24:12 +00:00
parent 0d9c871736
commit a0376e9ec2
1 changed files with 10 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

21
compiler/rustc_parse/src/lexer/mod.rs
View file

@ -166,10 +166,7 @@ impl<'a> StringReader<'a> {
continue; continue;
} }
rustc_lexer::TokenKind::Ident => { rustc_lexer::TokenKind::Ident => {
let sym = nfc_normalize(self.str_from(start)); self.ident(start)
let span = self.mk_sp(start, self.pos);
self.sess.symbol_gallery.insert(sym, span);
token::Ident(sym, false)
} }
rustc_lexer::TokenKind::RawIdent => { rustc_lexer::TokenKind::RawIdent => {
let sym = nfc_normalize(self.str_from(start + BytePos(2))); let sym = nfc_normalize(self.str_from(start + BytePos(2)));
@ -183,10 +180,7 @@ impl<'a> StringReader<'a> {
} }
rustc_lexer::TokenKind::UnknownPrefix => { rustc_lexer::TokenKind::UnknownPrefix => {
self.report_unknown_prefix(start); self.report_unknown_prefix(start);
let sym = nfc_normalize(self.str_from(start)); self.ident(start)
let span = self.mk_sp(start, self.pos);
self.sess.symbol_gallery.insert(sym, span);
token::Ident(sym, false)
} }
rustc_lexer::TokenKind::InvalidIdent rustc_lexer::TokenKind::InvalidIdent
// Do not recover an identifier with emoji if the codepoint is a confusable // Do not recover an identifier with emoji if the codepoint is a confusable
@ -222,10 +216,8 @@ impl<'a> StringReader<'a> {
self.cursor = Cursor::new(&str_before[prefix_len as usize..]); self.cursor = Cursor::new(&str_before[prefix_len as usize..]);
self.report_unknown_prefix(start); self.report_unknown_prefix(start);
let sym = nfc_normalize(self.str_from(start));
let prefix_span = self.mk_sp(start, lit_start); let prefix_span = self.mk_sp(start, lit_start);
self.sess.symbol_gallery.insert(sym, prefix_span); return (Token::new(self.ident(start), prefix_span), preceded_by_whitespace);
return (Token::new(token::Ident(sym, false), prefix_span), preceded_by_whitespace);
} }
rustc_lexer::TokenKind::Literal { kind, suffix_start } => { rustc_lexer::TokenKind::Literal { kind, suffix_start } => {
let suffix_start = start + BytePos(suffix_start); let suffix_start = start + BytePos(suffix_start);
@ -341,6 +333,13 @@ impl<'a> StringReader<'a> {
} }
} }
fn ident(&self, start: BytePos) -> TokenKind {
let sym = nfc_normalize(self.str_from(start));
let span = self.mk_sp(start, self.pos);
self.sess.symbol_gallery.insert(sym, span);
token::Ident(sym, false)
}
fn struct_fatal_span_char( fn struct_fatal_span_char(
&self, &self,
from_pos: BytePos, from_pos: BytePos,