bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Remove trivia tokens

Browse source
This commit is contained in:
Aleksey Kladov 2020-09-01 11:24:52 +02:00
parent 8f24c2ec9d
commit 5326361fc0
5 changed files with 5 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

2
compiler/rustc_parse/src/lexer/unicode_chars.rs
View file

@ -303,7 +303,7 @@ const UNICODE_ARRAY: &[(char, &str, char)] = &[
// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
// fancier error recovery to it, as there will be less overall work to do this way.
const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[
(' ', "Space", Some(token::Whitespace)),
(' ', "Space", None),
('_', "Underscore", Some(token::Ident(kw::Underscore, false))),
('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))),
(',', "Comma", Some(token::Comma)),

7
compiler/rustc_parse/src/lib.rs
View file

@ -348,9 +348,6 @@ pub fn tokenstream_probably_equal_for_proc_macro(
| token::CloseDelim(DelimToken::NoDelim)
// The pretty printer collapses many semicolons into one.
| token::Semi
// The pretty printer collapses whitespace arbitrarily and can
// introduce whitespace from `NoDelim`.
| token::Whitespace
// The pretty printer can turn `$crate` into `::crate_name`
| token::ModSep = token.kind {
return false;
@ -506,8 +503,6 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
| (&Pound, &Pound)
| (&Dollar, &Dollar)
| (&Question, &Question)
| (&Whitespace, &Whitespace)
| (&Comment, &Comment)
| (&Eof, &Eof) => true,
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
@ -516,8 +511,6 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
(&DocComment(a1, a2, a3), &DocComment(b1, b2, b3)) => a1 == b1 && a2 == b2 && a3 == b3,
(&Shebang(a), &Shebang(b)) => a == b,
(&Literal(a), &Literal(b)) => a == b,
(&Lifetime(a), &Lifetime(b)) => a == b,