diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index 0c9c9adcf9d..5cc8324b316 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -363,7 +363,8 @@ impl<'a> HashStable> for token::TokenKind { } token::DocComment(val) | - token::Shebang(val) => val.hash_stable(hcx, hasher), + token::Shebang(val) | + token::Unknown(val) => val.hash_stable(hcx, hasher), } } } diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 8132074d6e0..92d85ef9cac 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -237,7 +237,7 @@ impl<'a> Classifier<'a> { return Ok(()); }, - token::Whitespace => Class::None, + token::Whitespace | token::Unknown(..) => Class::None, token::Comment => Class::Comment, token::DocComment(..) => Class::DocComment, diff --git a/src/libsyntax/ext/proc_macro_server.rs b/src/libsyntax/ext/proc_macro_server.rs index 8d0023c9ab1..36621ce7775 100644 --- a/src/libsyntax/ext/proc_macro_server.rs +++ b/src/libsyntax/ext/proc_macro_server.rs @@ -184,7 +184,7 @@ impl FromInternal<(TreeAndJoint, &'_ ParseSess, &'_ mut Vec)> } OpenDelim(..) | CloseDelim(..) => unreachable!(), - Whitespace | Comment | Shebang(..) | Eof => unreachable!(), + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(), } } } diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index c209ae1cb9f..e86d4c7fde6 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -354,7 +354,7 @@ impl<'a> StringReader<'a> { // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, // as there will be less overall work to do this way. let token = unicode_chars::check_for_substitution(self, start, c, &mut err) - .unwrap_or(token::Whitespace); + .unwrap_or_else(|| token::Unknown(self.symbol_from(start))); err.emit(); token } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 830fbec58de..37e67a2729e 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -217,7 +217,7 @@ impl<'a> TokenTreesReader<'a> { loop { let token = self.string_reader.next_token(); match token.kind { - token::Whitespace | token::Comment | token::Shebang(_) => { + token::Whitespace | token::Comment | token::Shebang(_) | token::Unknown(_) => { self.joint_to_prev = NonJoint; } _ => { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 73adb5c947c..be800b4de66 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -255,6 +255,8 @@ pub enum TokenKind { /// A comment. Comment, Shebang(ast::Name), + /// A completely invalid token which should be skipped. + Unknown(ast::Name), Eof, } @@ -603,7 +605,7 @@ impl Token { DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | - Whitespace | Comment | Shebang(..) | Eof => return None, + Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => return None, }; Some(Token::new(kind, self.span.to(joint.span))) diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index 2ef8a919b9c..378ba1e4107 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -288,6 +288,7 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option) token::Whitespace => " ".to_string(), token::Comment => "/* */".to_string(), token::Shebang(s) => format!("/* shebang: {}*/", s), + token::Unknown(s) => s.to_string(), token::Interpolated(ref nt) => nonterminal_to_string(nt), }