1
Fork 0

Remove trivia tokens

This commit is contained in:
Aleksey Kladov 2020-09-01 11:24:52 +02:00
parent 8f24c2ec9d
commit 5326361fc0
5 changed files with 5 additions and 28 deletions

View file

@ -251,17 +251,6 @@ pub enum TokenKind {
/// similarly to symbols in string literal tokens.
DocComment(CommentKind, ast::AttrStyle, Symbol),
// Junk. These carry no data because we don't really care about the data
// they *would* carry, and don't really want to allocate a new ident for
// them. Instead, users could extract that from the associated span.
/// Whitespace.
Whitespace,
/// A comment.
Comment,
Shebang(Symbol),
/// A completely invalid token which should be skipped.
Unknown(Symbol),
Eof,
}
@ -331,7 +320,7 @@ impl Token {
/// Some token that will be thrown away later.
pub fn dummy() -> Self {
Token::new(TokenKind::Whitespace, DUMMY_SP)
Token::new(TokenKind::Question, DUMMY_SP)
}
/// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
@ -360,7 +349,7 @@ impl Token {
pub fn is_op(&self) -> bool {
match self.kind {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false,
| Lifetime(..) | Interpolated(..) | Eof => false,
_ => true,
}
}
@ -676,8 +665,7 @@ impl Token {
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
| DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment
| Shebang(..) | Unknown(..) | Eof => return None,
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
};
Some(Token::new(kind, self.span.to(joint.span)))