Remove token::{Open,Close}Delim.
By replacing them with `{Open,Close}{Param,Brace,Bracket,Invisible}`.
PR #137902 made `ast::TokenKind` more like `lexer::TokenKind` by
replacing the compound `BinOp{,Eq}(BinOpToken)` variants with fieldless
variants `Plus`, `Minus`, `Star`, etc. This commit does a similar thing
with delimiters. It also makes `ast::TokenKind` more similar to
`parser::TokenType`.
This requires a few new methods:
- `TokenKind::is_{,open_,close_}delim()` replace various kinds of
pattern matches.
- `Delimiter::as_{open,close}_token_kind` are used to convert
`Delimiter` values to `TokenKind`.
Despite these additions, it's a net reduction in lines of code. This is
because e.g. `token::OpenParen` is so much shorter than
`token::OpenDelim(Delimiter::Parenthesis)` that many multi-line forms
reduce to single line forms. And many places where the number of lines
doesn't change are still easier to read, just because the names are
shorter, e.g.:
```
- } else if self.token != token::CloseDelim(Delimiter::Brace) {
+ } else if self.token != token::CloseBrace {
```
This commit is contained in:
Nicholas Nethercote
parent
a15cce2690
commit
bf8ce32558
30 changed files with 456 additions and 498 deletions
|
|
@ -32,6 +32,18 @@ pub enum InvisibleOrigin {
|
|||
ProcMacro,
|
||||
}
|
||||
|
||||
impl InvisibleOrigin {
|
||||
// Should the parser skip these invisible delimiters? Ideally this function
|
||||
// will eventually disappear and no invisible delimiters will be skipped.
|
||||
#[inline]
|
||||
pub fn skip(&self) -> bool {
|
||||
match self {
|
||||
InvisibleOrigin::MetaVar(_) => false,
|
||||
InvisibleOrigin::ProcMacro => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for InvisibleOrigin {
|
||||
#[inline]
|
||||
fn eq(&self, _other: &InvisibleOrigin) -> bool {
|
||||
|
|
@ -125,8 +137,7 @@ impl Delimiter {
|
|||
pub fn skip(&self) -> bool {
|
||||
match self {
|
||||
Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false,
|
||||
Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false,
|
||||
Delimiter::Invisible(InvisibleOrigin::ProcMacro) => true,
|
||||
Delimiter::Invisible(origin) => origin.skip(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -140,6 +151,24 @@ impl Delimiter {
|
|||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_open_token_kind(&self) -> TokenKind {
|
||||
match *self {
|
||||
Delimiter::Parenthesis => OpenParen,
|
||||
Delimiter::Brace => OpenBrace,
|
||||
Delimiter::Bracket => OpenBracket,
|
||||
Delimiter::Invisible(origin) => OpenInvisible(origin),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_close_token_kind(&self) -> TokenKind {
|
||||
match *self {
|
||||
Delimiter::Parenthesis => CloseParen,
|
||||
Delimiter::Brace => CloseBrace,
|
||||
Delimiter::Bracket => CloseBracket,
|
||||
Delimiter::Invisible(origin) => CloseInvisible(origin),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Note that the suffix is *not* considered when deciding the `LitKind` in this
|
||||
|
|
@ -194,9 +223,9 @@ impl Lit {
|
|||
match token.uninterpolate().kind {
|
||||
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
|
||||
Literal(token_lit) => Some(token_lit),
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(
|
||||
MetaVarKind::Literal | MetaVarKind::Expr { .. },
|
||||
))) => {
|
||||
)) => {
|
||||
// Unreachable with the current test suite.
|
||||
panic!("from_token metavar");
|
||||
}
|
||||
|
|
@ -426,10 +455,22 @@ pub enum TokenKind {
|
|||
Question,
|
||||
/// Used by proc macros for representing lifetimes, not generated by lexer right now.
|
||||
SingleQuote,
|
||||
/// An opening delimiter (e.g., `{`).
|
||||
OpenDelim(Delimiter),
|
||||
/// A closing delimiter (e.g., `}`).
|
||||
CloseDelim(Delimiter),
|
||||
/// `(`
|
||||
OpenParen,
|
||||
/// `)`
|
||||
CloseParen,
|
||||
/// `{`
|
||||
OpenBrace,
|
||||
/// `}`
|
||||
CloseBrace,
|
||||
/// `[`
|
||||
OpenBracket,
|
||||
/// `]`
|
||||
CloseBracket,
|
||||
/// Invisible opening delimiter, produced by a macro.
|
||||
OpenInvisible(InvisibleOrigin),
|
||||
/// Invisible closing delimiter, produced by a macro.
|
||||
CloseInvisible(InvisibleOrigin),
|
||||
|
||||
/* Literals */
|
||||
Literal(Lit),
|
||||
|
|
@ -530,6 +571,37 @@ impl TokenKind {
|
|||
pub fn should_end_const_arg(&self) -> bool {
|
||||
matches!(self, Gt | Ge | Shr | ShrEq)
|
||||
}
|
||||
|
||||
pub fn is_delim(&self) -> bool {
|
||||
self.open_delim().is_some() || self.close_delim().is_some()
|
||||
}
|
||||
|
||||
pub fn open_delim(&self) -> Option<Delimiter> {
|
||||
match *self {
|
||||
OpenParen => Some(Delimiter::Parenthesis),
|
||||
OpenBrace => Some(Delimiter::Brace),
|
||||
OpenBracket => Some(Delimiter::Bracket),
|
||||
OpenInvisible(origin) => Some(Delimiter::Invisible(origin)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn close_delim(&self) -> Option<Delimiter> {
|
||||
match *self {
|
||||
CloseParen => Some(Delimiter::Parenthesis),
|
||||
CloseBrace => Some(Delimiter::Brace),
|
||||
CloseBracket => Some(Delimiter::Bracket),
|
||||
CloseInvisible(origin) => Some(Delimiter::Invisible(origin)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_close_delim_or_eof(&self) -> bool {
|
||||
match self {
|
||||
CloseParen | CloseBrace | CloseBracket | CloseInvisible(_) | Eof => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Token {
|
||||
|
|
@ -559,7 +631,8 @@ impl Token {
|
|||
| DotDotDot | DotDotEq | Comma | Semi | Colon | PathSep | RArrow | LArrow
|
||||
| FatArrow | Pound | Dollar | Question | SingleQuote => true,
|
||||
|
||||
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
|
||||
OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
|
||||
| OpenInvisible(_) | CloseInvisible(_) | Literal(..) | DocComment(..) | Ident(..)
|
||||
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
|
||||
}
|
||||
}
|
||||
|
|
@ -573,11 +646,12 @@ impl Token {
|
|||
/// **NB**: Take care when modifying this function, since it will change
|
||||
/// the stable set of tokens that are allowed to match an expr nonterminal.
|
||||
pub fn can_begin_expr(&self) -> bool {
|
||||
use Delimiter::*;
|
||||
match self.uninterpolate().kind {
|
||||
Ident(name, is_raw) =>
|
||||
ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
|
||||
OpenDelim(Parenthesis | Brace | Bracket) | // tuple, array or block
|
||||
OpenParen | // tuple
|
||||
OpenBrace | // block
|
||||
OpenBracket | // array
|
||||
Literal(..) | // literal
|
||||
Bang | // operator not
|
||||
Minus | // unary minus
|
||||
|
|
@ -591,12 +665,12 @@ impl Token {
|
|||
PathSep | // global path
|
||||
Lifetime(..) | // labeled loop
|
||||
Pound => true, // expression attributes
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(
|
||||
MetaVarKind::Block |
|
||||
MetaVarKind::Expr { .. } |
|
||||
MetaVarKind::Literal |
|
||||
MetaVarKind::Path
|
||||
))) => true,
|
||||
)) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
|
@ -608,8 +682,8 @@ impl Token {
|
|||
match &self.uninterpolate().kind {
|
||||
// box, ref, mut, and other identifiers (can stricten)
|
||||
Ident(..) | NtIdent(..) |
|
||||
OpenDelim(Delimiter::Parenthesis) | // tuple pattern
|
||||
OpenDelim(Delimiter::Bracket) | // slice pattern
|
||||
OpenParen | // tuple pattern
|
||||
OpenBracket | // slice pattern
|
||||
And | // reference
|
||||
Minus | // negative literal
|
||||
AndAnd | // double reference
|
||||
|
|
@ -620,14 +694,14 @@ impl Token {
|
|||
Lt | // path (UFCS constant)
|
||||
Shl => true, // path (double UFCS)
|
||||
Or => matches!(pat_kind, PatWithOr), // leading vert `|` or-pattern
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(
|
||||
MetaVarKind::Expr { .. } |
|
||||
MetaVarKind::Literal |
|
||||
MetaVarKind::Meta { .. } |
|
||||
MetaVarKind::Pat(_) |
|
||||
MetaVarKind::Path |
|
||||
MetaVarKind::Ty { .. }
|
||||
))) => true,
|
||||
)) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
|
@ -637,8 +711,8 @@ impl Token {
|
|||
match self.uninterpolate().kind {
|
||||
Ident(name, is_raw) =>
|
||||
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
|
||||
OpenDelim(Delimiter::Parenthesis) | // tuple
|
||||
OpenDelim(Delimiter::Bracket) | // array
|
||||
OpenParen | // tuple
|
||||
OpenBracket | // array
|
||||
Bang | // never
|
||||
Star | // raw pointer
|
||||
And | // reference
|
||||
|
|
@ -647,10 +721,10 @@ impl Token {
|
|||
Lifetime(..) | // lifetime bound in trait object
|
||||
Lt | Shl | // associated path
|
||||
PathSep => true, // global path
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(
|
||||
MetaVarKind::Ty { .. } |
|
||||
MetaVarKind::Path
|
||||
))) => true,
|
||||
)) => true,
|
||||
// For anonymous structs or unions, which only appear in specific positions
|
||||
// (type of struct fields or union fields), we don't consider them as regular types
|
||||
_ => false,
|
||||
|
|
@ -660,11 +734,11 @@ impl Token {
|
|||
/// Returns `true` if the token can appear at the start of a const param.
|
||||
pub fn can_begin_const_arg(&self) -> bool {
|
||||
match self.kind {
|
||||
OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
|
||||
OpenBrace | Literal(..) | Minus => true,
|
||||
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(
|
||||
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
|
||||
))) => true,
|
||||
)) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
|
@ -711,7 +785,7 @@ impl Token {
|
|||
match self.uninterpolate().kind {
|
||||
Literal(..) | Minus => true,
|
||||
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind {
|
||||
MetaVarKind::Literal => true,
|
||||
MetaVarKind::Expr { can_begin_literal_maybe_minus, .. } => {
|
||||
can_begin_literal_maybe_minus
|
||||
|
|
@ -725,7 +799,7 @@ impl Token {
|
|||
pub fn can_begin_string_literal(&self) -> bool {
|
||||
match self.uninterpolate().kind {
|
||||
Literal(..) => true,
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind))) => match mv_kind {
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(mv_kind)) => match mv_kind {
|
||||
MetaVarKind::Literal => true,
|
||||
MetaVarKind::Expr { can_begin_string_literal, .. } => can_begin_string_literal,
|
||||
_ => false,
|
||||
|
|
@ -892,7 +966,7 @@ impl Token {
|
|||
/// from an expanded metavar?
|
||||
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
|
||||
match self.kind {
|
||||
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
|
||||
OpenInvisible(InvisibleOrigin::MetaVar(kind)) => Some(kind),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
|
@ -970,7 +1044,8 @@ impl Token {
|
|||
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | PlusEq | MinusEq | StarEq | SlashEq
|
||||
| PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq
|
||||
| Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question
|
||||
| OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
|
||||
| OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
|
||||
| OpenInvisible(_) | CloseInvisible(_) | Literal(..) | Ident(..) | NtIdent(..)
|
||||
| Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof,
|
||||
_,
|
||||
) => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue