bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Remove NtIdent and NtLifetime.

Browse source 
The extra span is now recorded in the new `TokenKind::NtIdent` and
`TokenKind::NtLifetime`. These both consist of a single token, and so
there's no operator precedence problems with inserting them directly
into the token stream.

The other way to do this would be to wrap the ident/lifetime in invisible
delimiters, but there's a lot of code that assumes an interpolated
ident/lifetime fits in a single token, and changing all that code to work with
invisible delimiters would have been a pain. (Maybe it could be done in a
follow-up.)

This change might not seem like much of a win, but it's a first step toward the
much bigger and long-desired removal of `Nonterminal` and
`TokenKind::Interpolated`. That change is big and complex enough that it's
worth doing this piece separately. (Indeed, this commit is based on part of a
late commit in #114647, a prior attempt at that big and complex change.)
This commit is contained in:
Nicholas Nethercote 2024-04-22 19:46:51 +10:00
parent 9a63a42cb7
commit 95e519ecbf
11 changed files with 131 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_parse/src/parser/expr.rs
View file

@ -724,7 +724,9 @@ impl<'a> Parser<'a> {
/// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
match self.prev_token.kind {
TokenKind::Interpolated(..) => self.prev_token.span,
TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
self.prev_token.span
}
_ => expr.span,
}
}

6
compiler/rustc_parse/src/parser/mod.rs
View file

@ -407,6 +407,8 @@ pub(super) fn token_descr(token: &Token) -> String {
(Some(TokenDescription::Keyword), _) => Some("keyword"),
(Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
(Some(TokenDescription::DocComment), _) => Some("doc comment"),
(None, TokenKind::NtIdent(..)) => Some("identifier"),
(None, TokenKind::NtLifetime(..)) => Some("lifetime"),
(None, TokenKind::Interpolated(node)) => Some(node.descr()),
(None, _) => None,
};
@ -1633,5 +1635,9 @@ pub enum FlatToken {
#[derive(Clone, Debug)]
pub enum ParseNtResult {
Tt(TokenTree),
Ident(Ident, IdentIsRaw),
Lifetime(Ident),
/// This case will eventually be removed, along with `Token::Interpolate`.
Nt(Lrc<Nonterminal>),
}

53
compiler/rustc_parse/src/parser/nonterminal.rs
View file

@ -25,15 +25,13 @@ impl<'a> Parser<'a> {
| NtPat(_)
| NtExpr(_)
| NtTy(_)
| NtIdent(..)
| NtLiteral(_) // `true`, `false`
| NtMeta(_)
| NtPath(_) => true,
NtItem(_)
| NtBlock(_)
| NtVis(_)
| NtLifetime(_) => false,
| NtVis(_) => false,
}
}
@ -50,25 +48,30 @@ impl<'a> Parser<'a> {
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
NonterminalKind::Vis => match token.kind {
// The follow-set of :vis + "priv" keyword + interpolated
token::Comma | token::Ident(..) | token::Interpolated(_) => true,
token::Comma
| token::Ident(..)
| token::NtIdent(..)
| token::NtLifetime(..)
| token::Interpolated(_) => true,
_ => token.can_begin_type(),
},
NonterminalKind::Block => match &token.kind {
token::OpenDelim(Delimiter::Brace) => true,
token::NtLifetime(..) => true,
token::Interpolated(nt) => match &**nt {
NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_)
| NtVis(_) => false,
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
},
_ => false,
},
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
token::PathSep | token::Ident(..) => true,
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
token::Interpolated(nt) => may_be_ident(nt),
_ => false,
},
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind {
token::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
// box, ref, mut, and other identifiers (can stricten)
token::Ident(..) | token::NtIdent(..) |
token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern
token::OpenDelim(Delimiter::Bracket) | // slice pattern
token::BinOp(token::And) | // reference
@ -86,10 +89,7 @@ impl<'a> Parser<'a> {
_ => false,
},
NonterminalKind::Lifetime => match &token.kind {
token::Lifetime(_) => true,
token::Interpolated(nt) => {
matches!(&**nt, NtLifetime(_))
}
token::Lifetime(_) | token::NtLifetime(..) => true,
_ => false,
},
NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => {
@ -154,15 +154,16 @@ impl<'a> Parser<'a> {
}
// this could be handled like a token, since it is one
NonterminalKind::Ident if let Some((ident, is_raw)) = get_macro_ident(&self.token) => {
self.bump();
NtIdent(ident, is_raw)
}
NonterminalKind::Ident => {
return Err(self.dcx().create_err(UnexpectedNonterminal::Ident {
span: self.token.span,
token: self.token.clone(),
}));
return if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
self.bump();
Ok(ParseNtResult::Ident(ident, is_raw))
} else {
Err(self.dcx().create_err(UnexpectedNonterminal::Ident {
span: self.token.span,
token: self.token.clone(),
}))
};
}
NonterminalKind::Path => {
NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?))
@ -173,14 +174,14 @@ impl<'a> Parser<'a> {
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))
}
NonterminalKind::Lifetime => {
if self.check_lifetime() {
NtLifetime(self.expect_lifetime().ident)
return if self.check_lifetime() {
Ok(ParseNtResult::Lifetime(self.expect_lifetime().ident))
} else {
return Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime {
Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime {
span: self.token.span,
token: self.token.clone(),
}));
}
}))
};
}
};