bjoernager/rust
bjoernager/rust
1
Fork 0
Code Activity

Auto merge of #139897 - nnethercote:rm-OpenDelim-CloseDelim, r=petrochenkov

Browse source 
Remove `token::{Open,Close}Delim`

By replacing them with `{Open,Close}{Param,Brace,Bracket,Invisible}`.

PR #137902 made `ast::TokenKind` more like `lexer::TokenKind` by
replacing the compound `BinOp{,Eq}(BinOpToken)` variants with fieldless
variants `Plus`, `Minus`, `Star`, etc. This commit does a similar thing
with delimiters. It also makes `ast::TokenKind` more similar to
`parser::TokenType`.

This requires a few new methods:
- `TokenKind::is_{,open_,close_}delim()` replace various kinds of
  pattern matches.
- `Delimiter::as_{open,close}_token_kind` are used to convert
  `Delimiter` values to `TokenKind`.

Despite these additions, it's a net reduction in lines of code. This is
because e.g. `token::OpenParen` is so much shorter than
`token::OpenDelim(Delimiter::Parenthesis)` that many multi-line forms
reduce to single line forms. And many places where the number of lines
doesn't change are still easier to read, just because the names are
shorter, e.g.:
```
-   } else if self.token != token::CloseDelim(Delimiter::Brace) {
+   } else if self.token != token::CloseBrace {
```

r? `@petrochenkov`
This commit is contained in:
bors 2025-04-22 01:15:06 +00:00
commit fae7785b60
30 changed files with 456 additions and 498 deletions
Download patch file Download diff file Expand all files Collapse all files

5
compiler/rustc_expand/src/config.rs
View file

@ -237,10 +237,7 @@ impl<'a> StripUnconfigured<'a> {
inner = self.configure_tokens(&inner);
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner))
}
AttrTokenTree::Token(
Token { kind: TokenKind::OpenDelim(_) | TokenKind::CloseDelim(_), .. },
_,
) => {
AttrTokenTree::Token(Token { kind, .. }, _) if kind.is_delim() => {
panic!("Should be `AttrTokenTree::Delimited`, not delim tokens: {:?}", tree);
}
AttrTokenTree::Token(token, spacing) => Some(AttrTokenTree::Token(token, spacing)),

5
compiler/rustc_expand/src/expand.rs
View file

@ -7,13 +7,12 @@ use std::{iter, mem};
use rustc_ast as ast;
use rustc_ast::mut_visit::*;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter};
use rustc_ast::tokenstream::TokenStream;
use rustc_ast::visit::{self, AssocCtxt, Visitor, VisitorResult, try_visit, walk_list};
use rustc_ast::{
AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind,
HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner, MetaItemKind, ModKind,
NodeId, PatKind, StmtKind, TyKind,
NodeId, PatKind, StmtKind, TyKind, token,
};
use rustc_ast_pretty::pprust;
use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
@ -1004,7 +1003,7 @@ pub fn parse_ast_fragment<'a>(
AstFragmentKind::Stmts => {
let mut stmts = SmallVec::new();
// Won't make progress on a `}`.
while this.token != token::Eof && this.token != token::CloseDelim(Delimiter::Brace) {
while this.token != token::Eof && this.token != token::CloseBrace {
if let Some(stmt) = this.parse_full_stmt(AttemptLocalParseRecovery::Yes)? {
stmts.push(stmt);
}

6
compiler/rustc_expand/src/mbe/diagnostics.rs
View file

@ -1,6 +1,6 @@
use std::borrow::Cow;
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::token::{self, Token};
use rustc_ast::tokenstream::TokenStream;
use rustc_errors::{Applicability, Diag, DiagCtxtHandle, DiagMessage};
use rustc_macros::Subdiagnostic;
@ -66,8 +66,8 @@ pub(super) fn failed_to_match_macro(
}
if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
&& (matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
|| matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
&& (matches!(expected_token.kind, token::OpenInvisible(_))
|| matches!(token.kind, token::OpenInvisible(_)))
{
err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");

4
compiler/rustc_expand/src/mbe/macro_parser.rs
View file

@ -182,8 +182,8 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
locs.push(MatcherLoc::Token { token: *token });
}
TokenTree::Delimited(span, _, delimited) => {
let open_token = Token::new(token::OpenDelim(delimited.delim), span.open);
let close_token = Token::new(token::CloseDelim(delimited.delim), span.close);
let open_token = Token::new(delimited.delim.as_open_token_kind(), span.open);
let close_token = Token::new(delimited.delim.as_close_token_kind(), span.close);
locs.push(MatcherLoc::Delimited);
locs.push(MatcherLoc::Token { token: open_token });

24
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -6,7 +6,7 @@ use std::{mem, slice};
use ast::token::IdentIsRaw;
use rustc_ast::token::NtPatKind::*;
use rustc_ast::token::TokenKind::*;
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind};
use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::{self as ast, DUMMY_NODE_ID, NodeId};
use rustc_ast_pretty::pprust;
@ -784,7 +784,7 @@ impl<'tt> FirstSets<'tt> {
TokenTree::Delimited(span, _, delimited) => {
build_recur(sets, &delimited.tts);
first.replace_with(TtHandle::from_token_kind(
token::OpenDelim(delimited.delim),
delimited.delim.as_open_token_kind(),
span.open,
));
}
@ -852,7 +852,7 @@ impl<'tt> FirstSets<'tt> {
}
TokenTree::Delimited(span, _, delimited) => {
first.add_one(TtHandle::from_token_kind(
token::OpenDelim(delimited.delim),
delimited.delim.as_open_token_kind(),
span.open,
));
return first;
@ -1099,7 +1099,7 @@ fn check_matcher_core<'tt>(
}
TokenTree::Delimited(span, _, d) => {
let my_suffix = TokenSet::singleton(TtHandle::from_token_kind(
token::CloseDelim(d.delim),
d.delim.as_close_token_kind(),
span.close,
));
check_matcher_core(sess, node_id, first_sets, &d.tts, &my_suffix)?;
@ -1299,7 +1299,9 @@ enum IsInFollow {
fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
use mbe::TokenTree;
if let TokenTree::Token(Token { kind: token::CloseDelim(_), .. }) = *tok {
if let TokenTree::Token(Token { kind, .. }) = tok
&& kind.close_delim().is_some()
{
// closing a token tree can never be matched by any fragment;
// iow, we always require that `(` and `)` match, etc.
IsInFollow::Yes
@ -1358,16 +1360,8 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
];
match tok {
TokenTree::Token(token) => match token.kind {
OpenDelim(Delimiter::Brace)
| OpenDelim(Delimiter::Bracket)
| Comma
| FatArrow
| Colon
| Eq
| Gt
| Shr
| Semi
| Or => IsInFollow::Yes,
OpenBrace | OpenBracket | Comma | FatArrow | Colon | Eq | Gt | Shr
| Semi | Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
IsInFollow::Yes
}

8
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -181,7 +181,10 @@ fn parse_tree<'a>(
if delim != Delimiter::Parenthesis {
span_dollar_dollar_or_metavar_in_the_lhs_err(
sess,
&Token { kind: token::OpenDelim(delim), span: delim_span.entire() },
&Token {
kind: delim.as_open_token_kind(),
span: delim_span.entire(),
},
);
}
} else {
@ -217,7 +220,8 @@ fn parse_tree<'a>(
}
Delimiter::Parenthesis => {}
_ => {
let token = pprust::token_kind_to_string(&token::OpenDelim(delim));
let token =
pprust::token_kind_to_string(&delim.as_open_token_kind());
sess.dcx().emit_err(errors::ExpectedParenOrBrace {
span: delim_span.entire(),
token,

4
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -308,8 +308,8 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
}));
}
OpenDelim(..) | CloseDelim(..) => unreachable!(),
Eof => unreachable!(),
OpenParen | CloseParen | OpenBrace | CloseBrace | OpenBracket | CloseBracket
| OpenInvisible(_) | CloseInvisible(_) | Eof => unreachable!(),
}
}
trees