bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Replace ast::TokenKind::BinOp{,Eq} and remove BinOpToken.

Browse source 
`BinOpToken` is badly named, because it only covers the assignable
binary ops and excludes comparisons and `&&`/`||`. Its use in
`ast::TokenKind` does allow a small amount of code sharing, but it's a
clumsy factoring.

This commit removes `ast::TokenKind::BinOp{,Eq}`, replacing each one
with 10 individual variants. This makes `ast::TokenKind` more similar to
`rustc_lexer::TokenKind`, which has individual variants for all
operators.

Although the number of lines of code increases, the number of chars
decreases due to the frequent use of shorter names like `token::Plus`
instead of `token::BinOp(BinOpToken::Plus)`.
This commit is contained in:
Nicholas Nethercote 2024-12-20 07:28:16 +11:00
parent 7c4a55c2ac
commit 2a1e2e9632
19 changed files with 352 additions and 309 deletions
Download patch file Download diff file Expand all files Collapse all files

10
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -1135,7 +1135,7 @@ fn check_matcher_core<'tt>(
&& matches!(kind, NonterminalKind::Pat(PatParam { inferred: true }))
&& matches!(
next_token,
TokenTree::Token(token) if *token == BinOp(token::BinOpToken::Or)
TokenTree::Token(token) if *token == token::Or
)
{
// It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
@ -1177,7 +1177,7 @@ fn check_matcher_core<'tt>(
if kind == NonterminalKind::Pat(PatWithOr)
&& sess.psess.edition.at_least_rust_2021()
&& next_token.is_token(&BinOp(token::BinOpToken::Or))
&& next_token.is_token(&token::Or)
{
let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
span,
@ -1296,7 +1296,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"];
match tok {
TokenTree::Token(token) => match token.kind {
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
FatArrow | Comma | Eq | Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::If || name == kw::In => {
IsInFollow::Yes
}
@ -1332,9 +1332,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
| Colon
| Eq
| Gt
| BinOp(token::Shr)
| Shr
| Semi
| BinOp(token::Or) => IsInFollow::Yes,
| Or => IsInFollow::Yes,
Ident(name, IdentIsRaw::No) if name == kw::As || name == kw::Where => {
IsInFollow::Yes
}

4
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -302,8 +302,8 @@ fn parse_tree<'a>(
/// `None`.
fn kleene_op(token: &Token) -> Option<KleeneOp> {
match token.kind {
token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore),
token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore),
token::Star => Some(KleeneOp::ZeroOrMore),
token::Plus => Some(KleeneOp::OneOrMore),
token::Question => Some(KleeneOp::ZeroOrOne),
_ => None,
}

67
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -182,26 +182,26 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
OrOr => op("||"),
Not => op("!"),
Tilde => op("~"),
BinOp(Plus) => op("+"),
BinOp(Minus) => op("-"),
BinOp(Star) => op("*"),
BinOp(Slash) => op("/"),
BinOp(Percent) => op("%"),
BinOp(Caret) => op("^"),
BinOp(And) => op("&"),
BinOp(Or) => op("|"),
BinOp(Shl) => op("<<"),
BinOp(Shr) => op(">>"),
BinOpEq(Plus) => op("+="),
BinOpEq(Minus) => op("-="),
BinOpEq(Star) => op("*="),
BinOpEq(Slash) => op("/="),
BinOpEq(Percent) => op("%="),
BinOpEq(Caret) => op("^="),
BinOpEq(And) => op("&="),
BinOpEq(Or) => op("|="),
BinOpEq(Shl) => op("<<="),
BinOpEq(Shr) => op(">>="),
Plus => op("+"),
Minus => op("-"),
Star => op("*"),
Slash => op("/"),
Percent => op("%"),
Caret => op("^"),
And => op("&"),
Or => op("|"),
Shl => op("<<"),
Shr => op(">>"),
PlusEq => op("+="),
MinusEq => op("-="),
StarEq => op("*="),
SlashEq => op("/="),
PercentEq => op("%="),
CaretEq => op("^="),
AndEq => op("&="),
OrEq => op("|="),
ShlEq => op("<<="),
ShrEq => op(">>="),
At => op("@"),
Dot => op("."),
DotDot => op(".."),
@ -324,14 +324,14 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
b'>' => Gt,
b'!' => Not,
b'~' => Tilde,
b'+' => BinOp(Plus),
b'-' => BinOp(Minus),
b'*' => BinOp(Star),
b'/' => BinOp(Slash),
b'%' => BinOp(Percent),
b'^' => BinOp(Caret),
b'&' => BinOp(And),
b'|' => BinOp(Or),
b'+' => Plus,
b'-' => Minus,
b'*' => Star,
b'/' => Slash,
b'%' => Percent,
b'^' => Caret,
b'&' => And,
b'|' => Or,
b'@' => At,
b'.' => Dot,
b',' => Comma,
@ -372,10 +372,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(integer, span);
smallvec![a, b]
}
@ -385,10 +384,9 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
suffix,
span,
}) if symbol.as_str().starts_with('-') => {
let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
let a = tokenstream::TokenTree::token_joint_hidden(Minus, span);
let b = tokenstream::TokenTree::token_alone(float, span);
smallvec![a, b]
}
@ -599,10 +597,7 @@ impl server::TokenStream for Rustc<'_, '_> {
Ok(Self::TokenStream::from_iter([
// FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token_joint_hidden(
token::BinOp(token::Minus),
e.span,
),
tokenstream::TokenTree::token_joint_hidden(token::Minus, e.span),
tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
]))
}