bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

rustc_ast: Harmonize delimiter naming with proc_macro::Delimiter

Browse source
This commit is contained in:
Vadim Petrochenkov 2022-04-26 15:40:14 +03:00
parent 0e7915d11f
commit 2733ec1be3
41 changed files with 433 additions and 426 deletions
Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_expand/src/config.rs
View file

@ -1,7 +1,7 @@
//! Conditional compilation stripping.
use rustc_ast::ptr::P;
use rustc_ast::token::{DelimToken, Token, TokenKind};
use rustc_ast::token::{Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
use rustc_ast::tokenstream::{DelimSpan, Spacing};
use rustc_ast::tokenstream::{LazyTokenStream, TokenTree};
@ -418,7 +418,7 @@ impl<'a> StripUnconfigured<'a> {
// in `#[attr]`, so just use the span of the `#` token.
let bracket_group = AttrAnnotatedTokenTree::Delimited(
DelimSpan::from_single(pound_span),
DelimToken::Bracket,
Delimiter::Bracket,
item.tokens
.as_ref()
.unwrap_or_else(|| panic!("Missing tokens for {:?}", item))

4
compiler/rustc_expand/src/expand.rs
View file

@ -8,7 +8,7 @@ use crate::placeholders::{placeholder, PlaceholderExpander};
use rustc_ast as ast;
use rustc_ast::mut_visit::*;
use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::token::{self, Delimiter};
use rustc_ast::tokenstream::TokenStream;
use rustc_ast::visit::{self, AssocCtxt, Visitor};
use rustc_ast::{AssocItemKind, AstLike, AstLikeWrapper, AttrStyle, ExprKind, ForeignItemKind};
@ -884,7 +884,7 @@ pub fn parse_ast_fragment<'a>(
AstFragmentKind::Stmts => {
let mut stmts = SmallVec::new();
// Won't make progress on a `}`.
while this.token != token::Eof && this.token != token::CloseDelim(token::Brace) {
while this.token != token::Eof && this.token != token::CloseDelim(Delimiter::Brace) {
if let Some(stmt) = this.parse_full_stmt(AttemptLocalParseRecovery::Yes)? {
stmts.push(stmt);
}

8
compiler/rustc_expand/src/mbe.rs
View file

@ -11,16 +11,16 @@ crate mod quoted;
crate mod transcribe;
use metavar_expr::MetaVarExpr;
use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
use rustc_ast::tokenstream::DelimSpan;
use rustc_span::symbol::Ident;
use rustc_span::Span;
/// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`. The delimiters
/// might be `NoDelim`, but they are not represented explicitly.
/// Contains the sub-token-trees of a "delimited" token tree such as `(a b c)`.
/// The delimiters are not represented explicitly in the `tts` vector.
#[derive(PartialEq, Encodable, Decodable, Debug)]
struct Delimited {
delim: token::DelimToken,
delim: Delimiter,
/// FIXME: #67062 has details about why this is sub-optimal.
tts: Vec<TokenTree>,
}

8
compiler/rustc_expand/src/mbe/macro_check.rs
View file

@ -106,7 +106,7 @@
//! bound.
use crate::mbe::{KleeneToken, TokenTree};
use rustc_ast::token::{DelimToken, Token, TokenKind};
use rustc_ast::token::{Delimiter, Token, TokenKind};
use rustc_ast::{NodeId, DUMMY_NODE_ID};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::MultiSpan;
@ -439,7 +439,7 @@ fn check_nested_occurrences(
}
(NestedMacroState::MacroRulesNotName, &TokenTree::Delimited(_, ref del))
| (NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del))
if del.delim == DelimToken::Brace =>
if del.delim == Delimiter::Brace =>
{
let macro_rules = state == NestedMacroState::MacroRulesNotName;
state = NestedMacroState::Empty;
@ -469,7 +469,7 @@ fn check_nested_occurrences(
check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
}
(NestedMacroState::MacroName, &TokenTree::Delimited(_, ref del))
if del.delim == DelimToken::Paren =>
if del.delim == Delimiter::Parenthesis =>
{
state = NestedMacroState::MacroNameParen;
nested_binders = Binders::default();
@ -484,7 +484,7 @@ fn check_nested_occurrences(
);
}
(NestedMacroState::MacroNameParen, &TokenTree::Delimited(_, ref del))
if del.delim == DelimToken::Brace =>
if del.delim == Delimiter::Brace =>
{
state = NestedMacroState::Empty;
check_occurrences(

6
compiler/rustc_expand/src/mbe/macro_rules.rs
View file

@ -8,7 +8,7 @@ use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree, MatcherLoc};
use crate::mbe::transcribe::transcribe;
use rustc_ast as ast;
use rustc_ast::token::{self, NonterminalKind, Token, TokenKind, TokenKind::*};
use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*};
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::{NodeId, DUMMY_NODE_ID};
use rustc_ast_pretty::pprust;
@ -1250,8 +1250,8 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
];
match tok {
TokenTree::Token(token) => match token.kind {
OpenDelim(token::DelimToken::Brace)
| OpenDelim(token::DelimToken::Bracket)
OpenDelim(Delimiter::Brace)
| OpenDelim(Delimiter::Bracket)
| Comma
| FatArrow
| Colon

4
compiler/rustc_expand/src/mbe/metavar_expr.rs
View file

@ -1,4 +1,4 @@
use rustc_ast::token;
use rustc_ast::token::{self, Delimiter};
use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree};
use rustc_ast::{LitIntType, LitKind};
use rustc_ast_pretty::pprust;
@ -35,7 +35,7 @@ impl MetaVarExpr {
) -> PResult<'sess, MetaVarExpr> {
let mut tts = input.trees();
let ident = parse_ident(&mut tts, sess, outer_span)?;
let Some(TokenTree::Delimited(_, token::Paren, args)) = tts.next() else {
let Some(TokenTree::Delimited(_, Delimiter::Parenthesis, args)) = tts.next() else {
let msg = "meta-variable expression parameter must be wrapped in parentheses";
return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
};

12
compiler/rustc_expand/src/mbe/quoted.rs
View file

@ -1,7 +1,7 @@
use crate::mbe::macro_parser::count_metavar_decls;
use crate::mbe::{Delimited, KleeneOp, KleeneToken, MetaVarExpr, SequenceRepetition, TokenTree};
use rustc_ast::token::{self, Token};
use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::{tokenstream, NodeId};
use rustc_ast_pretty::pprust;
use rustc_feature::Features;
@ -147,11 +147,11 @@ fn parse_tree(
match tree {
// `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
// FIXME: Handle `None`-delimited groups in a more systematic way
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing.
let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
if let Some(tokenstream::TokenTree::Delimited(_, token::NoDelim, tts)) = next {
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
trees = Box::new(tts.into_trees());
next = trees.next();
} else {
@ -162,7 +162,7 @@ fn parse_tree(
// `tree` is followed by a delimited set of token trees.
Some(tokenstream::TokenTree::Delimited(delim_span, delim, tts)) => {
if parsing_patterns {
if delim != token::Paren {
if delim != Delimiter::Parenthesis {
span_dollar_dollar_or_metavar_in_the_lhs_err(
sess,
&Token { kind: token::OpenDelim(delim), span: delim_span.entire() },
@ -170,7 +170,7 @@ fn parse_tree(
}
} else {
match delim {
token::Brace => {
Delimiter::Brace => {
// The delimiter is `{`. This indicates the beginning
// of a meta-variable expression (e.g. `${count(ident)}`).
// Try to parse the meta-variable expression.
@ -191,7 +191,7 @@ fn parse_tree(
}
}
}
token::Paren => {}
Delimiter::Parenthesis => {}
_ => {
let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(` or `{{`, found `{}`", tok);

25
compiler/rustc_expand/src/mbe/transcribe.rs
View file

@ -2,7 +2,7 @@ use crate::base::ExtCtxt;
use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch};
use crate::mbe::{self, MetaVarExpr};
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Token, TokenKind};
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{pluralize, PResult};
@ -27,23 +27,14 @@ impl MutVisitor for Marker {
/// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
enum Frame<'a> {
Delimited {
tts: &'a [mbe::TokenTree],
idx: usize,
delim_token: token::DelimToken,
span: DelimSpan,
},
Sequence {
tts: &'a [mbe::TokenTree],
idx: usize,
sep: Option<Token>,
},
Delimited { tts: &'a [mbe::TokenTree], idx: usize, delim: Delimiter, span: DelimSpan },
Sequence { tts: &'a [mbe::TokenTree], idx: usize, sep: Option<Token> },
}
impl<'a> Frame<'a> {
/// Construct a new frame around the delimited set of tokens.
fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> {
Frame::Delimited { tts: &src.tts, idx: 0, delim_token: src.delim, span }
Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span }
}
}
@ -150,14 +141,14 @@ pub(super) fn transcribe<'a>(
// We are done processing a Delimited. If this is the top-level delimited, we are
// done. Otherwise, we unwind the result_stack to append what we have produced to
// any previous results.
Frame::Delimited { delim_token, span, .. } => {
Frame::Delimited { delim, span, .. } => {
if result_stack.is_empty() {
// No results left to compute! We are back at the top-level.
return Ok(TokenStream::new(result));
}
// Step back into the parent Delimited.
let tree = TokenTree::Delimited(span, delim_token, TokenStream::new(result));
let tree = TokenTree::Delimited(span, delim, TokenStream::new(result));
result = result_stack.pop().unwrap();
result.push(tree.into());
}
@ -240,7 +231,7 @@ pub(super) fn transcribe<'a>(
}
MatchedNonterminal(ref nt) => {
// Other variables are emitted into the output stream as groups with
// `Delimiter::None` to maintain parsing priorities.
// `Delimiter::Invisible` to maintain parsing priorities.
// `Interpolated` is currently used for such groups in rustc parser.
marker.visit_span(&mut sp);
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
@ -278,7 +269,7 @@ pub(super) fn transcribe<'a>(
mut_visit::visit_delim_span(&mut span, &mut marker);
stack.push(Frame::Delimited {
tts: &delimited.tts,
delim_token: delimited.delim,
delim: delimited.delim,
idx: 0,
span,
});

14
compiler/rustc_expand/src/parse/tests.rs
View file

@ -1,7 +1,7 @@
use crate::tests::{matches_codepattern, string_to_stream, with_error_checking_parse};
use rustc_ast::ptr::P;
use rustc_ast::token::{self, Token};
use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
use rustc_ast::visit;
use rustc_ast::{self as ast, PatKind};
@ -77,13 +77,14 @@ fn string_to_tts_macro() {
TokenTree::Delimited(_, first_delim, first_tts),
TokenTree::Token(Token { kind: token::FatArrow, .. }),
TokenTree::Delimited(_, second_delim, second_tts),
] if macro_delim == &token::Paren => {
] if macro_delim == &Delimiter::Parenthesis => {
let tts = &first_tts.trees().collect::<Vec<_>>();
match &tts[..] {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
] if first_delim == &token::Paren && name.as_str() == "a" => {}
] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => {
}
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
}
let tts = &second_tts.trees().collect::<Vec<_>>();
@ -91,7 +92,8 @@ fn string_to_tts_macro() {
[
TokenTree::Token(Token { kind: token::Dollar, .. }),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
] if second_delim == &token::Paren && name.as_str() == "a" => {}
] if second_delim == &Delimiter::Parenthesis
&& name.as_str() == "a" => {}
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
}
}
@ -113,7 +115,7 @@ fn string_to_tts_1() {
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(3, 4)).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
token::DelimToken::Paren,
Delimiter::Parenthesis,
TokenStream::new(vec![
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(6, 7)).into(),
TokenTree::token(token::Colon, sp(8, 9)).into(),
@ -124,7 +126,7 @@ fn string_to_tts_1() {
.into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
token::DelimToken::Brace,
Delimiter::Brace,
TokenStream::new(vec![
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(17, 18)).into(),
TokenTree::token(token::Semi, sp(18, 19)).into(),

30
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -28,24 +28,24 @@ trait ToInternal<T> {
fn to_internal(self) -> T;
}
impl FromInternal<token::DelimToken> for Delimiter {
fn from_internal(delim: token::DelimToken) -> Delimiter {
impl FromInternal<token::Delimiter> for Delimiter {
fn from_internal(delim: token::Delimiter) -> Delimiter {
match delim {
token::Paren => Delimiter::Parenthesis,
token::Brace => Delimiter::Brace,
token::Bracket => Delimiter::Bracket,
token::NoDelim => Delimiter::None,
token::Delimiter::Parenthesis => Delimiter::Parenthesis,
token::Delimiter::Brace => Delimiter::Brace,
token::Delimiter::Bracket => Delimiter::Bracket,
token::Delimiter::Invisible => Delimiter::None,
}
}
}
impl ToInternal<token::DelimToken> for Delimiter {
fn to_internal(self) -> token::DelimToken {
impl ToInternal<token::Delimiter> for Delimiter {
fn to_internal(self) -> token::Delimiter {
match self {
Delimiter::Parenthesis => token::Paren,
Delimiter::Brace => token::Brace,
Delimiter::Bracket => token::Bracket,
Delimiter::None => token::NoDelim,
Delimiter::Parenthesis => token::Delimiter::Parenthesis,
Delimiter::Brace => token::Delimiter::Brace,
Delimiter::Bracket => token::Delimiter::Bracket,
Delimiter::None => token::Delimiter::Invisible,
}
}
}
@ -61,7 +61,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
let joint = spacing == Joint;
let Token { kind, span } = match tree {
tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = Delimiter::from_internal(delim);
let delimiter = pm::Delimiter::from_internal(delim);
return TokenTree::Group(Group { delimiter, stream: tts, span, flatten: false });
}
tokenstream::TokenTree::Token(token) => token,
@ -164,7 +164,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
.map(|kind| tokenstream::TokenTree::token(kind, span))
.collect();
stack.push(TokenTree::Group(Group {
delimiter: Delimiter::Bracket,
delimiter: pm::Delimiter::Bracket,
stream,
span: DelimSpan::from_single(span),
flatten: false,
@ -181,7 +181,7 @@ impl FromInternal<(TreeAndSpacing, &'_ mut Vec<Self>, &mut Rustc<'_, '_>)>
Interpolated(nt) => {
let stream = nt_to_tokenstream(&nt, rustc.sess(), CanSynthesizeMissingTokens::No);
TokenTree::Group(Group {
delimiter: Delimiter::None,
delimiter: pm::Delimiter::None,
stream,
span: DelimSpan::from_single(span),
flatten: crate::base::pretty_printing_compatibility_hack(&nt, rustc.sess()),