1
Fork 0

Remove NtVis.

We now use invisible delimiters for expanded `vis` fragments, instead of
`Token::Interpolated`.
This commit is contained in:
Nicholas Nethercote 2024-04-17 12:17:09 +10:00
parent 5986ff05d8
commit c7981d6411
8 changed files with 110 additions and 18 deletions

View file

@ -206,7 +206,6 @@ impl HasTokens for Nonterminal {
Nonterminal::NtTy(ty) => ty.tokens(),
Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
Nonterminal::NtPath(path) => path.tokens(),
Nonterminal::NtVis(vis) => vis.tokens(),
Nonterminal::NtBlock(block) => block.tokens(),
}
}
@ -219,7 +218,6 @@ impl HasTokens for Nonterminal {
Nonterminal::NtTy(ty) => ty.tokens_mut(),
Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
Nonterminal::NtPath(path) => path.tokens_mut(),
Nonterminal::NtVis(vis) => vis.tokens_mut(),
Nonterminal::NtBlock(block) => block.tokens_mut(),
}
}

View file

@ -916,7 +916,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
visit_lazy_tts(vis, tokens);
}
token::NtPath(path) => vis.visit_path(path),
token::NtVis(visib) => vis.visit_vis(visib),
}
}

View file

@ -969,6 +969,15 @@ impl Token {
}
}
/// Is this an invisible open delimiter at the start of a token sequence
/// from an expanded metavar?
pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
match self.kind {
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
_ => None,
}
}
pub fn glue(&self, joint: &Token) -> Option<Token> {
let kind = match self.kind {
Eq => match joint.kind {
@ -1072,7 +1081,6 @@ pub enum Nonterminal {
/// Stuff inside brackets for attributes
NtMeta(P<ast::AttrItem>),
NtPath(P<ast::Path>),
NtVis(P<ast::Visibility>),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@ -1169,7 +1177,6 @@ impl Nonterminal {
NtTy(ty) => ty.span,
NtMeta(attr_item) => attr_item.span(),
NtPath(path) => path.span,
NtVis(vis) => vis.span,
}
}
@ -1184,7 +1191,6 @@ impl Nonterminal {
NtTy(..) => "type",
NtMeta(..) => "attribute",
NtPath(..) => "path",
NtVis(..) => "visibility",
}
}
}
@ -1211,7 +1217,6 @@ impl fmt::Debug for Nonterminal {
NtLiteral(..) => f.pad("NtLiteral(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtVis(..) => f.pad("NtVis(..)"),
}
}
}

View file

@ -472,7 +472,6 @@ impl TokenStream {
Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
Nonterminal::NtPath(path) => TokenStream::from_ast(path),
Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
}
}

View file

@ -3,7 +3,10 @@ use std::sync::Arc;
use rustc_ast::ExprKind;
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind};
use rustc_ast::token::{
self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token,
TokenKind,
};
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
@ -274,6 +277,33 @@ pub(super) fn transcribe<'a>(
// some of the unnecessary whitespace.
let ident = MacroRulesNormalizedIdent::new(original_ident);
if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) {
// We wrap the tokens in invisible delimiters, unless they are already wrapped
// in invisible delimiters with the same `MetaVarKind`. Because some proc
// macros can't multiple layers of invisible delimiters of the same
// `MetaVarKind`. This loses some span info, though it hopefully won't matter.
let mut mk_delimited = |mv_kind, mut stream: TokenStream| {
if stream.len() == 1 {
let tree = stream.iter().next().unwrap();
if let TokenTree::Delimited(_, _, delim, inner) = tree
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim
&& mv_kind == *mvk
{
stream = inner.clone();
}
}
// Emit as a token stream within `Delimiter::Invisible` to maintain
// parsing priorities.
marker.visit_span(&mut sp);
// Both the open delim and close delim get the same span, which covers the
// `$foo` in the decl macro RHS.
TokenTree::Delimited(
DelimSpan::from_single(sp),
DelimSpacing::new(Spacing::Alone, Spacing::Alone),
Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)),
stream,
)
};
let tt = match cur_matched {
MatchedSingle(ParseNtResult::Tt(tt)) => {
// `tt`s are emitted into the output stream directly as "raw tokens",
@ -292,6 +322,9 @@ pub(super) fn transcribe<'a>(
let kind = token::NtLifetime(*ident, *is_raw);
TokenTree::token_alone(kind, sp)
}
MatchedSingle(ParseNtResult::Vis(vis)) => {
mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis))
}
MatchedSingle(ParseNtResult::Nt(nt)) => {
// Other variables are emitted into the output stream as groups with
// `Delimiter::Invisible` to maintain parsing priorities.

View file

@ -721,6 +721,43 @@ impl<'a> Parser<'a> {
if !self.eat_keyword(exp) { self.unexpected() } else { Ok(()) }
}
/// Consume a sequence produced by a metavar expansion, if present.
fn eat_metavar_seq<T>(
&mut self,
mv_kind: MetaVarKind,
f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
) -> Option<T> {
self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f)
}
/// A slightly more general form of `eat_metavar_seq`, for use with the
/// `MetaVarKind` variants that have parameters, where an exact match isn't
/// desired.
fn eat_metavar_seq_with_matcher<T>(
&mut self,
match_mv_kind: impl Fn(MetaVarKind) -> bool,
mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
) -> Option<T> {
if let token::OpenDelim(delim) = self.token.kind
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
&& match_mv_kind(mv_kind)
{
self.bump();
let res = f(self).expect("failed to reparse {mv_kind:?}");
if let token::CloseDelim(delim) = self.token.kind
&& let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim
&& match_mv_kind(mv_kind)
{
self.bump();
Some(res)
} else {
panic!("no close delim when reparsing {mv_kind:?}");
}
} else {
None
}
}
/// Is the given keyword `kw` followed by a non-reserved identifier?
fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
@ -1455,7 +1492,11 @@ impl<'a> Parser<'a> {
/// so emit a proper diagnostic.
// Public for rustfmt usage.
pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
maybe_whole!(self, NtVis, |vis| vis.into_inner());
if let Some(vis) = self
.eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes))
{
return Ok(vis);
}
if !self.eat_keyword(exp!(Pub)) {
// We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
@ -1683,7 +1724,8 @@ pub enum ParseNtResult {
Tt(TokenTree),
Ident(Ident, IdentIsRaw),
Lifetime(Ident, IdentIsRaw),
Vis(P<ast::Visibility>),
/// This case will eventually be removed, along with `Token::Interpolate`.
/// This variant will eventually be removed, along with `Token::Interpolate`.
Nt(Arc<Nonterminal>),
}

View file

@ -56,9 +56,7 @@ impl<'a> Parser<'a> {
| NtMeta(_)
| NtPath(_) => true,
NtItem(_)
| NtBlock(_)
| NtVis(_) => false,
NtItem(_) | NtBlock(_) => false,
}
}
@ -88,7 +86,7 @@ impl<'a> Parser<'a> {
NonterminalKind::Ident => get_macro_ident(token).is_some(),
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
NonterminalKind::Vis => match token.kind {
// The follow-set of :vis + "priv" keyword + interpolated
// The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion.
token::Comma
| token::Ident(..)
| token::NtIdent(..)
@ -102,7 +100,7 @@ impl<'a> Parser<'a> {
token::NtLifetime(..) => true,
token::Interpolated(nt) => match &**nt {
NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) => false,
},
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
MetaVarKind::Block
@ -208,8 +206,9 @@ impl<'a> Parser<'a> {
}
NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(ForceCollect::Yes)?)),
NonterminalKind::Vis => {
NtVis(P(self
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))
return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| {
this.parse_visibility(FollowedByType::Yes)
})?)));
}
NonterminalKind::Lifetime => {
// We want to keep `'keyword` parsing, just like `keyword` is still

View file

@ -0,0 +1,17 @@
//@ check-pass
//
// A test case where a `block` fragment specifier is interpreted as an `expr`
// fragment specifier. It's an interesting case for the handling of invisible
// delimiters.
macro_rules! m_expr {
($e:expr) => { const _CURRENT: u32 = $e; };
}
macro_rules! m_block {
($b:block) => ( m_expr!($b); );
}
fn main() {
m_block!({ 1 });
}