diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index 60f8c6e1048..a2923d6448a 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -206,7 +206,6 @@ impl HasTokens for Nonterminal { Nonterminal::NtTy(ty) => ty.tokens(), Nonterminal::NtMeta(attr_item) => attr_item.tokens(), Nonterminal::NtPath(path) => path.tokens(), - Nonterminal::NtVis(vis) => vis.tokens(), Nonterminal::NtBlock(block) => block.tokens(), } } @@ -219,7 +218,6 @@ impl HasTokens for Nonterminal { Nonterminal::NtTy(ty) => ty.tokens_mut(), Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(), Nonterminal::NtPath(path) => path.tokens_mut(), - Nonterminal::NtVis(vis) => vis.tokens_mut(), Nonterminal::NtBlock(block) => block.tokens_mut(), } } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index de9f049704a..47cf77cefa7 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -916,7 +916,6 @@ fn visit_nonterminal(vis: &mut T, nt: &mut token::Nonterminal) { visit_lazy_tts(vis, tokens); } token::NtPath(path) => vis.visit_path(path), - token::NtVis(visib) => vis.visit_vis(visib), } } diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index 36a7b7d8789..661f2c3eca2 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -969,6 +969,15 @@ impl Token { } } + /// Is this an invisible open delimiter at the start of a token sequence + /// from an expanded metavar? + pub fn is_metavar_seq(&self) -> Option { + match self.kind { + OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind), + _ => None, + } + } + pub fn glue(&self, joint: &Token) -> Option { let kind = match self.kind { Eq => match joint.kind { @@ -1072,7 +1081,6 @@ pub enum Nonterminal { /// Stuff inside brackets for attributes NtMeta(P), NtPath(P), - NtVis(P), } #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)] @@ -1169,7 +1177,6 @@ impl Nonterminal { NtTy(ty) => ty.span, NtMeta(attr_item) => attr_item.span(), NtPath(path) => path.span, - NtVis(vis) => vis.span, } } @@ -1184,7 +1191,6 @@ impl Nonterminal { NtTy(..) => "type", NtMeta(..) => "attribute", NtPath(..) => "path", - NtVis(..) => "visibility", } } } @@ -1211,7 +1217,6 @@ impl fmt::Debug for Nonterminal { NtLiteral(..) => f.pad("NtLiteral(..)"), NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), - NtVis(..) => f.pad("NtVis(..)"), } } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 50f10d083a0..1b6d825a22c 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -472,7 +472,6 @@ impl TokenStream { Nonterminal::NtTy(ty) => TokenStream::from_ast(ty), Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr), Nonterminal::NtPath(path) => TokenStream::from_ast(path), - Nonterminal::NtVis(vis) => TokenStream::from_ast(vis), Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr), } } diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 595c8c3279f..1723b47798b 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use rustc_ast::ExprKind; use rustc_ast::mut_visit::{self, MutVisitor}; -use rustc_ast::token::{self, Delimiter, IdentIsRaw, Lit, LitKind, Nonterminal, Token, TokenKind}; +use rustc_ast::token::{ + self, Delimiter, IdentIsRaw, InvisibleOrigin, Lit, LitKind, MetaVarKind, Nonterminal, Token, + TokenKind, +}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize}; @@ -274,6 +277,33 @@ pub(super) fn transcribe<'a>( // some of the unnecessary whitespace. let ident = MacroRulesNormalizedIdent::new(original_ident); if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { + // We wrap the tokens in invisible delimiters, unless they are already wrapped + // in invisible delimiters with the same `MetaVarKind`. Because some proc + // macros can't multiple layers of invisible delimiters of the same + // `MetaVarKind`. This loses some span info, though it hopefully won't matter. + let mut mk_delimited = |mv_kind, mut stream: TokenStream| { + if stream.len() == 1 { + let tree = stream.iter().next().unwrap(); + if let TokenTree::Delimited(_, _, delim, inner) = tree + && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mvk)) = delim + && mv_kind == *mvk + { + stream = inner.clone(); + } + } + + // Emit as a token stream within `Delimiter::Invisible` to maintain + // parsing priorities. + marker.visit_span(&mut sp); + // Both the open delim and close delim get the same span, which covers the + // `$foo` in the decl macro RHS. + TokenTree::Delimited( + DelimSpan::from_single(sp), + DelimSpacing::new(Spacing::Alone, Spacing::Alone), + Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)), + stream, + ) + }; let tt = match cur_matched { MatchedSingle(ParseNtResult::Tt(tt)) => { // `tt`s are emitted into the output stream directly as "raw tokens", @@ -292,6 +322,9 @@ pub(super) fn transcribe<'a>( let kind = token::NtLifetime(*ident, *is_raw); TokenTree::token_alone(kind, sp) } + MatchedSingle(ParseNtResult::Vis(vis)) => { + mk_delimited(MetaVarKind::Vis, TokenStream::from_ast(vis)) + } MatchedSingle(ParseNtResult::Nt(nt)) => { // Other variables are emitted into the output stream as groups with // `Delimiter::Invisible` to maintain parsing priorities. diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 1de452dcf39..ecbc4e396ce 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -721,6 +721,43 @@ impl<'a> Parser<'a> { if !self.eat_keyword(exp) { self.unexpected() } else { Ok(()) } } + /// Consume a sequence produced by a metavar expansion, if present. + fn eat_metavar_seq( + &mut self, + mv_kind: MetaVarKind, + f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> Option { + self.eat_metavar_seq_with_matcher(|mvk| mvk == mv_kind, f) + } + + /// A slightly more general form of `eat_metavar_seq`, for use with the + /// `MetaVarKind` variants that have parameters, where an exact match isn't + /// desired. + fn eat_metavar_seq_with_matcher( + &mut self, + match_mv_kind: impl Fn(MetaVarKind) -> bool, + mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, + ) -> Option { + if let token::OpenDelim(delim) = self.token.kind + && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim + && match_mv_kind(mv_kind) + { + self.bump(); + let res = f(self).expect("failed to reparse {mv_kind:?}"); + if let token::CloseDelim(delim) = self.token.kind + && let Delimiter::Invisible(InvisibleOrigin::MetaVar(mv_kind)) = delim + && match_mv_kind(mv_kind) + { + self.bump(); + Some(res) + } else { + panic!("no close delim when reparsing {mv_kind:?}"); + } + } else { + None + } + } + /// Is the given keyword `kw` followed by a non-reserved identifier? fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool { self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident()) @@ -1455,7 +1492,11 @@ impl<'a> Parser<'a> { /// so emit a proper diagnostic. // Public for rustfmt usage. pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> { - maybe_whole!(self, NtVis, |vis| vis.into_inner()); + if let Some(vis) = self + .eat_metavar_seq(MetaVarKind::Vis, |this| this.parse_visibility(FollowedByType::Yes)) + { + return Ok(vis); + } if !self.eat_keyword(exp!(Pub)) { // We need a span for our `Spanned`, but there's inherently no @@ -1683,7 +1724,8 @@ pub enum ParseNtResult { Tt(TokenTree), Ident(Ident, IdentIsRaw), Lifetime(Ident, IdentIsRaw), + Vis(P), - /// This case will eventually be removed, along with `Token::Interpolate`. + /// This variant will eventually be removed, along with `Token::Interpolate`. Nt(Arc), } diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index eefdb641da2..f102821adf7 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -56,9 +56,7 @@ impl<'a> Parser<'a> { | NtMeta(_) | NtPath(_) => true, - NtItem(_) - | NtBlock(_) - | NtVis(_) => false, + NtItem(_) | NtBlock(_) => false, } } @@ -88,7 +86,7 @@ impl<'a> Parser<'a> { NonterminalKind::Ident => get_macro_ident(token).is_some(), NonterminalKind::Literal => token.can_begin_literal_maybe_minus(), NonterminalKind::Vis => match token.kind { - // The follow-set of :vis + "priv" keyword + interpolated + // The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion. token::Comma | token::Ident(..) | token::NtIdent(..) @@ -102,7 +100,7 @@ impl<'a> Parser<'a> { token::NtLifetime(..) => true, token::Interpolated(nt) => match &**nt { NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, - NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false, + NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) => false, }, token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k { MetaVarKind::Block @@ -208,8 +206,9 @@ impl<'a> Parser<'a> { } NonterminalKind::Meta => NtMeta(P(self.parse_attr_item(ForceCollect::Yes)?)), NonterminalKind::Vis => { - NtVis(P(self - .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?)) + return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| { + this.parse_visibility(FollowedByType::Yes) + })?))); } NonterminalKind::Lifetime => { // We want to keep `'keyword` parsing, just like `keyword` is still diff --git a/tests/ui/macros/block-to-expr-metavar.rs b/tests/ui/macros/block-to-expr-metavar.rs new file mode 100644 index 00000000000..04f10ad0f98 --- /dev/null +++ b/tests/ui/macros/block-to-expr-metavar.rs @@ -0,0 +1,17 @@ +//@ check-pass +// +// A test case where a `block` fragment specifier is interpreted as an `expr` +// fragment specifier. It's an interesting case for the handling of invisible +// delimiters. + +macro_rules! m_expr { + ($e:expr) => { const _CURRENT: u32 = $e; }; +} + +macro_rules! m_block { + ($b:block) => ( m_expr!($b); ); +} + +fn main() { + m_block!({ 1 }); +}