bjoernager/rust
bjoernager/rust
1
Fork 0
Code Issues Pull requests Activity

Remove NtBlock, Nonterminal, and TokenKind::Interpolated.

Browse source 
`NtBlock` is the last remaining variant of `Nonterminal`, so once it is
gone then `Nonterminal` can be removed as well.
This commit is contained in:
Nicholas Nethercote 2024-04-18 20:18:13 +10:00
parent 70dab5a27c
commit bb495d6d3e
18 changed files with 108 additions and 388 deletions
Download patch file Download diff file Expand all files Collapse all files

14
compiler/rustc_ast/src/ast_traits.rs
View file

@ -6,7 +6,6 @@ use std::fmt;
use std::marker::PhantomData;
use crate::ptr::P;
use crate::token::Nonterminal;
use crate::tokenstream::LazyAttrTokenStream;
use crate::{
Arm, AssocItem, AttrItem, AttrKind, AttrVec, Attribute, Block, Crate, Expr, ExprField,
@ -206,19 +205,6 @@ impl HasTokens for Attribute {
}
}
impl HasTokens for Nonterminal {
fn tokens(&self) -> Option<&LazyAttrTokenStream> {
match self {
Nonterminal::NtBlock(block) => block.tokens(),
}
}
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
match self {
Nonterminal::NtBlock(block) => block.tokens_mut(),
}
}
}
/// A trait for AST nodes having (or not having) attributes.
pub trait HasAttrs {
/// This is `true` if this `HasAttrs` might support 'custom' (proc-macro) inner

40
compiler/rustc_ast/src/mut_visit.rs
View file

@ -843,9 +843,9 @@ fn visit_lazy_tts<T: MutVisitor>(vis: &mut T, lazy_tts: &mut Option<LazyAttrToke
visit_lazy_tts_opt_mut(vis, lazy_tts.as_mut());
}
/// Applies ident visitor if it's an ident; applies other visits to interpolated nodes.
/// In practice the ident part is not actually used by specific visitors right now,
/// but there's a test below checking that it works.
/// Applies ident visitor if it's an ident. In practice this is not actually
/// used by specific visitors right now, but there's a test below checking that
/// it works.
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
let Token { kind, span } = t;
@ -863,45 +863,11 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
token::NtLifetime(ident, _is_raw) => {
vis.visit_ident(ident);
}
token::Interpolated(nt) => {
let nt = Arc::make_mut(nt);
visit_nonterminal(vis, nt);
}
_ => {}
}
vis.visit_span(span);
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
/// Applies the visitor to elements of interpolated nodes.
//
// N.B., this can occur only when applying a visitor to partially expanded
// code, where parsed pieces have gotten implanted ito *other* macro
// invocations. This is relevant for macro hygiene, but possibly not elsewhere.
//
// One problem here occurs because the types for flat_map_item, flat_map_stmt,
// etc., allow the visitor to return *multiple* items; this is a problem for the
// nodes here, because they insist on having exactly one piece. One solution
// would be to mangle the MutVisitor trait to include one-to-many and
// one-to-one versions of these entry points, but that would probably confuse a
// lot of people and help very few. Instead, I'm just going to put in dynamic
// checks. I think the performance impact of this will be pretty much
// nonexistent. The danger is that someone will apply a `MutVisitor` to a
// partially expanded node, and will be confused by the fact that their
// `flat_map_item` or `flat_map_stmt` isn't getting called on `NtItem` or `NtStmt`
// nodes. Hopefully they'll wind up reading this comment, and doing something
// appropriate.
//
// BTW, design choice: I considered just changing the type of, e.g., `NtItem` to
// contain multiple items, but decided against it when I looked at
// `parse_item_or_view_item` and tried to figure out what I would do with
// multiple items there....
fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
match nt {
token::NtBlock(block) => vis.visit_block(block),
}
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
fn visit_defaultness<T: MutVisitor>(vis: &mut T, defaultness: &mut Defaultness) {
match defaultness {

127
compiler/rustc_ast/src/token.rs
View file

@ -1,13 +1,10 @@
use std::borrow::Cow;
use std::fmt;
use std::sync::Arc;
pub use LitKind::*;
pub use Nonterminal::*;
pub use NtExprKind::*;
pub use NtPatKind::*;
pub use TokenKind::*;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_macros::{Decodable, Encodable, HashStable_Generic};
use rustc_span::edition::Edition;
use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym};
@ -16,7 +13,6 @@ use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, kw, sym};
use rustc_span::{Ident, Symbol};
use crate::ast;
use crate::ptr::P;
use crate::util::case::Case;
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
@ -35,8 +31,8 @@ pub enum InvisibleOrigin {
// `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
ProcMacro,
// Converted from `TokenKind::Interpolated` in
// `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
// Converted from `TokenKind::NtLifetime` in `TokenStream::flatten_token`.
// Treated similarly to `ProcMacro`.
FlattenToken,
}
@ -337,9 +333,7 @@ impl From<IdentIsRaw> for bool {
}
}
// SAFETY: due to the `Clone` impl below, all fields of all variants other than
// `Interpolated` must impl `Copy`.
#[derive(PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub enum TokenKind {
/* Expression-operator symbols. */
/// `=`
@ -468,21 +462,6 @@ pub enum TokenKind {
/// the `lifetime` metavariable in the macro's RHS.
NtLifetime(Ident, IdentIsRaw),
/// An embedded AST node, as produced by a macro. This only exists for
/// historical reasons. We'd like to get rid of it, for multiple reasons.
/// - It's conceptually very strange. Saying a token can contain an AST
/// node is like saying, in natural language, that a word can contain a
/// sentence.
/// - It requires special handling in a bunch of places in the parser.
/// - It prevents `Token` from implementing `Copy`.
/// It adds complexity and likely slows things down. Please don't add new
/// occurrences of this token kind!
///
/// The span in the surrounding `Token` is that of the metavariable in the
/// macro's RHS. The span within the Nonterminal is that of the fragment
/// passed to the macro at the call site.
Interpolated(Arc<Nonterminal>),
/// A doc comment token.
/// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
/// similarly to symbols in string literal tokens.
@ -492,19 +471,6 @@ pub enum TokenKind {
Eof,
}
impl Clone for TokenKind {
fn clone(&self) -> Self {
// `TokenKind` would impl `Copy` if it weren't for `Interpolated`. So
// for all other variants, this implementation of `clone` is just like
// a copy. This is faster than the `derive(Clone)` version which has a
// separate path for every variant.
match self {
Interpolated(nt) => Interpolated(Arc::clone(nt)),
_ => unsafe { std::ptr::read(self) },
}
}
}
#[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct Token {
pub kind: TokenKind,
@ -600,7 +566,7 @@ impl Token {
| FatArrow | Pound | Dollar | Question | SingleQuote => true,
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
| NtIdent(..) | Lifetime(..) | NtLifetime(..) | Eof => false,
}
}
@ -631,7 +597,6 @@ impl Token {
PathSep | // global path
Lifetime(..) | // labeled loop
Pound => true, // expression attributes
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Block |
MetaVarKind::Expr { .. } |
@ -703,7 +668,6 @@ impl Token {
match self.kind {
OpenDelim(Delimiter::Brace) | Literal(..) | Minus => true,
Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
Interpolated(ref nt) => matches!(&**nt, NtBlock(..)),
OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Expr { .. } | MetaVarKind::Block | MetaVarKind::Literal,
))) => true,
@ -831,31 +795,20 @@ impl Token {
/// Is this a pre-parsed expression dropped into the token stream
/// (which happens while parsing the result of macro expansion)?
pub fn is_metavar_expr(&self) -> bool {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let Interpolated(nt) = &self.kind
&& let NtBlock(_) = &**nt
{
true
} else if matches!(
matches!(
self.is_metavar_seq(),
Some(MetaVarKind::Expr { .. } | MetaVarKind::Literal | MetaVarKind::Path)
) {
true
} else {
matches!(self.is_metavar_seq(), Some(MetaVarKind::Path))
}
Some(
MetaVarKind::Expr { .. }
| MetaVarKind::Literal
| MetaVarKind::Path
| MetaVarKind::Block
)
)
}
/// Is the token an interpolated block (`$b:block`)?
pub fn is_whole_block(&self) -> bool {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let Interpolated(nt) = &self.kind
&& let NtBlock(..) = &**nt
{
return true;
}
false
/// Are we at a block from a metavar (`$b:block`)?
pub fn is_metavar_block(&self) -> bool {
matches!(self.is_metavar_seq(), Some(MetaVarKind::Block))
}
/// Returns `true` if the token is either the `mut` or `const` keyword.
@ -1024,7 +977,7 @@ impl Token {
| PercentEq | CaretEq | AndEq | OrEq | ShlEq | ShrEq | At | DotDotDot | DotDotEq
| Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question
| OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
| Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof,
| Lifetime(..) | NtLifetime(..) | DocComment(..) | Eof,
_,
) => {
return None;
@ -1063,12 +1016,6 @@ pub enum NtExprKind {
Expr2021 { inferred: bool },
}
#[derive(Clone, Encodable, Decodable)]
/// For interpolation during macro expansion.
pub enum Nonterminal {
NtBlock(P<ast::Block>),
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
pub enum NonterminalKind {
Item,
@ -1152,47 +1099,6 @@ impl fmt::Display for NonterminalKind {
}
}
impl Nonterminal {
pub fn use_span(&self) -> Span {
match self {
NtBlock(block) => block.span,
}
}
pub fn descr(&self) -> &'static str {
match self {
NtBlock(..) => "block",
}
}
}
impl PartialEq for Nonterminal {
fn eq(&self, _rhs: &Self) -> bool {
// FIXME: Assume that all nonterminals are not equal, we can't compare them
// correctly based on data from AST. This will prevent them from matching each other
// in macros. The comparison will become possible only when each nonterminal has an
// attached token stream from which it was parsed.
false
}
}
impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtBlock(..) => f.pad("NtBlock(..)"),
}
}
}
impl<CTX> HashStable<CTX> for Nonterminal
where
CTX: crate::HashStableContext,
{
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
panic!("interpolated tokens should not be present in the HIR")
}
}
// Some types are used a lot. Make sure they don't unintentionally get bigger.
#[cfg(target_pointer_width = "64")]
mod size_asserts {
@ -1202,7 +1108,6 @@ mod size_asserts {
// tidy-alphabetical-start
static_assert_size!(Lit, 12);
static_assert_size!(LitKind, 2);
static_assert_size!(Nonterminal, 8);
static_assert_size!(Token, 24);
static_assert_size!(TokenKind, 16);
// tidy-alphabetical-end

26
compiler/rustc_ast/src/tokenstream.rs
View file

@ -25,7 +25,7 @@ use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
use crate::ast::AttrStyle;
use crate::ast_traits::{HasAttrs, HasTokens};
use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
use crate::token::{self, Delimiter, InvisibleOrigin, Token, TokenKind};
use crate::{AttrVec, Attribute};
/// Part of a `TokenStream`.
@ -305,11 +305,6 @@ pub struct AttrsTarget {
}
/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
///
/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
/// instead of a representation of the abstract syntax tree.
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
/// backwards compatibility.
#[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
@ -476,12 +471,6 @@ impl TokenStream {
TokenStream::new(tts)
}
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
match nt {
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
}
}
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
match token.kind {
token::NtIdent(ident, is_raw) => {
@ -493,12 +482,6 @@ impl TokenStream {
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
),
token::Interpolated(ref nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
DelimSpacing::new(Spacing::JointHidden, spacing),
Delimiter::Invisible(InvisibleOrigin::FlattenToken),
TokenStream::from_nonterminal_ast(&nt).flattened(),
),
_ => TokenTree::Token(token.clone(), spacing),
}
}
@ -516,10 +499,9 @@ impl TokenStream {
pub fn flattened(&self) -> TokenStream {
fn can_skip(stream: &TokenStream) -> bool {
stream.iter().all(|tree| match tree {
TokenTree::Token(token, _) => !matches!(
token.kind,
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
),
TokenTree::Token(token, _) => {
!matches!(token.kind, token::NtIdent(..) | token::NtLifetime(..))
}
TokenTree::Delimited(.., inner) => can_skip(inner),
})
}

6
compiler/rustc_ast_pretty/src/pprust/mod.rs
View file

@ -5,14 +5,10 @@ pub mod state;
use std::borrow::Cow;
use rustc_ast as ast;
use rustc_ast::token::{Nonterminal, Token, TokenKind};
use rustc_ast::token::{Token, TokenKind};
use rustc_ast::tokenstream::{TokenStream, TokenTree};
pub use state::{AnnNode, Comments, PpAnn, PrintState, State, print_crate};
pub fn nonterminal_to_string(nt: &Nonterminal) -> String {
State::new().nonterminal_to_string(nt)
}
/// Print the token kind precisely, without converting `$crate` into its respective crate name.
pub fn token_kind_to_string(tok: &TokenKind) -> Cow<'static, str> {
State::new().token_kind_to_string(tok)

12
compiler/rustc_ast_pretty/src/pprust/state.rs
View file

@ -11,7 +11,7 @@ use std::sync::Arc;
use rustc_ast::attr::AttrIdGenerator;
use rustc_ast::ptr::P;
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind};
use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind};
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
use rustc_ast::util::classify;
use rustc_ast::util::comments::{Comment, CommentStyle};
@ -876,14 +876,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
}
}
fn nonterminal_to_string(&self, nt: &Nonterminal) -> String {
// We extract the token stream from the AST fragment and pretty print
// it, rather than using AST pretty printing, because `Nonterminal` is
// slated for removal in #124141. (This method will also then be
// removed.)
self.tts_to_string(&TokenStream::from_nonterminal_ast(nt))
}
/// Print the token kind precisely, without converting `$crate` into its respective crate name.
fn token_kind_to_string(&self, tok: &TokenKind) -> Cow<'static, str> {
self.token_kind_to_string_ext(tok, None)
@ -976,8 +968,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
doc_comment_to_string(comment_kind, attr_style, data).into()
}
token::Eof => "<eof>".into(),
token::Interpolated(ref nt) => self.nonterminal_to_string(&nt).into(),
}
}

20
compiler/rustc_attr_parsing/src/parser.rs
View file

@ -485,25 +485,7 @@ impl<'a> MetaItemListParserContext<'a> {
}
// or a path.
let path =
if let Some(TokenTree::Token(Token { kind: token::Interpolated(_), span, .. }, _)) =
self.inside_delimiters.peek()
{
self.inside_delimiters.next();
// We go into this path if an expr ended up in an attribute that
// expansion did not turn into a literal. Say, `#[repr(align(macro!()))]`
// where the macro didn't expand to a literal. An error is already given
// for this at this point, and then we do continue. This makes this path
// reachable...
let e = self.dcx.span_delayed_bug(
*span,
"expr in place where literal is expected (builtin attr parsing)",
);
return Some(MetaItemOrLitParser::Err(*span, e));
} else {
self.next_path()?
};
let path = self.next_path()?;
// Paths can be followed by:
// - `(more meta items)` (another list)

6
compiler/rustc_builtin_macros/src/cfg_eval.rs
View file

@ -92,11 +92,11 @@ impl CfgEval<'_> {
// the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization
// process is lossless, so this process is invisible to proc-macros.
// 'Flatten' all nonterminals (i.e. `TokenKind::Interpolated`)
// 'Flatten' all nonterminals (i.e. `TokenKind::Nt{Ident,Lifetime}`)
// to `None`-delimited groups containing the corresponding tokens. This
// is normally delayed until the proc-macro server actually needs to
// provide a `TokenKind::Interpolated` to a proc-macro. We do this earlier,
// so that we can handle cases like:
// provide tokens to a proc-macro. We do this earlier, so that we can
// handle cases like:
//
// ```rust
// #[cfg_eval] #[cfg] $item

8
compiler/rustc_expand/src/config.rs
View file

@ -238,13 +238,7 @@ impl<'a> StripUnconfigured<'a> {
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner))
}
AttrTokenTree::Token(
Token {
kind:
TokenKind::NtIdent(..)
| TokenKind::NtLifetime(..)
| TokenKind::Interpolated(..),
..
},
Token { kind: TokenKind::NtIdent(..) | TokenKind::NtLifetime(..), .. },
_,
) => {
panic!("Nonterminal should have been flattened: {:?}", tree);

4
compiler/rustc_expand/src/mbe/diagnostics.rs
View file

@ -66,9 +66,7 @@ pub(super) fn failed_to_match_macro(
}
if let MatcherLoc::Token { token: expected_token } = &remaining_matcher
&& (matches!(expected_token.kind, TokenKind::Interpolated(_))
|| matches!(token.kind, TokenKind::Interpolated(_))
|| matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
&& (matches!(expected_token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_)))
|| matches!(token.kind, TokenKind::OpenDelim(Delimiter::Invisible(_))))
{
err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");

19
compiler/rustc_expand/src/mbe/transcribe.rs
View file

@ -1,5 +1,4 @@
use std::mem;
use std::sync::Arc;
use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{
@ -307,7 +306,9 @@ pub(super) fn transcribe<'a>(
let tt = match cur_matched {
MatchedSingle(ParseNtResult::Tt(tt)) => {
// `tt`s are emitted into the output stream directly as "raw tokens",
// without wrapping them into groups.
// without wrapping them into groups. Other variables are emitted into
// the output stream as groups with `Delimiter::Invisible` to maintain
// parsing priorities.
maybe_use_metavar_location(psess, &stack, sp, tt, &mut marker)
}
MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => {
@ -325,6 +326,11 @@ pub(super) fn transcribe<'a>(
MatchedSingle(ParseNtResult::Item(item)) => {
mk_delimited(item.span, MetaVarKind::Item, TokenStream::from_ast(item))
}
MatchedSingle(ParseNtResult::Block(block)) => mk_delimited(
block.span,
MetaVarKind::Block,
TokenStream::from_ast(block),
),
MatchedSingle(ParseNtResult::Stmt(stmt)) => {
let stream = if let StmtKind::Empty = stmt.kind {
// FIXME: Properly collect tokens for empty statements.
@ -385,15 +391,6 @@ pub(super) fn transcribe<'a>(
MatchedSingle(ParseNtResult::Vis(vis)) => {
mk_delimited(vis.span, MetaVarKind::Vis, TokenStream::from_ast(vis))
}
MatchedSingle(ParseNtResult::Nt(nt)) => {
// Other variables are emitted into the output stream as groups with
// `Delimiter::Invisible` to maintain parsing priorities.
// `Interpolated` is currently used for such groups in rustc parser.
marker.visit_span(&mut sp);
let use_span = nt.use_span();
with_metavar_spans(|mspans| mspans.insert(use_span, sp));
TokenTree::token_alone(token::Interpolated(Arc::clone(nt)), sp)
}
MatchedSeq(..) => {
// We were unable to descend far enough. This is an error.
return Err(dcx.create_err(VarStillRepeating { span: sp, ident }));

9
compiler/rustc_expand/src/proc_macro_server.rs
View file

@ -309,15 +309,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
}));
}
Interpolated(nt) => {
let stream = TokenStream::from_nonterminal_ast(&nt);
trees.push(TokenTree::Group(Group {
delimiter: pm::Delimiter::None,
stream: Some(stream),
span: DelimSpan::from_single(span),
}))
}
OpenDelim(..) | CloseDelim(..) => unreachable!(),
Eof => unreachable!(),
}

38
compiler/rustc_parse/src/parser/expr.rs
View file

@ -637,9 +637,7 @@ impl<'a> Parser<'a> {
/// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
match self.prev_token.kind {
TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
self.prev_token.span
}
TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) => self.prev_token.span,
TokenKind::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
// `expr.span` is the interpolated span, because invisible open
// and close delims both get marked with the same span, one
@ -1386,15 +1384,7 @@ impl<'a> Parser<'a> {
maybe_recover_from_interpolated_ty_qpath!(self, true);
let span = self.token.span;
if let token::Interpolated(nt) = &self.token.kind {
match &**nt {
token::NtBlock(block) => {
let block = block.clone();
self.bump();
return Ok(self.mk_expr(self.prev_token.span, ExprKind::Block(block, None)));
}
};
} else if let Some(expr) = self.eat_metavar_seq_with_matcher(
if let Some(expr) = self.eat_metavar_seq_with_matcher(
|mv_kind| matches!(mv_kind, MetaVarKind::Expr { .. }),
|this| {
// Force collection (as opposed to just `parse_expr`) is required to avoid the
@ -1415,9 +1405,13 @@ impl<'a> Parser<'a> {
self.eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
{
return Ok(lit);
} else if let Some(path) = self.eat_metavar_seq(MetaVarKind::Path, |this| {
this.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))
}) {
} else if let Some(block) =
self.eat_metavar_seq(MetaVarKind::Block, |this| this.parse_block())
{
return Ok(self.mk_expr(span, ExprKind::Block(block, None)));
} else if let Some(path) =
self.eat_metavar_seq(MetaVarKind::Path, |this| this.parse_path(PathStyle::Type))
{
return Ok(self.mk_expr(span, ExprKind::Path(None, path)));
}
@ -1671,7 +1665,7 @@ impl<'a> Parser<'a> {
} else if self.eat_keyword(exp!(Loop)) {
self.parse_expr_loop(label, lo)
} else if self.check_noexpect(&token::OpenDelim(Delimiter::Brace))
|| self.token.is_whole_block()
|| self.token.is_metavar_block()
{
self.parse_expr_block(label, lo, BlockCheckMode::Default)
} else if !ate_colon
@ -2349,7 +2343,7 @@ impl<'a> Parser<'a> {
}
}
if self.token.is_whole_block() {
if self.token.is_metavar_block() {
self.dcx().emit_err(errors::InvalidBlockMacroSegment {
span: self.token.span,
context: lo.to(self.token.span),
@ -2472,7 +2466,7 @@ impl<'a> Parser<'a> {
if self.may_recover()
&& self.token.can_begin_expr()
&& !matches!(self.token.kind, TokenKind::OpenDelim(Delimiter::Brace))
&& !self.token.is_whole_block()
&& !self.token.is_metavar_block()
{
let snapshot = self.create_snapshot_for_diagnostic();
let restrictions =
@ -3519,7 +3513,7 @@ impl<'a> Parser<'a> {
self.token.is_keyword(kw::Do)
&& self.is_keyword_ahead(1, &[kw::Catch])
&& self
.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
.look_ahead(2, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_metavar_block())
&& !self.restrictions.contains(Restrictions::NO_STRUCT_LITERAL)
}
@ -3530,7 +3524,7 @@ impl<'a> Parser<'a> {
fn is_try_block(&self) -> bool {
self.token.is_keyword(kw::Try)
&& self
.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_metavar_block())
&& self.token_uninterpolated_span().at_least_rust_2018()
}
@ -3564,12 +3558,12 @@ impl<'a> Parser<'a> {
// `async move {`
self.is_keyword_ahead(lookahead + 1, &[kw::Move, kw::Use])
&& self.look_ahead(lookahead + 2, |t| {
*t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()
*t == token::OpenDelim(Delimiter::Brace) || t.is_metavar_block()
})
) || (
// `async {`
self.look_ahead(lookahead + 1, |t| {
*t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()
*t == token::OpenDelim(Delimiter::Brace) || t.is_metavar_block()
})
))
}

2
compiler/rustc_parse/src/parser/item.rs
View file

@ -2543,7 +2543,7 @@ impl<'a> Parser<'a> {
self.expect_semi()?;
*sig_hi = self.prev_token.span;
(AttrVec::new(), None)
} else if self.check(exp!(OpenBrace)) || self.token.is_whole_block() {
} else if self.check(exp!(OpenBrace)) || self.token.is_metavar_block() {
self.parse_block_common(self.token.span, BlockCheckMode::Default, None)
.map(|(attrs, body)| (attrs, Some(body)))?
} else if self.token == token::Eq {

33
compiler/rustc_parse/src/parser/mod.rs
View file

@ -13,7 +13,6 @@ mod ty;
use std::assert_matches::debug_assert_matches;
use std::ops::Range;
use std::sync::Arc;
use std::{fmt, mem, slice};
use attr_wrapper::{AttrWrapper, UsePreAttrPos};
@ -24,8 +23,8 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
use path::PathStyle;
use rustc_ast::ptr::P;
use rustc_ast::token::{
self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, NtExprKind, NtPatKind,
Token, TokenKind,
self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, NtExprKind, NtPatKind, Token,
TokenKind,
};
use rustc_ast::tokenstream::{AttrsTarget, Spacing, TokenStream, TokenTree};
use rustc_ast::util::case::Case;
@ -98,21 +97,6 @@ pub enum ForceCollect {
No,
}
#[macro_export]
macro_rules! maybe_whole {
($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
#[allow(irrefutable_let_patterns)] // FIXME: temporary
if let token::Interpolated(nt) = &$p.token.kind
&& let token::$constructor(x) = &**nt
{
#[allow(unused_mut)]
let mut $x = x.clone();
$p.bump();
return Ok($e);
}
};
}
/// If the next tokens are ill-formed `$ty::` recover them as `<$ty>::`.
#[macro_export]
macro_rules! maybe_recover_from_interpolated_ty_qpath {
@ -459,7 +443,6 @@ pub fn token_descr(token: &Token) -> String {
(Some(TokenDescription::MetaVar(kind)), _) => format!("`{kind}` metavariable"),
(None, TokenKind::NtIdent(..)) => format!("identifier `{s}`"),
(None, TokenKind::NtLifetime(..)) => format!("lifetime `{s}`"),
(None, TokenKind::Interpolated(node)) => format!("{} `{s}`", node.descr()),
(None, _) => format!("`{s}`"),
}
}
@ -828,8 +811,10 @@ impl<'a> Parser<'a> {
fn check_inline_const(&self, dist: usize) -> bool {
self.is_keyword_ahead(dist, &[kw::Const])
&& self.look_ahead(dist + 1, |t| match &t.kind {
token::Interpolated(nt) => matches!(&**nt, token::NtBlock(..)),
token::OpenDelim(Delimiter::Brace) => true,
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
MetaVarKind::Block,
))) => true,
_ => false,
})
}
@ -1375,7 +1360,7 @@ impl<'a> Parser<'a> {
// Avoid const blocks and const closures to be parsed as const items
if (self.check_const_closure() == is_closure)
&& !self
.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
.look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_metavar_block())
&& self.eat_keyword_case(exp!(Const), case)
{
Const::Yes(self.prev_token_uninterpolated_span())
@ -1732,7 +1717,6 @@ impl<'a> Parser<'a> {
pub fn token_uninterpolated_span(&self) -> Span {
match &self.token.kind {
token::NtIdent(ident, _) | token::NtLifetime(ident, _) => ident.span,
token::Interpolated(nt) => nt.use_span(),
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
self.look_ahead(1, |t| t.span)
}
@ -1744,7 +1728,6 @@ impl<'a> Parser<'a> {
pub fn prev_token_uninterpolated_span(&self) -> Span {
match &self.prev_token.kind {
token::NtIdent(ident, _) | token::NtLifetime(ident, _) => ident.span,
token::Interpolated(nt) => nt.use_span(),
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
self.look_ahead(0, |t| t.span)
}
@ -1801,6 +1784,7 @@ pub enum ParseNtResult {
Ident(Ident, IdentIsRaw),
Lifetime(Ident, IdentIsRaw),
Item(P<ast::Item>),
Block(P<ast::Block>),
Stmt(P<ast::Stmt>),
Pat(P<ast::Pat>, NtPatKind),
Expr(P<ast::Expr>, NtExprKind),
@ -1809,7 +1793,4 @@ pub enum ParseNtResult {
Meta(P<ast::AttrItem>),
Path(P<ast::Path>),
Vis(P<ast::Visibility>),
/// This variant will eventually be removed, along with `Token::Interpolate`.
Nt(Arc<Nonterminal>),
}

96
compiler/rustc_parse/src/parser/nonterminal.rs
View file

@ -1,14 +1,7 @@
use std::sync::Arc;
use rustc_ast::HasTokens;
use rustc_ast::ptr::P;
use rustc_ast::token::Nonterminal::*;
use rustc_ast::token::NtExprKind::*;
use rustc_ast::token::NtPatKind::*;
use rustc_ast::token::{
self, Delimiter, InvisibleOrigin, MetaVarKind, Nonterminal, NonterminalKind, Token,
};
use rustc_ast_pretty::pprust;
use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, NonterminalKind, Token};
use rustc_errors::PResult;
use rustc_span::{Ident, kw};
@ -45,13 +38,6 @@ impl<'a> Parser<'a> {
}
}
/// Old variant of `may_be_ident`. Being phased out.
fn nt_may_be_ident(nt: &Nonterminal) -> bool {
match nt {
NtBlock(_) => false,
}
}
match kind {
// `expr_2021` and earlier
NonterminalKind::Expr(Expr2021 { .. }) => {
@ -83,16 +69,12 @@ impl<'a> Parser<'a> {
| token::Ident(..)
| token::NtIdent(..)
| token::NtLifetime(..)
| token::Interpolated(_)
| token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
_ => token.can_begin_type(),
},
NonterminalKind::Block => match &token.kind {
token::OpenDelim(Delimiter::Brace) => true,
token::NtLifetime(..) => true,
token::Interpolated(nt) => match &**nt {
NtBlock(_) => true,
},
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
MetaVarKind::Block
| MetaVarKind::Stmt
@ -112,7 +94,6 @@ impl<'a> Parser<'a> {
},
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
token::Interpolated(nt) => nt_may_be_ident(nt),
token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
may_be_ident(*kind)
}
@ -136,33 +117,26 @@ impl<'a> Parser<'a> {
// A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
// which requires having captured tokens available. Since we cannot determine
// in advance whether or not a proc-macro will be (transitively) invoked,
// we always capture tokens for any `Nonterminal` which needs them.
let mut nt = match kind {
// we always capture tokens for any nonterminal that needs them.
match kind {
// Note that TT is treated differently to all the others.
NonterminalKind::TT => return Ok(ParseNtResult::Tt(self.parse_token_tree())),
NonterminalKind::TT => Ok(ParseNtResult::Tt(self.parse_token_tree())),
NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
Some(item) => return Ok(ParseNtResult::Item(item)),
None => {
return Err(self
.dcx()
.create_err(UnexpectedNonterminal::Item(self.token.span)));
}
Some(item) => Ok(ParseNtResult::Item(item)),
None => Err(self.dcx().create_err(UnexpectedNonterminal::Item(self.token.span))),
},
NonterminalKind::Block => {
// While a block *expression* may have attributes (e.g. `#[my_attr] { ... }`),
// the ':block' matcher does not support them
NtBlock(self.collect_tokens_no_attrs(|this| this.parse_block())?)
Ok(ParseNtResult::Block(self.collect_tokens_no_attrs(|this| this.parse_block())?))
}
NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? {
Some(stmt) => return Ok(ParseNtResult::Stmt(P(stmt))),
Some(stmt) => Ok(ParseNtResult::Stmt(P(stmt))),
None => {
return Err(self
.dcx()
.create_err(UnexpectedNonterminal::Statement(self.token.span)));
Err(self.dcx().create_err(UnexpectedNonterminal::Statement(self.token.span)))
}
},
NonterminalKind::Pat(pat_kind) => {
return Ok(ParseNtResult::Pat(
NonterminalKind::Pat(pat_kind) => Ok(ParseNtResult::Pat(
self.collect_tokens_no_attrs(|this| match pat_kind {
PatParam { .. } => this.parse_pat_no_top_alt(None, None),
PatWithOr => this.parse_pat_no_top_guard(
@ -173,25 +147,22 @@ impl<'a> Parser<'a> {
),
})?,
pat_kind,
));
}
)),
NonterminalKind::Expr(expr_kind) => {
return Ok(ParseNtResult::Expr(self.parse_expr_force_collect()?, expr_kind));
Ok(ParseNtResult::Expr(self.parse_expr_force_collect()?, expr_kind))
}
NonterminalKind::Literal => {
// The `:literal` matcher does not support attributes.
return Ok(ParseNtResult::Literal(
Ok(ParseNtResult::Literal(
self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?,
));
))
}
NonterminalKind::Ty => {
return Ok(ParseNtResult::Ty(
NonterminalKind::Ty => Ok(ParseNtResult::Ty(
self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?,
));
}
// this could be handled like a token, since it is one
)),
// This could be handled like a token, since it is one.
NonterminalKind::Ident => {
return if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
self.bump();
Ok(ParseNtResult::Ident(ident, is_raw))
} else {
@ -199,25 +170,22 @@ impl<'a> Parser<'a> {
span: self.token.span,
token: self.token.clone(),
}))
};
}
NonterminalKind::Path => {
return Ok(ParseNtResult::Path(P(
}
NonterminalKind::Path => Ok(ParseNtResult::Path(P(
self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?
)));
}
))),
NonterminalKind::Meta => {
return Ok(ParseNtResult::Meta(P(self.parse_attr_item(ForceCollect::Yes)?)));
Ok(ParseNtResult::Meta(P(self.parse_attr_item(ForceCollect::Yes)?)))
}
NonterminalKind::Vis => {
return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| {
this.parse_visibility(FollowedByType::Yes)
})?)));
Ok(ParseNtResult::Vis(P(self
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?)))
}
NonterminalKind::Lifetime => {
// We want to keep `'keyword` parsing, just like `keyword` is still
// an ident for nonterminal purposes.
return if let Some((ident, is_raw)) = self.token.lifetime() {
if let Some((ident, is_raw)) = self.token.lifetime() {
self.bump();
Ok(ParseNtResult::Lifetime(ident, is_raw))
} else {
@ -225,21 +193,9 @@ impl<'a> Parser<'a> {
span: self.token.span,
token: self.token.clone(),
}))
};
}
};
// If tokens are supported at all, they should be collected.
if matches!(nt.tokens_mut(), Some(None)) {
panic!(
"Missing tokens for nt {:?} at {:?}: {:?}",
nt,
nt.use_span(),
pprust::nonterminal_to_string(&nt)
);
}
Ok(ParseNtResult::Nt(Arc::new(nt)))
}
}
}

10
compiler/rustc_parse/src/parser/stmt.rs
View file

@ -23,8 +23,8 @@ use super::{
AttrWrapper, BlockMode, FnParseMode, ForceCollect, Parser, Restrictions, SemiColonMode,
Trailing, UsePreAttrPos,
};
use crate::errors::MalformedLoopLabel;
use crate::{errors, exp, maybe_whole};
use crate::errors::{self, MalformedLoopLabel};
use crate::exp;
impl<'a> Parser<'a> {
/// Parses a statement. This stops just before trailing semicolons on everything but items.
@ -681,7 +681,9 @@ impl<'a> Parser<'a> {
blk_mode: BlockCheckMode,
loop_header: Option<Span>,
) -> PResult<'a, (AttrVec, P<Block>)> {
maybe_whole!(self, NtBlock, |block| (AttrVec::new(), block));
if let Some(block) = self.eat_metavar_seq(MetaVarKind::Block, |this| this.parse_block()) {
return Ok((AttrVec::new(), block));
}
let maybe_ident = self.prev_token.clone();
self.maybe_recover_unexpected_block_label(loop_header);
@ -896,7 +898,7 @@ impl<'a> Parser<'a> {
{
if self.token == token::Colon
&& self.look_ahead(1, |token| {
token.is_whole_block()
token.is_metavar_block()
|| matches!(
token.kind,
token::Ident(

2
tests/ui/macros/macro-as-fn-body.rs
View file

@ -1,7 +1,7 @@
//
//@ run-pass
//
// Description - ensure Interpolated blocks can act as valid function bodies
// Description - ensure block metavariables can act as valid function bodies
// Covered cases: free functions, struct methods, and default trait functions
macro_rules! def_fn {