Auto merge of #138083 - nnethercote:rm-NtItem-NtStmt, r=petrochenkov

Remove `NtItem` and `NtStmt`

Another piece of #124141.

r? `@petrochenkov`

compiler/rustc_ast/src/ast_traits.rs

@@ -209,16 +209,12 @@ impl HasTokens for Attribute {
 impl HasTokens for Nonterminal {
     fn tokens(&self) -> Option<&LazyAttrTokenStream> {
         match self {
-            Nonterminal::NtItem(item) => item.tokens(),
-            Nonterminal::NtStmt(stmt) => stmt.tokens(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
             Nonterminal::NtBlock(block) => block.tokens(),
         }
     }
     fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
         match self {
-            Nonterminal::NtItem(item) => item.tokens_mut(),
-            Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
             Nonterminal::NtBlock(block) => block.tokens_mut(),
         }

compiler/rustc_ast/src/lib.rs

@@ -6,6 +6,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(deny(warnings)))

compiler/rustc_ast/src/mut_visit.rs

@@ -895,19 +895,7 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
 // multiple items there....
 fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
     match nt {
-        token::NtItem(item) => visit_clobber(item, |item| {
-            // This is probably okay, because the only visitors likely to
-            // peek inside interpolated nodes will be renamings/markings,
-            // which map single items to single items.
-            vis.flat_map_item(item).expect_one("expected visitor to produce exactly one item")
-        }),
         token::NtBlock(block) => vis.visit_block(block),
-        token::NtStmt(stmt) => visit_clobber(stmt, |stmt| {
-            // See reasoning above.
-            stmt.map(|stmt| {
-                vis.flat_map_stmt(stmt).expect_one("expected visitor to produce exactly one item")
-            })
-        }),
         token::NtExpr(expr) => vis.visit_expr(expr),
         token::NtLiteral(expr) => vis.visit_expr(expr),
     }

compiler/rustc_ast/src/token.rs

@@ -870,6 +870,7 @@ impl Token {
     /// Is this a pre-parsed expression dropped into the token stream
     /// (which happens while parsing the result of macro expansion)?
     pub fn is_whole_expr(&self) -> bool {
+        #[allow(irrefutable_let_patterns)] // FIXME: temporary
         if let Interpolated(nt) = &self.kind
             && let NtExpr(_) | NtLiteral(_) | NtBlock(_) = &**nt
         {
@@ -1103,9 +1104,7 @@ pub enum NtExprKind {
 #[derive(Clone, Encodable, Decodable)]
 /// For interpolation during macro expansion.
 pub enum Nonterminal {
-    NtItem(P<ast::Item>),
     NtBlock(P<ast::Block>),
-    NtStmt(P<ast::Stmt>),
     NtExpr(P<ast::Expr>),
     NtLiteral(P<ast::Expr>),
 }
@@ -1196,18 +1195,14 @@ impl fmt::Display for NonterminalKind {
 impl Nonterminal {
     pub fn use_span(&self) -> Span {
         match self {
-            NtItem(item) => item.span,
             NtBlock(block) => block.span,
-            NtStmt(stmt) => stmt.span,
             NtExpr(expr) | NtLiteral(expr) => expr.span,
         }
     }
 
     pub fn descr(&self) -> &'static str {
         match self {
-            NtItem(..) => "item",
             NtBlock(..) => "block",
-            NtStmt(..) => "statement",
             NtExpr(..) => "expression",
             NtLiteral(..) => "literal",
         }
@@ -1227,9 +1222,7 @@ impl PartialEq for Nonterminal {
 impl fmt::Debug for Nonterminal {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            NtItem(..) => f.pad("NtItem(..)"),
             NtBlock(..) => f.pad("NtBlock(..)"),
-            NtStmt(..) => f.pad("NtStmt(..)"),
             NtExpr(..) => f.pad("NtExpr(..)"),
             NtLiteral(..) => f.pad("NtLiteral(..)"),
         }

compiler/rustc_ast/src/tokenstream.rs

@@ -23,7 +23,7 @@ use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
 
-use crate::ast::{AttrStyle, StmtKind};
+use crate::ast::AttrStyle;
 use crate::ast_traits::{HasAttrs, HasTokens};
 use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
 use crate::{AttrVec, Attribute};
@@ -461,13 +461,7 @@ impl TokenStream {
 
     pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
         match nt {
-            Nonterminal::NtItem(item) => TokenStream::from_ast(item),
             Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
-            Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
-                // FIXME: Properly collect tokens for empty statements.
-                TokenStream::token_alone(token::Semi, stmt.span)
-            }
-            Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
             Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
         }
     }

compiler/rustc_ast_lowering/src/lib.rs

@@ -32,6 +32,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(rust_logo)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]

compiler/rustc_attr_parsing/src/lib.rs

@@ -77,6 +77,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(rust_logo)]
 #![feature(let_chains)]
 #![feature(rustdoc_internals)]

compiler/rustc_borrowck/src/lib.rs

@@ -2,6 +2,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(rust_logo)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -140,8 +140,9 @@ impl CfgEval<'_> {
                     Annotatable::ForeignItem(self.flat_map_foreign_item(item).pop().unwrap())
                 }
                 Annotatable::Stmt(_) => {
-                    let stmt =
+                    let stmt = parser
-                        parser.parse_stmt_without_recovery(false, ForceCollect::Yes)?.unwrap();
+                        .parse_stmt_without_recovery(false, ForceCollect::Yes, false)?
+                        .unwrap();
                     Annotatable::Stmt(P(self.flat_map_stmt(stmt).pop().unwrap()))
                 }
                 Annotatable::Expr(_) => {

compiler/rustc_builtin_macros/src/lib.rs

@@ -5,6 +5,7 @@
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]

compiler/rustc_codegen_ssa/src/lib.rs

@@ -2,6 +2,7 @@
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]

compiler/rustc_const_eval/src/lib.rs

@@ -1,6 +1,7 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(rust_logo)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]

compiler/rustc_driver/src/lib.rs

@@ -3,6 +3,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(rust_logo)]
 #![feature(rustdoc_internals)]
 // tidy-alphabetical-end

compiler/rustc_driver_impl/src/lib.rs

@@ -7,6 +7,7 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(decl_macro)]

compiler/rustc_expand/src/base.rs

@@ -7,7 +7,7 @@ use std::sync::Arc;
 
 use rustc_ast::attr::{AttributeExt, MarkedAttrs};
 use rustc_ast::ptr::P;
-use rustc_ast::token::Nonterminal;
+use rustc_ast::token::MetaVarKind;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{AssocCtxt, Visitor};
 use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, Item, NodeId, PatKind};
@@ -19,7 +19,7 @@ use rustc_feature::Features;
 use rustc_hir as hir;
 use rustc_lint_defs::{BufferedEarlyLint, RegisteredTools};
 use rustc_parse::MACRO_ARGUMENTS;
-use rustc_parse::parser::Parser;
+use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_session::config::CollapseMacroDebuginfo;
 use rustc_session::parse::ParseSess;
 use rustc_session::{Limit, Session};
@@ -1405,13 +1405,13 @@ pub fn parse_macro_name_and_helper_attrs(
 /// If this item looks like a specific enums from `rental`, emit a fatal error.
 /// See #73345 and #83125 for more details.
 /// FIXME(#73933): Remove this eventually.
-fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
+fn pretty_printing_compatibility_hack(item: &Item, psess: &ParseSess) {
     let name = item.ident.name;
     if name == sym::ProceduralMasqueradeDummyType
         && let ast::ItemKind::Enum(enum_def, _) = &item.kind
         && let [variant] = &*enum_def.variants
         && variant.ident.name == sym::Input
-        && let FileName::Real(real) = sess.source_map().span_to_filename(item.ident.span)
+        && let FileName::Real(real) = psess.source_map().span_to_filename(item.ident.span)
         && let Some(c) = real
             .local_path()
             .unwrap_or(Path::new(""))
@@ -1429,7 +1429,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
         };
 
         if crate_matches {
-            sess.dcx().emit_fatal(errors::ProcMacroBackCompat {
+            psess.dcx().emit_fatal(errors::ProcMacroBackCompat {
                 crate_name: "rental".to_string(),
                 fixed_version: "0.5.6".to_string(),
             });
@@ -1437,7 +1437,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &Session) {
     }
 }
 
-pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &Session) {
+pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, psess: &ParseSess) {
     let item = match ann {
         Annotatable::Item(item) => item,
         Annotatable::Stmt(stmt) => match &stmt.kind {
@@ -1446,17 +1446,36 @@ pub(crate) fn ann_pretty_printing_compatibility_hack(ann: &Annotatable, sess: &S
         },
         _ => return,
     };
-    pretty_printing_compatibility_hack(item, sess)
+    pretty_printing_compatibility_hack(item, psess)
 }
 
-pub(crate) fn nt_pretty_printing_compatibility_hack(nt: &Nonterminal, sess: &Session) {
+pub(crate) fn stream_pretty_printing_compatibility_hack(
-    let item = match nt {
+    kind: MetaVarKind,
-        Nonterminal::NtItem(item) => item,
+    stream: &TokenStream,
-        Nonterminal::NtStmt(stmt) => match &stmt.kind {
+    psess: &ParseSess,
-            ast::StmtKind::Item(item) => item,
+) {
-            _ => return,
+    let item = match kind {
-        },
+        MetaVarKind::Item => {
+            let mut parser = Parser::new(psess, stream.clone(), None);
+            // No need to collect tokens for this simple check.
+            parser
+                .parse_item(ForceCollect::No)
+                .expect("failed to reparse item")
+                .expect("an actual item")
+        }
+        MetaVarKind::Stmt => {
+            let mut parser = Parser::new(psess, stream.clone(), None);
+            // No need to collect tokens for this simple check.
+            let stmt = parser
+                .parse_stmt(ForceCollect::No)
+                .expect("failed to reparse")
+                .expect("an actual stmt");
+            match &stmt.kind {
+                ast::StmtKind::Item(item) => item.clone(),
+                _ => return,
+            }
+        }
         _ => return,
     };
-    pretty_printing_compatibility_hack(item, sess)
+    pretty_printing_compatibility_hack(&item, psess)
 }

compiler/rustc_expand/src/mbe/transcribe.rs

@@ -7,7 +7,7 @@ use rustc_ast::token::{
     TokenKind,
 };
 use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
-use rustc_ast::{ExprKind, TyKind};
+use rustc_ast::{ExprKind, StmtKind, TyKind};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Diag, DiagCtxtHandle, PResult, pluralize};
 use rustc_parse::lexer::nfc_normalize;
@@ -323,6 +323,18 @@ pub(super) fn transcribe<'a>(
                             let kind = token::NtLifetime(*ident, *is_raw);
                             TokenTree::token_alone(kind, sp)
                         }
+                        MatchedSingle(ParseNtResult::Item(item)) => {
+                            mk_delimited(item.span, MetaVarKind::Item, TokenStream::from_ast(item))
+                        }
+                        MatchedSingle(ParseNtResult::Stmt(stmt)) => {
+                            let stream = if let StmtKind::Empty = stmt.kind {
+                                // FIXME: Properly collect tokens for empty statements.
+                                TokenStream::token_alone(token::Semi, stmt.span)
+                            } else {
+                                TokenStream::from_ast(stmt)
+                            };
+                            mk_delimited(stmt.span, MetaVarKind::Stmt, stream)
+                        }
                         MatchedSingle(ParseNtResult::Pat(pat, pat_kind)) => mk_delimited(
                             pat.span,
                             MetaVarKind::Pat(*pat_kind),

compiler/rustc_expand/src/proc_macro.rs

@@ -122,7 +122,7 @@ impl MultiItemModifier for DeriveProcMacro {
         // We had a lint for a long time, but now we just emit a hard error.
         // Eventually we might remove the special case hard error check
         // altogether. See #73345.
-        crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess);
+        crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess);
         let input = item.to_tokens();
         let stream = {
             let _timer =

compiler/rustc_expand/src/proc_macro_server.rs

@@ -115,11 +115,43 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
 
         while let Some(tree) = iter.next() {
             let (Token { kind, span }, joint) = match tree.clone() {
-                tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
+                tokenstream::TokenTree::Delimited(span, _, mut delim, mut stream) => {
-                    let delimiter = pm::Delimiter::from_internal(delim);
+                    // We used to have an alternative behaviour for crates that
+                    // needed it: a hack used to pass AST fragments to
+                    // attribute and derive macros as a single nonterminal
+                    // token instead of a token stream. Such token needs to be
+                    // "unwrapped" and not represented as a delimited group. We
+                    // had a lint for a long time, but now we just emit a hard
+                    // error. Eventually we might remove the special case hard
+                    // error check altogether. See #73345.
+                    if let Delimiter::Invisible(InvisibleOrigin::MetaVar(kind)) = delim {
+                        crate::base::stream_pretty_printing_compatibility_hack(
+                            kind,
+                            &stream,
+                            rustc.psess(),
+                        );
+                    }
+
+                    // In `mk_delimited` we avoid nesting invisible delimited
+                    // of the same `MetaVarKind`. Here we do the same but
+                    // ignore the `MetaVarKind` because it is discarded when we
+                    // convert it to a `Group`.
+                    while let Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) = delim {
+                        if stream.len() == 1
+                            && let tree = stream.iter().next().unwrap()
+                            && let tokenstream::TokenTree::Delimited(_, _, delim2, stream2) = tree
+                            && let Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) = delim2
+                        {
+                            delim = *delim2;
+                            stream = stream2.clone();
+                        } else {
+                            break;
+                        }
+                    }
+
                     trees.push(TokenTree::Group(Group {
-                        delimiter,
+                        delimiter: pm::Delimiter::from_internal(delim),
-                        stream: Some(tts),
+                        stream: Some(stream),
                         span: DelimSpan {
                             open: span.open,
                             close: span.close,
@@ -279,15 +311,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
 
                 Interpolated(nt) => {
                     let stream = TokenStream::from_nonterminal_ast(&nt);
-                    // We used to have an alternative behaviour for crates that
-                    // needed it: a hack used to pass AST fragments to
-                    // attribute and derive macros as a single nonterminal
-                    // token instead of a token stream. Such token needs to be
-                    // "unwrapped" and not represented as a delimited group. We
-                    // had a lint for a long time, but now we just emit a hard
-                    // error. Eventually we might remove the special case hard
-                    // error check altogether. See #73345.
-                    crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.ecx.sess);
                     trees.push(TokenTree::Group(Group {
                         delimiter: pm::Delimiter::None,
                         stream: Some(stream),

compiler/rustc_hir/src/lib.rs

@@ -4,6 +4,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(associated_type_defaults)]
 #![feature(box_patterns)]
 #![feature(closure_track_caller)]

compiler/rustc_hir_analysis/src/lib.rs

@@ -59,6 +59,7 @@ This API is completely unstable and subject to change.
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]

compiler/rustc_hir_typeck/src/lib.rs

@@ -1,6 +1,7 @@
 // tidy-alphabetical-start
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(array_windows)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]

compiler/rustc_incremental/src/lib.rs

@@ -2,6 +2,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![deny(missing_docs)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]

compiler/rustc_lint/src/lib.rs

@@ -21,6 +21,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(array_windows)]

compiler/rustc_metadata/src/lib.rs

@@ -1,5 +1,6 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(coroutines)]

compiler/rustc_middle/src/lib.rs

@@ -29,6 +29,7 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::potential_query_instability)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(allocator_api)]

compiler/rustc_mir_build/src/lib.rs

@@ -3,6 +3,7 @@
 // tidy-alphabetical-start
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]

compiler/rustc_mir_dataflow/src/lib.rs

@@ -1,4 +1,5 @@
 // tidy-alphabetical-start
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(assert_matches)]
 #![feature(associated_type_defaults)]
 #![feature(box_patterns)]

compiler/rustc_mir_transform/src/lib.rs

@@ -1,4 +1,5 @@
 // tidy-alphabetical-start
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(array_windows)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]

compiler/rustc_monomorphize/src/lib.rs

@@ -1,4 +1,5 @@
 // tidy-alphabetical-start
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(array_windows)]
 #![feature(file_buffered)]
 #![feature(if_let_guard)]

compiler/rustc_parse/src/lib.rs

@@ -4,6 +4,7 @@
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(array_windows)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]

compiler/rustc_parse/src/parser/expr.rs

@@ -1364,7 +1364,6 @@ impl<'a> Parser<'a> {
                     self.bump();
                     return Ok(self.mk_expr(self.prev_token.span, ExprKind::Block(block, None)));
                 }
-                _ => {}
             };
         } else if let Some(path) = self.eat_metavar_seq(MetaVarKind::Path, |this| {
             this.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))
@@ -3064,7 +3063,7 @@ impl<'a> Parser<'a> {
             }
 
             self.restore_snapshot(pre_pat_snapshot);
-            match self.parse_stmt_without_recovery(true, ForceCollect::No) {
+            match self.parse_stmt_without_recovery(true, ForceCollect::No, false) {
                 // Consume statements for as long as possible.
                 Ok(Some(stmt)) => {
                     stmts.push(stmt);

compiler/rustc_parse/src/parser/item.rs

@@ -21,10 +21,10 @@ use super::diagnostics::{ConsumeClosingDelim, dummy_arg};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{
     AttrWrapper, ExpKeywordPair, ExpTokenPair, FollowedByType, ForceCollect, Parser, PathStyle,
-    Trailing, UsePreAttrPos,
+    Recovered, Trailing, UsePreAttrPos,
 };
 use crate::errors::{self, MacroExpandsToAdtField};
-use crate::{exp, fluent_generated as fluent, maybe_whole};
+use crate::{exp, fluent_generated as fluent};
 
 impl<'a> Parser<'a> {
     /// Parses a source module as a crate. This is the main entry point for the parser.
@@ -142,10 +142,13 @@ impl<'a> Parser<'a> {
         fn_parse_mode: FnParseMode,
         force_collect: ForceCollect,
     ) -> PResult<'a, Option<Item>> {
-        maybe_whole!(self, NtItem, |item| {
+        if let Some(item) =
+            self.eat_metavar_seq(MetaVarKind::Item, |this| this.parse_item(ForceCollect::Yes))
+        {
+            let mut item = item.expect("an actual item");
             attrs.prepend_to_nt_inner(&mut item.attrs);
-            Some(item.into_inner())
+            return Ok(Some(item.into_inner()));
-        });
+        }
 
         self.collect_tokens(None, attrs, force_collect, |this, mut attrs| {
             let lo = this.token.span;

compiler/rustc_parse/src/parser/mod.rs

@@ -1076,10 +1076,12 @@ impl<'a> Parser<'a> {
         let initial_semicolon = self.token.span;
 
         while self.eat(exp!(Semi)) {
-            let _ = self.parse_stmt_without_recovery(false, ForceCollect::No).unwrap_or_else(|e| {
+            let _ = self
-                e.cancel();
+                .parse_stmt_without_recovery(false, ForceCollect::No, false)
-                None
+                .unwrap_or_else(|e| {
-            });
+                    e.cancel();
+                    None
+                });
         }
 
         expect_err
@@ -1746,6 +1748,8 @@ pub enum ParseNtResult {
     Tt(TokenTree),
     Ident(Ident, IdentIsRaw),
     Lifetime(Ident, IdentIsRaw),
+    Item(P<ast::Item>),
+    Stmt(P<ast::Stmt>),
     Pat(P<ast::Pat>, NtPatKind),
     Ty(P<ast::Ty>),
     Meta(P<ast::AttrItem>),

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -48,12 +48,11 @@ impl<'a> Parser<'a> {
         /// Old variant of `may_be_ident`. Being phased out.
         fn nt_may_be_ident(nt: &Nonterminal) -> bool {
             match nt {
-                NtStmt(_)
+                NtExpr(_)
-                | NtExpr(_)
                 | NtLiteral(_) // `true`, `false`
                 => true,
 
-                NtItem(_) | NtBlock(_) => false,
+                NtBlock(_) => false,
             }
         }
 
@@ -96,8 +95,7 @@ impl<'a> Parser<'a> {
                 token::OpenDelim(Delimiter::Brace) => true,
                 token::NtLifetime(..) => true,
                 token::Interpolated(nt) => match &**nt {
-                    NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
+                    NtBlock(_) | NtExpr(_) | NtLiteral(_) => true,
-                    NtItem(_) => false,
                 },
                 token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
                     MetaVarKind::Block
@@ -147,7 +145,7 @@ impl<'a> Parser<'a> {
             // Note that TT is treated differently to all the others.
             NonterminalKind::TT => return Ok(ParseNtResult::Tt(self.parse_token_tree())),
             NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
-                Some(item) => NtItem(item),
+                Some(item) => return Ok(ParseNtResult::Item(item)),
                 None => {
                     return Err(self
                         .dcx()
@@ -160,7 +158,7 @@ impl<'a> Parser<'a> {
                 NtBlock(self.collect_tokens_no_attrs(|this| this.parse_block())?)
             }
             NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? {
-                Some(s) => NtStmt(P(s)),
+                Some(stmt) => return Ok(ParseNtResult::Stmt(P(stmt))),
                 None => {
                     return Err(self
                         .dcx()

compiler/rustc_parse/src/parser/stmt.rs

@@ -5,7 +5,7 @@ use std::ops::Bound;
 use ast::Label;
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter, TokenKind};
+use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, TokenKind};
 use rustc_ast::util::classify::{self, TrailingBrace};
 use rustc_ast::{
     AttrStyle, AttrVec, Block, BlockCheckMode, DUMMY_NODE_ID, Expr, ExprKind, HasAttrs, Local,
@@ -33,8 +33,8 @@ impl<'a> Parser<'a> {
     /// If `force_collect` is [`ForceCollect::Yes`], forces collection of tokens regardless of
     /// whether or not we have attributes.
     // Public for rustfmt usage.
-    pub(super) fn parse_stmt(&mut self, force_collect: ForceCollect) -> PResult<'a, Option<Stmt>> {
+    pub fn parse_stmt(&mut self, force_collect: ForceCollect) -> PResult<'a, Option<Stmt>> {
-        Ok(self.parse_stmt_without_recovery(false, force_collect).unwrap_or_else(|e| {
+        Ok(self.parse_stmt_without_recovery(false, force_collect, false).unwrap_or_else(|e| {
             e.emit();
             self.recover_stmt_(SemiColonMode::Break, BlockMode::Ignore);
             None
@@ -42,23 +42,27 @@ impl<'a> Parser<'a> {
     }
 
     /// If `force_collect` is [`ForceCollect::Yes`], forces collection of tokens regardless of
-    /// whether or not we have attributes.
+    /// whether or not we have attributes. If `force_full_expr` is true, parses the stmt without
-    // Public for `cfg_eval` macro expansion.
+    /// using `Restriction::STMT_EXPR`. Public for `cfg_eval` macro expansion.
     pub fn parse_stmt_without_recovery(
         &mut self,
         capture_semi: bool,
         force_collect: ForceCollect,
+        force_full_expr: bool,
     ) -> PResult<'a, Option<Stmt>> {
         let pre_attr_pos = self.collect_pos();
         let attrs = self.parse_outer_attributes()?;
         let lo = self.token.span;
 
-        maybe_whole!(self, NtStmt, |stmt| {
+        if let Some(stmt) = self.eat_metavar_seq(MetaVarKind::Stmt, |this| {
+            this.parse_stmt_without_recovery(false, ForceCollect::Yes, false)
+        }) {
+            let mut stmt = stmt.expect("an actual statement");
             stmt.visit_attrs(|stmt_attrs| {
                 attrs.prepend_to_nt_inner(stmt_attrs);
             });
-            Some(stmt.into_inner())
+            return Ok(Some(stmt));
-        });
+        }
 
         if self.token.is_keyword(kw::Mut) && self.is_keyword_ahead(1, &[kw::Let]) {
             self.bump();
@@ -147,12 +151,14 @@ impl<'a> Parser<'a> {
         } else if self.token != token::CloseDelim(Delimiter::Brace) {
             // Remainder are line-expr stmts. This is similar to the `parse_stmt_path_start` case
             // above.
+            let restrictions =
+                if force_full_expr { Restrictions::empty() } else { Restrictions::STMT_EXPR };
             let e = self.collect_tokens(
                 Some(pre_attr_pos),
                 AttrWrapper::empty(),
                 force_collect,
                 |this, _empty_attrs| {
-                    let (expr, _) = this.parse_expr_res(Restrictions::STMT_EXPR, attrs)?;
+                    let (expr, _) = this.parse_expr_res(restrictions, attrs)?;
                     Ok((expr, Trailing::No, UsePreAttrPos::Yes))
                 },
             )?;
@@ -229,11 +235,15 @@ impl<'a> Parser<'a> {
         let mac = P(MacCall { path, args });
 
         let kind = if (style == MacStmtStyle::Braces
-            && self.token != token::Dot
+            && !matches!(self.token.kind, token::Dot | token::Question))
-            && self.token != token::Question)
+            || matches!(
-            || self.token == token::Semi
+                self.token.kind,
-            || self.token == token::Eof
+                token::Semi
-        {
+                    | token::Eof
+                    | token::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+                        MetaVarKind::Stmt
+                    )))
+            ) {
             StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
         } else {
             // Since none of the above applied, this is an expression statement macro.
@@ -501,7 +511,7 @@ impl<'a> Parser<'a> {
         //      bar;
         //
         // which is valid in other languages, but not Rust.
-        match self.parse_stmt_without_recovery(false, ForceCollect::No) {
+        match self.parse_stmt_without_recovery(false, ForceCollect::No, false) {
             // If the next token is an open brace, e.g., we have:
             //
             //     if expr other_expr {
@@ -810,10 +820,24 @@ impl<'a> Parser<'a> {
         &mut self,
         recover: AttemptLocalParseRecovery,
     ) -> PResult<'a, Option<Stmt>> {
-        // Skip looking for a trailing semicolon when we have an interpolated statement.
+        // Skip looking for a trailing semicolon when we have a metavar seq.
-        maybe_whole!(self, NtStmt, |stmt| Some(stmt.into_inner()));
+        if let Some(stmt) = self.eat_metavar_seq(MetaVarKind::Stmt, |this| {
+            // Why pass `true` for `force_full_expr`? Statement expressions are less expressive
+            // than "full" expressions, due to the `STMT_EXPR` restriction, and sometimes need
+            // parentheses. E.g. the "full" expression `match paren_around_match {} | true` when
+            // used in statement context must be written `(match paren_around_match {} | true)`.
+            // However, if the expression we are parsing in this statement context was pasted by a
+            // declarative macro, it may have come from a "full" expression context, and lack
+            // these parentheses. So we lift the `STMT_EXPR` restriction to ensure the statement
+            // will reparse successfully.
+            this.parse_stmt_without_recovery(false, ForceCollect::No, true)
+        }) {
+            let stmt = stmt.expect("an actual statement");
+            return Ok(Some(stmt));
+        }
 
-        let Some(mut stmt) = self.parse_stmt_without_recovery(true, ForceCollect::No)? else {
+        let Some(mut stmt) = self.parse_stmt_without_recovery(true, ForceCollect::No, false)?
+        else {
             return Ok(None);
         };
 

compiler/rustc_passes/src/lib.rs

@@ -6,6 +6,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(let_chains)]

compiler/rustc_privacy/src/lib.rs

@@ -1,5 +1,6 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(associated_type_defaults)]

compiler/rustc_query_impl/src/lib.rs

@@ -3,6 +3,7 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![allow(unused_parens)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(min_specialization)]

compiler/rustc_sanitizers/src/lib.rs

@@ -4,6 +4,7 @@
 //! compiler.
 
 // tidy-alphabetical-start
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(let_chains)]
 #![warn(unreachable_pub)]
 // tidy-alphabetical-end

compiler/rustc_smir/src/lib.rs

@@ -9,6 +9,7 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![allow(rustc::usage_of_ty_tykind)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(allow(unused_variables), deny(warnings)))

compiler/rustc_symbol_mangling/src/lib.rs

@@ -89,6 +89,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(let_chains)]

compiler/rustc_transmute/src/lib.rs

@@ -1,4 +1,5 @@
 // tidy-alphabetical-start
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![feature(never_type)]
 #![warn(unreachable_pub)]
 // tidy-alphabetical-end

compiler/rustc_ty_utils/src/lib.rs

@@ -6,6 +6,7 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(doc, recursion_limit = "256")] // FIXME(nnethercote): will be removed by #124141
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]

tests/ui/macros/macro-stmt-2.rs

@@ -0,0 +1,11 @@
+//@ check-pass
+//
+// This shows a tricky case for #124141, where `declare!(_x)` was incorrectly
+// being categorised as a `StmtKind::Expr` instead of a `StmtKind::MacCall` in
+// `parse_stmt_mac`.
+
+macro_rules! as_stmt { ($s:stmt) => { $s }; }
+
+macro_rules! declare { ($name:ident) => { let $name = 0u32; }; }
+
+fn main() { as_stmt!(declare!(_x)); }

tests/ui/macros/nonterminal-matching.rs

@@ -16,7 +16,7 @@ macro complex_nonterminal($nt_item: item) {
         struct S;
     }
 
-    n!(a $nt_item b); //~ ERROR no rules expected item `enum E {}`
+    n!(a $nt_item b); //~ ERROR no rules expected `item` metavariable
 }
 
 simple_nonterminal!(a, 'a, (x, y, z)); // OK
@@ -32,7 +32,7 @@ macro_rules! foo {
     (expr $x:expr) => { bar!(expr $x); }; //~ ERROR: no rules expected expression `3`
     (literal $x:literal) => { bar!(literal $x); }; //~ ERROR: no rules expected literal `4`
     (path $x:path) => { bar!(path $x); }; //~ ERROR: no rules expected `path` metavariable
-    (stmt $x:stmt) => { bar!(stmt $x); }; //~ ERROR: no rules expected statement `let abc = 0`
+    (stmt $x:stmt) => { bar!(stmt $x); }; //~ ERROR: no rules expected `stmt` metavariable
 }
 
 macro_rules! bar {

tests/ui/macros/nonterminal-matching.stderr

@@ -1,4 +1,4 @@
-error: no rules expected item `enum E {}`
+error: no rules expected `item` metavariable
   --> $DIR/nonterminal-matching.rs:19:10
    |
 LL |     macro n(a $nt_item b) {
@@ -10,7 +10,7 @@ LL |     n!(a $nt_item b);
 LL | complex_nonterminal!(enum E {});
    | ------------------------------- in this macro invocation
    |
-note: while trying to match item `enum E {}`
+note: while trying to match `item` metavariable
   --> $DIR/nonterminal-matching.rs:15:15
    |
 LL |     macro n(a $nt_item b) {
@@ -89,7 +89,7 @@ LL |     (path a::b::c) => {};
    = help: try using `:tt` instead in the macro definition
    = note: this error originates in the macro `foo` (in Nightly builds, run with -Z macro-backtrace for more info)
 
-error: no rules expected statement `let abc = 0`
+error: no rules expected `stmt` metavariable
   --> $DIR/nonterminal-matching.rs:35:35
    |
 LL |     (stmt $x:stmt) => { bar!(stmt $x); };

tests/ui/proc-macro/expand-to-derive.stdout

@@ -44,52 +44,58 @@ PRINT-DERIVE INPUT (DEBUG): TokenStream [
                     Group {
                         delimiter: Brace,
                         stream: TokenStream [
-                            Punct {
-                                ch: '#',
-                                spacing: Alone,
-                                span: $DIR/expand-to-derive.rs:27:5: 27:6 (#0),
-                            },
                             Group {
-                                delimiter: Bracket,
+                                delimiter: None,
                                 stream: TokenStream [
+                                    Punct {
+                                        ch: '#',
+                                        spacing: Alone,
+                                        span: $DIR/expand-to-derive.rs:27:5: 27:6 (#0),
+                                    },
+                                    Group {
+                                        delimiter: Bracket,
+                                        stream: TokenStream [
+                                            Ident {
+                                                ident: "rustc_dummy",
+                                                span: $DIR/expand-to-derive.rs:27:28: 27:39 (#0),
+                                            },
+                                        ],
+                                        span: $DIR/expand-to-derive.rs:27:6: 27:41 (#0),
+                                    },
                                     Ident {
-                                        ident: "rustc_dummy",
+                                        ident: "struct",
-                                        span: $DIR/expand-to-derive.rs:27:28: 27:39 (#0),
+                                        span: $DIR/expand-to-derive.rs:28:5: 28:11 (#0),
+                                    },
+                                    Ident {
+                                        ident: "Inner",
+                                        span: $DIR/expand-to-derive.rs:28:12: 28:17 (#0),
+                                    },
+                                    Group {
+                                        delimiter: Brace,
+                                        stream: TokenStream [
+                                            Ident {
+                                                ident: "other_inner_field",
+                                                span: $DIR/expand-to-derive.rs:30:9: 30:26 (#0),
+                                            },
+                                            Punct {
+                                                ch: ':',
+                                                spacing: Alone,
+                                                span: $DIR/expand-to-derive.rs:30:26: 30:27 (#0),
+                                            },
+                                            Ident {
+                                                ident: "u8",
+                                                span: $DIR/expand-to-derive.rs:30:28: 30:30 (#0),
+                                            },
+                                            Punct {
+                                                ch: ',',
+                                                spacing: Alone,
+                                                span: $DIR/expand-to-derive.rs:30:30: 30:31 (#0),
+                                            },
+                                        ],
+                                        span: $DIR/expand-to-derive.rs:28:18: 31:6 (#0),
                                     },
                                 ],
-                                span: $DIR/expand-to-derive.rs:27:6: 27:41 (#0),
+                                span: $DIR/expand-to-derive.rs:19:17: 19:22 (#3),
-                            },
-                            Ident {
-                                ident: "struct",
-                                span: $DIR/expand-to-derive.rs:28:5: 28:11 (#0),
-                            },
-                            Ident {
-                                ident: "Inner",
-                                span: $DIR/expand-to-derive.rs:28:12: 28:17 (#0),
-                            },
-                            Group {
-                                delimiter: Brace,
-                                stream: TokenStream [
-                                    Ident {
-                                        ident: "other_inner_field",
-                                        span: $DIR/expand-to-derive.rs:30:9: 30:26 (#0),
-                                    },
-                                    Punct {
-                                        ch: ':',
-                                        spacing: Alone,
-                                        span: $DIR/expand-to-derive.rs:30:26: 30:27 (#0),
-                                    },
-                                    Ident {
-                                        ident: "u8",
-                                        span: $DIR/expand-to-derive.rs:30:28: 30:30 (#0),
-                                    },
-                                    Punct {
-                                        ch: ',',
-                                        spacing: Alone,
-                                        span: $DIR/expand-to-derive.rs:30:30: 30:31 (#0),
-                                    },
-                                ],
-                                span: $DIR/expand-to-derive.rs:28:18: 31:6 (#0),
                             },
                             Literal {
                                 kind: Integer,

tests/ui/proc-macro/nodelim-groups.rs

@@ -19,4 +19,17 @@ macro_rules! expand_it {
 fn main() {
     expand_it!(1 + (25) + 1);
     expand_it!(("hello".len()) ("world".len()));
+    f();
+}
+
+// The key thing here is to produce a single `None`-delimited `Group`, even
+// though there is multiple levels of macros.
+macro_rules! m5 { ($e:expr) => { print_bang_consume!($e) }; }
+macro_rules! m4 { ($e:expr) => { m5!($e); } }
+macro_rules! m3 { ($e:expr) => { m4!($e); } }
+macro_rules! m2 { ($e:expr) => { m3!($e); } }
+macro_rules! m1 { ($e:expr) => { m2!($e); } }
+
+fn f() {
+    m1!(123);
 }

tests/ui/proc-macro/nodelim-groups.stdout

@@ -165,3 +165,18 @@ PRINT-BANG INPUT (DEBUG): TokenStream [
         span: $DIR/nodelim-groups.rs:16:52: 16:59 (#8),
     },
 ]
+PRINT-BANG INPUT (DISPLAY): 123
+PRINT-BANG INPUT (DEBUG): TokenStream [
+    Group {
+        delimiter: None,
+        stream: TokenStream [
+            Literal {
+                kind: Integer,
+                symbol: "123",
+                suffix: None,
+                span: $DIR/nodelim-groups.rs:34:9: 34:12 (#0),
+            },
+        ],
+        span: $DIR/nodelim-groups.rs:27:54: 27:56 (#16),
+    },
+]